mirror of
https://github.com/hanzoai/gpt.git
synced 2026-10-30 18:35:47 +00:00
28 lines
682 B
Python
28 lines
682 B
Python
"""
|
|
Gpt2 model implementation
|
|
Parameters: {
|
|
"n_head": 12,
|
|
"n_embd": 768,
|
|
"params": "124M",
|
|
"n_layer": 12,
|
|
"block_size": 1024,
|
|
"moe": false
|
|
}
|
|
"""
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
from configs.gpt2_config import Gpt2Config
|
|
|
|
# Standard transformer implementation
|
|
class Gpt2Transformer(nn.Module):
|
|
def __init__(self, config):
|
|
super().__init__()
|
|
self.config = config
|
|
# Standard architecture
|
|
print("Initializing {} model with {} parameters".format(
|
|
"Gpt2", config.parameter_count))
|
|
|
|
def forward(self, x, **kwargs):
|
|
# This implementation would handle the standard transformer forward pass
|
|
pass |