mirror of
https://github.com/hanzoai/gpt.git
synced 2026-10-30 18:35:47 +00:00
28 lines
697 B
Python
28 lines
697 B
Python
"""
|
|
Gpt3_5 model implementation
|
|
Parameters: {
|
|
"n_head": 120,
|
|
"n_embd": 14336,
|
|
"params": "350B",
|
|
"n_layer": 120,
|
|
"block_size": 16384,
|
|
"moe": false
|
|
}
|
|
"""
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
from configs.gpt3_5_config import Gpt3_5Config
|
|
|
|
# Standard transformer implementation
|
|
class Gpt3_5Transformer(nn.Module):
|
|
def __init__(self, config):
|
|
super().__init__()
|
|
self.config = config
|
|
# Standard architecture
|
|
print("Initializing {} model with {} parameters".format(
|
|
"Gpt3_5", config.parameter_count))
|
|
|
|
def forward(self, x, **kwargs):
|
|
# This implementation would handle the standard transformer forward pass
|
|
pass |