File size: 1,709 Bytes
70a6fd7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
import torch
import torch.nn as nn
class TransformerBlock(nn.Module):
def __init__(self, embed_size, heads, ff_hidden_dim, dropout):
super().__init__()
self.attention = nn.MultiheadAttention(embed_dim=embed_size, num_heads=heads, batch_first=True)
self.norm1 = nn.LayerNorm(embed_size)
self.norm2 = nn.LayerNorm(embed_size)
self.ff = nn.Sequential(
nn.Linear(embed_size, ff_hidden_dim),
nn.ReLU(),
nn.Dropout(dropout),
nn.Linear(ff_hidden_dim, embed_size)
)
self.dropout = nn.Dropout(dropout)
def forward(self, x):
attn_output, _ = self.attention(x, x, x)
x = self.norm1(x + self.dropout(attn_output))
ff_output = self.ff(x)
x = self.norm2(x + self.dropout(ff_output))
return x
class TransformerModel(nn.Module):
def __init__(self, vocab_size, embed_size=512, num_heads=8, hidden_dim=2048, num_layers=6, max_len=512, dropout=0.1):
super().__init__()
self.embedding = nn.Embedding(vocab_size, embed_size)
self.pos_embedding = nn.Parameter(torch.zeros(1, max_len, embed_size))
self.transformer_blocks = nn.Sequential(
*[TransformerBlock(embed_size, num_heads, hidden_dim, dropout) for _ in range(num_layers)]
)
self.norm = nn.LayerNorm(embed_size)
self.output = nn.Linear(embed_size, vocab_size)
def forward(self, x):
seq_len = x.size(1)
positions = self.pos_embedding[:, :seq_len, :]
x = self.embedding(x) + positions
x = self.transformer_blocks(x)
x = self.norm(x)
return self.output(x)
|