Update.

[picoclvr.git] / mygpt.py

diff --git a/mygpt.py b/mygpt.py
index c93010a..0cf70e0 100755 (executable)
--- a/mygpt.py
+++ b/mygpt.py
@@ -45,6 +45,9 @@ class BracketedSequence:
     def slice(self):
         return self.x[:, self.first : self.first + self.nb]
 
+    def complete(self):
+        return self.first == 0 and self.nb == self.x.size(1)
+
 
 ######################################################################
 
@@ -62,9 +65,7 @@ class CacheWrapper(nn.Module):
         else:
             self.cache_y[:, bs.first : bs.first + bs.nb] = self.f(bs.slice())
 
-        bs.x = self.cache_y
-
-        return bs
+        return BracketedSequence(self.cache_y, bs.first, bs.nb)
 
 
 ##############################
@@ -76,8 +77,7 @@ class WithResidual(nn.Module):
         self.f = f[0] if len(f) == 1 else nn.Sequential(*f)
 
     def forward(self, bs):
-        bs.x = bs.x + self.f(bs).x
-        return bs
+        return BracketedSequence(bs.x + self.f(bs).x, bs.first, bs.nb)
 
 
 ##############################
@@ -108,9 +108,7 @@ class AddPositionalEncoding(nn.Module):
             bs.slice() + self.pe[bs.first : bs.first + bs.nb]
         )
 
-        bs.x = self.cache_y
-
-        return bs
+        return BracketedSequence(self.cache_y, bs.first, bs.nb)
 
 
 ##############################
@@ -118,7 +116,13 @@ class AddPositionalEncoding(nn.Module):
 
 class QKVAttention(nn.Module):
     def __init__(
-        self, dim_in, dim_qk, dim_v, nb_heads=1, causal=False, attention_dropout=0.0
+        self,
+        dim_in,
+        dim_qk,
+        dim_v,
+        nb_heads=1,
+        causal=False,
+        attention_dropout=0.0,
     ):
         super().__init__()
 
@@ -127,6 +131,7 @@ class QKVAttention(nn.Module):
 
         self.causal = causal
         self.attention_dropout = attention_dropout
+        self.record_attention = False
 
         self.w_q = randw(nb_heads, dim_qk, dim_in)
         self.w_k = randw(nb_heads, dim_qk, dim_in)
@@ -136,6 +141,10 @@ class QKVAttention(nn.Module):
     def forward(self, bs_q):
         x_q = bs_q.x
 
+        assert (
+            self.causal or bs_q.complete()
+        ), "Partial evaluation is only possible for causal models"
+
         if bs_q.first == 0:
             self.cache_k = x_q.new_zeros(
                 x_q.size(0), self.w_k.size(0), x_q.size(1), self.w_k.size(1)
@@ -148,6 +157,7 @@ class QKVAttention(nn.Module):
         q = torch.einsum(
             "ntc,hdc->nhtd", x_q[:, bs_q.first : bs_q.first + bs_q.nb], self.w_q
         )
+
         self.cache_k[:, :, bs_q.first : bs_q.first + bs_q.nb] = torch.einsum(
             "ntc,hdc->nhtd", x_q[:, bs_q.first : bs_q.first + bs_q.nb], self.w_k
         )
@@ -173,6 +183,10 @@ class QKVAttention(nn.Module):
             )
 
         a = a.softmax(dim=3)
+
+        if self.record_attention:
+            self.a = a
+
         a = F.dropout(a, self.attention_dropout, self.training)
 
         y = torch.einsum(
@@ -181,9 +195,7 @@ class QKVAttention(nn.Module):
 
         self.cache_y[:, bs_q.first : bs_q.first + bs_q.nb] = y @ self.w_o
 
-        bs_q.x = self.cache_y
-
-        return bs_q
+        return BracketedSequence(self.cache_y, bs_q.first, bs_q.nb)
 
 
 ##############################
@@ -252,7 +264,7 @@ class MyGPT(nn.Module):
                     m.weight.fill_(1.0)
 
     def forward(self, bs):
-        bs.x = F.pad(bs.x, (1, -1))
+        bs = BracketedSequence(F.pad(bs.x, (1, -1)), bs.first, bs.nb)
         bs = self.embedding(bs)
         bs = self.trunk(bs)
         bs = self.readout(bs)
@@ -282,33 +294,44 @@ class MyGPT(nn.Module):
                 t_next = dist.sample()
             input[:, s] = ar_mask[:, s] * t_next + (1 - ar_mask[:, s]) * input[:, s]
 
+    def record_attention(self, v=True):
+        for m in self.modules():
+            if isinstance(m, QKVAttention):
+                m.record_attention = v
+
+    def retrieve_attention(self):
+        a = []
+        for m in self.modules():
+            if isinstance(m, QKVAttention):
+                a.append(m.a)
+        return a
+
 
 ######################################################################
 
 if __name__ == "__main__":
     print("Basic check.")
 
-    vocabulary_size = 10
-    x = torch.randint(vocabulary_size, (9, 7))
+    vocabulary_size = 3
+    x = torch.randint(vocabulary_size, (1, 5))
 
     model = MyGPT(
         vocabulary_size=vocabulary_size,
-        dim_model=18,
-        dim_keys=50,
-        dim_hidden=100,
+        dim_model=4,
+        dim_keys=2,
+        dim_hidden=2,
         nb_heads=2,
-        nb_blocks=1,
+        nb_blocks=2,
         dropout=0.1,
+        causal=True,
     )
 
     model.eval()
-
     y1 = model(BracketedSequence(x)).x
-
     y2 = torch.randn_like(y1)
     for s in range(x.size(1)):
         z = model(BracketedSequence(x, s, 1))
-        y2[:, s] = z.x[:, s]
+        y2[:, s] = z.slice()
 
     print(f"error={((y1 - y2).norm() / (y1.norm() + y2.norm())).item()}")