Update.

[picoclvr.git] / tasks.py

diff --git a/tasks.py b/tasks.py
index 0c92af9..234e780 100755 (executable)
--- a/tasks.py
+++ b/tasks.py
@@ -140,7 +140,6 @@ class ProblemLevel2(Problem):
             num_classes=self.len_source,
         )
         source1 = torch.rand(nb, 10).sort(dim=1).indices[:, : self.len_source]
-        # source1 = torch.randint(10, (nb, self.len_source))
         marker1 = torch.full((nb, 1), 10)
         result1 = operators.bmm(source1[:, :, None]).squeeze(-1)
         marker2 = torch.full((nb, 1), 11)
@@ -1284,44 +1283,35 @@ class RPL(Task):
         )
 
         if save_attention_image is not None:
-            input = self.test_input[:1]
-            result = input.clone()
-            s = (result == self.t_prog).long()
-            ar_mask = (s.cumsum(dim=1) - s).clamp(min=0, max=1)
-            result = (1 - ar_mask) * result + ar_mask * self.t_nul
-
-            masked_inplace_autoregression(
-                model,
-                self.batch_size,
-                result,
-                ar_mask,
-                deterministic_synthesis,
-                device=self.device,
-            )
+            input = self.test_input[:1].clone()
+            last = (input != self.t_nul).max(0).values.nonzero().max() + 3
+            input = input[:, :last]
 
             with torch.autograd.no_grad():
                 t = model.training
                 model.eval()
                 model.record_attention(True)
-                model(BracketedSequence(result))
+                model(BracketedSequence(input))
                 model.train(t)
                 ram = model.retrieve_attention()
                 model.record_attention(False)
 
-            tokens_output = [self.id2token[i.item()] for i in result[0]]
+            tokens_output = [self.id2token[i.item()] for i in input[0]]
             tokens_input = ["n/a"] + tokens_output[:-1]
             for n_head in range(ram[0].size(1)):
-                filename = f"rpl_attention_{n_epoch}_h{n_head}.pdf"
+                filename = os.path.join(
+                    result_dir, f"rpl_attention_{n_epoch}_h{n_head}.pdf"
+                )
                 attention_matrices = [m[0, n_head] for m in ram]
                 save_attention_image(
                     filename,
                     tokens_input,
                     tokens_output,
                     attention_matrices,
-                    token_gap=12,
-                    layer_gap=50,
                     k_top=10,
                     # min_total_attention=0.9,
+                    token_gap=12,
+                    layer_gap=50,
                 )
                 logger(f"wrote {filename}")