Update.

diff --git a/expr.py b/expr.py
index d3883d5..baee502 100755 (executable)
--- a/expr.py
+++ b/expr.py
@@ -53,12 +53,15 @@ def generate_program(nb_variables, length):
     return s, variables
 
 
-def generate_sequences(nb, nb_variables=5, length=20):
+def generate_sequences(nb, nb_variables=5, length=20, randomize_length=False):
     sequences = []
     for n in range(nb):
         result = None
         while result == None or max(result.values()) > 100:
-            p, v = generate_program(nb_variables, length)
+            l = length
+            if l > 5 and randomize_length:
+                l = 5 + torch.randint(l-5, (1,)).item()
+            p, v = generate_program(nb_variables, l)
             v = ", ".join(['"' + v + '": ' + v for v in v])
             ldict = {}
             exec(p + "result={" + v + "}", globals(), ldict)
@@ -75,7 +78,7 @@ if __name__ == "__main__":
     import time
 
     start_time = time.perf_counter()
-    sequences = generate_sequences(1000)
+    sequences = generate_sequences(1000, randomize_length=True)
     end_time = time.perf_counter()
     for s in sequences[:10]:
         print(s)

diff --git a/main.py b/main.py
index b907e60..c52881b 100755 (executable)
--- a/main.py
+++ b/main.py
@@ -170,10 +170,10 @@ default_args = {
         "nb_test_samples": 1000,
     },
     "expr": {
-        "nb_epochs": 5,
+        "nb_epochs": 50,
         "batch_size": 25,
-        "nb_train_samples": 100000,
-        "nb_test_samples": 1000,
+        "nb_train_samples": 250000,
+        "nb_test_samples": 10000,
     },
 }
 
@@ -1028,10 +1028,10 @@ class TaskExpr(Task):
         self.device = device
 
         train_sequences = expr.generate_sequences(
-            nb_train_samples, nb_variables=nb_variables, length=sequence_length
+            nb_train_samples, nb_variables=nb_variables, length=2*sequence_length, randomize_length=True,
         )
         test_sequences = expr.generate_sequences(
-            nb_test_samples, nb_variables=nb_variables, length=sequence_length
+            nb_test_samples, nb_variables=nb_variables, length=sequence_length,
         )
         self.char2id = dict(
             [
@@ -1042,7 +1042,10 @@ class TaskExpr(Task):
             ]
         )
         self.id2char = dict([(n, c) for c, n in self.char2id.items()])
-        len_max = max([len(x) for x in train_sequences + test_sequences])
+
+        self.filler, self.space = self.char2id["#"], self.char2id[" "]
+
+        len_max = max([len(x) for x in train_sequences])
         self.train_input = torch.cat(
             [
                 torch.tensor(
@@ -1054,6 +1057,8 @@ class TaskExpr(Task):
             ],
             0,
         ).to(device)
+
+        len_max = max([len(x) for x in test_sequences])
         self.test_input = torch.cat(
             [
                 torch.tensor(
@@ -1065,6 +1070,7 @@ class TaskExpr(Task):
             ],
             0,
         ).to(device)
+
         self.nb_codes = max(self.train_input.max(), self.test_input.max()) + 1
 
     def batches(self, split="train", nb_to_use=-1, desc=None):
@@ -1077,11 +1083,17 @@ class TaskExpr(Task):
         for batch in tqdm.tqdm(
             input.split(self.batch_size), dynamic_ncols=True, desc=desc
         ):
+            if split == "train":
+                last=(batch!=self.filler).max(0).values.nonzero().max()+1
+                batch=batch[:,:last]
             yield batch
 
     def vocabulary_size(self):
         return self.nb_codes
 
+    def seq2str(self, s):
+        return "".join([self.id2char[k.item()] for k in s])
+
     def produce_results(self, n_epoch, model):
         with torch.autograd.no_grad():
             t = model.training
@@ -1089,15 +1101,14 @@ class TaskExpr(Task):
 
             def compute_nb_correct(input):
                 result = input.clone()
-                filler, space = self.char2id["#"], self.char2id[" "]
-                ar_mask = (result == space).long().cumsum(dim=1).clamp(max=1)
-                result = (1 - ar_mask) * result + ar_mask * filler
+                ar_mask = (result == self.space).long().cumsum(dim=1).clamp(max=1)
+                result = (1 - ar_mask) * result + ar_mask * self.filler
                 masked_inplace_autoregression(
                     model, self.batch_size, result, ar_mask, device=self.device
                 )
 
-                nb_total = ar_mask.sum()
-                nb_correct = ((input == result).long() * ar_mask).sum()
+                nb_total = input.size(0)
+                nb_correct = (input == result).long().min(1).values.sum()
 
                 return nb_total, nb_correct
 
@@ -1111,21 +1122,18 @@ class TaskExpr(Task):
             # Log a few generated sequences
             input = self.test_input[:10]
             result = input.clone()
-            filler, space = self.char2id["#"], self.char2id[" "]
-            ar_mask = (result == space).long().cumsum(dim=1).clamp(max=1)
-            result = (1 - ar_mask) * result + ar_mask * filler
+            ar_mask = (result == self.space).long().cumsum(dim=1).clamp(max=1)
+            result = (1 - ar_mask) * result + ar_mask * self.filler
             for n in range(result.size(0)):
-                s = "".join([self.id2char[k.item()] for k in result[n]])
-                log_string(f"test_before {s}")
+                log_string(f"test_before {self.seq2str(result[n])}")
             masked_inplace_autoregression(
                 model, self.batch_size, result, ar_mask, device=self.device
             )
-            correct = (1 - ar_mask) * space + ar_mask * input
+            correct = (1 - ar_mask) * self.space + ar_mask * input
             for n in range(result.size(0)):
-                s = "".join([self.id2char[k.item()] for k in result[n]])
-                log_string(f"test_after  {s}")
-                s = "".join([self.id2char[k.item()] for k in correct[n]])
-                log_string(f"correct     {s}")
+                comment="GOOD" if (result[n]-input[n]).abs().max()==0 else ""
+                log_string(f"test_after  {self.seq2str(result[n])} {comment}")
+                log_string(f"correct     {self.seq2str(correct[n])}")
             ##############################################################
 
             model.train(t)