Update.

diff --git a/expr.py b/expr.py
index 723022c..ca33daf 100755 (executable)
--- a/expr.py
+++ b/expr.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import math
+import math, re
 
 import torch, torchvision
 
@@ -53,6 +53,15 @@ def generate_program(nb_variables, length):
     return s, variables
 
 
+def extract_results(seq):
+    f = lambda a: (a[0], -1 if a[1] == "" else int(a[1]))
+    results = [
+        dict([f(tuple(x.split(":"))) for x in re.findall("[A-Z]:[0-9]*", s)])
+        for s in seq
+    ]
+    return results
+
+
 def generate_sequences(nb, nb_variables=5, length=20, randomize_length=False):
     sequences = []
     for n in range(nb):
@@ -78,8 +87,10 @@ if __name__ == "__main__":
     import time
 
     start_time = time.perf_counter()
-    sequences = generate_sequences(1000, randomize_length=True)
+    sequences = generate_sequences(1000)
     end_time = time.perf_counter()
     for s in sequences[:10]:
         print(s)
     print(f"{len(sequences) / (end_time - start_time):.02f} samples per second")
+
+    print(extract_results(sequences[:10]))

diff --git a/main.py b/main.py
index 35bf02c..e1f619c 100755 (executable)
--- a/main.py
+++ b/main.py
@@ -1030,8 +1030,9 @@ class TaskExpr(Task):
         train_sequences = expr.generate_sequences(
             nb_train_samples,
             nb_variables=nb_variables,
-            length=2 * sequence_length,
-            randomize_length=True,
+            length=sequence_length,
+            # length=2 * sequence_length,
+            # randomize_length=True,
         )
         test_sequences = expr.generate_sequences(
             nb_test_samples,
@@ -1115,6 +1116,24 @@ class TaskExpr(Task):
                 nb_total = input.size(0)
                 nb_correct = (input == result).long().min(1).values.sum()
 
+                values_input = expr.extract_results([self.seq2str(s) for s in input])
+                max_input = max([max(x.values()) for x in values_input])
+                values_result = expr.extract_results([self.seq2str(s) for s in result])
+                max_result = max(
+                    [-1 if len(x) == 0 else max(x.values()) for x in values_result]
+                )
+
+                nb_missing, nb_predicted = torch.zeros(max_input + 1), torch.zeros(
+                    max_input + 1, max_result + 1
+                )
+                for i, r in zip(values_input, values_result):
+                    for n, vi in i.items():
+                        vr = r.get(n)
+                        if vr is None or vr < 0:
+                            nb_missing[vi] += 1
+                        else:
+                            nb_predicted[vi, vr] += 1
+
                 return nb_total, nb_correct
 
             test_nb_total, test_nb_correct = compute_nb_correct(self.test_input[:1000])