From e6f60ce506f75da652d069a4926adfa4d92e3cbf Mon Sep 17 00:00:00 2001 From: Francois Fleuret Date: Tue, 26 Jul 2022 23:05:46 +0200 Subject: [PATCH] Update. --- main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.py b/main.py index cd0e1ea..82caebe 100755 --- a/main.py +++ b/main.py @@ -170,6 +170,7 @@ class TaskPicoCLVR(Task): descr = [ s.strip().split(' ') for s in descr ] l = max([ len(s) for s in descr ]) + #descr = [ [ '' ] * (l - len(s)) + s for s in descr ] descr = [ s + [ '' ] * (l - len(s)) for s in descr ] return descr @@ -191,6 +192,7 @@ class TaskPicoCLVR(Task): self.token2id = dict([ (t, n) for n, t in enumerate(tokens) ]) self.id2token = dict([ (n, t) for n, t in enumerate(tokens) ]) + # Tokenize the train and test sets t = [ [ self.token2id[u] for u in s ] for s in self.train_descr ] self.train_input = torch.tensor(t, device = self.device) t = [ [ self.token2id[u] for u in s ] for s in self.test_descr ] -- 2.20.1