- descr = [ s.strip().split(' ') for s in descr ]
- l = max([ len(s) for s in descr ])
- #descr = [ [ '<nul>' ] * (l - len(s)) + s for s in descr ]
- descr = [ s + [ '<nul>' ] * (l - len(s)) for s in descr ]
- t = [ [ self.token2id[u] for u in s ] for s in descr ]
- return torch.tensor(t, device = self.device)
+ token_descr = [ s.strip().split(' ') for s in descr ]
+ l = max([ len(s) for s in token_descr ])
+ #token_descr = [ [ '<nul>' ] * (l - len(s)) + s for s in token_descr ]
+ token_descr = [ s + [ '<nul>' ] * (l - len(s)) for s in token_descr ]
+ id_descr = [ [ self.token2id[u] for u in s ] for s in token_descr ]
+ return torch.tensor(id_descr, device = self.device)