+ dropout_head * (1 - epsilon - G.detach())
- dropout_tail * G.detach()
)
+
+######################################################################
+
+2024 Jan 18 07:39:29 (from mygpt.py)
+
+class Calibrator:
+ def __init__(self, w=None, b=None):
+ self.w = w
+ self.b = b
+ self.s, self.s_sq, self.n = 0, 0, 0
+ self.mean, self.std = 0, 0
+
+ def update(self, X):
+ X = X.detach()
+ self.s += X.sum(dim=0)
+ self.s_sq += X.pow(2).sum(dim=0)
+ self.n += X.size(0)
+
+ def moments(self):
+ mean = self.s / self.n
+ std = (self.s_sq / self.n - mean * mean).sqrt()
+ return mean, std
+
+ def normalize(self):
+ mean, std = self.moments()
+ if self.b is not None:
+ self.b.sub_(mean)
+ if self.w is not None:
+ self.w.div_(std)
+ result = mean - self.mean, std - self.std
+ self.mean, self.std = mean, std
+ self.s, self.s_sq, self.n = 0, 0, 0
+ return result
+
+
+
+######################################################################
+
+2024 Jan 18 07:39:34 (from mygpt.py)
+
+ # self.calibrator_G = Calibrator()
+ # self.calibrator_rec_V = Calibrator()
+ # self.calibrator_rec_K = Calibrator()
+
+
+######################################################################
+
+2024 Jan 18 07:39:37 (from mygpt.py)
+
+ # self.calibrator_G.update(G.reshape(-1, G.size(-1)))
+
+
+######################################################################
+
+2024 Jan 18 07:39:42 (from mygpt.py)
+
+ # self.calibrator_rec_V.update(
+ # next_V.permute(0, 1, 3, 2).reshape(-1, next_V.size(2))
+ # )
+ # self.calibrator_rec_K.update(
+ # next_K.permute(0, 1, 3, 2).reshape(-1, next_K.size(2))
+ # )
+
+
+######################################################################
+
+2024 Jan 18 07:47:12 (from mygpt.py)
+
+ ######################################################################
+ # Roll the gating indexes
+
+ # warnings.warn("rotating barrel", RuntimeWarning)
+
+ # r_barrel = torch.arange(R, device=G.device)[None, None, :, None]
+ # t_barrel = torch.arange(t1 - t0, device=G.device)[None, None, None, :]
+ # r_barrel = (r_barrel + (t_barrel + t0) // L) % R
+ # G = G.gather(dim=2, index=r_barrel.expand_as(G))
+
+
+######################################################################
+
+2024 Jan 18 07:47:25 (from mygpt.py)
+
+ # warnings.warn("harmonic recurrence", RuntimeWarning)
+ # har = torch.arange(t0, t1, device = G.device).float() + 1
+ # A = har / (har + 1)
+ # G = G / har
+
+
+######################################################################
+
+2024 Jan 18 08:46:18 (from mygpt.py)
+
+ # warnings.warn("softmax gating", RuntimeWarning)
+
+ # G = (
+ # torch.einsum("ntc,hrc->nhrt", X, self.w_G) + self.b_G[None, :, :, None]
+ # ).softmax(dim=2)
+
+######################################################################
+
+2024 Jan 21 16:55:24 (from main.py)
+
+ with open("test.dat", "a") as f:
+ for m filter(lambda m: isinstance(m,mygpt.Catenn.Linear),model.modules()):
+ for p in m.parameters() ]
+
+
+ for m in model.modules():
+ if isinstance(m, mygpt.Caterpillar):
+
+
+
+######################################################################
+
+2024 Feb 13 22:53:52 (from mygpt.py)
+
+ ######################################################################
+ # Prepare the keys
+
+ k_star = self.k_star[:, None, :].expand(-1, t1 - t0, -1)
+
+ warnings.warn("rotating key barrel", RuntimeWarning)
+ k_star = self.k_star[:, None, :].expand(-1, x_q.size(1), -1)
+ t_barrel = torch.arange(t0, t1, device=k_star.device)
+ t_barrel = t_barrel[None, :].expand(k_star.size(0), t1 - t0)
+ l_barrel = (
+ torch.arange(k_star.size(0), device=k_star.device)[:, None] + t_barrel
+ ) % k_star.size(0)
+ k_star = k_star[l_barrel, t_barrel]
+
+
+######################################################################
+
+2024 Feb 15 23:10:50 (from main.py)
+
+
+def add_memex_v4(batches, memex_proba, marker_token):
+ for input in batches:
+ if torch.rand(1).item() < memex_proba:
+ t = (
+ torch.arange(2 * input.size(1), device=input.device)[None, :]
+ .expand(input.size(0), -1)
+ .clone()
+ )
+
+ u = torch.rand(t.size(), device=t.device)
+ u[:, : input.size(1)] = 1.0
+ memex_v3_proba_fragment = 1 / 20
+ u = (u < memex_v3_proba_fragment).long()
+ v = u * torch.randint(input.size(1), u.size())
+ u[:, input.size(1) + 1 :] = v[:, input.size(1) + 1 :] - u[
+ :, : input.size(1) - 1
+ ] * input.size(1)
+ u = u.cumsum().clamp(min=0)
+
+ u0 = torch.randint(input.size(1), (input.size(0), 1), device=input.device)
+ caterpillar_length = args.nb_lines // args.caterpillar_height
+ u1 = (
+ u0
+ + torch.randint(
+ caterpillar_length, (input.size(0), 1), device=input.device
+ )
+ + 1
+ )
+
+ m0 = (t < u0).long()
+ m1 = (t >= u1).long() * (t < u1 + input.size(1)).long()
+
+ t = t * m0 + ((-1) * (1 - m0) * (1 - m1)) + (t - u1) * m1
+ m = (t < 0).long()
+ n = torch.arange(input.size(0), device=input.device)[:, None].expand(
+ -1, t.size(1)
+ )
+
+ new_input = input[n, t.clamp(min=0)]
+ new_input = (1 - m) * new_input + m * (marker_token)
+
+ yield new_input
+
+ yield input
+
+
+
+######################################################################
+
+2024 Feb 16 17:07:48 (from main.py)
+
+ # ||gn + lambda * gm|| = max(||gn||,||gm||)
+ # ||gn||^2 + lambda<gn,gm> + lambda^2||gm||^2 = max(||gn||^2,||gm||^2)
+ # A = ||gm||^2 B = <gn,gm> C = ||gn||^2 - max(||gn||^2, ||gm||^2)
+
+######################################################################
+
+2024 Feb 16 17:07:51 (from main.py)
+
+ # A,B,C = gmgm, gngm, gngn - max(gngn,gmgm)
+ # Delta = B*B - 4*A*C
+ # if(delta >= 0):
+ # l = ( -B - sqrt(Delta))/(2*A)
+ # ||gn||+l*rho*||gm|| = max(||gn||,rho*||gm||)
projects / mygptrnn.git / blobdiff