From c3621f9a75cd4d79410d90a29dc9fdec401eaa2d Mon Sep 17 00:00:00 2001 From: Francois Fleuret Date: Sun, 7 Aug 2022 21:50:15 +0200 Subject: [PATCH] Added the small-weight embedding initialization. --- mygpt.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mygpt.py b/mygpt.py index 7ff1035..3bce361 100755 --- a/mygpt.py +++ b/mygpt.py @@ -97,6 +97,10 @@ class MyGPT(nn.Module): AddPositionalEncoding(len_max), ) + # Small embedding initialization + with torch.no_grad(): + self.embedding[0].weight.normal_(0, 2e-2) + trunk_blocks = [ ] for _ in range(nb_blocks): -- 2.20.1