From 0c51561334475af559cda12627388c9d5567a55f Mon Sep 17 00:00:00 2001 From: Francois Fleuret Date: Wed, 27 Jul 2022 16:22:26 +0200 Subject: [PATCH] OCDC --- mygpt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mygpt.py b/mygpt.py index ab16e1e..43711b3 100755 --- a/mygpt.py +++ b/mygpt.py @@ -107,7 +107,8 @@ class MyGPT(nn.Module): nn.LayerNorm(dim_model), QKVAttention( dim_in = dim_model, - dim_qk = dim_keys, dim_v = dim_model // nb_heads, + dim_qk = dim_keys, + dim_v = dim_model // nb_heads, nb_heads = nb_heads, causal = True, attention_dropout = dropout ), -- 2.20.1