fix value residual

lucidrains · Oct 31, 2024 · 141239c · 141239c
1 parent 0b5c9b4
commit 141239c
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/setup.py b/setup.py
@@ -6,7 +6,7 @@
 setup(
   name = 'vit-pytorch',
   packages = find_packages(exclude=['examples']),
-  version = '1.8.6',
+  version = '1.8.7',
   license='MIT',
   description = 'Vision Transformer (ViT) - Pytorch',
   long_description=long_description,

diff --git a/vit_pytorch/simple_vit_with_value_residual.py b/vit_pytorch/simple_vit_with_value_residual.py
@@ -57,7 +57,7 @@ def forward(self, x, value_residual = None):
         q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.heads), qkv)
 
         if exists(value_residual):
-            v = v + value_residual
+            v = 0.5 * (v + value_residual)
 
         dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale