add additional stabilization technique, looking at ruDALL-E results

Author: lucidrains

dalle_pytorch/dalle_pytorch.py

@@ -557,6 +557,10 @@ def forward(
             seq_len -= 1
             tokens = tokens[:, :-1]
 
+        if self.stable:
+            alpha = 0.1
+            tokens = tokens * alpha + tokens.detach() * (1 - alpha)
+
         out = self.transformer(tokens)
 
         if self.stable:

setup.py

@@ -4,7 +4,7 @@
   name = 'dalle-pytorch',
   packages = find_packages(),
   include_package_data = True,
-  version = '1.1.0',
+  version = '1.1.1',
   license='MIT',
   description = 'DALL-E - Pytorch',
   author = 'Phil Wang',