latest transformer research shows bias is not needed and could be harmful

Author: lucidrains

se3_transformer_pytorch/se3_transformer_pytorch.py

@@ -124,7 +124,6 @@ def __init__(
         for degree, chan in fiber:
             self.transform[str(degree)] = nn.ParameterDict({
                 'scale': nn.Parameter(torch.ones(1, 1, chan)) if not gated_scale else None,
-                'bias': nn.Parameter(rand_uniform((1, 1, chan), -1e-3, 1e-3)),
                 'w_gate': nn.Parameter(rand_uniform((chan, chan), -1e-3, 1e-3)) if gated_scale else None
             })
 
@@ -137,14 +136,14 @@ def forward(self, features):
 
             # Transform on norms
             parameters = self.transform[degree]
-            gate_weights, bias, scale = parameters['w_gate'], parameters['bias'], parameters['scale']
+            gate_weights, scale = parameters['w_gate'], parameters['scale']
 
             transformed = rearrange(norm, '... () -> ...')
 
             if not exists(scale):
                 scale = einsum('b n d, d e -> b n e', transformed, gate_weights)
 
-            transformed = self.nonlin(transformed * scale + bias)
+            transformed = self.nonlin(transformed * scale)
             transformed = rearrange(transformed, '... -> ... ()')
 
             # Nonlinearity on norm

setup.py

@@ -4,7 +4,7 @@
   name = 'se3-transformer-pytorch',
   packages = find_packages(),
   include_package_data = True,
-  version = '0.8.13',
+  version = '0.9.0',
   license='MIT',
   description = 'SE3 Transformer - Pytorch',
   author = 'Phil Wang',