Skip to content

Commit 5f12a25

Browse files
committed
Add bias arg to Vitamin GeGLU
1 parent 5804d92 commit 5f12a25

File tree

1 file changed

+4
-3
lines changed

1 file changed

+4
-3
lines changed

timm/models/vitamin.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -259,16 +259,17 @@ def __init__(
259259
in_features,
260260
hidden_features,
261261
act_layer = 'gelu',
262+
bias = True,
262263
drop = 0.0,
263264
):
264265
super().__init__()
265266
norm_layer = partial(get_norm_layer('layernorm'), eps=1e-6)
266267

267268
self.norm = norm_layer(in_features)
268-
self.w0 = nn.Linear(in_features, hidden_features)
269+
self.w0 = nn.Linear(in_features, hidden_features, bias=bias)
269270
self.act = create_act_layer(act_layer)
270-
self.w1 = nn.Linear(in_features, hidden_features)
271-
self.w2 = nn.Linear(hidden_features, in_features)
271+
self.w1 = nn.Linear(in_features, hidden_features, bias=bias)
272+
self.w2 = nn.Linear(hidden_features, in_features, bias=bias)
272273

273274
def forward(self, x):
274275
x = self.norm(x)

0 commit comments

Comments
 (0)