commaai
/
tinygrad
mirror of https://github.com/commaai/tinygrad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

oops, forgot some stars

This commit is contained in:
George Hotz 2021-11-30 00:46:14 -05:00
parent 908db3bdea
commit c39824bc62
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
models/transformer.py
View File

@ -45,9 +45,9 @@ class TransformerBlock:
x = x + x.layernorm().linear(*self.ln2).linear(*self.ff1).gelu().linear(*self.ff2).dropout(0.1)
else:
x = x + self.attn(x).dropout(0.1)
x = x.layernorm().linear(self.ln1)
x = x + x.linear(self.ff1).relu().linear(self.ff2).dropout(0.1)
x = x.layernorm().linear(self.ln2)
x = x.layernorm().linear(*self.ln1)
x = x + x.linear(*self.ff1).relu().linear(*self.ff2).dropout(0.1)
x = x.layernorm().linear(*self.ln2)
return x
class Transformer: