adelorenzo/stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

modify feedforward net, move layer norm to start of thr block

Browse Source
This commit is contained in:
Yinon Polak 2023-03-19 17:03:36 +02:00
parent e08d8190ae
commit ddd1b5c0ff
1 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
freqtrade/freqai/prediction_models/PyTorchMLPModel.py
View File

@ -22,7 +22,7 @@ class PyTorchMLPModel(nn.Module):
def forward(self, x: Tensor) -> Tensor: def forward(self, x: Tensor) -> Tensor:
x = self.relu(self.input_layer(x)) x = self.relu(self.input_layer(x))
x = self.dropout(x) x = self.dropout(x)
x = self.relu(self.blocks(x)) x = self.blocks(x)
logits = self.output_layer(x) logits = self.output_layer(x)
return logits return logits
@ -35,8 +35,8 @@ class Block(nn.Module):
self.ln = nn.LayerNorm(hidden_dim) self.ln = nn.LayerNorm(hidden_dim)
def forward(self, x): def forward(self, x):
x = self.dropout(self.ff(x)) x = self.ff(self.ln(x))
x = self.ln(x) x = self.dropout(x)
return x return x
@ -46,7 +46,6 @@ class FeedForward(nn.Module):
self.net = nn.Sequential( self.net = nn.Sequential(
nn.Linear(hidden_dim, hidden_dim), nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(), nn.ReLU(),
nn.Linear(hidden_dim, hidden_dim),
) )
def forward(self, x): def forward(self, x):