change BertLayerNorm to torch.nn.LayerNorm

georgian-io · Oct 22, 2020 · f6aad22 · f6aad22
1 parent e0a2bb1
commit f6aad22
Show file tree

Hide file tree

Showing 3 changed files with 3,389 additions and 3,429 deletions.
diff --git a/multimodal_transformers/model/tabular_combiner.py b/multimodal_transformers/model/tabular_combiner.py
@@ -1,7 +1,6 @@
 import torch
 from torch import nn
 import torch.nn.functional as F
-from transformers.modeling_bert import BertLayerNorm
 
 from .layer_utils import calc_mlp_dims, create_act, glorot, zeros, MLP
 
@@ -229,7 +228,7 @@ def __init__(self, tabular_config):
                 self.weight_num = nn.Parameter(torch.rand(output_dim_num))
 
             self.act_func = create_act(self.mlp_act)
-            self.layer_norm = BertLayerNorm(self.text_out_dim)
+            self.layer_norm = nn.LayerNorm(self.text_out_dim)
             self.final_dropout = nn.Dropout(tabular_config.hidden_dropout_prob)
             self.final_out_dim = self.text_out_dim
 
@@ -330,7 +329,7 @@ def __init__(self, tabular_config):
                 self.h_num_layer = nn.Linear(min(self.text_out_dim, self.numerical_feat_dim),
                                              self.text_out_dim, bias=False)
             self.h_bias = nn.Parameter(torch.zeros(self.text_out_dim))
-            self.layer_norm = BertLayerNorm(self.text_out_dim)
+            self.layer_norm = nn.LayerNorm(self.text_out_dim)
             self.final_out_dim = self.text_out_dim
         else:
             raise ValueError(f'combine_feat_method {self.combine_feat_method} '