untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using cached model\n"
     ]
    }
   ],
   "source": [
    "#KoBERT\n",
    "from gluonnlp.data import SentencepieceTokenizer\n",
    "from kobert.utils import get_tokenizer\n",
    "tok_path = get_tokenizer()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using cached model\n",
      "using cached model\n",
      "using cached model\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from kobert.pytorch_kobert import get_pytorch_kobert_model\n",
    "bertmodel, vocab  = get_pytorch_kobert_model()\n",
    "tok_path = get_tokenizer()\n",
    "sp  = SentencepieceTokenizer(tok_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for tensorboard\n",
    "from tensorboardX import SummaryWriter\n",
    "summary = SummaryWriter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on BertModel in module transformers.modeling_bert object:\n",
      "\n",
      "class BertModel(BertPreTrainedModel)\n",
      " |  BertModel(config)\n",
      " |  \n",
      " |  The bare Bert Model transformer outputting raw hidden-states without any specific head on top.    The BERT model was proposed in\n",
      " |  `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_\n",
      " |  by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer\n",
      " |  pre-trained using a combination of masked language modeling objective and next sentence prediction\n",
      " |  on a large corpus comprising the Toronto Book Corpus and Wikipedia.\n",
      " |  \n",
      " |  This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and\n",
      " |  refer to the PyTorch documentation for all matter related to general usage and behavior.\n",
      " |  \n",
      " |  .. _`BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`:\n",
      " |      https://arxiv.org/abs/1810.04805\n",
      " |  \n",
      " |  .. _`torch.nn.Module`:\n",
      " |      https://pytorch.org/docs/stable/nn.html#module\n",
      " |  \n",
      " |  Parameters:\n",
      " |      config (:class:`~transformers.BertConfig`): Model configuration class with all the parameters of the model.\n",
      " |          Initializing with a config file does not load the weights associated with the model, only the configuration.\n",
      " |          Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights.\n",
      " |  \n",
      " |  Inputs:\n",
      " |      **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:\n",
      " |          Indices of input sequence tokens in the vocabulary.\n",
      " |          To match pre-training, BERT input sequence should be formatted with [CLS] and [SEP] tokens as follows:\n",
      " |  \n",
      " |          (a) For sequence pairs:\n",
      " |  \n",
      " |              ``tokens:         [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]``\n",
      " |  \n",
      " |              ``token_type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1``\n",
      " |  \n",
      " |          (b) For single sequences:\n",
      " |  \n",
      " |              ``tokens:         [CLS] the dog is hairy . [SEP]``\n",
      " |  \n",
      " |              ``token_type_ids:   0   0   0   0  0     0   0``\n",
      " |  \n",
      " |          Bert is a model with absolute position embeddings so it's usually advised to pad the inputs on\n",
      " |          the right rather than the left.\n",
      " |  \n",
      " |          Indices can be obtained using :class:`transformers.BertTokenizer`.\n",
      " |          See :func:`transformers.PreTrainedTokenizer.encode` and\n",
      " |          :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details.\n",
      " |      **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``:\n",
      " |          Mask to avoid performing attention on padding token indices.\n",
      " |          Mask values selected in ``[0, 1]``:\n",
      " |          ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.\n",
      " |      **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:\n",
      " |          Segment token indices to indicate first and second portions of the inputs.\n",
      " |          Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1``\n",
      " |          corresponds to a `sentence B` token\n",
      " |          (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details).\n",
      " |      **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``:\n",
      " |          Indices of positions of each input sequence tokens in the position embeddings.\n",
      " |          Selected in the range ``[0, config.max_position_embeddings - 1]``.\n",
      " |      **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``:\n",
      " |          Mask to nullify selected heads of the self-attention modules.\n",
      " |          Mask values selected in ``[0, 1]``:\n",
      " |          ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**.\n",
      " |      **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``:\n",
      " |          Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation.\n",
      " |          This is useful if you want more control over how to convert `input_ids` indices into associated vectors\n",
      " |          than the model's internal embedding lookup matrix.\n",
      " |      **encoder_hidden_states**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)``:\n",
      " |          Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model\n",
      " |          is configured as a decoder.\n",
      " |      **encoder_attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``:\n",
      " |          Mask to avoid performing attention on the padding token indices of the encoder input. This mask\n",
      " |          is used in the cross-attention if the model is configured as a decoder.\n",
      " |          Mask values selected in ``[0, 1]``:\n",
      " |          ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.\n",
      " |  \n",
      " |  Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:\n",
      " |      **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)``\n",
      " |          Sequence of hidden-states at the output of the last layer of the model.\n",
      " |      **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)``\n",
      " |          Last layer hidden-state of the first token of the sequence (classification token)\n",
      " |          further processed by a Linear layer and a Tanh activation function. The Linear\n",
      " |          layer weights are trained from the next sentence prediction (classification)\n",
      " |          objective during Bert pretraining. This output is usually *not* a good summary\n",
      " |          of the semantic content of the input, you're often better with averaging or pooling\n",
      " |          the sequence of hidden-states for the whole input sequence.\n",
      " |      **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)\n",
      " |          list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)\n",
      " |          of shape ``(batch_size, sequence_length, hidden_size)``:\n",
      " |          Hidden-states of the model at the output of each layer plus the initial embedding outputs.\n",
      " |      **attentions**: (`optional`, returned when ``config.output_attentions=True``)\n",
      " |          list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``:\n",
      " |          Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads.\n",
      " |  \n",
      " |  Examples::\n",
      " |  \n",
      " |      tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
      " |      model = BertModel.from_pretrained('bert-base-uncased')\n",
      " |      input_ids = torch.tensor(tokenizer.encode(\"Hello, my dog is cute\", add_special_tokens=True)).unsqueeze(0)  # Batch size 1\n",
      " |      outputs = model(input_ids)\n",
      " |      last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple\n",
      " |  \n",
      " |  Method resolution order:\n",
      " |      BertModel\n",
      " |      BertPreTrainedModel\n",
      " |      transformers.modeling_utils.PreTrainedModel\n",
      " |      torch.nn.modules.module.Module\n",
      " |      builtins.object\n",
      " |  \n",
      " |  Methods defined here:\n",
      " |  \n",
      " |  __init__(self, config)\n",
      " |      Initializes internal Module state, shared by both nn.Module and ScriptModule.\n",
      " |  \n",
      " |  forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, encoder_hidden_states=None, encoder_attention_mask=None)\n",
      " |      Forward pass on the Model.\n",
      " |      \n",
      " |      The model can behave as an encoder (with only self-attention) as well\n",
      " |      as a decoder, in which case a layer of cross-attention is added between\n",
      " |      the self-attention layers, following the architecture described in `Attention is all you need`_ by Ashish Vaswani,\n",
      " |      Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.\n",
      " |      \n",
      " |      To behave as an decoder the model needs to be initialized with the\n",
      " |      `is_decoder` argument of the configuration set to `True`; an\n",
      " |      `encoder_hidden_states` is expected as an input to the forward pass.\n",
      " |      \n",
      " |      .. _`Attention is all you need`:\n",
      " |          https://arxiv.org/abs/1706.03762\n",
      " |  \n",
      " |  get_input_embeddings(self)\n",
      " |      Get model's input embeddings\n",
      " |  \n",
      " |  set_input_embeddings(self, value)\n",
      " |      Set model's input embeddings\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from BertPreTrainedModel:\n",
      " |  \n",
      " |  load_tf_weights = load_tf_weights_in_bert(model, config, tf_checkpoint_path)\n",
      " |      Load tf checkpoints in a pytorch model.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data and other attributes inherited from BertPreTrainedModel:\n",
      " |  \n",
      " |  base_model_prefix = 'bert'\n",
      " |  \n",
      " |  config_class = <class 'transformers.configuration_bert.BertConfig'>\n",
      " |      :class:`~transformers.BertConfig` is the configuration class to store the configuration of a\n",
      " |      `BertModel`.\n",
      " |      \n",
      " |      \n",
      " |      Arguments:\n",
      " |          vocab_size: Vocabulary size of `inputs_ids` in `BertModel`.\n",
      " |          hidden_size: Size of the encoder layers and the pooler layer.\n",
      " |          num_hidden_layers: Number of hidden layers in the Transformer encoder.\n",
      " |          num_attention_heads: Number of attention heads for each attention layer in\n",
      " |              the Transformer encoder.\n",
      " |          intermediate_size: The size of the \"intermediate\" (i.e., feed-forward)\n",
      " |              layer in the Transformer encoder.\n",
      " |          hidden_act: The non-linear activation function (function or string) in the\n",
      " |              encoder and pooler. If string, \"gelu\", \"relu\", \"swish\" and \"gelu_new\" are supported.\n",
      " |          hidden_dropout_prob: The dropout probabilitiy for all fully connected\n",
      " |              layers in the embeddings, encoder, and pooler.\n",
      " |          attention_probs_dropout_prob: The dropout ratio for the attention\n",
      " |              probabilities.\n",
      " |          max_position_embeddings: The maximum sequence length that this model might\n",
      " |              ever be used with. Typically set this to something large just in case\n",
      " |              (e.g., 512 or 1024 or 2048).\n",
      " |          type_vocab_size: The vocabulary size of the `token_type_ids` passed into\n",
      " |              `BertModel`.\n",
      " |          initializer_range: The sttdev of the truncated_normal_initializer for\n",
      " |              initializing all weight matrices.\n",
      " |          layer_norm_eps: The epsilon used by LayerNorm.\n",
      " |  \n",
      " |  pretrained_model_archive_map = {'bert-base-cased': 'https://s3.amazona...\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from transformers.modeling_utils.PreTrainedModel:\n",
      " |  \n",
      " |  get_output_embeddings(self)\n",
      " |      Get model's output embeddings\n",
      " |      Return None if the model doesn't have output embeddings\n",
      " |  \n",
      " |  init_weights(self)\n",
      " |      Initialize and prunes weights if needed.\n",
      " |  \n",
      " |  prune_heads(self, heads_to_prune)\n",
      " |      Prunes heads of the base model.\n",
      " |      \n",
      " |      Arguments:\n",
      " |      \n",
      " |          heads_to_prune: dict with keys being selected layer indices (`int`) and associated values being the list of heads to prune in said layer (list of `int`).\n",
      " |          E.g. {1: [0, 2], 2: [2, 3]} will prune heads 0 and 2 on layer 1 and heads 2 and 3 on layer 2.\n",
      " |  \n",
      " |  resize_token_embeddings(self, new_num_tokens=None)\n",
      " |      Resize input token embeddings matrix of the model if new_num_tokens != config.vocab_size.\n",
      " |      Take care of tying weights embeddings afterwards if the model class has a `tie_weights()` method.\n",
      " |      \n",
      " |      Arguments:\n",
      " |      \n",
      " |          new_num_tokens: (`optional`) int:\n",
      " |              New number of tokens in the embedding matrix. Increasing the size will add newly initialized vectors at the end. Reducing the size will remove vectors from the end.\n",
      " |              If not provided or None: does nothing and just returns a pointer to the input tokens ``torch.nn.Embeddings`` Module of the model.\n",
      " |      \n",
      " |      Return: ``torch.nn.Embeddings``\n",
      " |          Pointer to the input tokens Embeddings Module of the model\n",
      " |  \n",
      " |  save_pretrained(self, save_directory)\n",
      " |      Save a model and its configuration file to a directory, so that it\n",
      " |      can be re-loaded using the `:func:`~transformers.PreTrainedModel.from_pretrained`` class method.\n",
      " |  \n",
      " |  tie_weights(self)\n",
      " |      Make sure we are sharing the input and output embeddings.\n",
      " |      Export to TorchScript can't handle parameter sharing so we are cloning them instead.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Class methods inherited from transformers.modeling_utils.PreTrainedModel:\n",
      " |  \n",
      " |  from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) from builtins.type\n",
      " |      Instantiate a pretrained pytorch model from a pre-trained model configuration.\n",
      " |      \n",
      " |      The model is set in evaluation mode by default using ``model.eval()`` (Dropout modules are deactivated)\n",
      " |      To train the model, you should first set it back in training mode with ``model.train()``\n",
      " |      \n",
      " |      The warning ``Weights from XXX not initialized from pretrained model`` means that the weights of XXX do not come pre-trained with the rest of the model.\n",
      " |      It is up to you to train those weights with a downstream fine-tuning task.\n",
      " |      \n",
      " |      The warning ``Weights from XXX not used in YYY`` means that the layer XXX is not used by YYY, therefore those weights are discarded.\n",
      " |      \n",
      " |      Parameters:\n",
      " |          pretrained_model_name_or_path: either:\n",
      " |      \n",
      " |              - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``.\n",
      " |              - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``.\n",
      " |              - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``.\n",
      " |              - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.\n",
      " |              - None if you are both providing the configuration and state dictionary (resp. with keyword arguments ``config`` and ``state_dict``)\n",
      " |      \n",
      " |          model_args: (`optional`) Sequence of positional arguments:\n",
      " |              All remaning positional arguments will be passed to the underlying model's ``__init__`` method\n",
      " |      \n",
      " |          config: (`optional`) one of:\n",
      " |                  - an instance of a class derived from :class:`~transformers.PretrainedConfig`, or\n",
      " |                  - a string valid as input to :func:`~transformers.PretrainedConfig.from_pretrained()`\n",
      " |              Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when:\n",
      " |      \n",
      " |              - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or\n",
      " |              - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory.\n",
      " |              - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory.\n",
      " |      \n",
      " |          state_dict: (`optional`) dict:\n",
      " |              an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file.\n",
      " |              This option can be used if you want to create a model from a pretrained configuration but load your own weights.\n",
      " |              In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option.\n",
      " |      \n",
      " |          cache_dir: (`optional`) string:\n",
      " |              Path to a directory in which a downloaded pre-trained model\n",
      " |              configuration should be cached if the standard cache should not be used.\n",
      " |      \n",
      " |          force_download: (`optional`) boolean, default False:\n",
      " |              Force to (re-)download the model weights and configuration files and override the cached versions if they exists.\n",
      " |      \n",
      " |          resume_download: (`optional`) boolean, default False:\n",
      " |              Do not delete incompletely recieved file. Attempt to resume the download if such a file exists.\n",
      " |      \n",
      " |          proxies: (`optional`) dict, default None:\n",
      " |              A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.\n",
      " |              The proxies are used on each request.\n",
      " |      \n",
      " |          output_loading_info: (`optional`) boolean:\n",
      " |              Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages.\n",
      " |      \n",
      " |          kwargs: (`optional`) Remaining dictionary of keyword arguments:\n",
      " |              Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded:\n",
      " |      \n",
      " |              - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done)\n",
      " |              - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function.\n",
      " |      \n",
      " |      Examples::\n",
      " |      \n",
      " |          model = BertModel.from_pretrained('bert-base-uncased')    # Download model and configuration from S3 and cache.\n",
      " |          model = BertModel.from_pretrained('./test/saved_model/')  # E.g. model was saved using `save_pretrained('./test/saved_model/')`\n",
      " |          model = BertModel.from_pretrained('bert-base-uncased', output_attention=True)  # Update configuration during loading\n",
      " |          assert model.config.output_attention == True\n",
      " |          # Loading from a TF checkpoint file instead of a PyTorch model (slower)\n",
      " |          config = BertConfig.from_json_file('./tf_model/my_tf_model_config.json')\n",
      " |          model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_tf=True, config=config)\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors inherited from transformers.modeling_utils.PreTrainedModel:\n",
      " |  \n",
      " |  base_model\n",
      " |  \n",
      " |  dummy_inputs\n",
      " |      Dummy inputs to do a forward pass in the network.\n",
      " |      \n",
      " |      Returns:\n",
      " |          torch.Tensor with dummy inputs\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from torch.nn.modules.module.Module:\n",
      " |  \n",
      " |  __call__(self, *input, **kwargs)\n",
      " |      Call self as a function.\n",
      " |  \n",
      " |  __delattr__(self, name)\n",
      " |      Implement delattr(self, name).\n",
      " |  \n",
      " |  __dir__(self)\n",
      " |      Default dir() implementation.\n",
      " |  \n",
      " |  __getattr__(self, name)\n",
      " |  \n",
      " |  __repr__(self)\n",
      " |      Return repr(self).\n",
      " |  \n",
      " |  __setattr__(self, name, value)\n",
      " |      Implement setattr(self, name, value).\n",
      " |  \n",
      " |  __setstate__(self, state)\n",
      " |  \n",
      " |  add_module(self, name, module)\n",
      " |      Adds a child module to the current module.\n",
      " |      \n",
      " |      The module can be accessed as an attribute using the given name.\n",
      " |      \n",
      " |      Args:\n",
      " |          name (string): name of the child module. The child module can be\n",
      " |              accessed from this module using the given name\n",
      " |          module (Module): child module to be added to the module.\n",
      " |  \n",
      " |  apply(self, fn)\n",
      " |      Applies ``fn`` recursively to every submodule (as returned by ``.children()``)\n",
      " |      as well as self. Typical use includes initializing the parameters of a model\n",
      " |      (see also :ref:`torch-nn-init`).\n",
      " |      \n",
      " |      Args:\n",
      " |          fn (:class:`Module` -> None): function to be applied to each submodule\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> def init_weights(m):\n",
      " |          >>>     print(m)\n",
      " |          >>>     if type(m) == nn.Linear:\n",
      " |          >>>         m.weight.data.fill_(1.0)\n",
      " |          >>>         print(m.weight)\n",
      " |          >>> net = nn.Sequential(nn.Linear(2, 2), nn.Linear(2, 2))\n",
      " |          >>> net.apply(init_weights)\n",
      " |          Linear(in_features=2, out_features=2, bias=True)\n",
      " |          Parameter containing:\n",
      " |          tensor([[ 1.,  1.],\n",
      " |                  [ 1.,  1.]])\n",
      " |          Linear(in_features=2, out_features=2, bias=True)\n",
      " |          Parameter containing:\n",
      " |          tensor([[ 1.,  1.],\n",
      " |                  [ 1.,  1.]])\n",
      " |          Sequential(\n",
      " |            (0): Linear(in_features=2, out_features=2, bias=True)\n",
      " |            (1): Linear(in_features=2, out_features=2, bias=True)\n",
      " |          )\n",
      " |          Sequential(\n",
      " |            (0): Linear(in_features=2, out_features=2, bias=True)\n",
      " |            (1): Linear(in_features=2, out_features=2, bias=True)\n",
      " |          )\n",
      " |  \n",
      " |  buffers(self, recurse=True)\n",
      " |      Returns an iterator over module buffers.\n",
      " |      \n",
      " |      Args:\n",
      " |          recurse (bool): if True, then yields buffers of this module\n",
      " |              and all submodules. Otherwise, yields only buffers that\n",
      " |              are direct members of this module.\n",
      " |      \n",
      " |      Yields:\n",
      " |          torch.Tensor: module buffer\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> for buf in model.buffers():\n",
      " |          >>>     print(type(buf.data), buf.size())\n",
      " |          <class 'torch.FloatTensor'> (20L,)\n",
      " |          <class 'torch.FloatTensor'> (20L, 1L, 5L, 5L)\n",
      " |  \n",
      " |  children(self)\n",
      " |      Returns an iterator over immediate children modules.\n",
      " |      \n",
      " |      Yields:\n",
      " |          Module: a child module\n",
      " |  \n",
      " |  cpu(self)\n",
      " |      Moves all model parameters and buffers to the CPU.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  cuda(self, device=None)\n",
      " |      Moves all model parameters and buffers to the GPU.\n",
      " |      \n",
      " |      This also makes associated parameters and buffers different objects. So\n",
      " |      it should be called before constructing optimizer if the module will\n",
      " |      live on GPU while being optimized.\n",
      " |      \n",
      " |      Arguments:\n",
      " |          device (int, optional): if specified, all parameters will be\n",
      " |              copied to that device\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  double(self)\n",
      " |      Casts all floating point parameters and buffers to ``double`` datatype.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  eval(self)\n",
      " |      Sets the module in evaluation mode.\n",
      " |      \n",
      " |      This has any effect only on certain modules. See documentations of\n",
      " |      particular modules for details of their behaviors in training/evaluation\n",
      " |      mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`,\n",
      " |      etc.\n",
      " |      \n",
      " |      This is equivalent with :meth:`self.train(False) <torch.nn.Module.train>`.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  extra_repr(self)\n",
      " |      Set the extra representation of the module\n",
      " |      \n",
      " |      To print customized extra information, you should reimplement\n",
      " |      this method in your own modules. Both single-line and multi-line\n",
      " |      strings are acceptable.\n",
      " |  \n",
      " |  float(self)\n",
      " |      Casts all floating point parameters and buffers to float datatype.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  half(self)\n",
      " |      Casts all floating point parameters and buffers to ``half`` datatype.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  load_state_dict(self, state_dict, strict=True)\n",
      " |      Copies parameters and buffers from :attr:`state_dict` into\n",
      " |      this module and its descendants. If :attr:`strict` is ``True``, then\n",
      " |      the keys of :attr:`state_dict` must exactly match the keys returned\n",
      " |      by this module's :meth:`~torch.nn.Module.state_dict` function.\n",
      " |      \n",
      " |      Arguments:\n",
      " |          state_dict (dict): a dict containing parameters and\n",
      " |              persistent buffers.\n",
      " |          strict (bool, optional): whether to strictly enforce that the keys\n",
      " |              in :attr:`state_dict` match the keys returned by this module's\n",
      " |              :meth:`~torch.nn.Module.state_dict` function. Default: ``True``\n",
      " |      \n",
      " |      Returns:\n",
      " |          ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields:\n",
      " |              * **missing_keys** is a list of str containing the missing keys\n",
      " |              * **unexpected_keys** is a list of str containing the unexpected keys\n",
      " |  \n",
      " |  modules(self)\n",
      " |      Returns an iterator over all modules in the network.\n",
      " |      \n",
      " |      Yields:\n",
      " |          Module: a module in the network\n",
      " |      \n",
      " |      Note:\n",
      " |          Duplicate modules are returned only once. In the following\n",
      " |          example, ``l`` will be returned only once.\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> l = nn.Linear(2, 2)\n",
      " |          >>> net = nn.Sequential(l, l)\n",
      " |          >>> for idx, m in enumerate(net.modules()):\n",
      " |                  print(idx, '->', m)\n",
      " |      \n",
      " |          0 -> Sequential(\n",
      " |            (0): Linear(in_features=2, out_features=2, bias=True)\n",
      " |            (1): Linear(in_features=2, out_features=2, bias=True)\n",
      " |          )\n",
      " |          1 -> Linear(in_features=2, out_features=2, bias=True)\n",
      " |  \n",
      " |  named_buffers(self, prefix='', recurse=True)\n",
      " |      Returns an iterator over module buffers, yielding both the\n",
      " |      name of the buffer as well as the buffer itself.\n",
      " |      \n",
      " |      Args:\n",
      " |          prefix (str): prefix to prepend to all buffer names.\n",
      " |          recurse (bool): if True, then yields buffers of this module\n",
      " |              and all submodules. Otherwise, yields only buffers that\n",
      " |              are direct members of this module.\n",
      " |      \n",
      " |      Yields:\n",
      " |          (string, torch.Tensor): Tuple containing the name and buffer\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> for name, buf in self.named_buffers():\n",
      " |          >>>    if name in ['running_var']:\n",
      " |          >>>        print(buf.size())\n",
      " |  \n",
      " |  named_children(self)\n",
      " |      Returns an iterator over immediate children modules, yielding both\n",
      " |      the name of the module as well as the module itself.\n",
      " |      \n",
      " |      Yields:\n",
      " |          (string, Module): Tuple containing a name and child module\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> for name, module in model.named_children():\n",
      " |          >>>     if name in ['conv4', 'conv5']:\n",
      " |          >>>         print(module)\n",
      " |  \n",
      " |  named_modules(self, memo=None, prefix='')\n",
      " |      Returns an iterator over all modules in the network, yielding\n",
      " |      both the name of the module as well as the module itself.\n",
      " |      \n",
      " |      Yields:\n",
      " |          (string, Module): Tuple of name and module\n",
      " |      \n",
      " |      Note:\n",
      " |          Duplicate modules are returned only once. In the following\n",
      " |          example, ``l`` will be returned only once.\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> l = nn.Linear(2, 2)\n",
      " |          >>> net = nn.Sequential(l, l)\n",
      " |          >>> for idx, m in enumerate(net.named_modules()):\n",
      " |                  print(idx, '->', m)\n",
      " |      \n",
      " |          0 -> ('', Sequential(\n",
      " |            (0): Linear(in_features=2, out_features=2, bias=True)\n",
      " |            (1): Linear(in_features=2, out_features=2, bias=True)\n",
      " |          ))\n",
      " |          1 -> ('0', Linear(in_features=2, out_features=2, bias=True))\n",
      " |  \n",
      " |  named_parameters(self, prefix='', recurse=True)\n",
      " |      Returns an iterator over module parameters, yielding both the\n",
      " |      name of the parameter as well as the parameter itself.\n",
      " |      \n",
      " |      Args:\n",
      " |          prefix (str): prefix to prepend to all parameter names.\n",
      " |          recurse (bool): if True, then yields parameters of this module\n",
      " |              and all submodules. Otherwise, yields only parameters that\n",
      " |              are direct members of this module.\n",
      " |      \n",
      " |      Yields:\n",
      " |          (string, Parameter): Tuple containing the name and parameter\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> for name, param in self.named_parameters():\n",
      " |          >>>    if name in ['bias']:\n",
      " |          >>>        print(param.size())\n",
      " |  \n",
      " |  parameters(self, recurse=True)\n",
      " |      Returns an iterator over module parameters.\n",
      " |      \n",
      " |      This is typically passed to an optimizer.\n",
      " |      \n",
      " |      Args:\n",
      " |          recurse (bool): if True, then yields parameters of this module\n",
      " |              and all submodules. Otherwise, yields only parameters that\n",
      " |              are direct members of this module.\n",
      " |      \n",
      " |      Yields:\n",
      " |          Parameter: module parameter\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> for param in model.parameters():\n",
      " |          >>>     print(type(param.data), param.size())\n",
      " |          <class 'torch.FloatTensor'> (20L,)\n",
      " |          <class 'torch.FloatTensor'> (20L, 1L, 5L, 5L)\n",
      " |  \n",
      " |  register_backward_hook(self, hook)\n",
      " |      Registers a backward hook on the module.\n",
      " |      \n",
      " |      The hook will be called every time the gradients with respect to module\n",
      " |      inputs are computed. The hook should have the following signature::\n",
      " |      \n",
      " |          hook(module, grad_input, grad_output) -> Tensor or None\n",
      " |      \n",
      " |      The :attr:`grad_input` and :attr:`grad_output` may be tuples if the\n",
      " |      module has multiple inputs or outputs. The hook should not modify its\n",
      " |      arguments, but it can optionally return a new gradient with respect to\n",
      " |      input that will be used in place of :attr:`grad_input` in subsequent\n",
      " |      computations.\n",
      " |      \n",
      " |      Returns:\n",
      " |          :class:`torch.utils.hooks.RemovableHandle`:\n",
      " |              a handle that can be used to remove the added hook by calling\n",
      " |              ``handle.remove()``\n",
      " |      \n",
      " |      .. warning ::\n",
      " |      \n",
      " |          The current implementation will not have the presented behavior\n",
      " |          for complex :class:`Module` that perform many operations.\n",
      " |          In some failure cases, :attr:`grad_input` and :attr:`grad_output` will only\n",
      " |          contain the gradients for a subset of the inputs and outputs.\n",
      " |          For such :class:`Module`, you should use :func:`torch.Tensor.register_hook`\n",
      " |          directly on a specific input or output to get the required gradients.\n",
      " |  \n",
      " |  register_buffer(self, name, tensor)\n",
      " |      Adds a persistent buffer to the module.\n",
      " |      \n",
      " |      This is typically used to register a buffer that should not to be\n",
      " |      considered a model parameter. For example, BatchNorm's ``running_mean``\n",
      " |      is not a parameter, but is part of the persistent state.\n",
      " |      \n",
      " |      Buffers can be accessed as attributes using given names.\n",
      " |      \n",
      " |      Args:\n",
      " |          name (string): name of the buffer. The buffer can be accessed\n",
      " |              from this module using the given name\n",
      " |          tensor (Tensor): buffer to be registered.\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> self.register_buffer('running_mean', torch.zeros(num_features))\n",
      " |  \n",
      " |  register_forward_hook(self, hook)\n",
      " |      Registers a forward hook on the module.\n",
      " |      \n",
      " |      The hook will be called every time after :func:`forward` has computed an output.\n",
      " |      It should have the following signature::\n",
      " |      \n",
      " |          hook(module, input, output) -> None or modified output\n",
      " |      \n",
      " |      The hook can modify the output. It can modify the input inplace but\n",
      " |      it will not have effect on forward since this is called after\n",
      " |      :func:`forward` is called.\n",
      " |      \n",
      " |      Returns:\n",
      " |          :class:`torch.utils.hooks.RemovableHandle`:\n",
      " |              a handle that can be used to remove the added hook by calling\n",
      " |              ``handle.remove()``\n",
      " |  \n",
      " |  register_forward_pre_hook(self, hook)\n",
      " |      Registers a forward pre-hook on the module.\n",
      " |      \n",
      " |      The hook will be called every time before :func:`forward` is invoked.\n",
      " |      It should have the following signature::\n",
      " |      \n",
      " |          hook(module, input) -> None or modified input\n",
      " |      \n",
      " |      The hook can modify the input. User can either return a tuple or a\n",
      " |      single modified value in the hook. We will wrap the value into a tuple\n",
      " |      if a single value is returned(unless that value is already a tuple).\n",
      " |      \n",
      " |      Returns:\n",
      " |          :class:`torch.utils.hooks.RemovableHandle`:\n",
      " |              a handle that can be used to remove the added hook by calling\n",
      " |              ``handle.remove()``\n",
      " |  \n",
      " |  register_parameter(self, name, param)\n",
      " |      Adds a parameter to the module.\n",
      " |      \n",
      " |      The parameter can be accessed as an attribute using given name.\n",
      " |      \n",
      " |      Args:\n",
      " |          name (string): name of the parameter. The parameter can be accessed\n",
      " |              from this module using the given name\n",
      " |          param (Parameter): parameter to be added to the module.\n",
      " |  \n",
      " |  requires_grad_(self, requires_grad=True)\n",
      " |      Change if autograd should record operations on parameters in this\n",
      " |      module.\n",
      " |      \n",
      " |      This method sets the parameters' :attr:`requires_grad` attributes\n",
      " |      in-place.\n",
      " |      \n",
      " |      This method is helpful for freezing part of the module for finetuning\n",
      " |      or training parts of a model individually (e.g., GAN training).\n",
      " |      \n",
      " |      Args:\n",
      " |          requires_grad (bool): whether autograd should record operations on\n",
      " |                                parameters in this module. Default: ``True``.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  share_memory(self)\n",
      " |  \n",
      " |  state_dict(self, destination=None, prefix='', keep_vars=False)\n",
      " |      Returns a dictionary containing a whole state of the module.\n",
      " |      \n",
      " |      Both parameters and persistent buffers (e.g. running averages) are\n",
      " |      included. Keys are corresponding parameter and buffer names.\n",
      " |      \n",
      " |      Returns:\n",
      " |          dict:\n",
      " |              a dictionary containing a whole state of the module\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> module.state_dict().keys()\n",
      " |          ['bias', 'weight']\n",
      " |  \n",
      " |  to(self, *args, **kwargs)\n",
      " |      Moves and/or casts the parameters and buffers.\n",
      " |      \n",
      " |      This can be called as\n",
      " |      \n",
      " |      .. function:: to(device=None, dtype=None, non_blocking=False)\n",
      " |      \n",
      " |      .. function:: to(dtype, non_blocking=False)\n",
      " |      \n",
      " |      .. function:: to(tensor, non_blocking=False)\n",
      " |      \n",
      " |      Its signature is similar to :meth:`torch.Tensor.to`, but only accepts\n",
      " |      floating point desired :attr:`dtype` s. In addition, this method will\n",
      " |      only cast the floating point parameters and buffers to :attr:`dtype`\n",
      " |      (if given). The integral parameters and buffers will be moved\n",
      " |      :attr:`device`, if that is given, but with dtypes unchanged. When\n",
      " |      :attr:`non_blocking` is set, it tries to convert/move asynchronously\n",
      " |      with respect to the host if possible, e.g., moving CPU Tensors with\n",
      " |      pinned memory to CUDA devices.\n",
      " |      \n",
      " |      See below for examples.\n",
      " |      \n",
      " |      .. note::\n",
      " |          This method modifies the module in-place.\n",
      " |      \n",
      " |      Args:\n",
      " |          device (:class:`torch.device`): the desired device of the parameters\n",
      " |              and buffers in this module\n",
      " |          dtype (:class:`torch.dtype`): the desired floating point type of\n",
      " |              the floating point parameters and buffers in this module\n",
      " |          tensor (torch.Tensor): Tensor whose dtype and device are the desired\n",
      " |              dtype and device for all parameters and buffers in this module\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |      \n",
      " |      Example::\n",
      " |      \n",
      " |          >>> linear = nn.Linear(2, 2)\n",
      " |          >>> linear.weight\n",
      " |          Parameter containing:\n",
      " |          tensor([[ 0.1913, -0.3420],\n",
      " |                  [-0.5113, -0.2325]])\n",
      " |          >>> linear.to(torch.double)\n",
      " |          Linear(in_features=2, out_features=2, bias=True)\n",
      " |          >>> linear.weight\n",
      " |          Parameter containing:\n",
      " |          tensor([[ 0.1913, -0.3420],\n",
      " |                  [-0.5113, -0.2325]], dtype=torch.float64)\n",
      " |          >>> gpu1 = torch.device(\"cuda:1\")\n",
      " |          >>> linear.to(gpu1, dtype=torch.half, non_blocking=True)\n",
      " |          Linear(in_features=2, out_features=2, bias=True)\n",
      " |          >>> linear.weight\n",
      " |          Parameter containing:\n",
      " |          tensor([[ 0.1914, -0.3420],\n",
      " |                  [-0.5112, -0.2324]], dtype=torch.float16, device='cuda:1')\n",
      " |          >>> cpu = torch.device(\"cpu\")\n",
      " |          >>> linear.to(cpu)\n",
      " |          Linear(in_features=2, out_features=2, bias=True)\n",
      " |          >>> linear.weight\n",
      " |          Parameter containing:\n",
      " |          tensor([[ 0.1914, -0.3420],\n",
      " |                  [-0.5112, -0.2324]], dtype=torch.float16)\n",
      " |  \n",
      " |  train(self, mode=True)\n",
      " |      Sets the module in training mode.\n",
      " |      \n",
      " |      This has any effect only on certain modules. See documentations of\n",
      " |      particular modules for details of their behaviors in training/evaluation\n",
      " |      mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`,\n",
      " |      etc.\n",
      " |      \n",
      " |      Args:\n",
      " |          mode (bool): whether to set training mode (``True``) or evaluation\n",
      " |                       mode (``False``). Default: ``True``.\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  type(self, dst_type)\n",
      " |      Casts all parameters and buffers to :attr:`dst_type`.\n",
      " |      \n",
      " |      Arguments:\n",
      " |          dst_type (type or string): the desired type\n",
      " |      \n",
      " |      Returns:\n",
      " |          Module: self\n",
      " |  \n",
      " |  zero_grad(self)\n",
      " |      Sets gradients of all model parameters to zero.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors inherited from torch.nn.modules.module.Module:\n",
      " |  \n",
      " |  __dict__\n",
      " |      dictionary for instance variables (if defined)\n",
      " |  \n",
      " |  __weakref__\n",
      " |      list of weak references to the object (if defined)\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data and other attributes inherited from torch.nn.modules.module.Module:\n",
      " |  \n",
      " |  dump_patches = False\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(bertmodel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'GeForce GTX 1060 3GB'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# install\n",
    "#!pip install pytorch-pretrained-bert pytorch-nlp\n",
    "\n",
    "# BERT imports\n",
    "from torch import nn\n",
    "from torch.nn import CrossEntropyLoss, MSELoss, NLLLoss\n",
    "import torch\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.model_selection import train_test_split\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertConfig\n",
    "from pytorch_pretrained_bert import BertAdam, BertForSequenceClassification\n",
    "from tqdm import tqdm, trange\n",
    "import pandas as pd\n",
    "import io\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "#% matplotlib inline\n",
    "\n",
    "# specify GPU device\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "n_gpu = torch.cuda.device_count()\n",
    "torch.cuda.get_device_name(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "#tonkenized_texts list\n",
    "input_ids = []\n",
    "f = open(\"data.txt\", \"r\")\n",
    "lines = f.readlines()\n",
    "\n",
    "sentdata = []\n",
    "labels = []\n",
    "for line in lines:\n",
    "    splited = line.split(',')\n",
    "    token_sent = sp(splited[0])\n",
    "    token_sent.insert(0,'[CLS]')\n",
    "    token_sent.append('[SEP]')\n",
    "    input_ids.append(vocab(token_sent))\n",
    "    labels.append(int(splited[1]))\n",
    "'''\n",
    "sentdata = ['B씨 모델을 공유합니다.', 'A씨 모델을 공유합니다.']\n",
    "for sent in sentdata:\n",
    "    token_sent = sp(sent)\n",
    "    token_sent.insert(0,'[CLS]')\n",
    "    token_sent.append('[SEP]')\n",
    "    input_ids.append(vocab(token_sent))\n",
    "\n",
    "#label\n",
    "labels = [2,1]\n",
    "'''\n",
    "print(sentdata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set the maximum sequence length. \n",
    "MAX_LEN = 64\n",
    "# Pad our input tokens\n",
    "#input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],\n",
    "#                          maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
    "# Use the BERT tokenizer to convert the tokens to their index numbers in the BERT vocabulary\n",
    "#input_ids = [tokenizer.convert_tokens_to_ids(x) for x in tokenized_texts]\n",
    "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[   2 1706 6213 ...    0    0    0]\n",
      " [   2 1706 6213 ...    0    0    0]\n",
      " [   2 1706 6213 ...    0    0    0]\n",
      " ...\n",
      " [   2 4902 2376 ...    0    0    0]\n",
      " [   2 4902 2145 ...    0    0    0]\n",
      " [   2 4902 2358 ...    0    0    0]]\n"
     ]
    }
   ],
   "source": [
    "print(input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertModel(\n",
       "  (embeddings): BertEmbeddings(\n",
       "    (word_embeddings): Embedding(8002, 768, padding_idx=0)\n",
       "    (position_embeddings): Embedding(512, 768)\n",
       "    (token_type_embeddings): Embedding(2, 768)\n",
       "    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (encoder): BertEncoder(\n",
       "    (layer): ModuleList(\n",
       "      (0): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (1): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (2): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (3): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (4): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (5): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (6): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (7): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (8): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (9): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (10): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (11): BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (pooler): BertPooler(\n",
       "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "    (activation): Tanh()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bertmodel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create attention masks\n",
    "attention_masks = []\n",
    "# Create a mask of 1s for each token followed by 0s for padding\n",
    "for seq in input_ids:\n",
    "  seq_mask = [float(i>0) for i in seq]\n",
    "  attention_masks.append(seq_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use train_test_split to split our data into train and validation sets for training\n",
    "train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels, \n",
    "                                                            random_state=2018, test_size=0.1)\n",
    "train_masks, validation_masks, _, _ = train_test_split(attention_masks, input_ids,\n",
    "                                             random_state=2018, test_size=0.1)\n",
    "                                             \n",
    "# Convert all of our data into torch tensors, the required datatype for our model\n",
    "train_inputs = torch.tensor(train_inputs)\n",
    "validation_inputs = torch.tensor(validation_inputs)\n",
    "train_labels = torch.tensor(train_labels)\n",
    "validation_labels = torch.tensor(validation_labels)\n",
    "train_masks = torch.tensor(train_masks)\n",
    "validation_masks = torch.tensor(validation_masks)\n",
    "\n",
    "# Select a batch size for training. \n",
    "batch_size = 4\n",
    "\n",
    "# Create an iterator of our data with torch DataLoader \n",
    "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
    "train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
    "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n",
    "validation_sampler = SequentialSampler(validation_data)\n",
    "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BERTClassifier(nn.Module):\n",
    "    def __init__(self,\n",
    "                 bert,\n",
    "                 hidden_size = 768,\n",
    "                 num_labels=17,\n",
    "                 dr_rate=None,\n",
    "                 params=None):\n",
    "        super(BERTClassifier, self).__init__()\n",
    "        self.bert = bert\n",
    "        self.dr_rate = dr_rate\n",
    "        self.num_labels=num_labels\n",
    "        if dr_rate:\n",
    "            self.dropout = nn.Dropout(p=dr_rate)\n",
    "        self.classifier = nn.Linear(hidden_size , num_labels)\n",
    "        \n",
    "    \n",
    "    def gen_attention_mask(self, token_ids, valid_length):\n",
    "        attention_mask = torch.zeros_like(token_ids)\n",
    "        for i, v in enumerate(valid_length):\n",
    "            attention_mask[i][:v] = 1\n",
    "        return attention_mask.float()\n",
    "\n",
    "    def forward(\n",
    "        self,\n",
    "        input_ids=None,\n",
    "        attention_mask=None,\n",
    "        token_type_ids=None,\n",
    "        position_ids=None,\n",
    "        head_mask=None,\n",
    "        inputs_embeds=None,\n",
    "        labels=None,\n",
    "    ):\n",
    "\n",
    "        outputs = self.bert(\n",
    "            input_ids,\n",
    "            attention_mask=attention_mask,\n",
    "            token_type_ids=token_type_ids,\n",
    "            position_ids=position_ids,\n",
    "            head_mask=head_mask,\n",
    "            inputs_embeds=inputs_embeds,\n",
    "        )\n",
    "\n",
    "        pooled_output = outputs[-1]\n",
    "\n",
    "        pooled_output = self.dropout(pooled_output)\n",
    "        logits = self.classifier(pooled_output)\n",
    "\n",
    "        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here\n",
    "\n",
    "        if labels is not None:\n",
    "            if self.num_labels == 1:\n",
    "                #  We are doing regression\n",
    "                loss_fct = MSELoss()\n",
    "                loss = loss_fct(logits.view(-1), labels.view(-1))\n",
    "            else:\n",
    "                loss_fct = CrossEntropyLoss()\n",
    "                \n",
    "                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))\n",
    "            outputs = (loss,) + outputs\n",
    "\n",
    "        return outputs  # (loss), logits, (hidden_states), (attentions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = BERTClassifier(bertmodel, num_labels=17, dr_rate=0.1).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<generator object Module.named_parameters at 0x7fdef5220bd0>\n"
     ]
    }
   ],
   "source": [
    "print(model.named_parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "load = True\n",
    "if load:\n",
    "    model = BERTClassifier(bertmodel, num_labels=17, dr_rate=0.1).to(device)\n",
    "    model.load_state_dict(torch.load('./modelsave/nonattentionmask100epoch.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "t_total value of -1 results in schedule not being applied\n",
      "Epoch:   0%|          | 0/3000 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 2.3042194843292236\n",
      "Validation Accuracy: 0.6128048780487805\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 1/3000 [00:19<16:25:33, 19.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 1.7209656238555908\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 2/3000 [00:37<15:50:32, 19.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.7408536585365854\n",
      "Train loss: 1.2139551639556885\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 3/3000 [00:53<15:17:26, 18.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.7103658536585366\n",
      "Train loss: 0.9421984553337097\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 4/3000 [01:10<14:53:16, 17.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.7652439024390244\n",
      "Train loss: 0.932296633720398\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 5/3000 [01:27<14:40:11, 17.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.7774390243902439\n",
      "Train loss: 1.4103471040725708\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 6/3000 [01:45<14:35:41, 17.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.5853658536585366\n",
      "Train loss: 1.3727397918701172\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 7/3000 [02:02<14:25:46, 17.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.7012195121951219\n",
      "Train loss: 1.1955183744430542\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 8/3000 [02:18<14:16:12, 17.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.698170731707317\n",
      "Train loss: 1.0929288864135742\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 9/3000 [02:35<14:09:24, 17.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.698170731707317\n",
      "Train loss: 1.038260579109192\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 10/3000 [02:52<14:04:41, 16.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.7012195121951219\n",
      "Train loss: 1.0035834312438965\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 11/3000 [03:09<14:02:29, 16.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.7073170731707317\n",
      "Train loss: 0.920626699924469\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 12/3000 [03:25<14:01:21, 16.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.7073170731707317\n",
      "Train loss: 0.8781470060348511\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 13/3000 [03:42<13:58:31, 16.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8048780487804879\n",
      "Train loss: 0.790408194065094\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 14/3000 [03:59<13:59:44, 16.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.850609756097561\n",
      "Train loss: 0.5924183130264282\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 15/3000 [04:16<13:58:37, 16.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8567073170731707\n",
      "Train loss: 0.441089004278183\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 16/3000 [04:33<13:59:17, 16.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8567073170731707\n",
      "Train loss: 0.45948395133018494\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 17/3000 [04:50<13:56:28, 16.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8445121951219512\n",
      "Train loss: 0.4319945275783539\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 18/3000 [05:07<14:01:19, 16.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8536585365853658\n",
      "Train loss: 0.7509341239929199\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 19/3000 [05:24<14:07:03, 17.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.7896341463414634\n",
      "Train loss: 0.6238254904747009\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 20/3000 [05:41<14:07:45, 17.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.7987804878048781\n",
      "Train loss: 1.1620601415634155\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 21/3000 [05:59<14:14:02, 17.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.7957317073170732\n",
      "Train loss: 0.8037079572677612\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 22/3000 [06:16<14:13:59, 17.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.7926829268292683\n",
      "Train loss: 0.6786804795265198\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 23/3000 [06:33<14:11:48, 17.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8536585365853658\n",
      "Train loss: 0.49312618374824524\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 24/3000 [06:51<14:17:34, 17.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8871951219512195\n",
      "Train loss: 0.3846803605556488\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 25/3000 [07:08<14:18:07, 17.31s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8841463414634146\n",
      "Train loss: 0.2723395824432373\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 26/3000 [07:25<14:11:13, 17.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9054878048780488\n",
      "Train loss: 0.31337982416152954\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 27/3000 [07:42<14:05:12, 17.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9024390243902439\n",
      "Train loss: 0.3110176920890808\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 28/3000 [07:58<14:02:24, 17.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9024390243902439\n",
      "Train loss: 0.31193292140960693\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 29/3000 [08:15<13:57:36, 16.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.899390243902439\n",
      "Train loss: 0.2935628592967987\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 30/3000 [08:32<13:54:15, 16.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9115853658536586\n",
      "Train loss: 0.23943932354450226\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 31/3000 [08:49<13:51:37, 16.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9115853658536586\n",
      "Train loss: 0.2752969563007355\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 32/3000 [09:05<13:50:47, 16.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9115853658536586\n",
      "Train loss: 0.23415607213974\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 33/3000 [09:22<13:52:29, 16.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9024390243902439\n",
      "Train loss: 0.2522006928920746\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 34/3000 [09:39<13:50:52, 16.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9115853658536586\n",
      "Train loss: 0.2439279705286026\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 35/3000 [09:56<13:50:59, 16.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9146341463414634\n",
      "Train loss: 0.29865148663520813\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 36/3000 [10:13<13:54:04, 16.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9085365853658537\n",
      "Train loss: 0.27919408679008484\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|          | 37/3000 [10:30<13:52:29, 16.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8567073170731707\n",
      "Train loss: 0.2968372702598572\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|▏         | 38/3000 [10:46<13:52:12, 16.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9237804878048781\n",
      "Train loss: 0.23495973646640778\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|▏         | 39/3000 [11:04<13:57:59, 16.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9176829268292683\n",
      "Train loss: 0.19470523297786713\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|▏         | 40/3000 [11:21<14:00:49, 17.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9390243902439024\n",
      "Train loss: 0.22527161240577698\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|▏         | 41/3000 [11:38<14:00:45, 17.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9176829268292683\n",
      "Train loss: 0.17422668635845184\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|▏         | 42/3000 [11:55<14:01:49, 17.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9359756097560976\n",
      "Train loss: 0.16609326004981995\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|▏         | 43/3000 [12:12<13:56:41, 16.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.926829268292683\n",
      "Train loss: 0.16831643879413605\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   1%|▏         | 44/3000 [12:29<13:53:00, 16.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.926829268292683\n",
      "Train loss: 0.18211089074611664\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 45/3000 [12:45<13:50:25, 16.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9237804878048781\n",
      "Train loss: 0.170633465051651\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 46/3000 [13:02<13:48:45, 16.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9329268292682927\n",
      "Train loss: 0.17481549084186554\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 47/3000 [13:19<13:48:11, 16.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9298780487804879\n",
      "Train loss: 0.462677925825119\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 48/3000 [13:36<13:47:58, 16.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8658536585365854\n",
      "Train loss: 0.5838956236839294\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 49/3000 [13:53<13:47:26, 16.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8170731707317073\n",
      "Train loss: 0.6248611807823181\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 50/3000 [14:09<13:47:37, 16.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.850609756097561\n",
      "Train loss: 0.3802024722099304\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 51/3000 [14:26<13:47:00, 16.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.875\n",
      "Train loss: 0.3310043513774872\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 52/3000 [14:43<13:46:21, 16.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8841463414634146\n",
      "Train loss: 0.39594462513923645\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 53/3000 [15:00<13:49:56, 16.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8689024390243902\n",
      "Train loss: 0.3362638056278229\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 54/3000 [15:17<13:54:15, 16.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8841463414634146\n",
      "Train loss: 0.36384111642837524\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 55/3000 [15:34<13:52:10, 16.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8262195121951219\n",
      "Train loss: 0.4967776834964752\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 56/3000 [15:51<13:54:05, 17.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8810975609756098\n",
      "Train loss: 0.38666829466819763\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 57/3000 [16:08<13:51:01, 16.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8689024390243902\n",
      "Train loss: 0.37313318252563477\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 58/3000 [16:25<13:48:47, 16.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8902439024390244\n",
      "Train loss: 0.34072059392929077\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 59/3000 [16:42<13:48:45, 16.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8810975609756098\n",
      "Train loss: 0.34235134720802307\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 60/3000 [16:59<13:53:46, 17.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.899390243902439\n",
      "Train loss: 0.23733605444431305\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 61/3000 [17:16<13:49:50, 16.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9207317073170732\n",
      "Train loss: 0.2456597536802292\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 62/3000 [17:33<13:46:17, 16.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.926829268292683\n",
      "Train loss: 0.2189531773328781\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 63/3000 [17:50<13:45:58, 16.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.926829268292683\n",
      "Train loss: 0.1984473615884781\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 64/3000 [18:07<13:49:19, 16.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.926829268292683\n",
      "Train loss: 0.17281579971313477\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 65/3000 [18:24<13:51:48, 17.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.926829268292683\n",
      "Train loss: 0.17668882012367249\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 66/3000 [18:41<13:48:50, 16.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.948170731707317\n",
      "Train loss: 0.15342167019844055\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 67/3000 [18:57<13:45:55, 16.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.948170731707317\n",
      "Train loss: 0.26944151520729065\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 68/3000 [19:14<13:45:27, 16.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.926829268292683\n",
      "Train loss: 0.277403324842453\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 69/3000 [19:32<13:52:14, 17.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9115853658536586\n",
      "Train loss: 0.18011774122714996\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 70/3000 [19:49<13:51:04, 17.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9359756097560976\n",
      "Train loss: 0.13750013709068298\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 71/3000 [20:05<13:48:37, 16.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8871951219512195\n",
      "Train loss: 0.2195323258638382\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 72/3000 [20:22<13:45:30, 16.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.926829268292683\n",
      "Train loss: 0.25972428917884827\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 73/3000 [20:39<13:48:22, 16.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.926829268292683\n",
      "Train loss: 0.2949536442756653\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▏         | 74/3000 [20:56<13:46:43, 16.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9329268292682927\n",
      "Train loss: 0.19811417162418365\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   2%|▎         | 75/3000 [21:13<13:48:07, 16.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9390243902439024\n",
      "Train loss: 0.19514885544776917\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 76/3000 [21:30<13:49:11, 17.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9390243902439024\n",
      "Train loss: 0.1422758847475052\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 77/3000 [21:48<13:55:16, 17.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.948170731707317\n",
      "Train loss: 0.12176356464624405\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 78/3000 [22:05<13:51:50, 17.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9420731707317073\n",
      "Train loss: 0.11605227738618851\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 79/3000 [22:22<13:49:12, 17.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9664634146341463\n",
      "Train loss: 0.10214927047491074\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 80/3000 [22:39<13:53:00, 17.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9664634146341463\n",
      "Train loss: 0.1361786425113678\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 81/3000 [22:56<13:54:41, 17.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9573170731707317\n",
      "Train loss: 0.12362705171108246\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 82/3000 [23:13<13:49:18, 17.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9664634146341463\n",
      "Train loss: 0.10073447227478027\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 83/3000 [23:30<13:45:40, 16.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9085365853658537\n",
      "Train loss: 0.18070442974567413\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 84/3000 [23:47<13:43:22, 16.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9390243902439024\n",
      "Train loss: 0.10441342741250992\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 85/3000 [24:04<13:41:30, 16.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9664634146341463\n",
      "Train loss: 0.10213388502597809\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 86/3000 [24:21<13:43:11, 16.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.948170731707317\n",
      "Train loss: 0.1318810135126114\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 87/3000 [24:37<13:40:07, 16.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9634146341463414\n",
      "Train loss: 0.10232438892126083\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 88/3000 [24:54<13:38:19, 16.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9695121951219512\n",
      "Train loss: 0.16038227081298828\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 89/3000 [25:11<13:37:35, 16.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9603658536585366\n",
      "Train loss: 0.15455977618694305\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 90/3000 [25:28<13:36:33, 16.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9176829268292683\n",
      "Train loss: 0.11972631514072418\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 91/3000 [25:45<13:36:11, 16.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9695121951219512\n",
      "Train loss: 0.07142649590969086\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 92/3000 [26:01<13:35:24, 16.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9878048780487805\n",
      "Train loss: 0.0612710602581501\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 93/3000 [26:18<13:34:52, 16.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9878048780487805\n",
      "Train loss: 0.11941895633935928\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 94/3000 [26:35<13:34:18, 16.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9847560975609756\n",
      "Train loss: 0.07119885087013245\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 95/3000 [26:52<13:40:59, 16.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.09228489547967911\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 96/3000 [27:09<13:38:21, 16.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9878048780487805\n",
      "Train loss: 0.14761681854724884\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 97/3000 [27:26<13:37:11, 16.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9420731707317073\n",
      "Train loss: 0.3825000822544098\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 98/3000 [27:44<13:46:46, 17.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9542682926829268\n",
      "Train loss: 0.22158876061439514\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 99/3000 [28:00<13:44:16, 17.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9664634146341463\n",
      "Train loss: 0.19980452954769135\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 100/3000 [28:18<13:44:34, 17.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9603658536585366\n",
      "Train loss: 0.16048651933670044\n",
      "Validation Accuracy: 0.975609756097561\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 101/3000 [28:35<13:50:36, 17.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.11902471631765366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 102/3000 [28:52<13:50:50, 17.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9634146341463414\n",
      "Train loss: 0.10512229800224304\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 103/3000 [29:10<13:52:27, 17.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.08734743297100067\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   3%|▎         | 104/3000 [29:27<13:53:43, 17.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9847560975609756\n",
      "Train loss: 0.06702959537506104\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▎         | 105/3000 [29:44<13:51:24, 17.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9847560975609756\n",
      "Train loss: 0.05703230947256088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▎         | 106/3000 [30:01<13:49:26, 17.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9847560975609756\n",
      "Train loss: 0.24654042720794678\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▎         | 107/3000 [30:18<13:44:49, 17.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.948170731707317\n",
      "Train loss: 0.1055610328912735\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▎         | 108/3000 [30:35<13:45:53, 17.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9847560975609756\n",
      "Train loss: 0.0733594074845314\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▎         | 109/3000 [30:52<13:40:34, 17.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.07229156792163849\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▎         | 110/3000 [31:09<13:38:10, 16.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.1071847528219223\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▎         | 111/3000 [31:26<13:37:22, 16.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9847560975609756\n",
      "Train loss: 0.06418139487504959\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▎         | 112/3000 [31:43<13:36:28, 16.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9847560975609756\n",
      "Train loss: 0.061387937515974045\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 113/3000 [32:00<13:34:27, 16.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9847560975609756\n",
      "Train loss: 0.06062395125627518\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 114/3000 [32:16<13:31:25, 16.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.06274056434631348\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 115/3000 [32:33<13:29:41, 16.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9512195121951219\n",
      "Train loss: 0.10670796781778336\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 116/3000 [32:50<13:28:39, 16.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.06926285475492477\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 117/3000 [33:07<13:27:46, 16.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9817073170731707\n",
      "Train loss: 0.03838729113340378\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 118/3000 [33:24<13:26:53, 16.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.11943437904119492\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 119/3000 [33:40<13:26:07, 16.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.1658555120229721\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 120/3000 [33:57<13:25:06, 16.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9512195121951219\n",
      "Train loss: 0.1693001240491867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 121/3000 [34:14<13:24:10, 16.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9390243902439024\n",
      "Train loss: 0.16410061717033386\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 122/3000 [34:31<13:24:03, 16.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9573170731707317\n",
      "Train loss: 0.17858076095581055\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 123/3000 [34:47<13:24:29, 16.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9085365853658537\n",
      "Train loss: 0.2702120840549469\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 124/3000 [35:04<13:24:53, 16.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9847560975609756\n",
      "Train loss: 0.23495472967624664\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 125/3000 [35:21<13:24:33, 16.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9451219512195121\n",
      "Train loss: 0.3005655109882355\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 126/3000 [35:38<13:25:02, 16.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.13081851601600647\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 127/3000 [35:55<13:24:30, 16.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9664634146341463\n",
      "Train loss: 0.13994546234607697\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 128/3000 [36:12<13:28:55, 16.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9786585365853658\n",
      "Train loss: 0.06998780369758606\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 129/3000 [36:29<13:35:47, 17.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9817073170731707\n",
      "Train loss: 0.06888125091791153\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 130/3000 [36:46<13:35:58, 17.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.027214540168642998\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 131/3000 [37:03<13:32:53, 17.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.024197988212108612\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 132/3000 [37:20<13:28:52, 16.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.018814001232385635\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 133/3000 [37:37<13:34:48, 17.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9878048780487805\n",
      "Train loss: 0.01803467981517315\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 134/3000 [37:54<13:34:39, 17.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.019253499805927277\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   4%|▍         | 135/3000 [38:11<13:30:23, 16.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.023123906925320625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▍         | 136/3000 [38:28<13:26:44, 16.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.01727246306836605\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▍         | 137/3000 [38:44<13:23:35, 16.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.016526363790035248\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▍         | 138/3000 [39:01<13:21:03, 16.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.01980302482843399\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▍         | 139/3000 [39:18<13:19:14, 16.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.018613064661622047\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▍         | 140/3000 [39:35<13:17:51, 16.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.01658466085791588\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▍         | 141/3000 [39:52<13:21:48, 16.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.02009524218738079\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▍         | 142/3000 [40:08<13:21:49, 16.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.016046322882175446\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▍         | 143/3000 [40:25<13:23:50, 16.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.017800211906433105\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▍         | 144/3000 [40:42<13:22:28, 16.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.016475072130560875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▍         | 145/3000 [40:59<13:22:04, 16.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.013841621577739716\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▍         | 146/3000 [41:16<13:20:23, 16.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.01686236262321472\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▍         | 147/3000 [41:33<13:18:40, 16.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.018255101516842842\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▍         | 148/3000 [41:49<13:19:16, 16.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.01658547669649124\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▍         | 149/3000 [42:06<13:19:57, 16.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.015083752572536469\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▌         | 150/3000 [42:23<13:19:02, 16.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.017414255067706108\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▌         | 151/3000 [42:40<13:20:00, 16.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.08392485976219177\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▌         | 152/3000 [42:57<13:20:01, 16.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9603658536585366\n",
      "Train loss: 0.2614085078239441\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▌         | 153/3000 [43:14<13:18:53, 16.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9542682926829268\n",
      "Train loss: 0.12716330587863922\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▌         | 154/3000 [43:30<13:17:48, 16.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9146341463414634\n",
      "Train loss: 0.14244885742664337\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▌         | 155/3000 [43:47<13:16:25, 16.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9573170731707317\n",
      "Train loss: 0.29457929730415344\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▌         | 156/3000 [44:04<13:15:53, 16.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9390243902439024\n",
      "Train loss: 0.45114418864250183\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▌         | 157/3000 [44:21<13:15:29, 16.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9359756097560976\n",
      "Train loss: 0.34518253803253174\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▌         | 158/3000 [44:37<13:14:46, 16.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.8628048780487805\n",
      "Train loss: 0.22538073360919952\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▌         | 159/3000 [44:54<13:14:06, 16.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9298780487804879\n",
      "Train loss: 0.11432037502527237\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▌         | 160/3000 [45:11<13:13:36, 16.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9695121951219512\n",
      "Train loss: 0.05560065433382988\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▌         | 161/3000 [45:28<13:12:45, 16.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9847560975609756\n",
      "Train loss: 0.10646472871303558\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▌         | 162/3000 [45:45<13:13:09, 16.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9725609756097561\n",
      "Train loss: 0.14665111899375916\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▌         | 163/3000 [46:01<13:14:43, 16.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9573170731707317\n",
      "Train loss: 0.15148015320301056\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   5%|▌         | 164/3000 [46:18<13:15:51, 16.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.15809403359889984\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 165/3000 [46:35<13:15:11, 16.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9542682926829268\n",
      "Train loss: 0.20879337191581726\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 166/3000 [46:52<13:14:29, 16.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9634146341463414\n",
      "Train loss: 0.13024458289146423\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 167/3000 [47:09<13:13:57, 16.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9390243902439024\n",
      "Train loss: 0.11327119916677475\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 168/3000 [47:26<13:14:53, 16.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9573170731707317\n",
      "Train loss: 0.1815134733915329\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 169/3000 [47:43<13:14:56, 16.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9573170731707317\n",
      "Train loss: 0.1402004063129425\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 170/3000 [47:59<13:14:56, 16.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9542682926829268\n",
      "Train loss: 0.18236804008483887\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 171/3000 [48:16<13:16:23, 16.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.18721090257167816\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 172/3000 [48:33<13:16:18, 16.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9695121951219512\n",
      "Train loss: 0.48284071683883667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 173/3000 [48:50<13:16:11, 16.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9115853658536586\n",
      "Train loss: 0.4188874661922455\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 174/3000 [49:07<13:15:46, 16.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9329268292682927\n",
      "Train loss: 0.17500588297843933\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 175/3000 [49:24<13:19:23, 16.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9603658536585366\n",
      "Train loss: 0.1013534888625145\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 176/3000 [49:41<13:19:25, 16.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.0915091261267662\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 177/3000 [49:58<13:15:42, 16.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.07453277707099915\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 178/3000 [50:15<13:13:10, 16.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.07917840033769608\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 179/3000 [50:32<13:14:03, 16.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.07742370665073395\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 180/3000 [50:48<13:12:33, 16.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9939024390243902\n",
      "Train loss: 0.0688202902674675\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 181/3000 [51:05<13:11:32, 16.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.09344286471605301\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 182/3000 [51:22<13:12:11, 16.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.07383431494235992\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 183/3000 [51:39<13:11:08, 16.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.0766979530453682\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 184/3000 [51:56<13:09:54, 16.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.975609756097561\n",
      "Train loss: 0.1173933744430542\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 185/3000 [52:13<13:08:45, 16.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9939024390243902\n",
      "Train loss: 0.133188396692276\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 186/3000 [52:29<13:07:43, 16.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9085365853658537\n",
      "Train loss: 0.2476331740617752\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▌         | 187/3000 [52:46<13:07:29, 16.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9817073170731707\n",
      "Train loss: 0.19004806876182556\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▋         | 188/3000 [53:03<13:07:34, 16.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9725609756097561\n",
      "Train loss: 0.12692025303840637\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▋         | 189/3000 [53:20<13:06:29, 16.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9817073170731707\n",
      "Train loss: 0.06966214627027512\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▋         | 190/3000 [53:37<13:08:07, 16.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.05231330171227455\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▋         | 191/3000 [53:53<13:06:42, 16.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9908536585365854\n",
      "Train loss: 0.019201034680008888\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▋         | 192/3000 [54:10<13:07:56, 16.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.005631439387798309\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▋         | 193/3000 [54:27<13:11:06, 16.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9969512195121951\n",
      "Train loss: 0.004535146988928318\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▋         | 194/3000 [54:44<13:09:24, 16.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9969512195121951\n",
      "Train loss: 0.003609568579122424\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   6%|▋         | 195/3000 [55:01<13:07:33, 16.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.003072671592235565\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 196/3000 [55:18<13:06:08, 16.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9969512195121951\n",
      "Train loss: 0.00270648836158216\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 197/3000 [55:34<13:05:13, 16.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9969512195121951\n",
      "Train loss: 0.002424798207357526\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 198/3000 [55:51<13:03:38, 16.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9969512195121951\n",
      "Train loss: 0.002146325306966901\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 199/3000 [56:08<13:03:16, 16.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9969512195121951\n",
      "Train loss: 0.0019162703538313508\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 200/3000 [56:25<13:03:18, 16.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9969512195121951\n",
      "Train loss: 0.0017604304011911154\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 201/3000 [56:42<13:09:14, 16.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9969512195121951\n",
      "Train loss: 0.001632385072298348\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 202/3000 [56:59<13:12:08, 16.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9969512195121951\n",
      "Train loss: 0.0014838270144537091\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 203/3000 [57:16<13:10:43, 16.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9969512195121951\n",
      "Train loss: 0.0013773426180705428\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 204/3000 [57:33<13:07:44, 16.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9969512195121951\n",
      "Train loss: 0.0012921326560899615\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 205/3000 [57:50<13:06:28, 16.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9969512195121951\n",
      "Train loss: 0.0011268025264143944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 206/3000 [58:06<13:05:42, 16.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 0.9969512195121951\n",
      "Train loss: 0.0010792277753353119\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 207/3000 [58:23<13:04:17, 16.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0010421653278172016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 208/3000 [58:40<13:03:08, 16.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0009526842623017728\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 209/3000 [58:57<13:01:47, 16.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0009665765101090074\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 210/3000 [59:14<13:00:51, 16.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0008762275101616979\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 211/3000 [59:31<13:03:50, 16.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0008180562872439623\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 212/3000 [59:47<13:02:34, 16.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0007646999438293278\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 213/3000 [1:00:04<13:04:13, 16.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0007371088140644133\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 214/3000 [1:00:21<13:05:20, 16.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0007523632375523448\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 215/3000 [1:00:38<13:06:34, 16.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.00067483214661479\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 216/3000 [1:00:55<13:08:38, 17.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.00067855668021366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 217/3000 [1:01:12<13:08:07, 16.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0005904959398321807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 218/3000 [1:01:29<13:05:09, 16.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0005542665021494031\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 219/3000 [1:01:46<13:03:19, 16.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0005373373278416693\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 220/3000 [1:02:03<13:04:45, 16.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.000523488677572459\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 221/3000 [1:02:20<13:02:00, 16.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0005233316915109754\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 222/3000 [1:02:37<12:59:49, 16.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.00048053552745841444\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 223/3000 [1:02:53<12:57:35, 16.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0004910346469841897\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   7%|▋         | 224/3000 [1:03:11<13:03:14, 16.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0004389591049402952\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   8%|▊         | 225/3000 [1:03:28<13:07:38, 17.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.00045791774755343795\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   8%|▊         | 226/3000 [1:03:45<13:08:55, 17.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0004091204609721899\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   8%|▊         | 227/3000 [1:04:02<13:10:03, 17.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0004095769254490733\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   8%|▊         | 228/3000 [1:04:19<13:07:39, 17.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.00039387037395499647\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   8%|▊         | 229/3000 [1:04:36<13:11:33, 17.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n",
      "Train loss: 0.0003489517548587173\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   8%|▊         | 230/3000 [1:04:54<13:12:50, 17.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 1.0\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-67a9d4b22142>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     48\u001b[0m     \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m     \u001b[0;31m# Update parameters and take a step using the computed gradient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m     \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     51\u001b[0m     \u001b[0;31m# Update tracking variables\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m     \u001b[0mtr_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/git/intentclassification/py37-venv/lib/python3.7/site-packages/pytorch_pretrained_bert/optimization.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    269\u001b[0m                 \u001b[0;31m# Add grad clipping\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    270\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'max_grad_norm'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 271\u001b[0;31m                     \u001b[0mclip_grad_norm_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'max_grad_norm'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    272\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    273\u001b[0m                 \u001b[0;31m# Decay the first and second moment running average coefficient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/git/intentclassification/py37-venv/lib/python3.7/site-packages/torch/nn/utils/clip_grad.py\u001b[0m in \u001b[0;36mclip_grad_norm_\u001b[0;34m(parameters, max_norm, norm_type)\u001b[0m\n\u001b[1;32m     31\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m             \u001b[0mparam_norm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnorm_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m             \u001b[0mtotal_norm\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mparam_norm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m**\u001b[0m \u001b[0mnorm_type\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m         \u001b[0mtotal_norm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtotal_norm\u001b[0m \u001b[0;34m**\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1.\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mnorm_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0mclip_coef\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax_norm\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtotal_norm\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1e-6\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# BERT fine-tuning parameters\n",
    "param_optimizer = list(model.named_parameters())\n",
    "no_decay = ['bias', 'gamma', 'beta']\n",
    "optimizer_grouped_parameters = [\n",
    "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
    "     'weight_decay_rate': 0.01},\n",
    "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
    "     'weight_decay_rate': 0.0}\n",
    "]\n",
    "\n",
    "optimizer = BertAdam(optimizer_grouped_parameters,\n",
    "                     lr=2e-5,\n",
    "                     warmup=.1)\n",
    "\n",
    "# Function to calculate the accuracy of our predictions vs labels\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
    "  \n",
    "# Store our loss and accuracy for plotting\n",
    "train_loss_set = []\n",
    "# Number of training epochs \n",
    "epochs = 3000\n",
    "\n",
    "# BERT training loop\n",
    "for _ in trange(epochs, desc=\"Epoch\"):  \n",
    "  \n",
    "  ## TRAINING\n",
    "  \n",
    "  # Set our model to training mode\n",
    "  model.train()  \n",
    "  # Tracking variables\n",
    "  tr_loss = 0\n",
    "  nb_tr_examples, nb_tr_steps = 0, 0\n",
    "  # Train the data for one epoch\n",
    "  for step, batch in enumerate(train_dataloader):\n",
    "    # Add batch to GPU\n",
    "    batch = tuple(t.to(device) for t in batch)\n",
    "    # Unpack the inputs from our dataloader\n",
    "    b_input_ids, b_input_mask, b_labels = batch\n",
    "    # Clear out the gradients (by default they accumulate)\n",
    "    optimizer.zero_grad()\n",
    "    # Forward pass\n",
    "    (loss, logit) = model(b_input_ids, token_type_ids=None, attention_mask=None, labels=b_labels)\n",
    "    train_loss_set.append(loss)    \n",
    "    # Backward pass\n",
    "    loss.backward()\n",
    "    # Update parameters and take a step using the computed gradient\n",
    "    optimizer.step()\n",
    "    # Update tracking variables\n",
    "    tr_loss += loss\n",
    "    nb_tr_examples += b_input_ids.size(0)\n",
    "    nb_tr_steps += 1\n",
    "  print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))    \n",
    "  summary.add_scalar('loss/train_loss', tr_loss/nb_tr_steps, _)\n",
    "    \n",
    "  ## VALIDATION\n",
    "\n",
    "  # Put model in evaluation mode\n",
    "  model.eval()\n",
    "  # Tracking variables \n",
    "  eval_loss, eval_accuracy = 0, 0\n",
    "  nb_eval_steps, nb_eval_examples = 0, 0\n",
    "  # Evaluate data for one epoch\n",
    "  for batch in train_dataloader:\n",
    "    # Add batch to GPU\n",
    "    batch = tuple(t.to(device) for t in batch)\n",
    "    # Unpack the inputs from our dataloader\n",
    "    b_input_ids, b_input_mask, b_labels = batch\n",
    "    # Telling the model not to compute or store gradients, saving memory and speeding up validation\n",
    "    with torch.no_grad():\n",
    "      # Forward pass, calculate logit predictions\n",
    "      logits = model(b_input_ids, token_type_ids=None, attention_mask=None)    \n",
    "    # Move logits and labels to CPU\n",
    "    logits = logits[0].detach().cpu().numpy()\n",
    "    label_ids = b_labels.to('cpu').numpy()\n",
    "    tmp_eval_accuracy = flat_accuracy(logits, label_ids)    \n",
    "    eval_accuracy += tmp_eval_accuracy\n",
    "    nb_eval_steps += 1\n",
    "  print(\"Validation Accuracy: {}\".format(eval_accuracy/nb_eval_steps))\n",
    "  summary.add_scalar('Validation Accuracy', eval_accuracy/nb_eval_steps, _)\n",
    "\n",
    "  if _%100 == 0:\n",
    "    torch.save(model.state_dict(), './modelsave/nonattentionmask'+str(_)+'epoch.pt')\n",
    "    \n",
    "# plot training performance\n",
    "plt.figure(figsize=(15,8))\n",
    "plt.title(\"Training loss\")\n",
    "plt.xlabel(\"Batch\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.plot(train_loss_set)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), './modelsave/abnormal3000epoch.pt')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}