NLP_LM_Finetuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 660
    },
    "colab_type": "code",
    "id": "ADKCnENMQdJE",
    "outputId": "7338f352-4aff-412c-cce2-a293d8a2d6e1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ee/fc/bd726a15ab2c66dc09306689d04da07a3770dad724f0883f0a4bfb745087/transformers-2.4.1-py3-none-any.whl (475kB)\n",
      "\u001b[K     |████████████████████████████████| 481kB 3.4MB/s \n",
      "\u001b[?25hCollecting sentencepiece\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
      "\u001b[K     |████████████████████████████████| 1.0MB 29.1MB/s \n",
      "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.21.0)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.17.5)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.28.1)\n",
      "Collecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/b4/7a41d630547a4afd58143597d5a49e07bfd4c42914d8335b2a5657efc14b/sacremoses-0.0.38.tar.gz (860kB)\n",
      "\u001b[K     |████████████████████████████████| 870kB 44.4MB/s \n",
      "\u001b[?25hCollecting tokenizers==0.0.11\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5e/36/7af38d572c935f8e0462ec7b4f7a46d73a2b3b1a938f50a5e8132d5b2dc5/tokenizers-0.0.11-cp36-cp36m-manylinux1_x86_64.whl (3.1MB)\n",
      "\u001b[K     |████████████████████████████████| 3.1MB 38.7MB/s \n",
      "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
      "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.11.15)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2019.11.28)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.8)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.0)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.1)\n",
      "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.3.3)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.9.4)\n",
      "Requirement already satisfied: botocore<1.15.0,>=1.14.15 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (1.14.15)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.15->boto3->transformers) (2.6.1)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.15->boto3->transformers) (0.15.2)\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.38-cp36-none-any.whl size=884628 sha256=3f8549be30e21c75144217ef9ec607f985c5b6d7c4ceef5ec75a3a9213264e33\n",
      "  Stored in directory: /root/.cache/pip/wheels/6d/ec/1a/21b8912e35e02741306f35f66c785f3afe94de754a0eaf1422\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: sentencepiece, sacremoses, tokenizers, transformers\n",
      "Successfully installed sacremoses-0.0.38 sentencepiece-0.1.85 tokenizers-0.0.11 transformers-2.4.1\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VuRLx3-IugqS"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 81
    },
    "colab_type": "code",
    "id": "u_nYLYAEucNR",
    "outputId": "75e2ba9f-20be-49ab-b94a-9acb7b728cbd"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<p style=\"color: red;\">\n",
       "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
       "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
       "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
       "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "from tqdm import tqdm\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "from sklearn.model_selection import train_test_split\n",
    "import torch\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from tqdm import  tqdm_notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IL-NKe90uf7-"
   },
   "outputs": [],
   "source": [
    "directory_path = '/content/drive/My Drive/nlp'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nqMyfhLQvfLk"
   },
   "outputs": [],
   "source": [
    "hotel_df = pd.read_csv(os.path.join(directory_path,'hotel.csv'))\n",
    "jokes_df = pd.read_csv(os.path.join(directory_path,'jokes.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "icA0x2dsvkfi",
    "outputId": "97f3d17f-187e-4dcc-b82a-6fab14d4bc70"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 25)"
      ]
     },
     "execution_count": 60,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hotel_df.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dCDJWiYdvrf9"
   },
   "outputs": [],
   "source": [
    "hotel_df['REVIEW'] = hotel_df['reviews.text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UAX1oxUBASyl"
   },
   "outputs": [],
   "source": [
    "hotel_df['REVIEW'] = hotel_df.REVIEW.str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EsFeGXrgvtrO"
   },
   "outputs": [],
   "source": [
    "hotel_df['REVIEW'] = hotel_df['REVIEW'].map(lambda k: '[REVIEW] '+str(k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "m0Zm-e5b4mxl"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "K15VAxRwv49Y"
   },
   "outputs": [],
   "source": [
    "changed_text=hotel_df.REVIEW.apply(lambda x:x+\"\\n\"+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "-oSgYg9hwAfq",
    "outputId": "fb3597f3-ed65-482d-9cc5-0820077f178a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3355809"
      ]
     },
     "execution_count": 67,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "open(os.path.join(directory_path,'data_lm_hotel.txt'), \"w\").write(''.join(changed_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 104
    },
    "colab_type": "code",
    "id": "TBqjX0ibwEBf",
    "outputId": "221e34b0-6db8-4179-8292-9437625bf6e4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['README.md',\n",
       " 'simple_lm_finetuning.py',\n",
       " 'pregenerate_training_data.py',\n",
       " 'finetune_on_pregenerated.py',\n",
       " 'data_lm_hotel.txt']"
      ]
     },
     "execution_count": 69,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#%cd /content/drive/My Drive/Bert Training/pytorch-transformers/examples/lm_finetuning\n",
    "import os\n",
    "os.listdir(os.getcwd())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 816
    },
    "colab_type": "code",
    "id": "HErmibFVwx7D",
    "outputId": "653f1097-8f77-4f05-dca4-a8e13a6b9de2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Dataset: 20000 lines [00:09, 2209.52 lines/s]\n",
      "Epoch:   0% 0/2 [00:00<?, ?it/s]\n",
      "Document:   0% 0/10000 [00:00<?, ?it/s]\u001b[A\n",
      "Document:   5% 505/10000 [00:00<00:01, 5043.48it/s]\u001b[A\n",
      "Document:  10% 1047/10000 [00:00<00:01, 5148.80it/s]\u001b[A\n",
      "Document:  15% 1473/10000 [00:00<00:01, 4844.25it/s]\u001b[A\n",
      "Document:  19% 1866/10000 [00:00<00:01, 4516.85it/s]\u001b[A\n",
      "Document:  23% 2331/10000 [00:00<00:01, 4555.82it/s]\u001b[A\n",
      "Document:  28% 2797/10000 [00:00<00:01, 4585.36it/s]\u001b[A\n",
      "Document:  33% 3315/10000 [00:00<00:01, 4748.24it/s]\u001b[A\n",
      "Document:  38% 3850/10000 [00:00<00:01, 4912.71it/s]\u001b[A\n",
      "Document:  43% 4341/10000 [00:00<00:01, 4910.91it/s]\u001b[A\n",
      "Document:  48% 4815/10000 [00:01<00:01, 4821.34it/s]\u001b[A\n",
      "Document:  54% 5352/10000 [00:01<00:00, 4973.76it/s]\u001b[A\n",
      "Document:  58% 5849/10000 [00:01<00:00, 4972.47it/s]\u001b[A\n",
      "Document:  63% 6342/10000 [00:01<00:00, 4957.64it/s]\u001b[A\n",
      "Document:  68% 6844/10000 [00:01<00:00, 4973.98it/s]\u001b[A\n",
      "Document:  73% 7345/10000 [00:01<00:00, 4983.13it/s]\u001b[A\n",
      "Document:  79% 7859/10000 [00:01<00:00, 5027.37it/s]\u001b[A\n",
      "Document:  84% 8361/10000 [00:01<00:00, 4923.47it/s]\u001b[A\n",
      "Document:  89% 8868/10000 [00:01<00:00, 4964.23it/s]\u001b[A\n",
      "Document:  94% 9373/10000 [00:01<00:00, 4988.85it/s]\u001b[A\n",
      "Document:  99% 9872/10000 [00:02<00:00, 4741.79it/s]\u001b[A\n",
      "Epoch:  50% 1/2 [00:02<00:02,  2.06s/it]\n",
      "Document:   0% 0/10000 [00:00<?, ?it/s]\u001b[A\n",
      "Document:   5% 513/10000 [00:00<00:01, 5124.78it/s]\u001b[A\n",
      "Document:  10% 1015/10000 [00:00<00:01, 5092.26it/s]\u001b[A\n",
      "Document:  14% 1435/10000 [00:00<00:01, 4784.17it/s]\u001b[A\n",
      "Document:  18% 1844/10000 [00:00<00:01, 4551.54it/s]\u001b[A\n",
      "Document:  23% 2318/10000 [00:00<00:01, 4606.33it/s]\u001b[A\n",
      "Document:  28% 2820/10000 [00:00<00:01, 4722.96it/s]\u001b[A\n",
      "Document:  33% 3330/10000 [00:00<00:01, 4828.63it/s]\u001b[A\n",
      "Document:  39% 3855/10000 [00:00<00:01, 4946.27it/s]\u001b[A\n",
      "Document:  43% 4327/10000 [00:00<00:01, 4873.59it/s]\u001b[A\n",
      "Document:  48% 4829/10000 [00:01<00:01, 4914.55it/s]\u001b[A\n",
      "Document:  54% 5355/10000 [00:01<00:00, 5011.67it/s]\u001b[A\n",
      "Document:  59% 5873/10000 [00:01<00:00, 5058.08it/s]\u001b[A\n",
      "Document:  64% 6386/10000 [00:01<00:00, 5078.28it/s]\u001b[A\n",
      "Document:  69% 6890/10000 [00:01<00:00, 5010.60it/s]\u001b[A\n",
      "Document:  74% 7390/10000 [00:01<00:00, 5007.37it/s]\u001b[A\n",
      "Document:  79% 7889/10000 [00:01<00:00, 4993.29it/s]\u001b[A\n",
      "Document:  84% 8394/10000 [00:01<00:00, 5007.12it/s]\u001b[A\n",
      "Document:  89% 8897/10000 [00:01<00:00, 5012.79it/s]\u001b[A\n",
      "Document:  94% 9398/10000 [00:01<00:00, 5004.09it/s]\u001b[A\n",
      "Document:  99% 9898/10000 [00:02<00:00, 4956.64it/s]\u001b[A\n",
      "Epoch: 100% 2/2 [00:04<00:00,  2.05s/it]\n"
     ]
    }
   ],
   "source": [
    "!python3 pregenerate_training_data.py --train_corpus data_lm_hotel.txt --bert_model bert-base-uncased --do_lower_case --output_dir training/ --epochs_to_generate 2 --max_seq_len 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 121
    },
    "colab_type": "code",
    "id": "W_em6w6RxEwQ",
    "outputId": "b99236a7-e893-4435-ffcf-ad81871bbb63"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['README.md',\n",
       " 'simple_lm_finetuning.py',\n",
       " 'pregenerate_training_data.py',\n",
       " 'finetune_on_pregenerated.py',\n",
       " 'data_lm_hotel.txt',\n",
       " 'training']"
      ]
     },
     "execution_count": 71,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir(os.getcwd())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 749
    },
    "colab_type": "code",
    "id": "-q-wxugGxMkx",
    "outputId": "1a812574-fcb0-48a2-c5c5-211191bb851d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-02-16 08:06:38,693: device: cuda n_gpu: 1, distributed training: False, 16-bits training: False\n",
      "2020-02-16 08:06:38,856: loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /root/.cache/torch/pytorch_transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
      "2020-02-16 08:06:39,021: loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-config.json from cache at /root/.cache/torch/pytorch_transformers/4dad0251492946e18ac39290fcfe91b89d370fee250efe9521476438fe8ca185.8f56353af4a709bf5ff0fbc915d8f5b42bfff892cbb6ac98c3c45f481a03c685\n",
      "2020-02-16 08:06:39,021: Model config {\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_labels\": 2,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"pruned_heads\": {},\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "2020-02-16 08:06:39,189: loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-pytorch_model.bin from cache at /root/.cache/torch/pytorch_transformers/aa1ef1aede4482d0dbcd4d52baad8ae300e60902e88fcb0bebdec09afd232066.36ca03ab34a1a5d5fa7bc3d03d55c4fa650fed07220e2eeebc06ce58d0e9a157\n",
      "2020-02-16 08:06:46,478: ***** Running training *****\n",
      "2020-02-16 08:06:46,478:   Num examples = 20000\n",
      "2020-02-16 08:06:46,478:   Batch size = 16\n",
      "2020-02-16 08:06:46,478:   Num steps = 1250\n",
      "2020-02-16 08:06:46,483: Loading training examples for epoch 0\n",
      "Training examples: 100% 10000/10000 [00:01<00:00, 6276.68it/s]\n",
      "2020-02-16 08:06:48,078: Loading complete!\n",
      "Epoch 0: 100% 625/625 [05:08<00:00,  1.95it/s, Loss: 2.30355]\n",
      "2020-02-16 08:11:56,262: Loading training examples for epoch 1\n",
      "Training examples: 100% 10000/10000 [00:01<00:00, 6287.76it/s]\n",
      "2020-02-16 08:11:57,854: Loading complete!\n",
      "Epoch 1: 100% 625/625 [05:19<00:00,  1.96it/s, Loss: 2.02588]\n",
      "2020-02-16 08:17:17,180: ** ** * Saving fine-tuned model ** ** * \n"
     ]
    }
   ],
   "source": [
    "!python3 finetune_on_pregenerated.py --pregenerated_data training/ --bert_model bert-base-uncased --do_lower_case --train_batch_size 16  --output_dir finetuned_lm/ --epochs 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ypk8HZG7xaiq"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "NLP - LM Finetuning",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "10ce723ecc384e0c86741114310eaa24": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "15ae6002d15b49c58c45c50a5bc94e10": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "22d851e7743a449a824f444aa663c9a2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "24dcd0943524418ea1a5b77f3988952b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2b79b98c9a454d2294b29d1bbbeb250c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading",
      "description_tooltip": null,
      "layout": "IPY_MODEL_24dcd0943524418ea1a5b77f3988952b",
      "max": 231508,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_22d851e7743a449a824f444aa663c9a2",
      "value": 231508
     }
    },
    "610b5c7c8a1a42f893d8adcc9c912910": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c5f399978c6249a0a84b6317a676067d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_610b5c7c8a1a42f893d8adcc9c912910",
      "placeholder": "​",
      "style": "IPY_MODEL_10ce723ecc384e0c86741114310eaa24",
      "value": "100% 232k/232k [00:00&lt;00:00, 2.67MB/s]"
     }
    },
    "f6196d39378e4406b9909e943d84d074": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_2b79b98c9a454d2294b29d1bbbeb250c",
       "IPY_MODEL_c5f399978c6249a0a84b6317a676067d"
      ],
      "layout": "IPY_MODEL_15ae6002d15b49c58c45c50a5bc94e10"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}