pretraining/pretrain-xlnet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import spacy\n",
    "import re\n",
    "from tqdm import tqdm\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/share/apps/python3/3.6.3/intel/lib/python3.6/site-packages/ipython-6.2.1-py3.6.egg/IPython/core/interactiveshell.py:2728: DtypeWarning: Columns (4,5) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "# STEP 1: load Note datasets\n",
    "df_notes = pd.read_csv('/scratch/kh2383/MIMIC/1.4/NOTEEVENTS.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# STEP 2: Generate the cohort to pretrain on (here e.g. we only pretrian on physician and nursing notes) \n",
    "df_notes = df_notes[df_notes.CATEGORY.isin(['Respiratory ','Nursing','Nursing/Others'])]\n",
    "\n",
    "# IMPORTANT: if you fine tune on the same dataset that you use for pretrain, you need to preclude the fine-tune test admissions \n",
    "df_test_ids = pd.read_csv('/scratch/kh2383/MechVent/data/test.csv').HADM_ID.unique()\n",
    "df_notes_fold = df_notes[~df_notes.HADM_ID.isin(df_test_ids)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# STEP 3: Preprocessing\n",
    "def preprocess1(x):\n",
    "    y=re.sub('\\\\[(.*?)\\\\]','',x) #remove de-identified brackets\n",
    "    y=re.sub('[0-9]+\\.','',y) #remove 1.2. since the segmenter segments based on this\n",
    "    y=re.sub('dr\\.','doctor',y)\n",
    "    y=re.sub('m\\.d\\.','md',y)\n",
    "    y=re.sub('admission date:','',y)\n",
    "    y=re.sub('discharge date:','',y)\n",
    "    y=re.sub('--|__|==','',y)\n",
    "    \n",
    "    # remove punctuation, digits, spaces\n",
    "    #y = y.translate(str.maketrans(\"\", \"\", string.punctuation))\n",
    "    y = y.translate(str.maketrans(\"\", \"\", string.digits))\n",
    "    y = \" \".join(y.split())\n",
    "    return y\n",
    "\n",
    "def preprocessing(df_notes): \n",
    "    df_notes['TEXT']=df_notes['TEXT'].fillna(' ')\n",
    "    df_notes['TEXT']=df_notes['TEXT'].str.replace('\\n',' ')\n",
    "    df_notes['TEXT']=df_notes['TEXT'].str.replace('\\r',' ')\n",
    "    df_notes['TEXT']=df_notes['TEXT'].apply(str.strip)\n",
    "    df_notes['TEXT']=df_notes['TEXT'].str.lower()\n",
    "\n",
    "    df_notes['TEXT']=df_notes['TEXT'].apply(lambda x: preprocess1(x))\n",
    "    \n",
    "    return df_notes\n",
    "\n",
    "df_notes_fold = preprocessing(df_notes_fold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n",
      "single word notes ignored\n"
     ]
    }
   ],
   "source": [
    "# STEP 4: Notes to Sentences\n",
    "from spacy.lang.en import English\n",
    "nlp = English()  # just the language with no model\n",
    "nlp.add_pipe(nlp.create_pipe('sentencizer'))\n",
    "\n",
    "# nlp praser may not work when there is only one token. In these cases, we just remove them as note that has length 1 usually is some random stuff\n",
    "\n",
    "def toSentence(x):\n",
    "    doc = nlp(x)\n",
    "    text=[]\n",
    "    try:\n",
    "        for sent in doc.sents:\n",
    "            st=str(sent).strip() \n",
    "            if len(st)<20:\n",
    "                #a lot of abbreviation is segmented as one line. But these are all describing the previous things\n",
    "                #so I attached it to the sentence before\n",
    "                if len(text)!=0:\n",
    "                    text[-1]=' '.join((text[-1],st))\n",
    "                else:\n",
    "                    text=[st]\n",
    "            else:\n",
    "                text.append((st))\n",
    "    except:\n",
    "        print('single word notes ignored')\n",
    "    return text\n",
    "\n",
    "pretrain_sent=df_notes_fold['TEXT'].apply(lambda x: toSentence(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# STEP 5: Create Pretraining File\n",
    "file=open('/scratch/kh2383/MechVent/data/clinical_sentences_pretrain_xlnet_total.txt','w')\n",
    "pretrain_sent = pretrain_sent.values\n",
    "for i in (range(len(pretrain_sent))):\n",
    "    if len(pretrain_sent[i]) > 0:\n",
    "        # remove the one token note\n",
    "        note = pretrain_sent[i]\n",
    "        for sent in note:\n",
    "            file.write(sent+'\\n')\n",
    "        file.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# STEP 6: Download the Implementations and Initial Checkpoint with the SentencePiece Model and Vocab \n",
    "# Github Repo:\n",
    "# XLNet: git clone https://github.com/zihangdai/xlnet.git\n",
    "# Model: \n",
    "# XLNet-Base: https://storage.googleapis.com/xlnet/released_models/cased_L-12_H-768_A-12.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I1102 19:06:04.042432 47659116077696 data_utils.py:211] Use glob: /scratch/kh2383/MechVent/data/clinical_sentences_pretrain_xlnet.txt\n",
      "I1102 19:06:04.043622 47659116077696 data_utils.py:212] Find 1 files: ['/scratch/kh2383/MechVent/data/clinical_sentences_pretrain_xlnet.txt']\n",
      "I1102 19:06:04.043815 47659116077696 data_utils.py:220] Task 0 process 1 files: ['/scratch/kh2383/MechVent/data/clinical_sentences_pretrain_xlnet.txt']\n",
      "I1102 19:06:04.164321 47659116077696 data_utils.py:89] Processing /scratch/kh2383/MechVent/data/clinical_sentences_pretrain_xlnet.txt\n",
      "I1102 19:06:04.184380 47659116077696 data_utils.py:92] Loading line 0\n",
      "I1102 19:06:16.002577 47659116077696 data_utils.py:92] Loading line 100000\n",
      "I1102 19:06:27.438141 47659116077696 data_utils.py:92] Loading line 200000\n",
      "I1102 19:06:38.681625 47659116077696 data_utils.py:92] Loading line 300000\n",
      "I1102 19:06:49.714661 47659116077696 data_utils.py:92] Loading line 400000\n",
      "I1102 19:07:00.752820 47659116077696 data_utils.py:92] Loading line 500000\n",
      "I1102 19:07:11.683559 47659116077696 data_utils.py:92] Loading line 600000\n",
      "I1102 19:07:22.531230 47659116077696 data_utils.py:92] Loading line 700000\n",
      "I1102 19:07:33.527734 47659116077696 data_utils.py:92] Loading line 800000\n",
      "I1102 19:07:44.728296 47659116077696 data_utils.py:92] Loading line 900000\n",
      "I1102 19:07:55.768712 47659116077696 data_utils.py:92] Loading line 1000000\n",
      "I1102 19:08:06.740363 47659116077696 data_utils.py:92] Loading line 1100000\n",
      "I1102 19:08:17.477593 47659116077696 data_utils.py:92] Loading line 1200000\n",
      "I1102 19:08:28.167937 47659116077696 data_utils.py:92] Loading line 1300000\n",
      "I1102 19:08:39.066353 47659116077696 data_utils.py:92] Loading line 1400000\n",
      "I1102 19:08:49.986129 47659116077696 data_utils.py:92] Loading line 1500000\n",
      "I1102 19:09:00.694377 47659116077696 data_utils.py:92] Loading line 1600000\n",
      "I1102 19:09:11.327736 47659116077696 data_utils.py:92] Loading line 1700000\n",
      "I1102 19:09:21.999844 47659116077696 data_utils.py:92] Loading line 1800000\n",
      "I1102 19:09:32.666931 47659116077696 data_utils.py:92] Loading line 1900000\n",
      "I1102 19:09:43.367851 47659116077696 data_utils.py:92] Loading line 2000000\n",
      "I1102 19:09:54.182780 47659116077696 data_utils.py:92] Loading line 2100000\n",
      "I1102 19:10:04.866650 47659116077696 data_utils.py:92] Loading line 2200000\n",
      "I1102 19:10:15.645810 47659116077696 data_utils.py:92] Loading line 2300000\n",
      "I1102 19:10:26.450424 47659116077696 data_utils.py:92] Loading line 2400000\n",
      "I1102 19:10:37.011532 47659116077696 data_utils.py:92] Loading line 2500000\n",
      "I1102 19:10:47.619202 47659116077696 data_utils.py:92] Loading line 2600000\n",
      "I1102 19:10:58.572218 47659116077696 data_utils.py:92] Loading line 2700000\n",
      "I1102 19:11:09.244786 47659116077696 data_utils.py:92] Loading line 2800000\n",
      "I1102 19:11:20.185267 47659116077696 data_utils.py:92] Loading line 2900000\n",
      "I1102 19:11:31.216450 47659116077696 data_utils.py:92] Loading line 3000000\n",
      "I1102 19:11:42.248136 47659116077696 data_utils.py:92] Loading line 3100000\n",
      "I1102 19:11:53.065721 47659116077696 data_utils.py:92] Loading line 3200000\n",
      "I1102 19:12:03.926588 47659116077696 data_utils.py:92] Loading line 3300000\n",
      "I1102 19:12:14.747720 47659116077696 data_utils.py:92] Loading line 3400000\n",
      "I1102 19:12:25.773216 47659116077696 data_utils.py:92] Loading line 3500000\n",
      "I1102 19:12:36.682801 47659116077696 data_utils.py:92] Loading line 3600000\n",
      "I1102 19:12:47.499281 47659116077696 data_utils.py:92] Loading line 3700000\n",
      "I1102 19:12:58.236128 47659116077696 data_utils.py:92] Loading line 3800000\n",
      "I1102 19:13:09.084016 47659116077696 data_utils.py:92] Loading line 3900000\n",
      "I1102 19:13:13.752708 47659116077696 data_utils.py:112] Finish with line 3942618\n",
      "I1102 19:13:23.840087 47659116077696 data_utils.py:122] [Task 0] Total number line: 3942618\n",
      "I1102 19:13:23.840455 47659116077696 data_utils.py:133] Using perm indices [0] for pass 0\n",
      "I1102 19:13:25.408652 47659116077696 data_utils.py:419] Raw data shape (8, 25279621).\n",
      "I1102 19:13:25.409985 47659116077696 data_utils.py:435] Start writing /scratch/kh2383/MechVent/data/xlnet_tfrecord/tfrecords/train-0-0.bsz-8.seqlen-512.reuse-256.uncased.bi.alpha-6.beta-1.fnp-85.tfrecords.\n",
      "I1102 19:13:25.410110 47659116077696 data_utils.py:450] Processing batch 0\n",
      "I1102 19:13:40.004443 47659116077696 data_utils.py:450] Processing batch 500\n",
      "I1102 19:13:54.596815 47659116077696 data_utils.py:450] Processing batch 1000\n",
      "I1102 19:14:09.168360 47659116077696 data_utils.py:450] Processing batch 1500\n",
      "I1102 19:14:23.782549 47659116077696 data_utils.py:450] Processing batch 2000\n",
      "I1102 19:14:38.445180 47659116077696 data_utils.py:450] Processing batch 2500\n",
      "I1102 19:14:53.053814 47659116077696 data_utils.py:450] Processing batch 3000\n",
      "I1102 19:15:07.652384 47659116077696 data_utils.py:450] Processing batch 3500\n",
      "I1102 19:15:22.238216 47659116077696 data_utils.py:450] Processing batch 4000\n",
      "I1102 19:15:36.857820 47659116077696 data_utils.py:450] Processing batch 4500\n",
      "I1102 19:15:51.472459 47659116077696 data_utils.py:450] Processing batch 5000\n",
      "I1102 19:16:06.121054 47659116077696 data_utils.py:450] Processing batch 5500\n",
      "I1102 19:16:20.759955 47659116077696 data_utils.py:450] Processing batch 6000\n",
      "I1102 19:16:35.382876 47659116077696 data_utils.py:450] Processing batch 6500\n",
      "I1102 19:16:49.990686 47659116077696 data_utils.py:450] Processing batch 7000\n",
      "I1102 19:17:04.599945 47659116077696 data_utils.py:450] Processing batch 7500\n",
      "I1102 19:17:19.200366 47659116077696 data_utils.py:450] Processing batch 8000\n",
      "I1102 19:17:33.814761 47659116077696 data_utils.py:450] Processing batch 8500\n",
      "I1102 19:17:48.400819 47659116077696 data_utils.py:450] Processing batch 9000\n",
      "I1102 19:18:03.001368 47659116077696 data_utils.py:450] Processing batch 9500\n",
      "I1102 19:18:17.635085 47659116077696 data_utils.py:450] Processing batch 10000\n",
      "I1102 19:18:32.251057 47659116077696 data_utils.py:450] Processing batch 10500\n",
      "I1102 19:18:46.856014 47659116077696 data_utils.py:450] Processing batch 11000\n",
      "I1102 19:19:01.468643 47659116077696 data_utils.py:450] Processing batch 11500\n",
      "I1102 19:19:16.070931 47659116077696 data_utils.py:450] Processing batch 12000\n",
      "I1102 19:19:30.675571 47659116077696 data_utils.py:450] Processing batch 12500\n",
      "I1102 19:19:45.298241 47659116077696 data_utils.py:450] Processing batch 13000\n",
      "I1102 19:19:59.866214 47659116077696 data_utils.py:450] Processing batch 13500\n",
      "I1102 19:20:14.455435 47659116077696 data_utils.py:450] Processing batch 14000\n",
      "I1102 19:20:29.086060 47659116077696 data_utils.py:450] Processing batch 14500\n",
      "I1102 19:20:43.674938 47659116077696 data_utils.py:450] Processing batch 15000\n",
      "I1102 19:20:58.258097 47659116077696 data_utils.py:450] Processing batch 15500\n",
      "I1102 19:21:12.872984 47659116077696 data_utils.py:450] Processing batch 16000\n",
      "I1102 19:21:27.448688 47659116077696 data_utils.py:450] Processing batch 16500\n",
      "I1102 19:21:42.056420 47659116077696 data_utils.py:450] Processing batch 17000\n",
      "I1102 19:21:56.656062 47659116077696 data_utils.py:450] Processing batch 17500\n",
      "I1102 19:22:11.259339 47659116077696 data_utils.py:450] Processing batch 18000\n",
      "I1102 19:22:25.834635 47659116077696 data_utils.py:450] Processing batch 18500\n",
      "I1102 19:22:40.414344 47659116077696 data_utils.py:450] Processing batch 19000\n",
      "I1102 19:22:55.021780 47659116077696 data_utils.py:450] Processing batch 19500\n",
      "I1102 19:23:09.619735 47659116077696 data_utils.py:450] Processing batch 20000\n",
      "I1102 19:23:24.236202 47659116077696 data_utils.py:450] Processing batch 20500\n",
      "I1102 19:23:38.820585 47659116077696 data_utils.py:450] Processing batch 21000\n",
      "I1102 19:23:53.435682 47659116077696 data_utils.py:450] Processing batch 21500\n",
      "I1102 19:24:08.003826 47659116077696 data_utils.py:450] Processing batch 22000\n",
      "I1102 19:24:22.598304 47659116077696 data_utils.py:450] Processing batch 22500\n",
      "I1102 19:24:37.199072 47659116077696 data_utils.py:450] Processing batch 23000\n",
      "I1102 19:24:51.823568 47659116077696 data_utils.py:450] Processing batch 23500\n",
      "I1102 19:25:06.405419 47659116077696 data_utils.py:450] Processing batch 24000\n",
      "I1102 19:25:21.023493 47659116077696 data_utils.py:450] Processing batch 24500\n",
      "I1102 19:25:35.640298 47659116077696 data_utils.py:450] Processing batch 25000\n",
      "I1102 19:25:50.204371 47659116077696 data_utils.py:450] Processing batch 25500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I1102 19:26:04.784489 47659116077696 data_utils.py:450] Processing batch 26000\n",
      "I1102 19:26:19.363143 47659116077696 data_utils.py:450] Processing batch 26500\n",
      "I1102 19:26:33.986644 47659116077696 data_utils.py:450] Processing batch 27000\n",
      "I1102 19:26:48.579244 47659116077696 data_utils.py:450] Processing batch 27500\n",
      "I1102 19:27:03.190010 47659116077696 data_utils.py:450] Processing batch 28000\n",
      "I1102 19:27:17.755956 47659116077696 data_utils.py:450] Processing batch 28500\n",
      "I1102 19:27:32.354958 47659116077696 data_utils.py:450] Processing batch 29000\n",
      "I1102 19:27:46.932151 47659116077696 data_utils.py:450] Processing batch 29500\n",
      "I1102 19:28:01.520366 47659116077696 data_utils.py:450] Processing batch 30000\n",
      "I1102 19:28:16.106199 47659116077696 data_utils.py:450] Processing batch 30500\n",
      "I1102 19:28:30.687078 47659116077696 data_utils.py:450] Processing batch 31000\n",
      "I1102 19:28:45.305592 47659116077696 data_utils.py:450] Processing batch 31500\n",
      "I1102 19:28:59.898533 47659116077696 data_utils.py:450] Processing batch 32000\n",
      "I1102 19:29:14.521198 47659116077696 data_utils.py:450] Processing batch 32500\n",
      "I1102 19:29:29.095923 47659116077696 data_utils.py:450] Processing batch 33000\n",
      "I1102 19:29:43.678377 47659116077696 data_utils.py:450] Processing batch 33500\n",
      "I1102 19:29:58.242813 47659116077696 data_utils.py:450] Processing batch 34000\n",
      "I1102 19:30:12.835708 47659116077696 data_utils.py:450] Processing batch 34500\n",
      "I1102 19:30:27.458497 47659116077696 data_utils.py:450] Processing batch 35000\n",
      "I1102 19:30:42.046772 47659116077696 data_utils.py:450] Processing batch 35500\n",
      "I1102 19:30:56.677594 47659116077696 data_utils.py:450] Processing batch 36000\n",
      "I1102 19:31:11.301242 47659116077696 data_utils.py:450] Processing batch 36500\n",
      "I1102 19:31:25.923048 47659116077696 data_utils.py:450] Processing batch 37000\n",
      "I1102 19:31:40.510117 47659116077696 data_utils.py:450] Processing batch 37500\n",
      "I1102 19:31:55.090274 47659116077696 data_utils.py:450] Processing batch 38000\n",
      "I1102 19:32:09.650711 47659116077696 data_utils.py:450] Processing batch 38500\n",
      "I1102 19:32:24.209658 47659116077696 data_utils.py:450] Processing batch 39000\n",
      "I1102 19:32:38.801931 47659116077696 data_utils.py:450] Processing batch 39500\n",
      "I1102 19:32:53.417537 47659116077696 data_utils.py:450] Processing batch 40000\n",
      "I1102 19:33:08.030093 47659116077696 data_utils.py:450] Processing batch 40500\n",
      "I1102 19:33:22.629601 47659116077696 data_utils.py:450] Processing batch 41000\n",
      "I1102 19:33:37.244094 47659116077696 data_utils.py:450] Processing batch 41500\n",
      "I1102 19:33:51.848951 47659116077696 data_utils.py:450] Processing batch 42000\n",
      "I1102 19:34:06.439668 47659116077696 data_utils.py:450] Processing batch 42500\n",
      "I1102 19:34:21.029874 47659116077696 data_utils.py:450] Processing batch 43000\n",
      "I1102 19:34:35.624371 47659116077696 data_utils.py:450] Processing batch 43500\n",
      "I1102 19:34:50.257631 47659116077696 data_utils.py:450] Processing batch 44000\n",
      "I1102 19:35:04.873391 47659116077696 data_utils.py:450] Processing batch 44500\n",
      "I1102 19:35:19.483936 47659116077696 data_utils.py:450] Processing batch 45000\n",
      "I1102 19:35:34.097113 47659116077696 data_utils.py:450] Processing batch 45500\n",
      "I1102 19:35:48.685784 47659116077696 data_utils.py:450] Processing batch 46000\n",
      "I1102 19:36:03.243218 47659116077696 data_utils.py:450] Processing batch 46500\n",
      "I1102 19:36:17.899717 47659116077696 data_utils.py:450] Processing batch 47000\n",
      "I1102 19:36:32.511772 47659116077696 data_utils.py:450] Processing batch 47500\n",
      "I1102 19:36:47.103621 47659116077696 data_utils.py:450] Processing batch 48000\n",
      "I1102 19:37:01.699763 47659116077696 data_utils.py:450] Processing batch 48500\n",
      "I1102 19:37:16.301657 47659116077696 data_utils.py:450] Processing batch 49000\n",
      "I1102 19:37:30.916049 47659116077696 data_utils.py:450] Processing batch 49500\n",
      "I1102 19:37:45.531858 47659116077696 data_utils.py:450] Processing batch 50000\n",
      "I1102 19:38:00.137943 47659116077696 data_utils.py:450] Processing batch 50500\n",
      "I1102 19:38:14.745145 47659116077696 data_utils.py:450] Processing batch 51000\n",
      "I1102 19:38:29.373739 47659116077696 data_utils.py:450] Processing batch 51500\n",
      "I1102 19:38:44.030983 47659116077696 data_utils.py:450] Processing batch 52000\n",
      "I1102 19:38:58.625360 47659116077696 data_utils.py:450] Processing batch 52500\n",
      "I1102 19:39:13.207233 47659116077696 data_utils.py:450] Processing batch 53000\n",
      "I1102 19:39:27.849495 47659116077696 data_utils.py:450] Processing batch 53500\n",
      "I1102 19:39:42.448796 47659116077696 data_utils.py:450] Processing batch 54000\n",
      "I1102 19:39:57.075474 47659116077696 data_utils.py:450] Processing batch 54500\n",
      "I1102 19:40:11.716294 47659116077696 data_utils.py:450] Processing batch 55000\n",
      "I1102 19:40:26.310037 47659116077696 data_utils.py:450] Processing batch 55500\n",
      "I1102 19:40:40.904569 47659116077696 data_utils.py:450] Processing batch 56000\n",
      "I1102 19:40:55.528607 47659116077696 data_utils.py:450] Processing batch 56500\n",
      "I1102 19:41:10.126410 47659116077696 data_utils.py:450] Processing batch 57000\n",
      "I1102 19:41:24.747827 47659116077696 data_utils.py:450] Processing batch 57500\n",
      "I1102 19:41:39.342072 47659116077696 data_utils.py:450] Processing batch 58000\n",
      "I1102 19:41:53.913545 47659116077696 data_utils.py:450] Processing batch 58500\n",
      "I1102 19:42:08.526705 47659116077696 data_utils.py:450] Processing batch 59000\n",
      "I1102 19:42:23.121922 47659116077696 data_utils.py:450] Processing batch 59500\n",
      "I1102 19:42:37.686701 47659116077696 data_utils.py:450] Processing batch 60000\n",
      "I1102 19:42:52.300849 47659116077696 data_utils.py:450] Processing batch 60500\n",
      "I1102 19:43:06.906001 47659116077696 data_utils.py:450] Processing batch 61000\n",
      "I1102 19:43:21.519745 47659116077696 data_utils.py:450] Processing batch 61500\n",
      "I1102 19:43:36.146413 47659116077696 data_utils.py:450] Processing batch 62000\n",
      "I1102 19:43:50.730404 47659116077696 data_utils.py:450] Processing batch 62500\n",
      "I1102 19:44:05.331679 47659116077696 data_utils.py:450] Processing batch 63000\n",
      "I1102 19:44:19.965342 47659116077696 data_utils.py:450] Processing batch 63500\n",
      "I1102 19:44:34.594332 47659116077696 data_utils.py:450] Processing batch 64000\n",
      "I1102 19:44:49.189826 47659116077696 data_utils.py:450] Processing batch 64500\n",
      "I1102 19:45:03.769603 47659116077696 data_utils.py:450] Processing batch 65000\n",
      "I1102 19:45:18.350382 47659116077696 data_utils.py:450] Processing batch 65500\n",
      "I1102 19:45:32.958289 47659116077696 data_utils.py:450] Processing batch 66000\n",
      "I1102 19:45:47.585225 47659116077696 data_utils.py:450] Processing batch 66500\n",
      "I1102 19:46:02.195524 47659116077696 data_utils.py:450] Processing batch 67000\n",
      "I1102 19:46:16.778967 47659116077696 data_utils.py:450] Processing batch 67500\n",
      "I1102 19:46:31.370540 47659116077696 data_utils.py:450] Processing batch 68000\n",
      "I1102 19:46:45.971815 47659116077696 data_utils.py:450] Processing batch 68500\n",
      "I1102 19:47:00.574565 47659116077696 data_utils.py:450] Processing batch 69000\n",
      "I1102 19:47:15.164706 47659116077696 data_utils.py:450] Processing batch 69500\n",
      "I1102 19:47:29.709546 47659116077696 data_utils.py:450] Processing batch 70000\n",
      "I1102 19:47:44.304301 47659116077696 data_utils.py:450] Processing batch 70500\n",
      "I1102 19:47:58.903832 47659116077696 data_utils.py:450] Processing batch 71000\n",
      "I1102 19:48:13.502887 47659116077696 data_utils.py:450] Processing batch 71500\n",
      "I1102 19:48:28.103515 47659116077696 data_utils.py:450] Processing batch 72000\n",
      "I1102 19:48:42.700702 47659116077696 data_utils.py:450] Processing batch 72500\n",
      "I1102 19:48:57.293021 47659116077696 data_utils.py:450] Processing batch 73000\n",
      "I1102 19:49:11.888250 47659116077696 data_utils.py:450] Processing batch 73500\n",
      "I1102 19:49:26.459937 47659116077696 data_utils.py:450] Processing batch 74000\n",
      "I1102 19:49:41.002057 47659116077696 data_utils.py:450] Processing batch 74500\n",
      "I1102 19:49:55.589848 47659116077696 data_utils.py:450] Processing batch 75000\n",
      "I1102 19:50:10.216760 47659116077696 data_utils.py:450] Processing batch 75500\n",
      "I1102 19:50:24.813657 47659116077696 data_utils.py:450] Processing batch 76000\n",
      "I1102 19:50:39.400761 47659116077696 data_utils.py:450] Processing batch 76500\n",
      "I1102 19:50:53.994918 47659116077696 data_utils.py:450] Processing batch 77000\n",
      "I1102 19:51:08.573266 47659116077696 data_utils.py:450] Processing batch 77500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I1102 19:51:23.170225 47659116077696 data_utils.py:450] Processing batch 78000\n",
      "I1102 19:51:37.759519 47659116077696 data_utils.py:450] Processing batch 78500\n",
      "I1102 19:51:52.327802 47659116077696 data_utils.py:450] Processing batch 79000\n",
      "I1102 19:52:06.934043 47659116077696 data_utils.py:450] Processing batch 79500\n",
      "I1102 19:52:21.534674 47659116077696 data_utils.py:450] Processing batch 80000\n",
      "I1102 19:52:36.132499 47659116077696 data_utils.py:450] Processing batch 80500\n",
      "I1102 19:52:50.702902 47659116077696 data_utils.py:450] Processing batch 81000\n",
      "I1102 19:53:05.284194 47659116077696 data_utils.py:450] Processing batch 81500\n",
      "I1102 19:53:19.899870 47659116077696 data_utils.py:450] Processing batch 82000\n",
      "I1102 19:53:34.507218 47659116077696 data_utils.py:450] Processing batch 82500\n",
      "I1102 19:53:49.163180 47659116077696 data_utils.py:450] Processing batch 83000\n",
      "I1102 19:54:03.766204 47659116077696 data_utils.py:450] Processing batch 83500\n",
      "I1102 19:54:18.361120 47659116077696 data_utils.py:450] Processing batch 84000\n",
      "I1102 19:54:32.959669 47659116077696 data_utils.py:450] Processing batch 84500\n",
      "I1102 19:54:47.583854 47659116077696 data_utils.py:450] Processing batch 85000\n",
      "I1102 19:55:02.170078 47659116077696 data_utils.py:450] Processing batch 85500\n",
      "I1102 19:55:16.741424 47659116077696 data_utils.py:450] Processing batch 86000\n",
      "I1102 19:55:31.373856 47659116077696 data_utils.py:450] Processing batch 86500\n",
      "I1102 19:55:45.975228 47659116077696 data_utils.py:450] Processing batch 87000\n",
      "I1102 19:56:00.600699 47659116077696 data_utils.py:450] Processing batch 87500\n",
      "I1102 19:56:15.193936 47659116077696 data_utils.py:450] Processing batch 88000\n",
      "I1102 19:56:29.817190 47659116077696 data_utils.py:450] Processing batch 88500\n",
      "I1102 19:56:44.424507 47659116077696 data_utils.py:450] Processing batch 89000\n",
      "I1102 19:56:59.038804 47659116077696 data_utils.py:450] Processing batch 89500\n",
      "I1102 19:57:13.657532 47659116077696 data_utils.py:450] Processing batch 90000\n",
      "I1102 19:57:28.244463 47659116077696 data_utils.py:450] Processing batch 90500\n",
      "I1102 19:57:42.851567 47659116077696 data_utils.py:450] Processing batch 91000\n",
      "I1102 19:57:57.464854 47659116077696 data_utils.py:450] Processing batch 91500\n",
      "I1102 19:58:12.090117 47659116077696 data_utils.py:450] Processing batch 92000\n",
      "I1102 19:58:26.688184 47659116077696 data_utils.py:450] Processing batch 92500\n",
      "I1102 19:58:41.302319 47659116077696 data_utils.py:450] Processing batch 93000\n",
      "I1102 19:58:55.974928 47659116077696 data_utils.py:450] Processing batch 93500\n",
      "I1102 19:59:10.612824 47659116077696 data_utils.py:450] Processing batch 94000\n",
      "I1102 19:59:25.247180 47659116077696 data_utils.py:450] Processing batch 94500\n",
      "I1102 19:59:39.849255 47659116077696 data_utils.py:450] Processing batch 95000\n",
      "I1102 19:59:54.454210 47659116077696 data_utils.py:450] Processing batch 95500\n",
      "I1102 20:00:09.079431 47659116077696 data_utils.py:450] Processing batch 96000\n",
      "I1102 20:00:23.713574 47659116077696 data_utils.py:450] Processing batch 96500\n",
      "I1102 20:00:38.383566 47659116077696 data_utils.py:450] Processing batch 97000\n",
      "I1102 20:00:52.981249 47659116077696 data_utils.py:450] Processing batch 97500\n",
      "I1102 20:01:07.615087 47659116077696 data_utils.py:450] Processing batch 98000\n",
      "I1102 20:01:22.255772 47659116077696 data_utils.py:450] Processing batch 98500\n",
      "I1102 20:01:29.467525 47659116077696 data_utils.py:523] Done writing /scratch/kh2383/MechVent/data/xlnet_tfrecord/tfrecords/train-0-0.bsz-8.seqlen-512.reuse-256.uncased.bi.alpha-6.beta-1.fnp-85.tfrecords. Num of batches: 98747\n"
     ]
    }
   ],
   "source": [
    "# STEP 7: Generate Pretraining Tensorflow TF_Records\n",
    "# For Clinical XLNet\n",
    "import os\n",
    "os.chdir('./xlnet/')\n",
    "!python data_utils.py \\\n",
    "    --bsz_per_host=8 \\\n",
    "    --num_core_per_host=1 \\\n",
    "    --seq_len=512 \\\n",
    "    --reuse_len=256 \\\n",
    "    --input_glob=/scratch/kh2383/MechVent/data/clinical_sentences_pretrain_xlnet.txt \\\n",
    "    --save_dir=/scratch/kh2383/MechVent/data/xlnet_tfrecord/ \\\n",
    "    --num_passes=5 \\\n",
    "    --bi_data=True \\\n",
    "    --sp_path=/scratch/kh2383/clibert/xlnet_cased_L-12_H-768_A-12/spiece.model \\\n",
    "    --mask_alpha=6 \\\n",
    "    --mask_beta=1 \\\n",
    "    --num_predict=85"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I1102 20:03:23.792002 47329276351104 train_gpu.py:328] n_token 32000\n",
      "I1102 20:03:23.792614 47329276351104 data_utils.py:795] Use the following tfrecord dirs: ['/scratch/kh2383/MechVent/data/xlnet_tfrecord/tfrecords/']\n",
      "I1102 20:03:23.792743 47329276351104 data_utils.py:799] [0] Record glob: /scratch/kh2383/MechVent/data/xlnet_tfrecord/tfrecords/record_info-train-*.bsz-8.seqlen-512.reuse-256.bi.alpha-6.beta-1.fnp-85.json\n",
      "I1102 20:03:23.793778 47329276351104 data_utils.py:809] ['/scratch/kh2383/MechVent/data/xlnet_tfrecord/tfrecords/record_info-train-0-0.bsz-8.seqlen-512.reuse-256.uncased.bi.alpha-6.beta-1.fnp-85.json']\n",
      "I1102 20:03:23.793876 47329276351104 data_utils.py:812] [0] Num of record info path: 1\n",
      "I1102 20:03:23.794657 47329276351104 data_utils.py:845] [Dir 0] Number of chosen batches: 98747\n",
      "I1102 20:03:23.794754 47329276351104 data_utils.py:847] [Dir 0] Number of chosen files: 1\n",
      "I1102 20:03:23.794821 47329276351104 data_utils.py:848] ['/scratch/kh2383/MechVent/data/xlnet_tfrecord/tfrecords/train-0-0.bsz-8.seqlen-512.reuse-256.uncased.bi.alpha-6.beta-1.fnp-85.tfrecords']\n",
      "I1102 20:03:23.794884 47329276351104 data_utils.py:855] Total number of batches: 98747\n",
      "I1102 20:03:23.795356 47329276351104 data_utils.py:857] Total number of files: 1\n",
      "I1102 20:03:23.795424 47329276351104 data_utils.py:858] ['/scratch/kh2383/MechVent/data/xlnet_tfrecord/tfrecords/train-0-0.bsz-8.seqlen-512.reuse-256.uncased.bi.alpha-6.beta-1.fnp-85.tfrecords']\n",
      "I1102 20:03:23.795488 47329276351104 train_gpu.py:205] num of batches 98747\n",
      "I1102 20:03:23.795562 47329276351104 data_utils.py:555] Host 0 handles 1 files\n",
      "I1102 20:03:23.977332 47329276351104 data_utils.py:744] label: Tensor(\"Cast_6:0\", shape=(1,), dtype=int32)\n",
      "I1102 20:03:23.977522 47329276351104 data_utils.py:744] seg_id: Tensor(\"Cast_7:0\", shape=(512,), dtype=int32)\n",
      "I1102 20:03:23.977641 47329276351104 data_utils.py:744] target_mapping: Tensor(\"Reshape_4:0\", shape=(85, 512), dtype=float32)\n",
      "I1102 20:03:23.977757 47329276351104 data_utils.py:744] target: Tensor(\"Cast_8:0\", shape=(85,), dtype=int32)\n",
      "I1102 20:03:23.977863 47329276351104 data_utils.py:744] target_mask: Tensor(\"Reshape_6:0\", shape=(85,), dtype=float32)\n",
      "I1102 20:03:23.977966 47329276351104 data_utils.py:744] perm_mask: Tensor(\"Reshape_7:0\", shape=(512, 512), dtype=float32)\n",
      "I1102 20:03:23.978072 47329276351104 data_utils.py:744] input_k: Tensor(\"Cast_9:0\", shape=(512,), dtype=int32)\n",
      "I1102 20:03:23.978173 47329276351104 data_utils.py:744] input_q: Tensor(\"Reshape_9:0\", shape=(512,), dtype=float32)\n",
      "I1102 20:03:24.041713 47329276351104 modeling.py:453] memory input [<tf.Tensor 'Placeholder:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_1:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_2:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_3:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_4:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_5:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_6:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_7:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_8:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_9:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_10:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_11:0' shape=(384, 4, 768) dtype=float32>]\n",
      "I1102 20:03:24.041826 47329276351104 modeling.py:455] Use float type <dtype: 'float32'>\n",
      "W1102 20:03:24.047688 47329276351104 deprecation.py:323] From /home/kh2383/pytorch-gpu/py3.6.3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "W1102 20:03:24.117021 47329276351104 deprecation.py:323] From /home/kh2383/MechVent/pretraining/xlnet/modeling.py:535: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dropout instead.\n",
      "W1102 20:03:24.118338 47329276351104 deprecation.py:506] From /home/kh2383/pytorch-gpu/py3.6.3/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "W1102 20:03:25.650289 47329276351104 deprecation.py:323] From /home/kh2383/MechVent/pretraining/xlnet/modeling.py:67: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "I1102 20:03:30.717067 47329276351104 train_gpu.py:143] #params: 116750336\n",
      "W1102 20:03:30.826201 47329276351104 deprecation.py:323] From /home/kh2383/pytorch-gpu/py3.6.3/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "I1102 20:03:37.937689 47329276351104 modeling.py:453] memory input [<tf.Tensor 'Placeholder_12:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_13:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_14:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_15:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_16:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_17:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_18:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_19:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_20:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_21:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_22:0' shape=(384, 4, 768) dtype=float32>, <tf.Tensor 'Placeholder_23:0' shape=(384, 4, 768) dtype=float32>]\n",
      "I1102 20:03:37.937863 47329276351104 modeling.py:455] Use float type <dtype: 'float32'>\n",
      "I1102 20:03:42.510440 47329276351104 train_gpu.py:143] #params: 116750336\n",
      "W1102 20:03:50.018595 47329276351104 deprecation.py:323] From /home/kh2383/pytorch-gpu/py3.6.3/lib/python3.6/site-packages/tensorflow/python/training/learning_rate_decay_v2.py:321: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "I1102 20:03:52.895934 47329276351104 model_utils.py:71] Initialize from the ckpt /scratch/kh2383/clibert/xlnet_cased_L-12_H-768_A-12/xlnet_model.ckpt\n",
      "I1102 20:03:53.496299 47329276351104 model_utils.py:85] **** Global Variables ****\n",
      "I1102 20:03:53.496516 47329276351104 model_utils.py:91]   name = model/transformer/r_w_bias:0, shape = (12, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.496614 47329276351104 model_utils.py:91]   name = model/transformer/r_r_bias:0, shape = (12, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.496698 47329276351104 model_utils.py:91]   name = model/transformer/word_embedding/lookup_table:0, shape = (32000, 768), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.496777 47329276351104 model_utils.py:91]   name = model/transformer/mask_emb/mask_emb:0, shape = (1, 1, 768), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.496854 47329276351104 model_utils.py:91]   name = model/transformer/r_s_bias:0, shape = (12, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.496928 47329276351104 model_utils.py:91]   name = model/transformer/seg_embed:0, shape = (12, 2, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.497006 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/k/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.497082 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/v/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.497158 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/r/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.497233 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/q/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.497315 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/o/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.497391 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.497462 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.497533 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/layer_1/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.497608 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/layer_1/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.497678 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/layer_2/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.497752 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/layer_2/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.497823 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.497894 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.497963 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/k/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.498038 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/v/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.498112 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/r/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.498186 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/q/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.498265 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/o/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.498342 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.498412 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.498483 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/layer_1/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.498561 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/layer_1/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.498632 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/layer_2/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.498709 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/layer_2/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.498780 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.498850 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.498919 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/k/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.498994 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/v/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.499067 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/r/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.499140 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/q/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.499214 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/o/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.499295 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.499366 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.499436 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/layer_1/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.499509 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/layer_1/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.499578 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/layer_2/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.499654 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/layer_2/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.499724 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.499794 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.499863 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/k/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.499937 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/v/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.500011 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/r/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.500086 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/q/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.500160 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/o/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.500234 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.500309 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.500379 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/layer_1/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.500451 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/layer_1/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.500520 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/layer_2/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.500593 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/layer_2/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.500663 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.500732 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.500801 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/k/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.500875 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/v/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.500949 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/r/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.501022 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/q/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.501096 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/o/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.501169 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.501238 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.501313 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/layer_1/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.501385 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/layer_1/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.501455 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/layer_2/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.501527 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/layer_2/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.501595 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.501665 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.501733 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/k/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.501806 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/v/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.501879 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/r/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.501952 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/q/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.502024 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/o/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.502097 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.502166 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.502236 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/layer_1/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.502316 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/layer_1/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.502386 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/layer_2/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.502458 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/layer_2/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.502526 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.502595 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.502664 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/k/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.502741 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/v/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.502815 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/r/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.502888 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/q/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.502962 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/o/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.503036 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.503105 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.503174 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/layer_1/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.503247 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/layer_1/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.503322 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/layer_2/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.503404 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/layer_2/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.503473 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.503542 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.503612 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/k/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.503686 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/v/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.503760 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/r/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.503833 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/q/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.503907 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/o/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.503981 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.504050 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.504119 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/layer_1/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.504190 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/layer_1/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.504263 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/layer_2/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.504336 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/layer_2/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.504404 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.504472 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.504540 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/k/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.504613 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/v/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.504686 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/r/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.504760 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/q/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.504833 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/o/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.504906 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.504978 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.505047 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/layer_1/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.505119 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/layer_1/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.505187 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/layer_2/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.505263 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/layer_2/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.505333 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.505401 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.505469 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/k/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.505541 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/v/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.505614 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/r/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.505687 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/q/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.505759 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/o/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.505833 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.505902 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.505971 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/layer_1/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.506043 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/layer_1/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.506111 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/layer_2/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.506182 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/layer_2/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.506251 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.506325 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.506394 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/k/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.506469 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/v/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.506544 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/r/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.506618 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/q/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.506692 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/o/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.506767 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.506837 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.506910 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/layer_1/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.506984 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/layer_1/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.507055 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/layer_2/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.507129 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/layer_2/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.507199 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.507276 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.507346 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/k/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.507421 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/v/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.507499 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/r/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.507573 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/q/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.507647 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/o/kernel:0, shape = (768, 12, 64), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.507721 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.507791 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.507861 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/layer_1/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.507935 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/layer_1/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.508004 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/layer_2/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.508078 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/layer_2/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.508147 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.508216 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.508292 47329276351104 model_utils.py:91]   name = model/lm_loss/bias:0, shape = (32000,), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.508364 47329276351104 model_utils.py:91]   name = global_step:0, shape = (), *INIT_FROM_CKPT*\n",
      "I1102 20:03:53.508441 47329276351104 model_utils.py:91]   name = beta1_power:0, shape = ()\n",
      "I1102 20:03:53.508515 47329276351104 model_utils.py:91]   name = beta2_power:0, shape = ()\n",
      "I1102 20:03:53.508588 47329276351104 model_utils.py:91]   name = model/transformer/r_w_bias/Adam:0, shape = (12, 12, 64)\n",
      "I1102 20:03:53.508667 47329276351104 model_utils.py:91]   name = model/transformer/r_w_bias/Adam_1:0, shape = (12, 12, 64)\n",
      "I1102 20:03:53.508744 47329276351104 model_utils.py:91]   name = model/transformer/r_r_bias/Adam:0, shape = (12, 12, 64)\n",
      "I1102 20:03:53.508822 47329276351104 model_utils.py:91]   name = model/transformer/r_r_bias/Adam_1:0, shape = (12, 12, 64)\n",
      "I1102 20:03:53.508898 47329276351104 model_utils.py:91]   name = model/transformer/word_embedding/lookup_table/Adam:0, shape = (32000, 768)\n",
      "I1102 20:03:53.508974 47329276351104 model_utils.py:91]   name = model/transformer/word_embedding/lookup_table/Adam_1:0, shape = (32000, 768)\n",
      "I1102 20:03:53.509050 47329276351104 model_utils.py:91]   name = model/transformer/mask_emb/mask_emb/Adam:0, shape = (1, 1, 768)\n",
      "I1102 20:03:53.509127 47329276351104 model_utils.py:91]   name = model/transformer/mask_emb/mask_emb/Adam_1:0, shape = (1, 1, 768)\n",
      "I1102 20:03:53.509204 47329276351104 model_utils.py:91]   name = model/transformer/r_s_bias/Adam:0, shape = (12, 12, 64)\n",
      "I1102 20:03:53.509288 47329276351104 model_utils.py:91]   name = model/transformer/r_s_bias/Adam_1:0, shape = (12, 12, 64)\n",
      "I1102 20:03:53.509366 47329276351104 model_utils.py:91]   name = model/transformer/seg_embed/Adam:0, shape = (12, 2, 12, 64)\n",
      "I1102 20:03:53.509445 47329276351104 model_utils.py:91]   name = model/transformer/seg_embed/Adam_1:0, shape = (12, 2, 12, 64)\n",
      "I1102 20:03:53.509521 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/k/kernel/Adam:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.509600 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/k/kernel/Adam_1:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.509678 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/v/kernel/Adam:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.509755 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/v/kernel/Adam_1:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.509832 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/r/kernel/Adam:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.509910 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/r/kernel/Adam_1:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.509988 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/q/kernel/Adam:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.510064 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/q/kernel/Adam_1:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.510142 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/o/kernel/Adam:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.510217 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/o/kernel/Adam_1:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.510296 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/LayerNorm/beta/Adam:0, shape = (768,)\n",
      "I1102 20:03:53.510369 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/LayerNorm/beta/Adam_1:0, shape = (768,)\n",
      "I1102 20:03:53.510445 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/LayerNorm/gamma/Adam:0, shape = (768,)\n",
      "I1102 20:03:53.510516 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/rel_attn/LayerNorm/gamma/Adam_1:0, shape = (768,)\n",
      "I1102 20:03:53.510587 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/layer_1/kernel/Adam:0, shape = (768, 3072)\n",
      "I1102 20:03:53.510664 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/layer_1/kernel/Adam_1:0, shape = (768, 3072)\n",
      "I1102 20:03:53.510740 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/layer_1/bias/Adam:0, shape = (3072,)\n",
      "I1102 20:03:53.510813 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/layer_1/bias/Adam_1:0, shape = (3072,)\n",
      "I1102 20:03:53.510885 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/layer_2/kernel/Adam:0, shape = (3072, 768)\n",
      "I1102 20:03:53.510962 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/layer_2/kernel/Adam_1:0, shape = (3072, 768)\n",
      "I1102 20:03:53.511036 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/layer_2/bias/Adam:0, shape = (768,)\n",
      "I1102 20:03:53.511108 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/layer_2/bias/Adam_1:0, shape = (768,)\n",
      "I1102 20:03:53.511177 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/LayerNorm/beta/Adam:0, shape = (768,)\n",
      "I1102 20:03:53.511247 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/LayerNorm/beta/Adam_1:0, shape = (768,)\n",
      "I1102 20:03:53.511324 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/LayerNorm/gamma/Adam:0, shape = (768,)\n",
      "I1102 20:03:53.511395 47329276351104 model_utils.py:91]   name = model/transformer/layer_0/ff/LayerNorm/gamma/Adam_1:0, shape = (768,)\n",
      "I1102 20:03:53.511466 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/k/kernel/Adam:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.511543 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/k/kernel/Adam_1:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.511619 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/v/kernel/Adam:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.511696 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/v/kernel/Adam_1:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.511773 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/r/kernel/Adam:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.511854 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/r/kernel/Adam_1:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.511931 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/q/kernel/Adam:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.512008 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/q/kernel/Adam_1:0, shape = (768, 12, 64)\n",
      "I1102 20:03:53.512086 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/o/kernel/Adam:0, shape = (768, 12, 64)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I1102 20:03:53.512163 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/o/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.512239 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.512314 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.512386 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.512457 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/rel_attn/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.512529 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/layer_1/kernel/Adam:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.512602 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/layer_1/kernel/Adam_1:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.512678 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/layer_1/bias/Adam:0, shape = (3072,)\r\n",
      "I1102 20:03:53.512750 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/layer_1/bias/Adam_1:0, shape = (3072,)\r\n",
      "I1102 20:03:53.512822 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/layer_2/kernel/Adam:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.512898 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/layer_2/kernel/Adam_1:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.512973 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/layer_2/bias/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.513043 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/layer_2/bias/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.513114 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.513185 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.513255 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.513331 47329276351104 model_utils.py:91]   name = model/transformer/layer_1/ff/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.513401 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/k/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.513479 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/k/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.513557 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/v/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.513634 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/v/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.513711 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/r/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.513788 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/r/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.513864 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/q/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.513941 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/q/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.514017 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/o/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.514095 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/o/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.514171 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.514243 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.514320 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.514391 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/rel_attn/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.514464 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/layer_1/kernel/Adam:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.514541 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/layer_1/kernel/Adam_1:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.514617 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/layer_1/bias/Adam:0, shape = (3072,)\r\n",
      "I1102 20:03:53.514693 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/layer_1/bias/Adam_1:0, shape = (3072,)\r\n",
      "I1102 20:03:53.514764 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/layer_2/kernel/Adam:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.514841 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/layer_2/kernel/Adam_1:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.514917 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/layer_2/bias/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.514989 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/layer_2/bias/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.515061 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.515132 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.515202 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.515279 47329276351104 model_utils.py:91]   name = model/transformer/layer_2/ff/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.515350 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/k/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.515426 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/k/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.515503 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/v/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.515580 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/v/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.515662 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/r/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.515739 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/r/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.515816 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/q/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.515893 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/q/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.515969 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/o/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.516046 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/o/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.516123 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.516196 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.516272 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.516345 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/rel_attn/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.516417 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/layer_1/kernel/Adam:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.516494 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/layer_1/kernel/Adam_1:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.516571 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/layer_1/bias/Adam:0, shape = (3072,)\r\n",
      "I1102 20:03:53.516643 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/layer_1/bias/Adam_1:0, shape = (3072,)\r\n",
      "I1102 20:03:53.516715 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/layer_2/kernel/Adam:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.516789 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/layer_2/kernel/Adam_1:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.516865 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/layer_2/bias/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.516937 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/layer_2/bias/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.517006 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.517077 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.517147 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.517217 47329276351104 model_utils.py:91]   name = model/transformer/layer_3/ff/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.517292 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/k/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.517370 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/k/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.517447 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/v/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.517524 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/v/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.517600 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/r/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.517677 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/r/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.517753 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/q/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.517829 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/q/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.517906 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/o/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.517984 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/o/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.518059 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.518131 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.518201 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.518278 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/rel_attn/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.518350 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/layer_1/kernel/Adam:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.518424 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/layer_1/kernel/Adam_1:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.518501 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/layer_1/bias/Adam:0, shape = (3072,)\r\n",
      "I1102 20:03:53.518573 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/layer_1/bias/Adam_1:0, shape = (3072,)\r\n",
      "I1102 20:03:53.518644 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/layer_2/kernel/Adam:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.518720 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/layer_2/kernel/Adam_1:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.518794 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/layer_2/bias/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.518865 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/layer_2/bias/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.518936 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.519006 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.519077 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.519147 47329276351104 model_utils.py:91]   name = model/transformer/layer_4/ff/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.519217 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/k/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.519299 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/k/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.519377 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/v/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.519454 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/v/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.519531 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/r/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.519607 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/r/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.519682 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/q/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.519758 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/q/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.519834 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/o/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.519910 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/o/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.519986 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.520058 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.520128 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.520199 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/rel_attn/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.520274 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/layer_1/kernel/Adam:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.520351 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/layer_1/kernel/Adam_1:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.520426 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/layer_1/bias/Adam:0, shape = (3072,)\r\n",
      "I1102 20:03:53.520499 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/layer_1/bias/Adam_1:0, shape = (3072,)\r\n",
      "I1102 20:03:53.520572 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/layer_2/kernel/Adam:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.520646 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/layer_2/kernel/Adam_1:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.520720 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/layer_2/bias/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.520791 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/layer_2/bias/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.520861 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.520933 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.521003 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.521074 47329276351104 model_utils.py:91]   name = model/transformer/layer_5/ff/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.521145 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/k/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.521221 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/k/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.521303 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/v/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.521380 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/v/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.521456 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/r/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.521532 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/r/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.521611 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/q/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.521687 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/q/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.521764 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/o/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.521841 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/o/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.521915 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.521987 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.522058 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.522128 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/rel_attn/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.522200 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/layer_1/kernel/Adam:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.522277 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/layer_1/kernel/Adam_1:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.522354 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/layer_1/bias/Adam:0, shape = (3072,)\r\n",
      "I1102 20:03:53.522427 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/layer_1/bias/Adam_1:0, shape = (3072,)\r\n",
      "I1102 20:03:53.522499 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/layer_2/kernel/Adam:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.522576 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/layer_2/kernel/Adam_1:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.522650 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/layer_2/bias/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.522722 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/layer_2/bias/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.522794 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.522866 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.522936 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.523007 47329276351104 model_utils.py:91]   name = model/transformer/layer_6/ff/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.523077 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/k/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.523154 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/k/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.523231 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/v/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.523313 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/v/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.523391 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/r/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.523469 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/r/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.523546 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/q/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.523622 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/q/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.523699 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/o/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.523776 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/o/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.523853 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.523926 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.523996 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.524066 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/rel_attn/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.524138 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/layer_1/kernel/Adam:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.524213 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/layer_1/kernel/Adam_1:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.524291 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/layer_1/bias/Adam:0, shape = (3072,)\r\n",
      "I1102 20:03:53.524366 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/layer_1/bias/Adam_1:0, shape = (3072,)\r\n",
      "I1102 20:03:53.524436 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/layer_2/kernel/Adam:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.524511 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/layer_2/kernel/Adam_1:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.524585 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/layer_2/bias/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.524657 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/layer_2/bias/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.524727 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.524797 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.524868 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.524938 47329276351104 model_utils.py:91]   name = model/transformer/layer_7/ff/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.525009 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/k/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.525090 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/k/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.525167 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/v/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.525243 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/v/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.525326 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/r/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.525404 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/r/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.525483 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/q/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.525561 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/q/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.525638 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/o/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.525722 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/o/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.525798 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.525873 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.525945 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.526016 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/rel_attn/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.526088 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/layer_1/kernel/Adam:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.526163 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/layer_1/kernel/Adam_1:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.526239 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/layer_1/bias/Adam:0, shape = (3072,)\r\n",
      "I1102 20:03:53.526317 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/layer_1/bias/Adam_1:0, shape = (3072,)\r\n",
      "I1102 20:03:53.526388 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/layer_2/kernel/Adam:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.526464 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/layer_2/kernel/Adam_1:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.526540 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/layer_2/bias/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.526613 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/layer_2/bias/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.526684 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.526756 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.526829 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.526901 47329276351104 model_utils.py:91]   name = model/transformer/layer_8/ff/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.526971 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/k/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.527050 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/k/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.527127 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/v/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.527204 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/v/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.527286 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/r/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.527363 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/r/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.527440 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/q/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.527516 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/q/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.527595 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/o/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.527671 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/o/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.527749 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.527821 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.527892 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.527963 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/rel_attn/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.528035 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/layer_1/kernel/Adam:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.528109 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/layer_1/kernel/Adam_1:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.528185 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/layer_1/bias/Adam:0, shape = (3072,)\r\n",
      "I1102 20:03:53.528257 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/layer_1/bias/Adam_1:0, shape = (3072,)\r\n",
      "I1102 20:03:53.528335 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/layer_2/kernel/Adam:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.528411 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/layer_2/kernel/Adam_1:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.528487 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/layer_2/bias/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.528560 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/layer_2/bias/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.528630 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.528700 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.528771 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.528842 47329276351104 model_utils.py:91]   name = model/transformer/layer_9/ff/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.528913 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/k/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.528990 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/k/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.529067 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/v/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.529143 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/v/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.529220 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/r/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.529302 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/r/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.529379 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/q/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.529457 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/q/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.529533 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/o/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.529609 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/o/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.529684 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.529755 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.529827 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.529898 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/rel_attn/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.529969 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/layer_1/kernel/Adam:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.530043 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/layer_1/kernel/Adam_1:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.530118 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/layer_1/bias/Adam:0, shape = (3072,)\r\n",
      "I1102 20:03:53.530187 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/layer_1/bias/Adam_1:0, shape = (3072,)\r\n",
      "I1102 20:03:53.530261 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/layer_2/kernel/Adam:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.530337 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/layer_2/kernel/Adam_1:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.530412 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/layer_2/bias/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.530484 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/layer_2/bias/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.530556 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.530628 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.530698 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.530769 47329276351104 model_utils.py:91]   name = model/transformer/layer_10/ff/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.530839 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/k/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.530915 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/k/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.530991 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/v/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.531068 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/v/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.531145 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/r/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.531222 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/r/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.531302 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/q/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.531380 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/q/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.531456 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/o/kernel/Adam:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.531532 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/o/kernel/Adam_1:0, shape = (768, 12, 64)\r\n",
      "I1102 20:03:53.531607 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.531679 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.531754 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.531826 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/rel_attn/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.531895 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/layer_1/kernel/Adam:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.531971 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/layer_1/kernel/Adam_1:0, shape = (768, 3072)\r\n",
      "I1102 20:03:53.532047 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/layer_1/bias/Adam:0, shape = (3072,)\r\n",
      "I1102 20:03:53.532120 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/layer_1/bias/Adam_1:0, shape = (3072,)\r\n",
      "I1102 20:03:53.532191 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/layer_2/kernel/Adam:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.532271 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/layer_2/kernel/Adam_1:0, shape = (3072, 768)\r\n",
      "I1102 20:03:53.532351 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/layer_2/bias/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.532423 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/layer_2/bias/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.532493 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/LayerNorm/beta/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.532563 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/LayerNorm/beta/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.532634 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/LayerNorm/gamma/Adam:0, shape = (768,)\r\n",
      "I1102 20:03:53.532703 47329276351104 model_utils.py:91]   name = model/transformer/layer_11/ff/LayerNorm/gamma/Adam_1:0, shape = (768,)\r\n",
      "I1102 20:03:53.532774 47329276351104 model_utils.py:91]   name = model/lm_loss/bias/Adam:0, shape = (32000,)\r\n",
      "I1102 20:03:53.532844 47329276351104 model_utils.py:91]   name = model/lm_loss/bias/Adam_1:0, shape = (32000,)\r\n",
      "2019-11-02 20:03:53.533055: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\r\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-11-02 20:03:53.773087: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0xcf98410 executing computations on platform CUDA. Devices:\n",
      "2019-11-02 20:03:53.773126: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): Tesla P100-PCIE-16GB, Compute Capability 6.0\n",
      "2019-11-02 20:03:53.773138: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (1): Tesla P100-PCIE-16GB, Compute Capability 6.0\n",
      "2019-11-02 20:03:53.800345: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2600220000 Hz\n",
      "2019-11-02 20:03:53.800589: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0xd009100 executing computations on platform Host. Devices:\n",
      "2019-11-02 20:03:53.800614: I tensorflow/compiler/xla/service/service.cc:158]   StreamExecutor device (0): <undefined>, <undefined>\n",
      "2019-11-02 20:03:53.801383: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1433] Found device 0 with properties: \n",
      "name: Tesla P100-PCIE-16GB major: 6 minor: 0 memoryClockRate(GHz): 1.3285\n",
      "pciBusID: 0000:05:00.0\n",
      "totalMemory: 15.90GiB freeMemory: 15.64GiB\n",
      "2019-11-02 20:03:53.802075: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1433] Found device 1 with properties: \n",
      "name: Tesla P100-PCIE-16GB major: 6 minor: 0 memoryClockRate(GHz): 1.3285\n",
      "pciBusID: 0000:85:00.0\n",
      "totalMemory: 15.90GiB freeMemory: 15.64GiB\n",
      "2019-11-02 20:03:53.802130: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1512] Adding visible gpu devices: 0, 1\n",
      "2019-11-02 20:03:53.805022: I tensorflow/core/common_runtime/gpu/gpu_device.cc:984] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2019-11-02 20:03:53.805043: I tensorflow/core/common_runtime/gpu/gpu_device.cc:990]      0 1 \n",
      "2019-11-02 20:03:53.805052: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 0:   N N \n",
      "2019-11-02 20:03:53.805058: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1003] 1:   N N \n",
      "2019-11-02 20:03:53.806474: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 15216 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:05:00.0, compute capability: 6.0)\n",
      "2019-11-02 20:03:53.806868: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:1 with 15216 MB memory) -> physical GPU (device: 1, name: Tesla P100-PCIE-16GB, pci bus id: 0000:85:00.0, compute capability: 6.0)\n",
      "2019-11-02 20:04:16.101450: I tensorflow/stream_executor/dso_loader.cc:152] successfully opened CUDA library libcublas.so.10.0 locally\n",
      "I1102 20:04:23.785169 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "0\n",
      "I1102 20:13:27.694873 47329276351104 train_gpu.py:302] [500] | gnorm 9.14 lr 0.000100 | loss 2.67 | pplx   14.39, bpc  3.8469\n",
      "I1102 20:22:31.301795 47329276351104 train_gpu.py:302] [1000] | gnorm 7.03 lr 0.000100 | loss 1.91 | pplx    6.75, bpc  2.7549\n",
      "I1102 20:31:35.122084 47329276351104 train_gpu.py:302] [1500] | gnorm 8.28 lr 0.000099 | loss 1.72 | pplx    5.59, bpc  2.4823\n",
      "I1102 20:40:39.089791 47329276351104 train_gpu.py:302] [2000] | gnorm 8.58 lr 0.000099 | loss 1.61 | pplx    5.00, bpc  2.3231\n",
      "I1102 20:49:43.063775 47329276351104 train_gpu.py:302] [2500] | gnorm 7.27 lr 0.000099 | loss 1.55 | pplx    4.70, bpc  2.2328\n",
      "I1102 20:58:48.017932 47329276351104 train_gpu.py:302] [3000] | gnorm 7.58 lr 0.000099 | loss 1.50 | pplx    4.49, bpc  2.1676\n",
      "I1102 21:07:52.006306 47329276351104 train_gpu.py:302] [3500] | gnorm 7.18 lr 0.000098 | loss 1.45 | pplx    4.26, bpc  2.0921\n",
      "I1102 21:16:56.223449 47329276351104 train_gpu.py:302] [4000] | gnorm 7.60 lr 0.000098 | loss 1.45 | pplx    4.26, bpc  2.0907\n",
      "I1102 21:26:00.139831 47329276351104 train_gpu.py:302] [4500] | gnorm 6.30 lr 0.000098 | loss 1.41 | pplx    4.11, bpc  2.0384\n",
      "I1102 21:35:04.257607 47329276351104 train_gpu.py:302] [5000] | gnorm 6.42 lr 0.000098 | loss 1.38 | pplx    3.97, bpc  1.9878\n",
      "I1102 21:35:08.001638 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "5000\n",
      "I1102 21:44:12.031682 47329276351104 train_gpu.py:302] [5500] | gnorm 6.03 lr 0.000097 | loss 1.35 | pplx    3.85, bpc  1.9460\n",
      "I1102 21:53:15.808625 47329276351104 train_gpu.py:302] [6000] | gnorm 7.09 lr 0.000097 | loss 1.33 | pplx    3.77, bpc  1.9139\n",
      "I1102 22:02:19.982499 47329276351104 train_gpu.py:302] [6500] | gnorm 7.31 lr 0.000097 | loss 1.31 | pplx    3.70, bpc  1.8880\n",
      "I1102 22:11:23.881095 47329276351104 train_gpu.py:302] [7000] | gnorm 7.05 lr 0.000097 | loss 1.29 | pplx    3.64, bpc  1.8655\n",
      "I1102 22:20:28.275289 47329276351104 train_gpu.py:302] [7500] | gnorm 6.33 lr 0.000096 | loss 1.29 | pplx    3.64, bpc  1.8628\n",
      "I1102 22:29:32.534976 47329276351104 train_gpu.py:302] [8000] | gnorm 6.87 lr 0.000096 | loss 1.30 | pplx    3.67, bpc  1.8748\n",
      "I1102 22:38:36.741188 47329276351104 train_gpu.py:302] [8500] | gnorm 6.39 lr 0.000096 | loss 1.27 | pplx    3.57, bpc  1.8358\n",
      "I1102 22:47:40.792493 47329276351104 train_gpu.py:302] [9000] | gnorm 7.08 lr 0.000096 | loss 1.25 | pplx    3.49, bpc  1.8013\n",
      "I1102 22:56:45.297809 47329276351104 train_gpu.py:302] [9500] | gnorm 7.06 lr 0.000095 | loss 1.22 | pplx    3.39, bpc  1.7623\n",
      "I1102 23:05:49.442130 47329276351104 train_gpu.py:302] [10000] | gnorm 6.07 lr 0.000095 | loss 1.26 | pplx    3.53, bpc  1.8177\n",
      "I1102 23:05:53.480527 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "10000\n",
      "I1102 23:14:57.385156 47329276351104 train_gpu.py:302] [10500] | gnorm 6.24 lr 0.000095 | loss 1.22 | pplx    3.39, bpc  1.7615\n",
      "I1102 23:24:02.343833 47329276351104 train_gpu.py:302] [11000] | gnorm 6.05 lr 0.000095 | loss 1.22 | pplx    3.40, bpc  1.7646\n",
      "I1102 23:33:06.177774 47329276351104 train_gpu.py:302] [11500] | gnorm 6.03 lr 0.000094 | loss 1.23 | pplx    3.44, bpc  1.7808\n",
      "I1102 23:42:10.263979 47329276351104 train_gpu.py:302] [12000] | gnorm 5.90 lr 0.000094 | loss 1.18 | pplx    3.27, bpc  1.7095\n",
      "I1102 23:51:14.742236 47329276351104 train_gpu.py:302] [12500] | gnorm 5.88 lr 0.000094 | loss 1.20 | pplx    3.32, bpc  1.7330\n",
      "I1103 00:00:18.906215 47329276351104 train_gpu.py:302] [13000] | gnorm 5.67 lr 0.000094 | loss 1.17 | pplx    3.23, bpc  1.6908\n",
      "I1103 00:09:22.919795 47329276351104 train_gpu.py:302] [13500] | gnorm 5.61 lr 0.000093 | loss 1.16 | pplx    3.19, bpc  1.6747\n",
      "I1103 00:18:27.708111 47329276351104 train_gpu.py:302] [14000] | gnorm 7.32 lr 0.000093 | loss 1.15 | pplx    3.16, bpc  1.6582\n",
      "I1103 00:27:32.099175 47329276351104 train_gpu.py:302] [14500] | gnorm 6.42 lr 0.000093 | loss 1.15 | pplx    3.14, bpc  1.6524\n",
      "I1103 00:36:36.831947 47329276351104 train_gpu.py:302] [15000] | gnorm 6.66 lr 0.000093 | loss 1.14 | pplx    3.13, bpc  1.6440\n",
      "I1103 00:36:40.616602 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "15000\n",
      "I1103 00:45:45.379650 47329276351104 train_gpu.py:302] [15500] | gnorm 6.20 lr 0.000092 | loss 1.16 | pplx    3.19, bpc  1.6721\n",
      "I1103 00:54:49.422549 47329276351104 train_gpu.py:302] [16000] | gnorm 5.91 lr 0.000092 | loss 1.18 | pplx    3.24, bpc  1.6978\n",
      "I1103 01:03:53.344932 47329276351104 train_gpu.py:302] [16500] | gnorm 6.56 lr 0.000092 | loss 1.17 | pplx    3.22, bpc  1.6858\n",
      "I1103 01:12:57.749879 47329276351104 train_gpu.py:302] [17000] | gnorm 6.74 lr 0.000092 | loss 1.15 | pplx    3.16, bpc  1.6616\n",
      "I1103 01:22:02.013714 47329276351104 train_gpu.py:302] [17500] | gnorm 6.49 lr 0.000091 | loss 1.16 | pplx    3.20, bpc  1.6761\n",
      "I1103 01:31:06.318476 47329276351104 train_gpu.py:302] [18000] | gnorm 6.62 lr 0.000091 | loss 1.16 | pplx    3.17, bpc  1.6665\n",
      "I1103 01:40:13.065076 47329276351104 train_gpu.py:302] [18500] | gnorm 5.83 lr 0.000091 | loss 1.15 | pplx    3.16, bpc  1.6587\n",
      "I1103 01:49:20.912509 47329276351104 train_gpu.py:302] [19000] | gnorm 6.02 lr 0.000091 | loss 1.13 | pplx    3.09, bpc  1.6271\n",
      "I1103 01:58:28.755146 47329276351104 train_gpu.py:302] [19500] | gnorm 5.48 lr 0.000090 | loss 1.13 | pplx    3.11, bpc  1.6353\n",
      "I1103 01:07:39.113584 47329276351104 train_gpu.py:302] [20000] | gnorm 6.10 lr 0.000090 | loss 1.11 | pplx    3.03, bpc  1.5972\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I1103 01:07:42.963507 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "20000\n",
      "I1103 01:16:52.842212 47329276351104 train_gpu.py:302] [20500] | gnorm 5.85 lr 0.000090 | loss 1.14 | pplx    3.11, bpc  1.6377\n",
      "I1103 01:26:03.266377 47329276351104 train_gpu.py:302] [21000] | gnorm 6.19 lr 0.000090 | loss 1.14 | pplx    3.14, bpc  1.6492\n",
      "I1103 01:35:13.472800 47329276351104 train_gpu.py:302] [21500] | gnorm 6.27 lr 0.000089 | loss 1.13 | pplx    3.08, bpc  1.6250\n",
      "I1103 01:44:22.371781 47329276351104 train_gpu.py:302] [22000] | gnorm 6.33 lr 0.000089 | loss 1.10 | pplx    3.01, bpc  1.5897\n",
      "I1103 01:53:31.341625 47329276351104 train_gpu.py:302] [22500] | gnorm 6.36 lr 0.000089 | loss 1.07 | pplx    2.92, bpc  1.5476\n",
      "I1103 02:02:39.925597 47329276351104 train_gpu.py:302] [23000] | gnorm 6.28 lr 0.000089 | loss 1.12 | pplx    3.06, bpc  1.6130\n",
      "I1103 02:11:49.331837 47329276351104 train_gpu.py:302] [23500] | gnorm 5.83 lr 0.000088 | loss 1.10 | pplx    3.01, bpc  1.5921\n",
      "I1103 02:20:58.480799 47329276351104 train_gpu.py:302] [24000] | gnorm 5.55 lr 0.000088 | loss 1.11 | pplx    3.03, bpc  1.6015\n",
      "I1103 02:30:07.698126 47329276351104 train_gpu.py:302] [24500] | gnorm 5.42 lr 0.000088 | loss 1.08 | pplx    2.94, bpc  1.5540\n",
      "I1103 02:39:17.648651 47329276351104 train_gpu.py:302] [25000] | gnorm 5.61 lr 0.000088 | loss 1.08 | pplx    2.96, bpc  1.5634\n",
      "I1103 02:39:21.746845 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "25000\n",
      "I1103 02:48:30.985956 47329276351104 train_gpu.py:302] [25500] | gnorm 5.12 lr 0.000087 | loss 1.07 | pplx    2.91, bpc  1.5420\n",
      "I1103 02:57:40.001794 47329276351104 train_gpu.py:302] [26000] | gnorm 5.82 lr 0.000087 | loss 1.08 | pplx    2.96, bpc  1.5653\n",
      "I1103 03:06:49.969990 47329276351104 train_gpu.py:302] [26500] | gnorm 5.60 lr 0.000087 | loss 1.10 | pplx    3.02, bpc  1.5928\n",
      "I1103 03:15:58.926578 47329276351104 train_gpu.py:302] [27000] | gnorm 5.78 lr 0.000087 | loss 1.08 | pplx    2.95, bpc  1.5598\n",
      "I1103 03:25:07.691721 47329276351104 train_gpu.py:302] [27500] | gnorm 5.27 lr 0.000086 | loss 1.07 | pplx    2.91, bpc  1.5394\n",
      "I1103 03:34:16.405913 47329276351104 train_gpu.py:302] [28000] | gnorm 5.80 lr 0.000086 | loss 1.08 | pplx    2.96, bpc  1.5653\n",
      "I1103 03:43:26.786362 47329276351104 train_gpu.py:302] [28500] | gnorm 5.15 lr 0.000086 | loss 1.07 | pplx    2.90, bpc  1.5368\n",
      "I1103 03:52:36.471677 47329276351104 train_gpu.py:302] [29000] | gnorm 6.16 lr 0.000086 | loss 1.06 | pplx    2.88, bpc  1.5283\n",
      "I1103 04:01:45.502117 47329276351104 train_gpu.py:302] [29500] | gnorm 5.81 lr 0.000085 | loss 1.10 | pplx    2.99, bpc  1.5825\n",
      "I1103 04:10:56.036396 47329276351104 train_gpu.py:302] [30000] | gnorm 5.86 lr 0.000085 | loss 1.07 | pplx    2.92, bpc  1.5477\n",
      "I1103 04:11:00.057079 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "30000\n",
      "I1103 04:20:09.052572 47329276351104 train_gpu.py:302] [30500] | gnorm 6.35 lr 0.000085 | loss 1.05 | pplx    2.87, bpc  1.5215\n",
      "I1103 04:29:17.800527 47329276351104 train_gpu.py:302] [31000] | gnorm 5.51 lr 0.000085 | loss 1.09 | pplx    2.97, bpc  1.5707\n",
      "I1103 04:38:27.831639 47329276351104 train_gpu.py:302] [31500] | gnorm 6.37 lr 0.000084 | loss 1.05 | pplx    2.86, bpc  1.5138\n",
      "I1103 04:47:36.850785 47329276351104 train_gpu.py:302] [32000] | gnorm 5.33 lr 0.000084 | loss 1.06 | pplx    2.89, bpc  1.5320\n",
      "I1103 04:56:45.650449 47329276351104 train_gpu.py:302] [32500] | gnorm 6.38 lr 0.000084 | loss 1.08 | pplx    2.93, bpc  1.5514\n",
      "I1103 05:05:54.284387 47329276351104 train_gpu.py:302] [33000] | gnorm 6.22 lr 0.000084 | loss 1.07 | pplx    2.92, bpc  1.5477\n",
      "I1103 05:15:04.052057 47329276351104 train_gpu.py:302] [33500] | gnorm 4.98 lr 0.000083 | loss 1.03 | pplx    2.81, bpc  1.4898\n",
      "I1103 05:24:13.025470 47329276351104 train_gpu.py:302] [34000] | gnorm 5.43 lr 0.000083 | loss 1.06 | pplx    2.88, bpc  1.5240\n",
      "I1103 05:33:21.128034 47329276351104 train_gpu.py:302] [34500] | gnorm 6.45 lr 0.000083 | loss 1.05 | pplx    2.87, bpc  1.5208\n",
      "I1103 05:42:30.687968 47329276351104 train_gpu.py:302] [35000] | gnorm 6.08 lr 0.000083 | loss 1.01 | pplx    2.75, bpc  1.4569\n",
      "I1103 05:42:34.821827 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "35000\n",
      "I1103 05:51:43.592861 47329276351104 train_gpu.py:302] [35500] | gnorm 4.98 lr 0.000082 | loss 1.05 | pplx    2.87, bpc  1.5204\n",
      "I1103 06:00:52.668806 47329276351104 train_gpu.py:302] [36000] | gnorm 5.05 lr 0.000082 | loss 1.03 | pplx    2.81, bpc  1.4892\n",
      "I1103 06:10:02.014807 47329276351104 train_gpu.py:302] [36500] | gnorm 5.97 lr 0.000082 | loss 1.04 | pplx    2.84, bpc  1.5065\n",
      "I1103 06:19:11.302567 47329276351104 train_gpu.py:302] [37000] | gnorm 5.58 lr 0.000082 | loss 1.03 | pplx    2.81, bpc  1.4888\n",
      "I1103 06:28:20.515439 47329276351104 train_gpu.py:302] [37500] | gnorm 6.27 lr 0.000081 | loss 1.03 | pplx    2.80, bpc  1.4870\n",
      "I1103 06:37:29.650047 47329276351104 train_gpu.py:302] [38000] | gnorm 5.95 lr 0.000081 | loss 1.04 | pplx    2.84, bpc  1.5042\n",
      "I1103 06:46:39.165226 47329276351104 train_gpu.py:302] [38500] | gnorm 5.37 lr 0.000081 | loss 1.05 | pplx    2.86, bpc  1.5147\n",
      "I1103 06:55:48.457707 47329276351104 train_gpu.py:302] [39000] | gnorm 5.86 lr 0.000081 | loss 1.05 | pplx    2.84, bpc  1.5079\n",
      "I1103 07:04:57.876276 47329276351104 train_gpu.py:302] [39500] | gnorm 5.90 lr 0.000080 | loss 1.03 | pplx    2.81, bpc  1.4909\n",
      "I1103 07:14:07.952716 47329276351104 train_gpu.py:302] [40000] | gnorm 5.53 lr 0.000080 | loss 1.02 | pplx    2.77, bpc  1.4677\n",
      "I1103 07:14:12.230807 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "40000\n",
      "I1103 07:23:21.568805 47329276351104 train_gpu.py:302] [40500] | gnorm 5.43 lr 0.000080 | loss 1.01 | pplx    2.74, bpc  1.4548\n",
      "I1103 07:32:30.270199 47329276351104 train_gpu.py:302] [41000] | gnorm 5.28 lr 0.000080 | loss 1.01 | pplx    2.76, bpc  1.4634\n",
      "I1103 07:41:40.864822 47329276351104 train_gpu.py:302] [41500] | gnorm 5.23 lr 0.000079 | loss 1.04 | pplx    2.82, bpc  1.4969\n",
      "I1103 07:50:49.837156 47329276351104 train_gpu.py:302] [42000] | gnorm 5.63 lr 0.000079 | loss 1.02 | pplx    2.77, bpc  1.4710\n",
      "I1103 07:59:59.013343 47329276351104 train_gpu.py:302] [42500] | gnorm 6.50 lr 0.000079 | loss 1.04 | pplx    2.83, bpc  1.5014\n",
      "I1103 08:09:07.761728 47329276351104 train_gpu.py:302] [43000] | gnorm 5.66 lr 0.000079 | loss 1.03 | pplx    2.81, bpc  1.4901\n",
      "I1103 08:18:17.245540 47329276351104 train_gpu.py:302] [43500] | gnorm 5.72 lr 0.000078 | loss 1.01 | pplx    2.75, bpc  1.4609\n",
      "I1103 08:27:26.356765 47329276351104 train_gpu.py:302] [44000] | gnorm 5.75 lr 0.000078 | loss 1.01 | pplx    2.74, bpc  1.4529\n",
      "I1103 08:36:35.227875 47329276351104 train_gpu.py:302] [44500] | gnorm 6.24 lr 0.000078 | loss 0.99 | pplx    2.70, bpc  1.4322\n",
      "I1103 08:45:45.402005 47329276351104 train_gpu.py:302] [45000] | gnorm 6.04 lr 0.000078 | loss 1.03 | pplx    2.80, bpc  1.4864\n",
      "I1103 08:45:49.491271 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "45000\n",
      "I1103 08:54:57.620000 47329276351104 train_gpu.py:302] [45500] | gnorm 6.05 lr 0.000077 | loss 1.02 | pplx    2.79, bpc  1.4781\n",
      "I1103 09:04:05.545472 47329276351104 train_gpu.py:302] [46000] | gnorm 5.21 lr 0.000077 | loss 1.02 | pplx    2.77, bpc  1.4675\n",
      "I1103 09:13:13.615065 47329276351104 train_gpu.py:302] [46500] | gnorm 6.21 lr 0.000077 | loss 1.02 | pplx    2.78, bpc  1.4754\n",
      "I1103 09:22:22.073972 47329276351104 train_gpu.py:302] [47000] | gnorm 6.25 lr 0.000077 | loss 0.98 | pplx    2.67, bpc  1.4164\n",
      "I1103 09:31:30.153419 47329276351104 train_gpu.py:302] [47500] | gnorm 5.83 lr 0.000076 | loss 1.00 | pplx    2.73, bpc  1.4474\n",
      "I1103 09:40:38.482858 47329276351104 train_gpu.py:302] [48000] | gnorm 5.65 lr 0.000076 | loss 1.00 | pplx    2.72, bpc  1.4432\n",
      "I1103 09:49:46.424936 47329276351104 train_gpu.py:302] [48500] | gnorm 6.00 lr 0.000076 | loss 1.01 | pplx    2.75, bpc  1.4576\n",
      "I1103 09:58:54.084217 47329276351104 train_gpu.py:302] [49000] | gnorm 6.06 lr 0.000076 | loss 0.98 | pplx    2.67, bpc  1.4151\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I1103 10:08:01.925547 47329276351104 train_gpu.py:302] [49500] | gnorm 5.30 lr 0.000075 | loss 0.97 | pplx    2.64, bpc  1.3997\n",
      "I1103 10:17:10.813624 47329276351104 train_gpu.py:302] [50000] | gnorm 5.69 lr 0.000075 | loss 0.90 | pplx    2.46, bpc  1.2983\n",
      "I1103 10:17:14.763897 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "50000\n",
      "I1103 10:26:22.588086 47329276351104 train_gpu.py:302] [50500] | gnorm 6.05 lr 0.000075 | loss 0.93 | pplx    2.52, bpc  1.3356\n",
      "I1103 10:35:30.125064 47329276351104 train_gpu.py:302] [51000] | gnorm 5.75 lr 0.000075 | loss 0.92 | pplx    2.52, bpc  1.3330\n",
      "I1103 10:44:37.771480 47329276351104 train_gpu.py:302] [51500] | gnorm 5.18 lr 0.000074 | loss 0.94 | pplx    2.56, bpc  1.3581\n",
      "I1103 10:53:45.779591 47329276351104 train_gpu.py:302] [52000] | gnorm 6.38 lr 0.000074 | loss 0.92 | pplx    2.51, bpc  1.3287\n",
      "I1103 11:02:54.049780 47329276351104 train_gpu.py:302] [52500] | gnorm 6.30 lr 0.000074 | loss 0.94 | pplx    2.55, bpc  1.3520\n",
      "I1103 11:12:02.148454 47329276351104 train_gpu.py:302] [53000] | gnorm 5.82 lr 0.000074 | loss 0.95 | pplx    2.60, bpc  1.3771\n",
      "I1103 11:21:10.445872 47329276351104 train_gpu.py:302] [53500] | gnorm 6.61 lr 0.000073 | loss 0.98 | pplx    2.65, bpc  1.4068\n",
      "I1103 11:30:18.170047 47329276351104 train_gpu.py:302] [54000] | gnorm 6.19 lr 0.000073 | loss 0.97 | pplx    2.62, bpc  1.3923\n",
      "I1103 11:39:25.937669 47329276351104 train_gpu.py:302] [54500] | gnorm 6.13 lr 0.000073 | loss 0.93 | pplx    2.54, bpc  1.3463\n",
      "I1103 11:48:33.814721 47329276351104 train_gpu.py:302] [55000] | gnorm 6.80 lr 0.000073 | loss 0.94 | pplx    2.55, bpc  1.3494\n",
      "I1103 11:48:37.864858 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "55000\n",
      "I1103 11:57:44.760095 47329276351104 train_gpu.py:302] [55500] | gnorm 5.28 lr 0.000072 | loss 0.95 | pplx    2.58, bpc  1.3666\n",
      "I1103 12:06:52.694161 47329276351104 train_gpu.py:302] [56000] | gnorm 5.56 lr 0.000072 | loss 0.94 | pplx    2.56, bpc  1.3552\n",
      "I1103 12:16:00.290042 47329276351104 train_gpu.py:302] [56500] | gnorm 6.13 lr 0.000072 | loss 0.96 | pplx    2.62, bpc  1.3879\n",
      "I1103 12:25:07.779495 47329276351104 train_gpu.py:302] [57000] | gnorm 5.93 lr 0.000072 | loss 0.93 | pplx    2.54, bpc  1.3446\n",
      "I1103 12:34:14.470006 47329276351104 train_gpu.py:302] [57500] | gnorm 5.45 lr 0.000071 | loss 0.92 | pplx    2.52, bpc  1.3336\n",
      "I1103 12:43:21.998704 47329276351104 train_gpu.py:302] [58000] | gnorm 5.64 lr 0.000071 | loss 0.95 | pplx    2.58, bpc  1.3677\n",
      "I1103 12:52:29.658291 47329276351104 train_gpu.py:302] [58500] | gnorm 5.93 lr 0.000071 | loss 0.94 | pplx    2.56, bpc  1.3549\n",
      "I1103 13:01:37.084585 47329276351104 train_gpu.py:302] [59000] | gnorm 5.32 lr 0.000071 | loss 0.94 | pplx    2.56, bpc  1.3564\n",
      "I1103 13:10:45.217329 47329276351104 train_gpu.py:302] [59500] | gnorm 5.57 lr 0.000070 | loss 0.95 | pplx    2.59, bpc  1.3707\n",
      "I1103 13:19:53.014325 47329276351104 train_gpu.py:302] [60000] | gnorm 5.14 lr 0.000070 | loss 0.94 | pplx    2.55, bpc  1.3505\n",
      "I1103 13:19:56.915286 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "60000\n",
      "I1103 13:29:04.919598 47329276351104 train_gpu.py:302] [60500] | gnorm 6.22 lr 0.000070 | loss 0.93 | pplx    2.53, bpc  1.3375\n",
      "I1103 13:38:12.238294 47329276351104 train_gpu.py:302] [61000] | gnorm 5.14 lr 0.000070 | loss 0.94 | pplx    2.57, bpc  1.3625\n",
      "I1103 13:47:20.182641 47329276351104 train_gpu.py:302] [61500] | gnorm 5.69 lr 0.000069 | loss 0.93 | pplx    2.52, bpc  1.3360\n",
      "I1103 13:56:28.321769 47329276351104 train_gpu.py:302] [62000] | gnorm 5.48 lr 0.000069 | loss 0.94 | pplx    2.56, bpc  1.3569\n",
      "I1103 14:05:37.236435 47329276351104 train_gpu.py:302] [62500] | gnorm 6.12 lr 0.000069 | loss 0.95 | pplx    2.58, bpc  1.3699\n",
      "I1103 14:14:45.742495 47329276351104 train_gpu.py:302] [63000] | gnorm 6.09 lr 0.000069 | loss 0.93 | pplx    2.53, bpc  1.3374\n",
      "I1103 14:23:54.198764 47329276351104 train_gpu.py:302] [63500] | gnorm 5.59 lr 0.000068 | loss 0.94 | pplx    2.55, bpc  1.3500\n",
      "I1103 14:33:02.066759 47329276351104 train_gpu.py:302] [64000] | gnorm 5.83 lr 0.000068 | loss 0.93 | pplx    2.54, bpc  1.3440\n",
      "I1103 14:42:09.606336 47329276351104 train_gpu.py:302] [64500] | gnorm 6.00 lr 0.000068 | loss 0.94 | pplx    2.55, bpc  1.3527\n",
      "I1103 14:51:17.739230 47329276351104 train_gpu.py:302] [65000] | gnorm 6.50 lr 0.000068 | loss 0.93 | pplx    2.54, bpc  1.3469\n",
      "I1103 14:51:21.553015 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "65000\n",
      "I1103 15:00:29.781442 47329276351104 train_gpu.py:302] [65500] | gnorm 5.43 lr 0.000067 | loss 0.92 | pplx    2.52, bpc  1.3318\n",
      "I1103 15:09:37.195111 47329276351104 train_gpu.py:302] [66000] | gnorm 6.37 lr 0.000067 | loss 0.94 | pplx    2.55, bpc  1.3491\n",
      "I1103 15:18:44.939859 47329276351104 train_gpu.py:302] [66500] | gnorm 6.25 lr 0.000067 | loss 0.94 | pplx    2.57, bpc  1.3606\n",
      "I1103 15:27:53.093615 47329276351104 train_gpu.py:302] [67000] | gnorm 5.60 lr 0.000067 | loss 0.93 | pplx    2.54, bpc  1.3475\n",
      "I1103 15:37:01.176233 47329276351104 train_gpu.py:302] [67500] | gnorm 4.76 lr 0.000066 | loss 0.93 | pplx    2.54, bpc  1.3440\n",
      "I1103 15:46:08.637166 47329276351104 train_gpu.py:302] [68000] | gnorm 5.10 lr 0.000066 | loss 0.92 | pplx    2.50, bpc  1.3210\n",
      "I1103 15:55:17.259878 47329276351104 train_gpu.py:302] [68500] | gnorm 6.14 lr 0.000066 | loss 0.95 | pplx    2.58, bpc  1.3675\n",
      "I1103 16:04:25.007515 47329276351104 train_gpu.py:302] [69000] | gnorm 4.81 lr 0.000066 | loss 0.95 | pplx    2.58, bpc  1.3678\n",
      "I1103 16:13:33.873629 47329276351104 train_gpu.py:302] [69500] | gnorm 5.02 lr 0.000065 | loss 0.93 | pplx    2.55, bpc  1.3489\n",
      "I1103 16:22:42.598965 47329276351104 train_gpu.py:302] [70000] | gnorm 6.09 lr 0.000065 | loss 0.94 | pplx    2.56, bpc  1.3584\n",
      "I1103 16:22:46.704836 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "70000\n",
      "I1103 16:31:56.567209 47329276351104 train_gpu.py:302] [70500] | gnorm 6.47 lr 0.000065 | loss 0.93 | pplx    2.53, bpc  1.3395\n",
      "I1103 16:41:05.198556 47329276351104 train_gpu.py:302] [71000] | gnorm 5.25 lr 0.000065 | loss 0.92 | pplx    2.52, bpc  1.3311\n",
      "I1103 16:50:13.690127 47329276351104 train_gpu.py:302] [71500] | gnorm 5.81 lr 0.000064 | loss 0.92 | pplx    2.52, bpc  1.3333\n",
      "I1103 16:59:23.260864 47329276351104 train_gpu.py:302] [72000] | gnorm 5.60 lr 0.000064 | loss 0.91 | pplx    2.49, bpc  1.3135\n",
      "I1103 17:08:34.243763 47329276351104 train_gpu.py:302] [72500] | gnorm 5.86 lr 0.000064 | loss 0.90 | pplx    2.46, bpc  1.2974\n",
      "I1103 17:17:45.468695 47329276351104 train_gpu.py:302] [73000] | gnorm 5.56 lr 0.000064 | loss 0.90 | pplx    2.47, bpc  1.3035\n",
      "I1103 17:26:57.023956 47329276351104 train_gpu.py:302] [73500] | gnorm 6.14 lr 0.000063 | loss 0.93 | pplx    2.53, bpc  1.3419\n",
      "I1103 17:36:08.325029 47329276351104 train_gpu.py:302] [74000] | gnorm 6.82 lr 0.000063 | loss 0.91 | pplx    2.48, bpc  1.3122\n",
      "I1103 17:45:20.357656 47329276351104 train_gpu.py:302] [74500] | gnorm 6.13 lr 0.000063 | loss 0.93 | pplx    2.53, bpc  1.3415\n",
      "I1103 17:54:32.004908 47329276351104 train_gpu.py:302] [75000] | gnorm 6.30 lr 0.000063 | loss 0.92 | pplx    2.51, bpc  1.3271\n",
      "I1103 17:54:35.873255 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "75000\n",
      "I1103 18:03:47.942958 47329276351104 train_gpu.py:302] [75500] | gnorm 5.90 lr 0.000062 | loss 0.91 | pplx    2.47, bpc  1.3073\n",
      "I1103 18:12:59.271747 47329276351104 train_gpu.py:302] [76000] | gnorm 5.34 lr 0.000062 | loss 0.93 | pplx    2.55, bpc  1.3479\n",
      "I1103 18:22:09.277253 47329276351104 train_gpu.py:302] [76500] | gnorm 5.80 lr 0.000062 | loss 0.92 | pplx    2.51, bpc  1.3276\n",
      "I1103 18:31:18.262335 47329276351104 train_gpu.py:302] [77000] | gnorm 5.59 lr 0.000062 | loss 0.91 | pplx    2.47, bpc  1.3071\n",
      "I1103 18:40:26.977657 47329276351104 train_gpu.py:302] [77500] | gnorm 5.06 lr 0.000061 | loss 0.91 | pplx    2.48, bpc  1.3123\n",
      "I1103 18:49:36.469582 47329276351104 train_gpu.py:302] [78000] | gnorm 6.01 lr 0.000061 | loss 0.92 | pplx    2.51, bpc  1.3284\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I1103 18:58:45.751961 47329276351104 train_gpu.py:302] [78500] | gnorm 5.63 lr 0.000061 | loss 0.93 | pplx    2.52, bpc  1.3351\n",
      "I1103 19:07:57.169102 47329276351104 train_gpu.py:302] [79000] | gnorm 6.15 lr 0.000061 | loss 0.94 | pplx    2.56, bpc  1.3545\n",
      "I1103 19:17:08.623243 47329276351104 train_gpu.py:302] [79500] | gnorm 5.65 lr 0.000060 | loss 0.92 | pplx    2.50, bpc  1.3215\n",
      "I1103 19:26:20.618197 47329276351104 train_gpu.py:302] [80000] | gnorm 5.62 lr 0.000060 | loss 0.91 | pplx    2.50, bpc  1.3192\n",
      "I1103 19:26:24.568567 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "80000\n",
      "I1103 19:35:36.019667 47329276351104 train_gpu.py:302] [80500] | gnorm 6.06 lr 0.000060 | loss 0.93 | pplx    2.54, bpc  1.3470\n",
      "I1103 19:44:46.459486 47329276351104 train_gpu.py:302] [81000] | gnorm 5.29 lr 0.000060 | loss 0.92 | pplx    2.52, bpc  1.3312\n",
      "I1103 19:53:57.337118 47329276351104 train_gpu.py:302] [81500] | gnorm 5.89 lr 0.000059 | loss 0.94 | pplx    2.56, bpc  1.3560\n",
      "I1103 20:03:03.639304 47329276351104 train_gpu.py:302] [82000] | gnorm 5.27 lr 0.000059 | loss 0.92 | pplx    2.51, bpc  1.3302\n",
      "I1103 20:12:11.247814 47329276351104 train_gpu.py:302] [82500] | gnorm 5.52 lr 0.000059 | loss 0.91 | pplx    2.49, bpc  1.3181\n",
      "I1103 20:24:44.855023 47329276351104 train_gpu.py:302] [83000] | gnorm 6.01 lr 0.000059 | loss 0.90 | pplx    2.45, bpc  1.2943\n",
      "I1103 20:34:09.295480 47329276351104 train_gpu.py:302] [83500] | gnorm 5.26 lr 0.000058 | loss 0.91 | pplx    2.49, bpc  1.3175\n",
      "I1103 20:43:23.143309 47329276351104 train_gpu.py:302] [84000] | gnorm 5.68 lr 0.000058 | loss 0.88 | pplx    2.42, bpc  1.2751\n",
      "I1103 20:52:49.122251 47329276351104 train_gpu.py:302] [84500] | gnorm 5.71 lr 0.000058 | loss 0.91 | pplx    2.47, bpc  1.3061\n",
      "I1103 21:07:30.426741 47329276351104 train_gpu.py:302] [85000] | gnorm 4.95 lr 0.000058 | loss 0.90 | pplx    2.47, bpc  1.3049\n",
      "I1103 21:07:44.247541 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "85000\n",
      "I1103 21:16:54.408882 47329276351104 train_gpu.py:302] [85500] | gnorm 5.09 lr 0.000057 | loss 0.89 | pplx    2.43, bpc  1.2782\n",
      "I1103 21:29:09.538335 47329276351104 train_gpu.py:302] [86000] | gnorm 5.73 lr 0.000057 | loss 0.90 | pplx    2.46, bpc  1.2994\n",
      "I1103 21:44:24.803336 47329276351104 train_gpu.py:302] [86500] | gnorm 5.08 lr 0.000057 | loss 0.89 | pplx    2.44, bpc  1.2871\n",
      "I1103 22:07:14.059116 47329276351104 train_gpu.py:302] [87000] | gnorm 5.44 lr 0.000057 | loss 0.93 | pplx    2.52, bpc  1.3349\n",
      "I1103 22:25:11.858206 47329276351104 train_gpu.py:302] [87500] | gnorm 5.76 lr 0.000056 | loss 0.93 | pplx    2.54, bpc  1.3427\n",
      "I1103 22:38:12.156380 47329276351104 train_gpu.py:302] [88000] | gnorm 5.32 lr 0.000056 | loss 0.90 | pplx    2.45, bpc  1.2957\n",
      "I1103 22:47:21.932763 47329276351104 train_gpu.py:302] [88500] | gnorm 6.31 lr 0.000056 | loss 0.93 | pplx    2.53, bpc  1.3411\n",
      "I1103 22:57:00.128980 47329276351104 train_gpu.py:302] [89000] | gnorm 5.28 lr 0.000056 | loss 0.91 | pplx    2.48, bpc  1.3121\n",
      "I1103 23:30:05.822106 47329276351104 train_gpu.py:302] [89500] | gnorm 5.80 lr 0.000055 | loss 0.90 | pplx    2.46, bpc  1.3012\n",
      "I1103 23:41:19.243920 47329276351104 train_gpu.py:302] [90000] | gnorm 6.00 lr 0.000055 | loss 0.92 | pplx    2.50, bpc  1.3216\n",
      "I1103 23:41:41.509939 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "90000\n",
      "I1103 23:50:52.616744 47329276351104 train_gpu.py:302] [90500] | gnorm 6.03 lr 0.000055 | loss 0.92 | pplx    2.52, bpc  1.3330\n",
      "I1104 00:00:01.709030 47329276351104 train_gpu.py:302] [91000] | gnorm 5.73 lr 0.000055 | loss 0.92 | pplx    2.51, bpc  1.3299\n",
      "I1104 00:09:10.551975 47329276351104 train_gpu.py:302] [91500] | gnorm 5.31 lr 0.000054 | loss 0.91 | pplx    2.48, bpc  1.3103\n",
      "I1104 00:18:41.928662 47329276351104 train_gpu.py:302] [92000] | gnorm 5.82 lr 0.000054 | loss 0.90 | pplx    2.46, bpc  1.2989\n",
      "I1104 00:27:52.805831 47329276351104 train_gpu.py:302] [92500] | gnorm 5.60 lr 0.000054 | loss 0.89 | pplx    2.45, bpc  1.2909\n",
      "I1104 00:37:02.828892 47329276351104 train_gpu.py:302] [93000] | gnorm 5.46 lr 0.000054 | loss 0.90 | pplx    2.47, bpc  1.3016\n",
      "I1104 00:46:13.312255 47329276351104 train_gpu.py:302] [93500] | gnorm 5.29 lr 0.000053 | loss 0.89 | pplx    2.45, bpc  1.2905\n",
      "I1104 00:55:23.149668 47329276351104 train_gpu.py:302] [94000] | gnorm 6.00 lr 0.000053 | loss 0.92 | pplx    2.50, bpc  1.3223\n",
      "I1104 01:04:33.838020 47329276351104 train_gpu.py:302] [94500] | gnorm 5.93 lr 0.000053 | loss 0.91 | pplx    2.48, bpc  1.3088\n",
      "I1104 01:33:20.079618 47329276351104 train_gpu.py:302] [95000] | gnorm 5.76 lr 0.000053 | loss 0.91 | pplx    2.48, bpc  1.3106\n",
      "I1104 01:39:54.055290 47329276351104 train_gpu.py:308] Model saved in path: /scratch/kh2383/MechVent/model/xlnet_model/model.ckpt\n",
      "200000\n",
      "95000\n",
      "I1104 01:49:04.292728 47329276351104 train_gpu.py:302] [95500] | gnorm 6.30 lr 0.000052 | loss 0.90 | pplx    2.47, bpc  1.3041\n"
     ]
    }
   ],
   "source": [
    "!python train_gpu.py \\\n",
    "    --record_info_dir=/scratch/kh2383/MechVent/data/xlnet_tfrecord/tfrecords/ \\\n",
    "    --model_dir=/scratch/kh2383/MechVent/model/xlnet_model/ \\\n",
    "    --init_checkpoint=/scratch/kh2383/MechVent/model/xlnet_model/model.ckpt \\\n",
    "    --train_batch_size=8 \\\n",
    "    --seq_len=512 \\\n",
    "    --reuse_len=256 \\\n",
    "    --mem_len=384 \\\n",
    "    --perm_size=256 \\\n",
    "    --n_layer=12 \\\n",
    "    --d_model=768 \\\n",
    "    --d_embed=768 \\\n",
    "    --n_head=12 \\\n",
    "    --d_head=64 \\\n",
    "    --d_inner=3072 \\\n",
    "    --untie_r=True \\\n",
    "    --mask_alpha=6 \\\n",
    "    --mask_beta=1 \\\n",
    "    --num_predict=85 \\\n",
    "    --num_hosts=1 \\\n",
    "    --num_core_per_host=2 \\\n",
    "    --train_steps=200000 \\\n",
    "    --iterations=500 \\\n",
    "    --save_steps=5000 "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}