Dataset production (UTH-CCB).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this produces & persists datasets for the UTH-CCB model\n",
    "\n",
    "import scipy\n",
    "from scipy.sparse import csr_matrix\n",
    "\n",
    "experiment=9 #8=referral, 9 = primary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# this is a reproduction of :\n",
    "\n",
    "# Xu, Jun, Yaoyun Zhang, Jingqi Wang, et al. \n",
    "# 2015UTH-CCB: The Participation of the SemEval 2015 Challenge–Task 14. Proceedings of SemEval-2015. \n",
    "# http://www.anthology.aclweb.org/S/S15/S15-2.pdf#page=353, accessed December 21, 2016.\n",
    "\n",
    "\n",
    "\n",
    "import negation_data\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sklearn\n",
    "from collections import Counter\n",
    "import gensim\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from pycorenlp import StanfordCoreNLP # requires the stanford CoreNLP server to be running\n",
    "\n",
    "corenlp = StanfordCoreNLP('http://localhost:9001')\n",
    "\n",
    "# 2 passes.  \n",
    "# 1 : extract features\n",
    "# 2 : create vectors\n",
    "\n",
    "\n",
    "# a feature extraction function is a function from a sentence + metadata to a dictionary of string -> real\n",
    "\n",
    "def extract_unigrams(study_id,tokens):\n",
    "\n",
    "    features=Counter()\n",
    "    for token in tokens:\n",
    "        features[token] += 1.0\n",
    "    return features\n",
    "\n",
    "def extract_bigrams(study_id,tokens):\n",
    "    stop=\"STOP\"\n",
    "\n",
    "    features=Counter()\n",
    "\n",
    "    # handle first and last tokens\n",
    "    features[stop + ' ' + tokens[0]] = 1.0\n",
    "    last_token = tokens[len(tokens) - 1]\n",
    "    features[last_token + '_' + stop] = 1.0\n",
    "    for idx, token in enumerate(tokens):\n",
    "        feature=\"\"\n",
    "        if idx==len(tokens) - 1:\n",
    "            break\n",
    "        feature = token + ' ' + tokens[idx + 1]\n",
    "        features[feature] += 1.0\n",
    "    \n",
    "    return features\n",
    "\n",
    "# regexes = list of tuples regex -> numpy array of feature representation\n",
    "# n = number of surrounding tokens to include in regex matching for phrase matching, ie n=1 compares token + 1 tokens either seide of token\n",
    "def extract_group_features(study_phrases, study_id, tokens):\n",
    "\n",
    "    import re\n",
    "    # handling multi token regex ie 'ruled out' is tricky with regards to tokenisation\n",
    "    # we proceed as follows:\n",
    "    # recombine tokens into a sentence \n",
    "    # match each regex to the sentence, and then determine if a subsequent match covers the span containing the token.\n",
    "    # nb only matches first instance of a regex match.  I can't find a way of matching all in python!!!\n",
    "\n",
    "\n",
    "    sentence = ' '.join(tokens)\n",
    "\n",
    "    # a list of regex matches -> feature vectors iif the regex matched the sentence\n",
    "    matches = list(filter(lambda x:x, [re.search(regex, sentence) for regex in study_phrases]))\n",
    "    features = Counter() # a dict of our features\n",
    "    token_start_position_in_sentence = 0 # where the token starts in the sentence\n",
    "    for idx, token in enumerate(tokens):\n",
    "        # print()\n",
    "        token_end_position_in_sentence = (token_start_position_in_sentence + len(token)) - 1\n",
    "        token_matched = False\n",
    "        # print('%s %s' % (token_start_position_in_sentence, token_end_position_in_sentence))\n",
    "        # print('considering %s' % token)\n",
    "        for match in matches:\n",
    "            start, end = match.span()\n",
    "            # print(match)\n",
    "            # print('span %s %s' % (start,end))\n",
    "            if token_start_position_in_sentence >= start and  token_end_position_in_sentence < end:\n",
    "                # token inside match, add the 8 surrounding tokens\n",
    "                # print('token ' + token + ' is matched')\n",
    "\n",
    "                # add the 8 preceding tokens as feature groups\n",
    "                for n in range(idx - 1, max(-1,idx-8), -1):\n",
    "                    token=tokens[n]\n",
    "                    group_label = -1 # the paper groups tokens into either L1, L4 or L8 labels based on how lcose the surrounding token is to the matched disease\n",
    "                    if idx-n == 1:\n",
    "                        group_label = '_L1'\n",
    "                    elif idx - n > 1 and idx - n  <= 4:\n",
    "                        group_label = '_L4'\n",
    "                    else:\n",
    "                        group_label = '_L8'\n",
    "\n",
    "                    feature = tokens[n] + group_label # ie 'said_L4' when 'said' within 4 tokens to the left of the match, but not the immeidately preceding token as this would get labelled _1\n",
    "                    features[feature] += 1.0\n",
    "\n",
    "                # add the 8 subsequent tokens as feature groups\n",
    "                for n in range(idx + 1, min(len(tokens),idx + 8)):\n",
    "                    token=tokens[n]\n",
    "                    group_label = -1 # the paper groups tokens into either L1, L4 or L8 labels based on how lcose the surrounding token is to the matched disease\n",
    "                    if n-idx == 1:\n",
    "                        group_label = '_R1'\n",
    "                    elif n-idx  > 1 and n-idx <= 4:\n",
    "                        group_label = '_R4'\n",
    "                    else:\n",
    "                        group_label = '_R8'\n",
    "\n",
    "                    feature = tokens[n] + group_label # ie 'said_L4' when 'said' within 4 tokens to the left of the match, but not the immeidately preceding token as this would get labelled _1\n",
    "                    features[feature] += 1.0 \n",
    "\n",
    "                token_matched = True\n",
    "                break\n",
    "\n",
    "        # we only perform feature extraction on the first token to be matched\n",
    "        if token_matched:\n",
    "            break\n",
    "\n",
    "        token_start_position_in_sentence +=len(token) + 1 # set for next token\n",
    "    return features\n",
    "\n",
    "# regex features = (regex, feature vector, feature type) feature_vector is ignored by uth-ccb\n",
    "\n",
    "def extract_ConText_features(regex_features, tokens):\n",
    "    import re\n",
    "    # uth-ccb paper is underspecified unfortinately!  here is what they say\n",
    "    # \"3) Lexicon features, including word lists for ne- gation, pseudo-negation, conjunction, condition, uncertainty, subject, severity, and course\"\n",
    "\n",
    "    # So, I will do something similar to negation.py : mark the presence or absence of the ConText classes\n",
    "\n",
    "    sentence = ' '.join(tokens)\n",
    "\n",
    "    # a list of regex matches -> feature vectors iif the regex matched the sentence\n",
    "    matches = list(filter(lambda x:x[0], [(re.search(regex, sentence),feature_type) for (regex, regex_feature_vector,feature_type) in regex_features]))\n",
    "    features=Counter()\n",
    "\n",
    "    for match, feature_type in matches:\n",
    "        features[feature_type] += 1.0\n",
    "    return features\n",
    "\n",
    "# compile the regex, otherwise this is very slow\n",
    "import re\n",
    "compiled_study_phrases={}\n",
    "for studyid, regexes in negation_data.get_study_phrases().items():\n",
    "    compiled_study_phrases[studyid]=[map(lambda x:re.compile(x),regexes)]\n",
    "\n",
    "def extract_dependency_features(studyid, tokens):\n",
    "    import re\n",
    "    # the paper is a bit underspecified, here is what it says:\n",
    "    # 4). Dependency relation features. We used the Stanford Parser to generate dependency relations of a sentence. We only counted dependency relations where a \n",
    "    # target disorder is the governor or the de- pendent in the relation. We extracted all these syn- tactic relations as features.\n",
    "    # The most sensible thing seems to be:\n",
    "    #   match the glosses for governor & dependencies to the study regex, if matched add a feature uniquely identifying the glosses & their relationship\n",
    "\n",
    "    features=Counter()\n",
    "    \n",
    "    # just ignore any parsing errors, some sentences seem to be unparsable, and an error state in the output\n",
    "    try:\n",
    "        # uses stanford corenlp \n",
    "        sentence = ' '.join(tokens)\n",
    "        output = corenlp.annotate(sentence, properties={'annotators':'depparse','outputFormat':'json'})\n",
    "        dependencies = output['sentences'][0]['basicDependencies']\n",
    "\n",
    "        \n",
    "        for reg in compiled_study_phrases[studyid]:\n",
    "            for dependency in dependencies:\n",
    "                if reg.search(' %s ' % dependency['governorGloss']):\n",
    "                    # disease is a governor\n",
    "                    features['governor_%s_->_%s' % (dependency['governorGloss'],dependency['dependentGloss'])]+=1\n",
    "                if reg.search(' %s ' % dependency['dependentGloss']):\n",
    "                    # disease is a dependent\n",
    "                    features['dependent_%s_->_%s' % (dependency['governorGloss'],dependency['dependentGloss'])]+=1\n",
    "    except:\n",
    "        return features\n",
    "    return features\n",
    "\n",
    "def single_sentence_file_generator(file='negation_detection_sentences_train.txt'):\n",
    "    df=pd.read_csv(negation_data.data_processed_folder + '/' + file, sep=',',header=0)\n",
    "    for index, row in df.iterrows():\n",
    "        yield index,row\n",
    "\n",
    "def two_sentence_file_generator(file_1='negation_detection_sentences_train.txt',file_2='negation_detection_sentences_train.txt'):\n",
    "    df=pd.read_csv(negation_data.data_processed_folder + '/' + file_1, sep=',',header=0)\n",
    "    for index, row in df.iterrows():\n",
    "        yield index,row\n",
    "    df=pd.read_csv(negation_data.data_processed_folder + '/' + file_2, sep=',',header=0)\n",
    "    for index, row in df.iterrows():\n",
    "        yield index,row\n",
    "        \n",
    "\n",
    "def sentences_to_uth_ccb(word2vec_model, sentence_generator,feature_vectoriser=None):\n",
    "    \n",
    "    # Prepare for ConText features\n",
    "    regex_features, no_match_vector = negation_data.load_context_feature_definitions()\n",
    "\n",
    "    # cache the study phrases\n",
    "    study_phrases = negation_data.get_study_phrases()\n",
    "    \n",
    "    # track corespondence between studies  & sentences : studyid-> list of indices\n",
    "    study_indices={}\n",
    "\n",
    "    feature_multisets=list() \n",
    "    y=list()\n",
    "    n=0\n",
    "    for index, row in sentence_generator:\n",
    "        StudyId,PatientID,NoteId,CaseLabel,Sublabel,SentenceLabel,DiagnosisDate,SourceNoteRecordedDate,Sentence = row \n",
    "        n=n+1\n",
    "        if n % 1000 == 0:\n",
    "            print('finished %s sentences' % n)\n",
    "\n",
    "        StudyId = str(StudyId) # lol python typing.  It seems to be typed as an integer from the file, and a string in the map.\n",
    "        \n",
    "        indices=study_indices.get(StudyId,[])\n",
    "        indices.append(index)\n",
    "        study_indices[StudyId]=indices\n",
    "        \n",
    "        # create input vector, drop tokens which are out of vocab\n",
    "        tokens = negation_data.tokenize(Sentence)\n",
    "        tokens = list(filter(lambda x: x in word2vec_model, tokens))\n",
    "        # combine all features extracted from tokens\n",
    "        combined_features = extract_dependency_features(study_phrases[StudyId], tokens) + extract_unigrams(StudyId, tokens) + extract_bigrams(StudyId, tokens) + extract_group_features(study_phrases[StudyId], StudyId, tokens) + extract_ConText_features(regex_features, tokens)\n",
    "        \n",
    "        # add the training example and loop\n",
    "        feature_multisets.append(combined_features)\n",
    "        y.append(SentenceLabel)\n",
    "\n",
    "    if feature_vectoriser == None:\n",
    "        feature_vectoriser = DictVectorizer()\n",
    "        x = feature_vectoriser.fit_transform(feature_multisets)\n",
    "        return (x,y,feature_vectoriser,study_indices)\n",
    "    else:\n",
    "        x = feature_vectoriser.transform(feature_multisets)\n",
    "        return (x,y,feature_vectoriser,study_indices)\n",
    "    \n",
    "def hyperparameter_search(x_train,y_train,x_dev,y_dev,C=0.000001):\n",
    "    from sklearn import preprocessing\n",
    "    from sklearn.svm import LinearSVC\n",
    "\n",
    "    svm = LinearSVC(C=C)\n",
    "    svm.fit(x_train,y_train)\n",
    "    y_pred=svm.decision_function(x_dev)\n",
    "    auc=negation_data.auc(y_dev,y_pred)\n",
    "    print('C=%s, AUC=%s' % (C,auc))\n",
    "    return svm\n",
    "\n",
    "# this combines primary and referral into 1 dataset\n",
    "def run_uth_ccb_primary_and_referral_combined():\n",
    "    from sklearn import preprocessing\n",
    "    from sklearn.svm import LinearSVC\n",
    "\n",
    "    word2vec_model = negation_data.load_word2vec_sg() # used to ensure tokens are the same across both experiments\n",
    "    \n",
    "    def get_generator(dataset='train'):\n",
    "        return two_sentence_file_generator(file_1='negation_detection_sentences_experiment_%s_%s.txt' % (8,dataset),\n",
    "                                           file_2='negation_detection_sentences_experiment_%s_%s.txt' % (9,dataset))\n",
    "    # gen features & data sets\n",
    "    (x_train,y_train,feature_vectoriser_train,study_indices_train) = sentences_to_uth_ccb(word2vec_model,get_generator('train'))\n",
    "    (x_dev,y_dev,_,study_indices_dev) = sentences_to_uth_ccb(word2vec_model,get_generator('dev'),feature_vectoriser=feature_vectoriser_train)\n",
    "    (x_test,y_test,feature_vectoriser_test,study_indices_test) = sentences_to_uth_ccb(word2vec_model,get_generator('test'),feature_vectoriser=feature_vectoriser_train)\n",
    "\n",
    "    # how to eval on gold standard\n",
    "    #(x_test_0,y_test_0, feature_vectoriser_gold_0) = sentences_to_uth_ccb(word2vec_model,file='gold standard 2 is 0.csv',feature_vectoriser=feature_vectoriser_train)\n",
    "    #(x_test_1,y_test_1, feature_vectoriser_gold_1) = sentences_to_uth_ccb(word2vec_model,file='gold standard 2 is 1.csv',feature_vectoriser=feature_vectoriser_train)\n",
    "    \n",
    "\n",
    "    # scale features for SVM\n",
    "    scaler = preprocessing.StandardScaler(with_mean=False).partial_fit(x_train)\n",
    "    x_train = scaler.transform(x_train)\n",
    "    x_dev = scaler.transform(x_dev)\n",
    "    x_test=scaler.transform(x_test)\n",
    "    #x_gold_0 = scaler.transform(x_gold_0)\n",
    "    #x_gold_1 = scaler.transform(x_gold_1)\n",
    "\n",
    "    # train\n",
    "    #svm = LinearSVC(C=0.000001) # best results on dev set obtained with this value of C\n",
    "    #svm.fit(x_train,y_train)\n",
    "\n",
    "    # predict and Roc\n",
    "    #y_pred = svm.decision_function(x_dev)\n",
    "    #negation_data.roc(y_dev,y_pred, 'ROC dev set')\n",
    "    #egation_data.roc(y_gold_0,svm.decision_function(x_gold_0), 'ROC: likely are 0')\n",
    "    #negation_data.roc(y_gold_1,svm.decision_function(x_gold_1), 'ROC: likely are 1')\n",
    "\n",
    "    return (word2vec_model,x_train,y_train,x_dev,y_dev,x_test,y_test,feature_vectoriser_train, study_indices_train,study_indices_dev,study_indices_test) \n",
    "\n",
    "def run_uth_ccb(experiment=9):\n",
    "    from sklearn import preprocessing\n",
    "    from sklearn.svm import LinearSVC\n",
    "\n",
    "    word2vec_model = negation_data.load_word2vec_sg() # used to ensure tokens are the same across both experiments\n",
    "\n",
    "    # gen features & data sets\n",
    "    (x_train,y_train,feature_vectoriser_train,_) = sentences_to_uth_ccb(word2vec_model,file='negation_detection_sentences_experiment_%s_train.txt' % experiment)\n",
    "    (x_dev,y_dev,_,_) = sentences_to_uth_ccb(word2vec_model,file='negation_detection_sentences_experiment_%s_dev.txt' %  experiment,feature_vectoriser=feature_vectoriser_train)\n",
    "    (x_test,y_test,_,_) = sentences_to_uth_ccb(word2vec_model,file='negation_detection_sentences_experiment_%s_test.txt' % experiment,feature_vectoriser=feature_vectoriser_train)\n",
    "    \n",
    "    # how to eval on gold standard\n",
    "    #(x_test_0,y_test_0, feature_vectoriser_gold_0) = sentences_to_uth_ccb(word2vec_model,file='gold standard 2 is 0.csv',feature_vectoriser=feature_vectoriser_train)\n",
    "    #(x_test_1,y_test_1, feature_vectoriser_gold_1) = sentences_to_uth_ccb(word2vec_model,file='gold standard 2 is 1.csv',feature_vectoriser=feature_vectoriser_train)\n",
    "    \n",
    "\n",
    "    # scale features for SVM\n",
    "    scaler = preprocessing.StandardScaler(with_mean=False).partial_fit(x_train)\n",
    "    x_train = scaler.transform(x_train)\n",
    "    x_dev = scaler.transform(x_dev)\n",
    "    x_test=scaler.transform(x_test)\n",
    "    #x_gold_0 = scaler.transform(x_gold_0)\n",
    "    #x_gold_1 = scaler.transform(x_gold_1)\n",
    "\n",
    "    # train\n",
    "    svm = LinearSVC(C=0.000001) # best results on dev set obtained with this value of C\n",
    "    svm.fit(x_train,y_train)\n",
    "\n",
    "    # predict and Roc\n",
    "    #y_pred = svm.decision_function(x_dev)\n",
    "    #negation_data.roc(y_dev,y_pred, 'ROC dev set')\n",
    "    #egation_data.roc(y_gold_0,svm.decision_function(x_gold_0), 'ROC: likely are 0')\n",
    "    #negation_data.roc(y_gold_1,svm.decision_function(x_gold_1), 'ROC: likely are 1')\n",
    "\n",
    "    return (word2vec_model,x_train,y_train,x_dev,y_dev,x_test,y_test,feature_vectoriser_train, svm,scaler) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/noel/tensorflow/lib/python3.5/site-packages/ipykernel_launcher.py:5: DtypeWarning: Columns (6) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  \"\"\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished 1000 sentences\n",
      "finished 2000 sentences\n",
      "finished 3000 sentences\n",
      "finished 4000 sentences\n",
      "finished 5000 sentences\n",
      "finished 6000 sentences\n",
      "finished 7000 sentences\n",
      "finished 8000 sentences\n",
      "finished 9000 sentences\n",
      "finished 10000 sentences\n",
      "finished 11000 sentences\n",
      "finished 12000 sentences\n",
      "finished 13000 sentences\n",
      "finished 14000 sentences\n",
      "finished 15000 sentences\n",
      "finished 16000 sentences\n",
      "finished 17000 sentences\n",
      "finished 18000 sentences\n",
      "finished 19000 sentences\n",
      "finished 20000 sentences\n",
      "finished 21000 sentences\n",
      "finished 22000 sentences\n",
      "finished 23000 sentences\n",
      "finished 24000 sentences\n",
      "finished 25000 sentences\n",
      "finished 26000 sentences\n",
      "finished 27000 sentences\n",
      "finished 28000 sentences\n",
      "finished 29000 sentences\n",
      "finished 30000 sentences\n",
      "finished 31000 sentences\n",
      "finished 32000 sentences\n",
      "finished 33000 sentences\n",
      "finished 34000 sentences\n",
      "finished 35000 sentences\n",
      "finished 36000 sentences\n",
      "finished 37000 sentences\n",
      "finished 38000 sentences\n",
      "finished 39000 sentences\n",
      "finished 40000 sentences\n",
      "finished 41000 sentences\n",
      "finished 42000 sentences\n",
      "finished 43000 sentences\n",
      "finished 44000 sentences\n",
      "finished 45000 sentences\n",
      "finished 46000 sentences\n",
      "finished 47000 sentences\n",
      "finished 48000 sentences\n",
      "finished 49000 sentences\n",
      "finished 50000 sentences\n",
      "finished 51000 sentences\n",
      "finished 52000 sentences\n",
      "finished 53000 sentences\n",
      "finished 54000 sentences\n",
      "finished 55000 sentences\n",
      "finished 56000 sentences\n",
      "finished 57000 sentences\n",
      "finished 58000 sentences\n",
      "finished 59000 sentences\n",
      "finished 60000 sentences\n",
      "finished 61000 sentences\n",
      "finished 62000 sentences\n",
      "finished 63000 sentences\n",
      "finished 64000 sentences\n",
      "finished 65000 sentences\n",
      "finished 66000 sentences\n",
      "finished 67000 sentences\n",
      "finished 68000 sentences\n",
      "finished 69000 sentences\n",
      "finished 70000 sentences\n",
      "finished 71000 sentences\n",
      "finished 72000 sentences\n",
      "finished 73000 sentences\n",
      "finished 74000 sentences\n",
      "finished 75000 sentences\n",
      "finished 76000 sentences\n",
      "finished 77000 sentences\n",
      "finished 78000 sentences\n",
      "finished 79000 sentences\n",
      "finished 80000 sentences\n",
      "finished 81000 sentences\n",
      "finished 82000 sentences\n",
      "finished 83000 sentences\n",
      "finished 84000 sentences\n",
      "finished 85000 sentences\n",
      "finished 86000 sentences\n",
      "finished 87000 sentences\n",
      "finished 88000 sentences\n",
      "finished 89000 sentences\n",
      "finished 90000 sentences\n",
      "finished 91000 sentences\n",
      "finished 92000 sentences\n",
      "finished 93000 sentences\n",
      "finished 94000 sentences\n",
      "finished 95000 sentences\n",
      "finished 96000 sentences\n",
      "finished 97000 sentences\n",
      "finished 98000 sentences\n",
      "finished 99000 sentences\n",
      "finished 100000 sentences\n",
      "finished 101000 sentences\n",
      "finished 102000 sentences\n",
      "finished 103000 sentences\n",
      "finished 104000 sentences\n",
      "finished 105000 sentences\n",
      "finished 106000 sentences\n",
      "finished 107000 sentences\n",
      "finished 108000 sentences\n",
      "finished 109000 sentences\n",
      "finished 110000 sentences\n",
      "finished 111000 sentences\n",
      "finished 112000 sentences\n",
      "finished 113000 sentences\n",
      "finished 114000 sentences\n",
      "finished 115000 sentences\n",
      "finished 116000 sentences\n",
      "finished 117000 sentences\n",
      "finished 118000 sentences\n",
      "finished 119000 sentences\n",
      "finished 120000 sentences\n",
      "finished 121000 sentences\n",
      "finished 122000 sentences\n",
      "finished 123000 sentences\n",
      "finished 124000 sentences\n",
      "finished 125000 sentences\n",
      "finished 126000 sentences\n",
      "finished 127000 sentences\n",
      "finished 128000 sentences\n",
      "finished 129000 sentences\n",
      "finished 130000 sentences\n",
      "finished 131000 sentences\n",
      "finished 132000 sentences\n",
      "finished 133000 sentences\n",
      "finished 134000 sentences\n",
      "finished 135000 sentences\n",
      "finished 136000 sentences\n",
      "finished 137000 sentences\n",
      "finished 138000 sentences\n",
      "finished 139000 sentences\n",
      "finished 140000 sentences\n",
      "finished 141000 sentences\n",
      "finished 142000 sentences\n",
      "finished 143000 sentences\n",
      "finished 144000 sentences\n",
      "finished 145000 sentences\n",
      "finished 146000 sentences\n",
      "finished 147000 sentences\n",
      "finished 148000 sentences\n",
      "finished 149000 sentences\n",
      "finished 150000 sentences\n",
      "finished 151000 sentences\n",
      "finished 152000 sentences\n",
      "finished 153000 sentences\n",
      "finished 154000 sentences\n",
      "finished 155000 sentences\n",
      "finished 156000 sentences\n",
      "finished 157000 sentences\n",
      "finished 158000 sentences\n",
      "finished 159000 sentences\n",
      "finished 160000 sentences\n",
      "finished 161000 sentences\n",
      "finished 162000 sentences\n",
      "finished 163000 sentences\n",
      "finished 164000 sentences\n",
      "finished 165000 sentences\n",
      "finished 166000 sentences\n",
      "finished 167000 sentences\n",
      "finished 168000 sentences\n",
      "finished 169000 sentences\n",
      "finished 170000 sentences\n",
      "finished 171000 sentences\n",
      "finished 172000 sentences\n",
      "finished 173000 sentences\n",
      "finished 174000 sentences\n",
      "finished 175000 sentences\n",
      "finished 176000 sentences\n",
      "finished 177000 sentences\n",
      "finished 178000 sentences\n",
      "finished 179000 sentences\n",
      "finished 180000 sentences\n",
      "finished 181000 sentences\n",
      "finished 182000 sentences\n",
      "finished 183000 sentences\n",
      "finished 184000 sentences\n",
      "finished 185000 sentences\n",
      "finished 186000 sentences\n",
      "finished 187000 sentences\n",
      "finished 188000 sentences\n",
      "finished 189000 sentences\n",
      "finished 190000 sentences\n",
      "finished 191000 sentences\n",
      "finished 192000 sentences\n",
      "finished 193000 sentences\n",
      "finished 194000 sentences\n",
      "finished 195000 sentences\n",
      "finished 196000 sentences\n",
      "finished 197000 sentences\n",
      "finished 198000 sentences\n",
      "finished 199000 sentences\n",
      "finished 200000 sentences\n",
      "finished 201000 sentences\n",
      "finished 202000 sentences\n",
      "finished 203000 sentences\n",
      "finished 204000 sentences\n",
      "finished 205000 sentences\n",
      "finished 206000 sentences\n",
      "finished 207000 sentences\n",
      "finished 208000 sentences\n",
      "finished 209000 sentences\n",
      "finished 210000 sentences\n",
      "finished 211000 sentences\n",
      "finished 212000 sentences\n",
      "finished 213000 sentences\n",
      "finished 214000 sentences\n",
      "finished 215000 sentences\n",
      "finished 216000 sentences\n",
      "finished 217000 sentences\n",
      "finished 218000 sentences\n",
      "finished 219000 sentences\n",
      "finished 220000 sentences\n",
      "finished 221000 sentences\n",
      "finished 222000 sentences\n",
      "finished 223000 sentences\n",
      "finished 224000 sentences\n",
      "finished 225000 sentences\n",
      "finished 226000 sentences\n",
      "finished 227000 sentences\n",
      "finished 228000 sentences\n",
      "finished 229000 sentences\n",
      "finished 230000 sentences\n",
      "finished 231000 sentences\n",
      "finished 232000 sentences\n",
      "finished 233000 sentences\n",
      "finished 234000 sentences\n",
      "finished 235000 sentences\n",
      "finished 236000 sentences\n",
      "finished 237000 sentences\n",
      "finished 238000 sentences\n",
      "finished 239000 sentences\n",
      "finished 240000 sentences\n",
      "finished 241000 sentences\n",
      "finished 242000 sentences\n",
      "finished 243000 sentences\n",
      "finished 244000 sentences\n",
      "finished 245000 sentences\n",
      "finished 246000 sentences\n",
      "finished 247000 sentences\n",
      "finished 248000 sentences\n",
      "finished 249000 sentences\n",
      "finished 250000 sentences\n",
      "finished 251000 sentences\n",
      "finished 252000 sentences\n",
      "finished 253000 sentences\n",
      "finished 254000 sentences\n",
      "finished 255000 sentences\n",
      "finished 256000 sentences\n",
      "finished 257000 sentences\n",
      "finished 258000 sentences\n",
      "finished 259000 sentences\n",
      "finished 260000 sentences\n",
      "finished 261000 sentences\n",
      "finished 262000 sentences\n",
      "finished 263000 sentences\n",
      "finished 264000 sentences\n",
      "finished 265000 sentences\n",
      "finished 266000 sentences\n",
      "finished 267000 sentences\n",
      "finished 268000 sentences\n",
      "finished 269000 sentences\n",
      "finished 270000 sentences\n",
      "finished 271000 sentences\n",
      "finished 272000 sentences\n",
      "finished 273000 sentences\n",
      "finished 274000 sentences\n",
      "finished 275000 sentences\n",
      "finished 276000 sentences\n",
      "finished 277000 sentences\n",
      "finished 278000 sentences\n",
      "finished 279000 sentences\n",
      "finished 280000 sentences\n",
      "finished 281000 sentences\n",
      "finished 282000 sentences\n",
      "finished 283000 sentences\n",
      "finished 284000 sentences\n",
      "finished 285000 sentences\n",
      "finished 286000 sentences\n",
      "finished 287000 sentences\n",
      "finished 288000 sentences\n",
      "finished 289000 sentences\n",
      "finished 290000 sentences\n",
      "finished 291000 sentences\n",
      "finished 292000 sentences\n",
      "finished 293000 sentences\n",
      "finished 294000 sentences\n",
      "finished 295000 sentences\n",
      "finished 296000 sentences\n",
      "finished 297000 sentences\n",
      "finished 298000 sentences\n",
      "finished 299000 sentences\n",
      "finished 300000 sentences\n",
      "finished 301000 sentences\n",
      "finished 302000 sentences\n",
      "finished 303000 sentences\n",
      "finished 304000 sentences\n",
      "finished 305000 sentences\n",
      "finished 306000 sentences\n",
      "finished 307000 sentences\n",
      "finished 308000 sentences\n",
      "finished 309000 sentences\n",
      "finished 310000 sentences\n",
      "finished 311000 sentences\n",
      "finished 312000 sentences\n",
      "finished 313000 sentences\n",
      "finished 314000 sentences\n",
      "finished 315000 sentences\n",
      "finished 316000 sentences\n",
      "finished 317000 sentences\n",
      "finished 318000 sentences\n",
      "finished 319000 sentences\n",
      "finished 320000 sentences\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished 321000 sentences\n",
      "finished 322000 sentences\n",
      "finished 323000 sentences\n",
      "finished 324000 sentences\n",
      "finished 325000 sentences\n",
      "finished 326000 sentences\n",
      "finished 327000 sentences\n",
      "finished 328000 sentences\n",
      "finished 329000 sentences\n",
      "finished 330000 sentences\n",
      "finished 331000 sentences\n",
      "finished 332000 sentences\n",
      "finished 333000 sentences\n",
      "finished 334000 sentences\n",
      "finished 335000 sentences\n",
      "finished 336000 sentences\n",
      "finished 337000 sentences\n",
      "finished 338000 sentences\n",
      "finished 339000 sentences\n",
      "finished 340000 sentences\n",
      "finished 341000 sentences\n",
      "finished 342000 sentences\n",
      "finished 343000 sentences\n",
      "finished 344000 sentences\n",
      "finished 345000 sentences\n",
      "finished 346000 sentences\n",
      "finished 347000 sentences\n",
      "finished 348000 sentences\n",
      "finished 349000 sentences\n",
      "finished 350000 sentences\n",
      "finished 351000 sentences\n",
      "finished 352000 sentences\n",
      "finished 353000 sentences\n",
      "finished 354000 sentences\n",
      "finished 355000 sentences\n",
      "finished 356000 sentences\n",
      "finished 357000 sentences\n",
      "finished 358000 sentences\n",
      "finished 359000 sentences\n",
      "finished 360000 sentences\n",
      "finished 361000 sentences\n",
      "finished 362000 sentences\n",
      "finished 363000 sentences\n",
      "finished 364000 sentences\n",
      "finished 365000 sentences\n",
      "finished 366000 sentences\n",
      "finished 367000 sentences\n",
      "finished 368000 sentences\n",
      "finished 369000 sentences\n",
      "finished 370000 sentences\n",
      "finished 371000 sentences\n",
      "finished 372000 sentences\n",
      "finished 373000 sentences\n",
      "finished 374000 sentences\n",
      "finished 375000 sentences\n",
      "finished 376000 sentences\n",
      "finished 377000 sentences\n",
      "finished 378000 sentences\n",
      "finished 379000 sentences\n",
      "finished 380000 sentences\n",
      "finished 381000 sentences\n",
      "finished 382000 sentences\n",
      "finished 383000 sentences\n",
      "finished 384000 sentences\n",
      "finished 385000 sentences\n",
      "finished 386000 sentences\n",
      "finished 387000 sentences\n",
      "finished 388000 sentences\n",
      "finished 389000 sentences\n",
      "finished 390000 sentences\n",
      "finished 391000 sentences\n",
      "finished 392000 sentences\n",
      "finished 393000 sentences\n",
      "finished 394000 sentences\n",
      "finished 395000 sentences\n",
      "finished 396000 sentences\n",
      "finished 397000 sentences\n",
      "finished 398000 sentences\n",
      "finished 399000 sentences\n",
      "finished 400000 sentences\n",
      "finished 401000 sentences\n",
      "finished 402000 sentences\n",
      "finished 403000 sentences\n",
      "finished 404000 sentences\n",
      "finished 405000 sentences\n",
      "finished 406000 sentences\n",
      "finished 407000 sentences\n",
      "finished 408000 sentences\n",
      "finished 409000 sentences\n",
      "finished 410000 sentences\n",
      "finished 411000 sentences\n",
      "finished 412000 sentences\n",
      "finished 413000 sentences\n",
      "finished 414000 sentences\n",
      "finished 415000 sentences\n",
      "finished 416000 sentences\n",
      "finished 417000 sentences\n",
      "finished 418000 sentences\n",
      "finished 419000 sentences\n",
      "finished 420000 sentences\n",
      "finished 421000 sentences\n",
      "finished 422000 sentences\n",
      "finished 423000 sentences\n",
      "finished 424000 sentences\n",
      "finished 425000 sentences\n",
      "finished 426000 sentences\n",
      "finished 427000 sentences\n",
      "finished 428000 sentences\n",
      "finished 429000 sentences\n",
      "finished 430000 sentences\n",
      "finished 431000 sentences\n",
      "finished 432000 sentences\n",
      "finished 433000 sentences\n",
      "finished 434000 sentences\n",
      "finished 435000 sentences\n",
      "finished 436000 sentences\n",
      "finished 437000 sentences\n",
      "finished 438000 sentences\n",
      "finished 439000 sentences\n",
      "finished 440000 sentences\n",
      "finished 441000 sentences\n",
      "finished 442000 sentences\n",
      "finished 443000 sentences\n",
      "finished 444000 sentences\n",
      "finished 445000 sentences\n",
      "finished 446000 sentences\n",
      "finished 447000 sentences\n",
      "finished 448000 sentences\n",
      "finished 449000 sentences\n",
      "finished 450000 sentences\n",
      "finished 451000 sentences\n",
      "finished 452000 sentences\n",
      "finished 453000 sentences\n",
      "finished 454000 sentences\n",
      "finished 455000 sentences\n",
      "finished 456000 sentences\n",
      "finished 457000 sentences\n",
      "finished 458000 sentences\n",
      "finished 459000 sentences\n",
      "finished 460000 sentences\n",
      "finished 461000 sentences\n",
      "finished 462000 sentences\n",
      "finished 463000 sentences\n",
      "finished 464000 sentences\n",
      "finished 465000 sentences\n",
      "finished 466000 sentences\n",
      "finished 467000 sentences\n",
      "finished 468000 sentences\n",
      "finished 469000 sentences\n",
      "finished 470000 sentences\n",
      "finished 471000 sentences\n",
      "finished 472000 sentences\n",
      "finished 473000 sentences\n",
      "finished 474000 sentences\n",
      "finished 475000 sentences\n",
      "finished 476000 sentences\n",
      "finished 477000 sentences\n",
      "finished 478000 sentences\n",
      "finished 479000 sentences\n",
      "finished 480000 sentences\n",
      "finished 481000 sentences\n",
      "finished 482000 sentences\n",
      "finished 483000 sentences\n",
      "finished 484000 sentences\n",
      "finished 485000 sentences\n",
      "finished 486000 sentences\n",
      "finished 487000 sentences\n",
      "finished 488000 sentences\n",
      "finished 489000 sentences\n",
      "finished 490000 sentences\n",
      "finished 491000 sentences\n",
      "finished 492000 sentences\n",
      "finished 493000 sentences\n",
      "finished 494000 sentences\n",
      "finished 495000 sentences\n",
      "finished 496000 sentences\n",
      "finished 497000 sentences\n",
      "finished 498000 sentences\n",
      "finished 499000 sentences\n",
      "finished 500000 sentences\n",
      "finished 501000 sentences\n",
      "finished 502000 sentences\n",
      "finished 503000 sentences\n",
      "finished 504000 sentences\n",
      "finished 505000 sentences\n",
      "finished 506000 sentences\n",
      "finished 507000 sentences\n",
      "finished 508000 sentences\n",
      "finished 509000 sentences\n",
      "finished 510000 sentences\n",
      "finished 511000 sentences\n",
      "finished 512000 sentences\n",
      "finished 513000 sentences\n",
      "finished 514000 sentences\n",
      "finished 515000 sentences\n",
      "finished 516000 sentences\n",
      "finished 517000 sentences\n",
      "finished 518000 sentences\n",
      "finished 519000 sentences\n",
      "finished 520000 sentences\n",
      "finished 1000 sentences\n",
      "finished 2000 sentences\n",
      "finished 3000 sentences\n",
      "finished 4000 sentences\n",
      "finished 5000 sentences\n",
      "finished 6000 sentences\n",
      "finished 7000 sentences\n",
      "finished 8000 sentences\n",
      "finished 9000 sentences\n",
      "finished 10000 sentences\n",
      "finished 11000 sentences\n",
      "finished 12000 sentences\n",
      "finished 13000 sentences\n",
      "finished 14000 sentences\n",
      "finished 15000 sentences\n",
      "finished 16000 sentences\n",
      "finished 17000 sentences\n",
      "finished 18000 sentences\n",
      "finished 19000 sentences\n",
      "finished 20000 sentences\n",
      "finished 21000 sentences\n",
      "finished 22000 sentences\n",
      "finished 23000 sentences\n",
      "finished 24000 sentences\n",
      "finished 25000 sentences\n",
      "finished 26000 sentences\n",
      "finished 27000 sentences\n",
      "finished 28000 sentences\n",
      "finished 29000 sentences\n",
      "finished 30000 sentences\n",
      "finished 31000 sentences\n",
      "finished 32000 sentences\n",
      "finished 33000 sentences\n",
      "finished 34000 sentences\n",
      "finished 35000 sentences\n",
      "finished 36000 sentences\n",
      "finished 37000 sentences\n",
      "finished 38000 sentences\n",
      "finished 39000 sentences\n",
      "finished 40000 sentences\n",
      "finished 41000 sentences\n",
      "finished 42000 sentences\n",
      "finished 43000 sentences\n",
      "finished 44000 sentences\n",
      "finished 45000 sentences\n",
      "finished 46000 sentences\n",
      "finished 47000 sentences\n",
      "finished 48000 sentences\n",
      "finished 49000 sentences\n",
      "finished 50000 sentences\n",
      "finished 51000 sentences\n",
      "finished 52000 sentences\n",
      "finished 53000 sentences\n",
      "finished 54000 sentences\n",
      "finished 55000 sentences\n",
      "finished 56000 sentences\n",
      "finished 57000 sentences\n",
      "finished 58000 sentences\n",
      "finished 59000 sentences\n",
      "finished 60000 sentences\n",
      "finished 61000 sentences\n",
      "finished 62000 sentences\n",
      "finished 63000 sentences\n",
      "finished 64000 sentences\n",
      "finished 65000 sentences\n",
      "finished 1000 sentences\n",
      "finished 2000 sentences\n",
      "finished 3000 sentences\n",
      "finished 4000 sentences\n",
      "finished 5000 sentences\n",
      "finished 6000 sentences\n",
      "finished 7000 sentences\n",
      "finished 8000 sentences\n",
      "finished 9000 sentences\n",
      "finished 10000 sentences\n",
      "finished 11000 sentences\n",
      "finished 12000 sentences\n",
      "finished 13000 sentences\n",
      "finished 14000 sentences\n",
      "finished 15000 sentences\n",
      "finished 16000 sentences\n",
      "finished 17000 sentences\n",
      "finished 18000 sentences\n",
      "finished 19000 sentences\n",
      "finished 20000 sentences\n",
      "finished 21000 sentences\n",
      "finished 22000 sentences\n",
      "finished 23000 sentences\n",
      "finished 24000 sentences\n",
      "finished 25000 sentences\n",
      "finished 26000 sentences\n",
      "finished 27000 sentences\n",
      "finished 28000 sentences\n",
      "finished 29000 sentences\n",
      "finished 30000 sentences\n",
      "finished 31000 sentences\n",
      "finished 32000 sentences\n",
      "finished 33000 sentences\n",
      "finished 34000 sentences\n",
      "finished 35000 sentences\n",
      "finished 36000 sentences\n",
      "finished 37000 sentences\n",
      "finished 38000 sentences\n",
      "finished 39000 sentences\n",
      "finished 40000 sentences\n",
      "finished 41000 sentences\n",
      "finished 42000 sentences\n",
      "finished 43000 sentences\n",
      "finished 44000 sentences\n",
      "finished 45000 sentences\n",
      "finished 46000 sentences\n",
      "finished 47000 sentences\n",
      "finished 48000 sentences\n",
      "finished 49000 sentences\n",
      "finished 50000 sentences\n",
      "finished 51000 sentences\n",
      "finished 52000 sentences\n",
      "finished 53000 sentences\n",
      "finished 54000 sentences\n",
      "finished 55000 sentences\n",
      "finished 56000 sentences\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished 57000 sentences\n",
      "finished 58000 sentences\n",
      "finished 59000 sentences\n",
      "finished 60000 sentences\n",
      "finished 61000 sentences\n",
      "finished 62000 sentences\n",
      "finished 63000 sentences\n",
      "finished 64000 sentences\n",
      "finished 65000 sentences\n"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "import numpy as np\n",
    "\n",
    "#(_,x_train,y_train,x_dev,y_dev,x_test,y_test,feature_vectoriser_train, svm,scaler) = run_uth_ccb(experiment=experiment)\n",
    "(_,x_train,y_train,x_dev,y_dev,x_test,y_test,feature_vectoriser_train, study_indices_train,study_indices_dev,study_indices_test) = run_uth_ccb_primary_and_referral_combined()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# persist the x datasets (y datasets are the same across models, so they are saved by the CNN scripts)\n",
    "# we are using sparse matrices here, so need to save in a special way\n",
    "\n",
    "#https://stackoverflow.com/questions/8955448/save-load-scipy-sparse-csr-matrix-in-portable-data-format\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "def save_sparse_csr(filename, array):\n",
    "    np.savez(filename, data=array.data, indices=array.indices,\n",
    "             indptr=array.indptr, shape=array.shape)\n",
    "\n",
    "def load_sparse_csr(filename):\n",
    "    loader = np.load(filename)\n",
    "    return csr_matrix((loader['data'], loader['indices'], loader['indptr']),\n",
    "                      shape=loader['shape'])\n",
    "\n",
    "\n",
    "def save_x(x,dataset='train',experiment=8):\n",
    "    save_sparse_csr('data/datasets/x_%s_uth_exp%s.npy' % (dataset,experiment),x)\n",
    " \n",
    "def save_study_sentence_dictionary(study_sentence_dictionary, dataset='train',experiment=8):\n",
    "    with open('data/datasets/study_sentence_dictionary_UTH_%s_exp_%s.json' % (dataset, experiment), 'w') as file:\n",
    "            json.dump(study_sentence_dictionary, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "#save_x(x_train,dataset='train',experiment=experiment)\n",
    "#save_x(x_dev,dataset='dev',experiment=experiment)\n",
    "#save_x(x_test,dataset='test',experiment=experiment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "experiment = 'pri_ref'\n",
    "\n",
    "save_x(x_train,dataset='train',experiment=experiment)\n",
    "save_x(x_dev,dataset='dev',experiment=experiment)\n",
    "save_x(x_test,dataset='test',experiment=experiment)\n",
    "\n",
    "save_study_sentence_dictionary(study_indices_train,dataset='train',experiment=experiment)\n",
    "save_study_sentence_dictionary(study_indices_dev,dataset='dev',experiment=experiment)\n",
    "save_study_sentence_dictionary(study_indices_test,dataset='test',experiment=experiment)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = svm.decision_function(x_dev)\n",
    "negation_data.auc(y_dev,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}