data_analysis_and_exploration_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "\n",
    "try:\n",
    "  # %tensorflow_version only exists in Colab.\n",
    "  %tensorflow_version 2.x\n",
    "except Exception:\n",
    "  pass\n",
    "import tensorflow as tf\n",
    "\n",
    "from imp import reload\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import IPython.display as ipd\n",
    "\n",
    "from tensorflow.keras import datasets, layers, models\n",
    "\n",
    "import textwrap\n",
    "import random\n",
    "import json\n",
    "\n",
    "from simpletransformers.classification import ClassificationModel\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../noncomm_use_subset/pmc_json/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load up files from a directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_path = '../noncomm_use_subset/pmc_json_test/'\n",
    "filename = 'PMC1616946.xml.json'\n",
    "\n",
    "print(os.listdir(dir_path))\n",
    "\n",
    "print(\"---------\" + dir_path + filename)\n",
    "print()\n",
    "\n",
    "articles = []\n",
    "\n",
    "for file in os.listdir(dir_path):\n",
    "    filepath = dir_path + file\n",
    "    print(\"++---------\" + filepath)\n",
    "    with open(filepath, 'r') as infile:\n",
    "        json_object = json.load(infile)\n",
    "        #print(json.dumps(json_object, indent=2))\n",
    "        a = Article(json_object)\n",
    "        articles.append(a)\n",
    "        \n",
    "# Check result        \n",
    "for a in articles:\n",
    "    print(\"@@@@@@\\n\\n\")\n",
    "    print(len(a.get_text()))\n",
    "    \n",
    "            #df = pd.read_json(infile,\n",
    "            #                 lines=True,\n",
    "            #                 orient='columns')\n",
    "    #df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "articles[1].sections.sections[2].get_text()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a class object to represent papers/articles with contained methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords = []\n",
    "\n",
    "def normalize(words):\n",
    "    \n",
    "    out_words = [word.lower() for word in words if word not in stopwords]\n",
    "    \n",
    "    return out_words\n",
    "\n",
    "#######################################################################\n",
    "# Class method for representing Section\n",
    "#######################################################################\n",
    "\n",
    "# Section class encapsulates loading, displaying, and simple helper operations \n",
    "# e.g. return text or bag of words representation\n",
    "class Section:\n",
    "    # Initialize with key article fields from JSON object\n",
    "    # body_text (should preserve section type)\n",
    "    def __init__(self, json_object):#, label_map):\n",
    "        \n",
    "        #print(json_input)\n",
    "        \n",
    "        # Extract\n",
    "        self.type = json_object.get('section', '')\n",
    "        self.text= json_object.get('text', '')\n",
    "        self.normalized_text=normalize(self.text.split())\n",
    "        self.bow = set(self.normalized_text)\n",
    "        \n",
    "    def get_text(self):\n",
    "        return self.text\n",
    "        \n",
    "        \n",
    "# Encapsulates list of sections, takes abstract and body_text JSON list \n",
    "# and generates subsections in order\n",
    "class Sections:\n",
    "    # Initialize with key article fields from JSON object\n",
    "    # body_text (should preserve section type)\n",
    "    def __init__(self, abstract=[], body_text=[]):#, label_map):\n",
    "        \n",
    "        self.sections=[]\n",
    "        \n",
    "        # Create a section from the abstract\n",
    "        if abstract:\n",
    "            self.sections.append(Section(abstract))\n",
    "        \n",
    "        # Iterate through the elements of the body_text and create sections, \n",
    "        # combining sequential items of the same type\n",
    "        if body_text:\n",
    "            current_type = ''\n",
    "            current_text = ''\n",
    "            delim_text = ''\n",
    "            for paragraph in body_text:\n",
    "                temp_type = paragraph.get('section', '')\n",
    "                temp_text = paragraph.get('text', '')\n",
    "                #print(temp_text)\n",
    "                if temp_type and temp_type != current_type:\n",
    "                    # Start new section\n",
    "                    self.sections.append(Section({'section': current_type, 'text': current_text}))\n",
    "                    current_type = temp_type\n",
    "                    current_text = temp_text\n",
    "                \n",
    "                # Continue adding to current section data\n",
    "                current_text += delim_text + temp_text\n",
    "                \n",
    "                if delim_text == '':\n",
    "                    delim_text = ' '\n",
    "                    \n",
    "            # If anything remaining, add section\n",
    "            if current_type or current_text:\n",
    "                self.sections.append(Section({'section': current_type, 'text': current_text}))\n",
    "        \n",
    "        # Extract\n",
    "        self.normalized_text=[]\n",
    "        \n",
    "    def __iter__(self):\n",
    "        for s in self.sections:\n",
    "            yield s\n",
    "    \n",
    "    def get_text(self):\n",
    "        return '\\n'.join([s.get_text() for s in self.sections])\n",
    "        \n",
    "\n",
    "#######################################################################\n",
    "# Class method for representing Article (list of Sections)\n",
    "#######################################################################\n",
    "\n",
    "# Article class encapsulates loading, displaying, and simple helper operations \n",
    "# e.g. return text or bag of words representation\n",
    "class Article:\n",
    "    # Initialize with key article fields from JSON representation\n",
    "    # paper_id, title, authors, abstract, body_text, sections\n",
    "    def __init__(self, json_input):#, label_map):\n",
    "        \n",
    "        # Extract\n",
    "        self.paperID=json_object['paper_id'] #Paper ID\n",
    "        in_abs = json_object.get('abstract', {})\n",
    "        self.abstract=Section(in_abs)\n",
    "\n",
    "        self.sections = Sections(in_abs, json_object.get('body_text', []))\n",
    "\n",
    "    # \n",
    "    def get_text(self):\n",
    "        return self.sections.get_text()\n",
    "    \n",
    "    \n",
    "    #def get_bow(self, stopwords=[]):\n",
    "        #return self.card + ' ' + self.answer\n",
    "    \n",
    "    \n",
    "    # Add methods for running ML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LDA methods below will take a list of docs where each is the unicode string including newlines etc. for each article"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Update to larger dataset for better run now that it's debugged\n",
    "dir_path = '../noncomm_use_subset/pmc_json/'\n",
    "filename = 'PMC1616946.xml.json'\n",
    "\n",
    "print(len(os.listdir(dir_path)))\n",
    "print(os.listdir(dir_path))\n",
    "\n",
    "print(\"---------\" + dir_path + filename)\n",
    "print()\n",
    "\n",
    "articles = []\n",
    "\n",
    "num_articles = 0\n",
    "for file in os.listdir(dir_path):\n",
    "    filepath = dir_path + file\n",
    "    print(\"++---------\" + filepath)\n",
    "    with open(filepath, 'r') as infile:\n",
    "        json_object = json.load(infile)\n",
    "        #print(json.dumps(json_object, indent=2))\n",
    "        a = Article(json_object)\n",
    "        articles.append(a)\n",
    "        \n",
    "    #if num_articles > 100:\n",
    "    #    break\n",
    "    # Use all 2093 articles    \n",
    "    #num_articles += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "section_texts = []\n",
    "sec_types = []\n",
    "for a in articles:\n",
    "    section_texts.extend(a.sections)\n",
    "doc_texts = [s.get_text() for s in section_texts]\n",
    "sec_types = [s.section for s in section_texts]\n",
    "\n",
    "# Filter empty docs (sections)\n",
    "meta_texts = [(t, doc) for t, doc in zip(sec_types, doc_texts) if doc]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(doc_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_texts[len(docs)+1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_texts[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sections = []\n",
    "for a in articles:\n",
    "    sections.extend(a.sections)\n",
    "docs = [s.get_text() for s in sections]\n",
    "\n",
    "# Filter empty docs (sections)\n",
    "docs = [d for d in docs if d]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "docs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tokenize the documents.\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "\n",
    "# Split the documents into tokens.\n",
    "tokenizer = RegexpTokenizer(r'\\w+')\n",
    "for idx in range(len(docs)):\n",
    "    docs[idx] = docs[idx].lower()  # Convert to lowercase.\n",
    "    docs[idx] = tokenizer.tokenize(docs[idx])  # Split into words.\n",
    "\n",
    "# Remove numbers, but not words that contain numbers.\n",
    "docs = [[token for token in doc if not token.isnumeric()] for doc in docs]\n",
    "\n",
    "# Remove words that are only one character.\n",
    "docs = [[token for token in doc if len(token) > 2] for doc in docs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "nltk.download('wordnet')\n",
    "import gensim # conda install gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lemmatize the documents.\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "docs = [[lemmatizer.lemmatize(token) for token in doc] for doc in docs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute bigrams.\n",
    "from gensim.models import Phrases\n",
    "\n",
    "# Add bigrams and trigrams to docs (only ones that appear 20 times or more).\n",
    "bigram = Phrases(docs, min_count=20)\n",
    "for idx in range(len(docs)):\n",
    "    for token in bigram[docs[idx]]:\n",
    "        if '_' in token:\n",
    "            # Token is a bigram, add to document.\n",
    "            docs[idx].append(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove rare and common tokens.\n",
    "from gensim.corpora import Dictionary\n",
    "\n",
    "# Create a dictionary representation of the documents.\n",
    "dictionary = Dictionary(docs)\n",
    "\n",
    "# Filter out words that occur less than 20 documents, or more than 50% of the documents.\n",
    "dictionary.filter_extremes(no_below=20, no_above=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bag-of-words representation of the documents.\n",
    "corpus = [dictionary.doc2bow(doc) for doc in docs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Number of unique tokens: %d' % len(dictionary))\n",
    "print('Number of documents: %d' % len(corpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train LDA model.\n",
    "#from gensim.models import LdaModel\n",
    "from gensim.models.ldamulticore import LdaMulticore\n",
    "\n",
    "# Set training parameters.\n",
    "num_topics = 30\n",
    "chunksize = 2000\n",
    "passes = 20\n",
    "iterations = 400\n",
    "eval_every = None  # Don't evaluate model perplexity, takes too much time.\n",
    "\n",
    "# Make a index to word dictionary.\n",
    "temp = dictionary[0]  # This is only to \"load\" the dictionary.\n",
    "id2word = dictionary.id2token\n",
    "\n",
    "model = LdaMulticore(\n",
    "    corpus=corpus,\n",
    "    id2word=id2word,\n",
    "    chunksize=chunksize,\n",
    "    alpha='symmetric',\n",
    "    eta='auto',\n",
    "    iterations=iterations,\n",
    "    num_topics=num_topics,\n",
    "    passes=passes,\n",
    "    eval_every=eval_every,\n",
    "    workers=5\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_topics = model.top_topics(corpus) #, num_words=20)\n",
    "\n",
    "# Average topic coherence is the sum of topic coherences of all topics, divided by the number of topics.\n",
    "avg_topic_coherence = sum([t[1] for t in top_topics]) / num_topics\n",
    "print('Average topic coherence: %.4f.' % avg_topic_coherence)\n",
    "\n",
    "from pprint import pprint\n",
    "pprint(top_topics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate vectors and metadata file for loading and visualizing in embedding projector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.get_document_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vecs = []\n",
    "for d in docs:\n",
    "    vecs.append(model.get_document_topics(d))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(doc_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_texts[0][0:300]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vecs = []\n",
    "meta_data = []\n",
    "counter = 0\n",
    "for c in corpus:\n",
    "    vecs.append(gensim.matutils.sparse2full(model.get_document_topics(c), num_topics))\n",
    "    meta_data.append(doc_texts[counter][0:300])\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_vecs_filename = 'output_vecs_v1.tsv'\n",
    "with open(out_vecs_filename, 'w') as outfile:\n",
    "    for v in vecs:\n",
    "        outfile.write('\\t'.join(v)+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}