rej analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import mailbox\n",
    "import email.utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime\n",
    "import plotly.express as px"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "mbox = mailbox.mbox('Rej.mbox')\n",
    "mbox2 = mailbox.mbox('rej2.mbox')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Email Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import mailbox\n",
    "import bs4\n",
    "\n",
    "def get_html_text(html):\n",
    "    try:\n",
    "        return bs4.BeautifulSoup(html, \"html5lib\").body.get_text(' ', strip=True)\n",
    "    except AttributeError: # message contents empty\n",
    "        return None\n",
    "\n",
    "class GmailMboxMessage():\n",
    "    def __init__(self, email_data):\n",
    "        if not isinstance(email_data, mailbox.mboxMessage):\n",
    "            raise TypeError('Variable must be type mailbox.mboxMessage')\n",
    "        self.email_data = email_data\n",
    "        self.labels= self.date= self.efrom= self.eto= self.subject= self.text=None\n",
    "        \n",
    "\n",
    "    def parse_email(self):\n",
    "        email_labels = self.email_data['X-Gmail-Labels']\n",
    "        email_date = self.email_data['Date']\n",
    "        email_from = self.email_data['From']\n",
    "        email_to = self.email_data['To']\n",
    "        email_subject = self.email_data['Subject']\n",
    "        email_text = self.read_email_payload() \n",
    "        \n",
    "        self.labels=email_labels\n",
    "        self.date=email_date\n",
    "        self.efrom=email_from\n",
    "        self.eto=email_to\n",
    "        self.subject=email_subject\n",
    "        self.text=email_text\n",
    "        \n",
    "\n",
    "    def read_email_payload(self):\n",
    "        email_payload = self.email_data.get_payload()\n",
    "        if self.email_data.is_multipart():\n",
    "            email_messages = list(self._get_email_messages(email_payload))\n",
    "        else:\n",
    "            email_messages = [email_payload]\n",
    "        return [self._read_email_text(msg) for msg in email_messages]\n",
    "\n",
    "    def _get_email_messages(self, email_payload):\n",
    "        for msg in email_payload:\n",
    "            if isinstance(msg, (list,tuple)):\n",
    "                for submsg in self._get_email_messages(msg):\n",
    "                    yield submsg\n",
    "            elif msg.is_multipart():\n",
    "                for submsg in self._get_email_messages(msg.get_payload()):\n",
    "                    yield submsg\n",
    "            else:\n",
    "                yield msg\n",
    "\n",
    "    def _read_email_text(self, msg):\n",
    "        content_type = 'NA' if isinstance(msg, str) else msg.get_content_type()\n",
    "        encoding = 'NA' if isinstance(msg, str) else msg.get('Content-Transfer-Encoding', 'NA')\n",
    "        if 'text/plain' in content_type and 'base64' not in encoding:\n",
    "            msg_text = msg.get_payload()\n",
    "        elif 'text/html' in content_type and 'base64' not in encoding:\n",
    "            msg_text = get_html_text(msg.get_payload())\n",
    "        elif content_type == 'NA':\n",
    "            msg_text = get_html_text(msg)\n",
    "        else:\n",
    "            msg_text = None\n",
    "        return (content_type, encoding, msg_text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "emails=[]\n",
    "num_entries = len(mbox)\n",
    "for idx, email_obj in enumerate(mbox):\n",
    "    email_data = GmailMboxMessage(email_obj)\n",
    "    email_data.parse_email()\n",
    "    emails.append(email_data)\n",
    "#     print('Parsing email {0} of {1}'.format(idx, num_entries))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "num_entries = len(mbox2)\n",
    "for idx, email_obj in enumerate(mbox2):\n",
    "    email_data = GmailMboxMessage(email_obj)\n",
    "    email_data.parse_email()\n",
    "    emails.append(email_data)\n",
    "#     print('Parsing email {0} of {1}'.format(idx, num_entries))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# construct the dataframe\n",
    "email_df= pd.DataFrame()\n",
    "for e in emails:\n",
    "    email_df=email_df.append([{'date':e.date,'from':e.efrom,'to':e.eto,'subject':e.subject,'text':e.text}])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytz import timezone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dateutil.parser import parse\n",
    "from datetime import datetime as dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "email_df['date_n']=pd.to_datetime(email_df.date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "email_df['date_es']=email_df['date_n'].apply(lambda x: x.astimezone(timezone('US/Eastern')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "email_df['weekdays']=email_df.date_es.apply(lambda x: dt.strftime(x, \"%A\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "email_df['hour']=email_df.date_es.apply(\n",
    "    lambda x: dt.strftime(x, \"%I %p\")\n",
    ") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "day_list = [\n",
    "    \"Monday\",\n",
    "    \"Tuesday\",\n",
    "    \"Wednesday\",\n",
    "    \"Thursday\",\n",
    "    \"Friday\",\n",
    "    \"Saturday\",\n",
    "    \"Sunday\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Mining"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Reich\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import re, string\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "\n",
    "\n",
    "stop = list(stopwords.words('english'))\n",
    "stop.extend(['yukun','yukun yang','yang','data','scientist'])\n",
    "\n",
    "def extract_text(text_list):\n",
    "    \n",
    "    tags=[component[0] for component in text_list]\n",
    "    \n",
    "    real_content=None\n",
    "    if 'text/html' in tags:\n",
    "        ind=[component[0] for component in text_list].index('text/html')\n",
    "        real_content=text_list[ind][-1]\n",
    "        if real_content=='None':\n",
    "            real_content=None\n",
    "\n",
    "    elif 'text/plain' in tags:\n",
    "        ind=[component[0] for component in text_list].index('text/plain')\n",
    "        real_content=text_list[ind][-1]\n",
    "    elif 'NA' in tags:\n",
    "        ind=[component[0] for component in text_list].index('NA')\n",
    "        real_content=text_list[ind][-1]\n",
    "    \n",
    "    if (real_content is not None):\n",
    "            if len(real_content)>10000:\n",
    "                real_content=None\n",
    "    \n",
    "    return real_content\n",
    "  \n",
    "        \n",
    "def clean_text(text):\n",
    "    \n",
    "    if text is not None:\n",
    "    \n",
    "        text=re.sub('=\\n', '', text) \n",
    "\n",
    "        text=re.sub('\\S*@\\S*\\s?', '',  text)\n",
    "\n",
    "        text=' '.join(word.strip(string.punctuation) for word in text.split())\n",
    "\n",
    "        text=re.sub(r'((http|https)\\:\\/\\/)?[a-zA-Z0-9\\.\\/\\?\\:@\\-_=#]+\\.([a-zA-Z]){2,6}([a-zA-Z0-9\\.\\&\\/\\?\\:@\\-_=#])*', '', text, flags=re.MULTILINE)\n",
    "\n",
    "    #     text=re.sub(r'\\..*\\..* ?', '', text, flags=re.MULTILINE)\n",
    "\n",
    "        text=re.sub(r\"\\d+\", \"\", text)\n",
    "\n",
    "        text=re.sub(r\"={1}.{2}\", \"\", text)\n",
    "        \n",
    "        text=text.replace('size','').replace('text size', '').replace('adjust','').replace('td','')\n",
    "        \n",
    "        \n",
    "        \n",
    "        return text\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "email_df['extracted']=email_df.text.apply(extract_text)\n",
    "email_df['cleaned']=email_df.extracted.apply(clean_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "  \n",
    "    \n",
    "def important_words(metric, ranks):\n",
    "\n",
    "    \n",
    "    import nltk\n",
    "    from nltk.corpus import stopwords\n",
    "    nltk.download('stopwords')\n",
    "\n",
    "#     stop = list(stopwords.words('english'))\n",
    "#     stop.extend(['yukun','yukun yang','yang','data','scientist'])\n",
    "    \n",
    "    if metric=='tf':\n",
    "        vectorizer= CountVectorizer(ngram_range=(1,3),stop_words=stop)\n",
    "        vectors = vectorizer.fit_transform(email_df.dropna().cleaned.to_list())\n",
    "    elif metric=='tfidf':\n",
    "        vectorizer = TfidfVectorizer(ngram_range=(1,3),stop_words=stop)\n",
    "#         tf_vectorizer= CountVectorizer(ngram_range=(1,3),stop_words=stop)\n",
    "        vectors = vectorizer.fit_transform(email_df.dropna().cleaned.to_list())\n",
    "    \n",
    "    \n",
    "    #making df\n",
    "    rankings=pd.DataFrame(vectors.todense().tolist(), columns=vectorizer.get_feature_names()).sum().reset_index().rename(columns={'index':'word',0:'value'})\n",
    "    \n",
    "#     if rankings.value.dtype !='int':\n",
    "    rankings['value']=rankings['value'].round(2)\n",
    "    #making distinguish\n",
    "    rankings['type']=None\n",
    "    for ind, row in rankings.iterrows():\n",
    "            num=len(row['word'].split())\n",
    "            if num==1:\n",
    "                rankings.loc[ind,'type']='unigram'\n",
    "            elif num==2:\n",
    "                rankings.loc[ind,'type']='bigram'\n",
    "            elif num==3:\n",
    "                rankings.loc[ind,'type']='trigram'\n",
    "    \n",
    "    return rankings.sort_values('value', ascending=False).head(ranks)\n",
    "\n",
    "def make_important_graphs(df):\n",
    "    \n",
    "    \n",
    "    bar=px.bar(df, y='value', x='word', text='value', color='type', template='seaborn', title='Important Terms Bar Chart')\n",
    "    bar.update_layout(xaxis_categoryorder = 'total descending')\n",
    "    \n",
    "    tree=px.treemap(df,path=['type','word'],values='value', template='seaborn', title='Important Terms Tree Map')\n",
    "    return bar,tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.collocations import *\n",
    "from collections import Counter\n",
    "\n",
    "def collo(metric):\n",
    "    \n",
    "\n",
    "    bigram_measures = nltk.collocations.BigramAssocMeasures()\n",
    "    # trigram_measures = nltk.collocations.TrigramAssocMeasures()\n",
    "\n",
    "    text=' '.join(i for i in email_df.dropna().cleaned.to_list())\n",
    "    words=[word for word in text.lower().split() if word not in stop]\n",
    "\n",
    "    # change this to read in your data\n",
    "    finder = BigramCollocationFinder.from_words(words)\n",
    "\n",
    "    # only bigrams that appear 3+ times\n",
    "    finder.apply_freq_filter(3)\n",
    "\n",
    "    # return the 10 n-grams with the highest PMI\n",
    "    # finder.nbest(bigram_measures.pmi, 15)\n",
    "#     finder.nbest(bigram_measures.likelihood_ratio, 15)\n",
    "    if metric=='pmi':\n",
    "        coli=finder.score_ngrams(bigram_measures.pmi)\n",
    "    elif metric=='chisquare':\n",
    "        coli=finder.score_ngrams(bigram_measures.chi_sq)\n",
    "    elif metric=='likelihood_ratio':\n",
    "        coli=finder.score_ngrams(bigram_measures.likelihood_ratio)\n",
    "    \n",
    "    return coli\n",
    "\n",
    "def make_circos(test_co):\n",
    "    \n",
    "    colors=['#fff1d6','#c9a03d','#02b1a0','#848484','#cfcfcf','#a2e8eb','#ecd1fc',\"#f0b3c5\",\"#c4e5d6\",\"#d7f2fd\",'#feb408',\n",
    "            '#f09654','#ee6f37','#6da393','#007890','#e8c8ee','#ffa3a3','#f6e777']\n",
    "    \n",
    "\n",
    "    colors.extend(colors)\n",
    "    \n",
    "    top20=test_co[:10]\n",
    "    workcounter=Counter()\n",
    "    for i in top20:\n",
    "        workcounter.update(i[0])\n",
    "\n",
    "\n",
    "    ideogram=[]\n",
    "    for tu in workcounter.most_common():\n",
    "        ideogram.append({'id':tu[0], 'label':tu[0], 'color':colors.pop(), 'len':tu[-1]})\n",
    "\n",
    "    ribbons=[]\n",
    "    for co in top20:\n",
    "        ribbons.append({'color':'#9ecaf6', \n",
    "                        'source':{'id':co[0][0],'start':0,'end':1},\n",
    "                       'target':{'id':co[0][-1],'start':0,'end':1}})\n",
    "        \n",
    "    return  dashbio.Circos(\n",
    "                id='circos',\n",
    "                layout=ideogram,\n",
    "        size=600,\n",
    "                selectEvent={'hover': \"hover\", 'click': \"click\", 2: \"both\"},\n",
    "#                 eventDatum={\"0\": \"hover\", \"1\": \"click\", \"2\": \"both\"},\n",
    "                config= {'ticks': {'display': False},'labels':{'size': 10,'position': 'center'}},\n",
    "                tracks=[{\n",
    "                    'type': 'CHORDS',\n",
    "                    'data': ribbons,\n",
    "                     'selectEvent':{\"0\": \"hover\", \"1\": \"click\", \"2\": \"both\"},\n",
    "                     'tooltipContent': {\n",
    "                                        'source': 'source',\n",
    "                                        'sourceID': 'id',\n",
    "                                        'target': 'target',\n",
    "                                        'targetID': 'id',\n",
    "                                        'targetEnd': 'end'\n",
    "                                    }\n",
    "                                }\n",
    "                ]\n",
    "            )\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "import spacy\n",
    "def sent_to_words(sentences):\n",
    "    for sentence in sentences:\n",
    "        yield(gensim.utils.simple_preprocess(str(sentence), deacc=True))\n",
    "def remove_stopwords(texts):\n",
    "\n",
    "    new_docs=[]\n",
    "    for doc in texts:\n",
    "        new_docs.append([word for word in doc if word not in stop])\n",
    "    return new_docs    \n",
    "    \n",
    "\n",
    "def make_bigrams(texts):\n",
    "    return [bigram_mod[doc] for doc in texts]\n",
    "\n",
    "def make_trigrams(texts):\n",
    "    return [trigram_mod[bigram_mod[doc]] for doc in texts]\n",
    "\n",
    "def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):\n",
    "    \"\"\"https://spacy.io/api/annotation\"\"\"\n",
    "    texts_out = []\n",
    "    for sent in texts:\n",
    "        doc = nlp(\" \".join(sent)) \n",
    "        texts_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags])\n",
    "    return texts_out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=email_df[email_df.cleaned.notna()].cleaned.values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_words = list(sent_to_words(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigram = gensim.models.Phrases(data_words, min_count=5, threshold=100) # higher threshold fewer phrases.\n",
    "trigram = gensim.models.Phrases(bigram[data_words], threshold=100)  \n",
    "\n",
    "bigram_mod = gensim.models.phrases.Phraser(bigram)\n",
    "trigram_mod = gensim.models.phrases.Phraser(trigram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove Stop Words\n",
    "data_words_nostops = remove_stopwords(data_words)\n",
    "\n",
    "# Form Bigrams\n",
    "data_words_bigrams = make_bigrams(data_words_nostops)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en', disable=['parser', 'ner'])\n",
    "\n",
    "# Do lemmatization keeping only noun, adj, vb, adv\n",
    "data_lemmatized = lemmatization(data_words_bigrams, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])\n",
    "\n",
    "# print(data_lemmatized[:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "import gensim.corpora as corpora\n",
    "from gensim.utils import simple_preprocess\n",
    "from gensim.models import CoherenceModel\n",
    "# Create Dictionary\n",
    "id2word = corpora.Dictionary(data_lemmatized)\n",
    "\n",
    "# Create Corpus\n",
    "texts = data_lemmatized\n",
    "\n",
    "# Term Document Frequency\n",
    "corpus = [id2word.doc2bow(text) for text in texts]\n",
    "\n",
    "# View\n",
    "# print(corpus[:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_coherence_values(dictionary, corpus, texts, limit, start=2, step=3):\n",
    "    \"\"\"\n",
    "    Compute c_v coherence for various number of topics\n",
    "\n",
    "    Parameters:\n",
    "    ----------\n",
    "    dictionary : Gensim dictionary\n",
    "    corpus : Gensim corpus\n",
    "    texts : List of input texts\n",
    "    limit : Max num of topics\n",
    "\n",
    "    Returns:\n",
    "    -------\n",
    "    model_list : List of LDA topic models\n",
    "    coherence_values : Coherence values corresponding to the LDA model with respective number of topics\n",
    "    \"\"\"\n",
    "    coherence_values = []\n",
    "    model_list = []\n",
    "    for num_topics in range(start, limit, step):\n",
    "        model = gensim.models.ldamodel.LdaModel(corpus=corpus, num_topics=num_topics, id2word=id2word,random_state=2020)\n",
    "        model_list.append(model)\n",
    "        coherencemodel = CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence='c_v')\n",
    "        coherence_values.append(coherencemodel.get_coherence())\n",
    "\n",
    "    return model_list, coherence_values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_list, coherence_values = compute_coherence_values(dictionary=id2word, corpus=corpus, texts=data_lemmatized, start=2, limit=40, step=2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "x=range(2,40,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "choose_k=pd.DataFrame({'# of Topics':x,'coherence':coherence_values})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_topics_sentences(ldamodel=None, corpus=corpus, texts=data):\n",
    "    # Init output\n",
    "    sent_topics_df = pd.DataFrame()\n",
    "\n",
    "    # Get main topic in each document\n",
    "    for i, row_list in enumerate(ldamodel[corpus]):\n",
    "        row = row_list[0] if ldamodel.per_word_topics else row_list            \n",
    "        # print(row)\n",
    "        row = sorted(row, key=lambda x: (x[1]), reverse=True)\n",
    "        # Get the Dominant topic, Perc Contribution and Keywords for each document\n",
    "        for j, (topic_num, prop_topic) in enumerate(row):\n",
    "            if j == 0:  # => dominant topic\n",
    "                wp = ldamodel.show_topic(topic_num)\n",
    "                topic_keywords = \", \".join([word for word, prop in wp])\n",
    "                sent_topics_df = sent_topics_df.append(pd.Series([int(topic_num), round(prop_topic,4), topic_keywords]), ignore_index=True)\n",
    "            else:\n",
    "                break\n",
    "    sent_topics_df.columns = ['Dominant_Topic', 'Perc_Contribution', 'Topic_Keywords']\n",
    "\n",
    "    # Add original text to the end of the output\n",
    "    contents = pd.Series(texts)\n",
    "    sent_topics_df = pd.concat([sent_topics_df, contents], axis=1)\n",
    "    return(sent_topics_df)\n",
    "\n",
    "\n",
    "# df_topic_sents_keywords = format_topics_sentences(ldamodel=lda_model, corpus=corpus, texts=data_lemmatized)\n",
    "\n",
    "# # Format\n",
    "# df_dominant_topic = df_topic_sents_keywords.reset_index()\n",
    "# df_dominant_topic.columns = ['Document_No', 'Dominant_Topic', 'Topic_Perc_Contrib', 'Keywords', 'Text']\n",
    "# df_dominant_topic.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define all functions for the app"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def intro():\n",
    "    \"\"\"\n",
    "    :return: A Div containing dashboard title & descriptions.\n",
    "    \"\"\"\n",
    "    return html.Div(\n",
    "        id=\"description-card\",\n",
    "        children=[\n",
    "            html.H3(\"Rejection Email Analytics\"),\n",
    "            #             html.H3(\"Welcome to the Clinical Analytics Dashboard\"),\n",
    "            html.Blockquote(\n",
    "                id=\"intro\",\n",
    "                children=[\"Rejection hurts. Yes but yet, I've met nobody who has not been rejected.\\\n",
    "                It is a part of life and instead of drowning in the sorrow and somberness of being rejected, we can make it fun and try to try to analyze it.\",\n",
    "                         ]\n",
    "            ),\n",
    "            html.Div(children=['👋 My name is ',\n",
    "                               html.A(\"Yukun\", href='#contact_info'),\n",
    "                               \", I graduated in this crazy time of the year and have collected 100 rejection emails from all kinds of employers during these 3 months.\\\n",
    "            Here I am applying my Data Science skills in Interactive Data Viz, Temporal Point Process Modelling, and Text Mining to analyze these emails. Hope you enjoy it!😀\"]),\n",
    "            html.Br(),\n",
    "            html.H4('Instructions'),\n",
    "            html.Div(children=[\n",
    "                html.P(\"This dashboard enables multiple ways for you to interact with the plots. Every plot can be zoomed and selected, along with hovering tooltips. Despite these basics, it also supports the following interactions:\"),\n",
    "                html.Li(\n",
    "                    'Subsetting the dataset by selecting the time range, the weekdays, and the hours.'),\n",
    "                html.Li('Showing specific data entries by clicking on the Heatmap.'),\n",
    "                html.Div('Other interaction options are detailed in the corresponding part')])\n",
    "#                 html.Li(\n",
    "#                     'Change the Solver of the Temporal Process and the number of days to prdict.'),\n",
    "#                 html.Li('Change the metric to rank the important terms, and the number of topics to model.')])\n",
    "        ],\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def control_card():\n",
    "    return html.Div(\n",
    "        id=\"control-card\",\n",
    "        children=[\n",
    "            html.Strong(\"Select Date Range\"),\n",
    "            dcc.DatePickerRange(\n",
    "                id='my-date-picker-range',\n",
    "                min_date_allowed=email_df['date_es'].min().date(),\n",
    "                max_date_allowed=email_df['date_es'].max().date(),\n",
    "                initial_visible_month=dt(2020, 4, 5),\n",
    "                start_date=email_df['date_es'].min().date(),\n",
    "                end_date=email_df['date_es'].max().date()\n",
    "            ),\n",
    "            html.Br(),\n",
    "            html.Br(),\n",
    "            html.Strong(\"Select the Day of a Week\"),\n",
    "            dcc.Dropdown(\n",
    "                id='weekdays',\n",
    "                options=[{'label': day, 'value': day} for day in day_list],\n",
    "                value=day_list,\n",
    "                multi=True\n",
    "#                 style=dict(\n",
    "# #                     height='100%',\n",
    "#                     display='block',\n",
    "#                     verticalAlign=\"middle\"\n",
    "#                 )\n",
    "            ),\n",
    "            html.Br(),\n",
    "            html.Br(),\n",
    "            html.Strong(\"Select Specific Hours\"),\n",
    "            dcc.Checklist(\n",
    "                id='time',\n",
    "                options=[{'label': t, 'value': t}\n",
    "                         for t in[datetime.time(i).strftime(\"%I %p\") for i in range(24)]],\n",
    "                value=[datetime.time(i).strftime(\"%I %p\") for i in range(24)],\n",
    "                labelStyle={'display': 'inline-block'}\n",
    "            ),\n",
    "        ],\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_df(start,end,weekdays,time):\n",
    "    filtered_df = email_df.sort_values(\"date_es\").set_index(\"date_es\")[\n",
    "        start.astimezone(timezone('US/Eastern')):end.astimezone(timezone('US/Eastern'))\n",
    "    ]\n",
    "    \n",
    "    filtered_df= filtered_df[filtered_df.weekdays.isin(weekdays)]\n",
    "    filtered_df= filtered_df[filtered_df.hour.isin(time)]\n",
    "    return filtered_df.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_paco(filtered):\n",
    "    \n",
    "    filtered.loc[filtered.date_es.dt.weekday.isin([6,5]),'Is Weekend']=True\n",
    "    filtered['Is Weekend']=filtered['Is Weekend'].fillna(False)\n",
    "    filtered['Day Time']=((filtered.date_es.dt.hour % 24 + 4) // 4).map({1: 'Late Night',\n",
    "                      2: 'Early Morning',\n",
    "                      3: 'Morning',\n",
    "                      4: 'Noon',\n",
    "                      5: 'Evening',\n",
    "                      6: 'Night'})\n",
    "    \n",
    "    groupedby=filtered.groupby(['Is Weekend','weekdays','Day Time','hour'])['from'].count().reset_index()\n",
    "    \n",
    "    new_df=pd.merge(left=filtered,right=groupedby, left_on=['Is Weekend','weekdays','Day Time','hour'],right_on=['Is Weekend','weekdays','Day Time','hour'])\n",
    "\n",
    "    fig=px.parallel_categories(data_frame=new_df, \n",
    "                               dimensions=['Is Weekend','weekdays','Day Time','hour'],\n",
    "                               color='from_y',\n",
    "                               labels={'weekdays':'Day in the Week','hour':'Hour in the Day'},\n",
    "                              color_continuous_scale=px.colors.sequential.dense)\n",
    "    \n",
    "    fig.layout['coloraxis']['colorbar']['title']['text']='Count'\n",
    "    fig.update_layout({'height':600})\n",
    "    \n",
    "    fig.layout.margin={'t':30, 'l':10, 'r':10,'b':20}\n",
    "    return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# f=generate_paco(email_df)\n",
    "# f.layout.margin=['t':30, 'l':10, 'r':10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_patient_volume_heatmap(start, end, hm_click, reset, weekdays, time):\n",
    "    \"\"\"\n",
    "    :param: start: start date from selection.\n",
    "    :param: end: end date from selection.\n",
    "    :param: clinic: clinic from selection.\n",
    "    :param: hm_click: clickData from heatmap.\n",
    "    :param: admit_type: admission type from selection.\n",
    "    :param: reset (boolean): reset heatmap graph if True.\n",
    "    :return: Patient volume annotated heatmap.\n",
    "    \"\"\"\n",
    "\n",
    "#     filtered_df = df[\n",
    "#         (df[\"Clinic Name\"] == clinic) & (df[\"Admit Source\"].isin(admit_type))\n",
    "#     ]\n",
    "    filtered_df = email_df.sort_values(\"date_es\").set_index(\"date_es\")[\n",
    "        start.astimezone(timezone('US/Eastern')):end.astimezone(timezone('US/Eastern'))\n",
    "    ]\n",
    "\n",
    "    filtered_df = filtered_df[filtered_df.weekdays.isin(weekdays)]\n",
    "    filtered_df = filtered_df[filtered_df.hour.isin(time)]\n",
    "\n",
    "    x_axis = [datetime.time(i).strftime(\"%I %p\")\n",
    "              for i in range(24)]  # 24hr time list\n",
    "    y_axis = day_list\n",
    "\n",
    "    hour_of_day = \"\"\n",
    "    weekday = \"\"\n",
    "    shapes = []\n",
    "\n",
    "\n",
    "    if hm_click is not None:\n",
    "        hour_of_day = hm_click[\"points\"][0][\"x\"]\n",
    "        weekday = hm_click[\"points\"][0][\"y\"]\n",
    "\n",
    "        # Add shapes\n",
    "        x0 = x_axis.index(hour_of_day) / 24\n",
    "        x1 = x0 + 1 / 24\n",
    "        y0 = y_axis.index(weekday) / 7\n",
    "        y1 = y0 + 1 / 7\n",
    "\n",
    "        shapes = [\n",
    "            dict(\n",
    "                type=\"rect\",\n",
    "                xref=\"paper\",\n",
    "                yref=\"paper\",\n",
    "                x0=x0,\n",
    "                x1=x1,\n",
    "                y0=y0,\n",
    "                y1=y1,\n",
    "                line=dict(color=\"#ff6347\"),\n",
    "            )\n",
    "        ]\n",
    "\n",
    "    z = np.zeros((7, 24))\n",
    "    annotations = []\n",
    "\n",
    "    for ind_y, day in enumerate(y_axis):\n",
    "        filtered_day = filtered_df[filtered_df[\"weekdays\"] == day]\n",
    "        for ind_x, x_val in enumerate(x_axis):\n",
    "            sum_of_record = len(filtered_day[filtered_day[\"hour\"] == x_val])\n",
    "            z[ind_y][ind_x] = sum_of_record\n",
    "\n",
    "            annotation_dict = dict(\n",
    "                showarrow=False,\n",
    "                text=\"<b>\" + str(sum_of_record) + \"<b>\",\n",
    "                xref=\"x\",\n",
    "                yref=\"y\",\n",
    "                x=x_val,\n",
    "                y=day,\n",
    "                font=dict(family=\"sans-serif\"),\n",
    "            )\n",
    "            # Highlight annotation text by self-click\n",
    "            if x_val == hour_of_day and day == weekday:\n",
    "                if not reset:\n",
    "                    annotation_dict.update(size=15, font=dict(color=\"#ff6347\"))\n",
    "\n",
    "            annotations.append(annotation_dict)\n",
    "\n",
    "    hovertemplate = \"<b> %{y}  %{x} <br><br> %{z} Emails\"\n",
    "    data = [\n",
    "        dict(\n",
    "            x=x_axis,\n",
    "            y=y_axis,\n",
    "            z=z,\n",
    "            type=\"heatmap\",\n",
    "            name=\"\",\n",
    "            hovertemplate=hovertemplate,\n",
    "            showscale=False,\n",
    "            colorscale=[[0, \"#caf3ff\"], [1, \"#2c82ff\"]],\n",
    "        )\n",
    "    ]\n",
    "\n",
    "    layout = dict(\n",
    "        margin=dict(l=70, b=30, t=50, r=50),\n",
    "        modebar={\"orientation\": \"v\"},\n",
    "        font=dict(family=\"Open Sans\"),\n",
    "        annotations=annotations,\n",
    "        shapes=shapes,\n",
    "        xaxis=dict(\n",
    "            side=\"top\",\n",
    "            ticks=\"\",\n",
    "            ticklen=2,\n",
    "            tickfont=dict(family=\"sans-serif\"),\n",
    "            tickcolor=\"#ffffff\",\n",
    "        ),\n",
    "        yaxis=dict(\n",
    "            side=\"left\", ticks=\"\", tickfont=dict(family=\"sans-serif\"), ticksuffix=\" \"\n",
    "        ),\n",
    "        hovermode=\"closest\",\n",
    "        showlegend=False,\n",
    "    )\n",
    "    return {\"data\": data, \"layout\": layout}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_hist(start,end, weekdays,time):\n",
    "    \n",
    "    df=email_df.copy()\n",
    "    df['date_pure']=df.date_es.dt.date\n",
    "    df=df.sort_values(\"date_pure\").set_index(\"date_pure\")\n",
    "    \n",
    "    df['selected']=False\n",
    "    \n",
    "    \n",
    "#     print(df)\n",
    "#     df.loc[~df.weekdays.isin(weekdays),'selected']=False\n",
    "#     df.loc[~df.hour.isin(time),'selected']=False\n",
    "\n",
    "    df.loc[pd.to_datetime(start).date():pd.to_datetime(end).date(),'selected']=True\n",
    "    \n",
    "    df.loc[~df.weekdays.isin(weekdays),'selected']=False\n",
    "    df.loc[~df.hour.isin(time),'selected']=False\n",
    "    df=df.reset_index()\n",
    "    fig=px.histogram(df, \"date_es\", color='selected', marginal=\"rug\", nbins=12,\n",
    "                         height=400,\n",
    "                   color_discrete_map={\n",
    "                True: \"rgb(166,206,227)\", False: \"rgb(31,120,180)\"\n",
    "            },\n",
    "                  )\n",
    "    fig.update_layout(margin=dict(l=2, r=2, t=2, b=2), height=200)\n",
    "    fig.layout.xaxis.title.text=None\n",
    "    fig.layout.yaxis.title.text='Count'\n",
    "#     fig.layout.legend['orientation']='h'\n",
    "#     fig.update_layout(legend=dict(x=0.25, y=-0.25))\n",
    "    fig.data[0]['nbinsx']=20\n",
    "    return fig\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fig=generate_hist('2020-03-15', '2020-06-15', day_list, ['12 PM'])\n",
    "# fig.layout.xaxis.title.text=None\n",
    "# fig.layout.margin.l=30\n",
    "# fig.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tick.hawkes import (SimuHawkes, HawkesKernelTimeFunc, HawkesKernelExp,\n",
    "                         HawkesEM, SimuHawkesSumExpKernels, HawkesSumExpKern,HawkesExpKern)\n",
    "from tick.plot import plot_point_process\n",
    "import itertools\n",
    "import plotly.tools as tls\n",
    "\n",
    "\n",
    "def haweks(learner, pre_days):\n",
    "\n",
    "    test_time = (email_df.date_es.sort_values() -\n",
    "                 email_df.date_es.min()).astype('timedelta64[h]')/24.0\n",
    "    timestamps = [test_time.to_numpy(dtype='double')]\n",
    "\n",
    "    best_score = -1e100\n",
    "    decay_candidates = np.logspace(0, 6, 20)\n",
    "\n",
    "    if learner == 'Exponential':\n",
    "\n",
    "        for i, decay in enumerate(decay_candidates):\n",
    "            hawkes_learner = HawkesExpKern(decay, verbose=False, max_iter=10000,\n",
    "                                           tol=1e-10)\n",
    "    #         hawkes_learner = HawkesSumExpKern(decays=[6])\n",
    "            hawkes_learner.fit(timestamps)\n",
    "\n",
    "            hawkes_score = hawkes_learner.score()\n",
    "            if hawkes_score > best_score:\n",
    "                print('obtained {}\\n with {}\\n'.format(hawkes_score, decay))\n",
    "                best_hawkes = hawkes_learner\n",
    "                best_score = hawkes_score\n",
    "\n",
    "    elif learner == 'ExponentialSum':\n",
    "        decay_candidates = np.logspace(0, 3, 10)\n",
    "        for i, decays in enumerate(itertools.combinations(decay_candidates, 3)):\n",
    "            # Each time we test a different set of 3 decays.\n",
    "            decays = np.array(decays)\n",
    "            hawkes_learner = HawkesSumExpKern(decays, verbose=False, max_iter=10000,\n",
    "                                              tol=1e-10)\n",
    "#             hawkes_learner._prox_obj.positive = False\n",
    "            hawkes_learner.fit(timestamps)\n",
    "\n",
    "            hawkes_score = hawkes_learner.score()\n",
    "            if hawkes_score > best_score:\n",
    "                print('obtained {}\\n with {}\\n'.format(hawkes_score, decays))\n",
    "                best_hawkes = hawkes_learner\n",
    "                best_score = hawkes_score\n",
    "\n",
    "\n",
    "    simu = best_hawkes._corresponding_simu()\n",
    "    simu.seed=2020\n",
    "    simu.track_intensity(0.01)\n",
    "    simu.set_timestamps([test_time.to_numpy(dtype='double')])\n",
    "    simu.end_time = 100+pre_days\n",
    "    simu.simulate()\n",
    "\n",
    "\n",
    "# process = plot_point_process(simu, plot_intensity=True)\n",
    "\n",
    "    plotly_fig = tls.mpl_to_plotly(plot_point_process(simu, plot_intensity=True))\n",
    "\n",
    "    return seperate(plotly_fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly.graph_objects as go\n",
    "def seperate(f):\n",
    "    original_f= go.Figure(f)\n",
    "    cop_f= go.Figure(f)\n",
    "    \n",
    "    new_color='rgba(63, 191, 63, 0.4)'\n",
    "    \n",
    "    cop_f.data[0]['x']=tuple(filter(lambda x: x > 100, cop_f.data[0]['x']))\n",
    "    cop_f.data[0]['y']=cop_f.data[0]['y'][len(cop_f.data[0]['y'])-len(cop_f.data[0]['x']):]\n",
    "    \n",
    "        \n",
    "    cop_f.data[1]['x']=tuple(filter(lambda x: x > 100, cop_f.data[1]['x']))\n",
    "    cop_f.data[1]['y']=cop_f.data[1]['y'][len(cop_f.data[1]['y'])-len(cop_f.data[1]['x']):]\n",
    "    \n",
    "    \n",
    "    cop_f.data[1]['marker']['color']=new_color\n",
    "    cop_f.data[1]['marker']['line']['color']=new_color\n",
    "    \n",
    "    cop_f.data[0]['line']['color']=new_color\n",
    "    \n",
    "    \n",
    "    original_f.data[0]['x']=tuple(filter(lambda x: x <= 100, original_f.data[0]['x']))\n",
    "    \n",
    "    original_f.data[0]['y']=original_f.data[0]['y'][:len(original_f.data[0]['x'])]\n",
    "    \n",
    "        \n",
    "    original_f.data[1]['x']=tuple(filter(lambda x: x <= 100, original_f.data[1]['x']))\n",
    "    original_f.data[1]['y']=original_f.data[1]['y'][:len(original_f.data[1]['x'])]\n",
    "    \n",
    "    \n",
    "    \n",
    "    original_f.add_traces([i for i in cop_f['data']])\n",
    "    \n",
    "    original_f.update_layout({'height':400})\n",
    "#     original_f.layout.margin['l']=25\n",
    "    original_f.layout.margin={\n",
    "    'b': 50, 'l': 25, 'r': 30, 't': 50\n",
    "}\n",
    "    \n",
    "    \n",
    "    original_f.layout['xaxis']['title']={'font': {'color': '#000000', 'size': 13.0}, 'text': 'No. of Days since the 1st Rej Letter'}\n",
    "    \n",
    "    \n",
    "    original_f.update_layout(showlegend=True)\n",
    "    original_f.update_layout(legend_orientation=\"h\")\n",
    "    original_f.update_layout(legend=dict(x=0.25, y=-0.25))\n",
    "\n",
    "    original_f.add_shape(\n",
    "        # Line Vertical\n",
    "        dict(\n",
    "            type=\"line\",\n",
    "            x0=100,\n",
    "            y0=0,\n",
    "            x1=100,\n",
    "            y1=3,\n",
    "            line=dict(\n",
    "                color=\"RoyalBlue\",\n",
    "                width=2,\n",
    "                 dash=\"dashdot\"\n",
    "            )\n",
    "))\n",
    "    \n",
    "    \n",
    "    original_f['data'][0]['name']='Estimated Intensity of Original Events'\n",
    "    original_f['data'][1]['name']='Original Events'\n",
    "\n",
    "    original_f['data'][2]['name']='Estimated Intensity of Predicted Events'\n",
    "    original_f['data'][3]['name']='Predicted Events'\n",
    "\n",
    "    \n",
    "    original_f.layout['title']={'font': {'color': 'rgb(87, 145, 203)', 'size': 17}, \n",
    "                                'text': 'Hawekes Modelling Results','xanchor': 'center',\n",
    "        'yanchor': 'top','x':0.5}\n",
    "    original_f.layout.margin.l=30\n",
    "#     original_f.layout.width=1000\n",
    "    original_f.layout.autosize=True\n",
    "    \n",
    "    \n",
    "    return original_f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dash app running on http://127.0.0.1:8070/\n"
     ]
    }
   ],
   "source": [
    "import dash\n",
    "import plotly.express as px\n",
    "from jupyter_dash import JupyterDash\n",
    "import dash_bootstrap_components as dbc\n",
    "import dash_core_components as dcc\n",
    "import dash_html_components as html\n",
    "from dash.dependencies import Input, Output\n",
    "from dash import no_update\n",
    "import dash_table\n",
    "import dash_bio as dashbio\n",
    "import textstat\n",
    "import lexicalrichness\n",
    "import textblob\n",
    "\n",
    "\n",
    "def cal_slider(start_date, end_date):\n",
    "    start_value = (dt.strptime(start_date, \"%Y-%m-%d\") -\n",
    "                   (email_df['date_es'].min().tz_convert(None))).days+1\n",
    "    time_delta = (dt.strptime(end_date, \"%Y-%m-%d\")) - \\\n",
    "        (dt.strptime(start_date, \"%Y-%m-%d\"))\n",
    "    end_value = time_delta.days\n",
    "    return [start_value, start_value+end_value]\n",
    "\n",
    "\n",
    "def cal_range(value):\n",
    "    start_value, end_value = value\n",
    "    start_date = email_df['date_es'].min().date() + \\\n",
    "        datetime.timedelta(start_value)\n",
    "    end_date = start_date+datetime.timedelta(end_value)\n",
    "    return start_date, end_date\n",
    "\n",
    "\n",
    "app = JupyterDash(__name__, external_stylesheets=[\"https://codepen.io/chriddyp/pen/bWLwgP.css\",\n",
    "                                                  \"https://dash-gallery.plotly.host/dash-oil-and-gas/assets/styles.css?m=1590087908.0\"])\n",
    "# app = JupyterDash(__name__,external_stylesheets=[\"https://dash-gallery.plotly.host/dash-oil-and-gas/assets/styles.css?m=1590087908.0\"])\n",
    "\n",
    "\n",
    "app.layout = html.Div(\n",
    "    id=\"app-container\",\n",
    "    children=[\n",
    "        # Banner\n",
    "        html.Div(\n",
    "            id=\"banner\",\n",
    "            className=\"banner\"\n",
    "        ),  # Left column\n",
    "        html.Div(\n",
    "            id=\"left-column \",\n",
    "            style={},\n",
    "            className=\"four columns pretty_container \",\n",
    "            children=[intro(), control_card()],\n",
    "        ),\n",
    "        html.Div(\n",
    "            id=\"right-column\",\n",
    "            className=\"eight columns\",\n",
    "            style={},\n",
    "            children=[html.Div(\n",
    "                [\n",
    "                    html.Div(\n",
    "                        [html.H6(),\n",
    "                         html.P(\"No. of Days Selected\"),\n",
    "                         html.Strong(id=\"days_selected\")],\n",
    "                        id=\"days\",\n",
    "                        className=\"mini_container\",\n",
    "                    ),\n",
    "                    html.Div(\n",
    "                        [html.H6(), html.P(\n",
    "                            \"No. of Letters Received in the Period\"),\n",
    "                         html.Strong(id=\"total\")],\n",
    "\n",
    "                        className=\"mini_container\",\n",
    "                    ),\n",
    "                    html.Div(\n",
    "                        [html.H6(), html.P(\n",
    "                            \"Peak Day and Hour\"),\n",
    "                         html.Strong(id=\"peak_date\")],\n",
    "\n",
    "                        className=\"mini_container\",\n",
    "                    ),\n",
    "                    html.Div(\n",
    "                        [html.H6(id=\"pn\"), html.P(\n",
    "                            \"Rej Letters Peak Volume\"),\n",
    "                         html.Strong(id=\"peak_num\")],\n",
    "\n",
    "                        className=\"mini_container\",\n",
    "                    ),\n",
    "                ],\n",
    "                id=\"info-container\",\n",
    "                className=\"row container-display\",\n",
    "            ),\n",
    "                # Patient Volume Heatmap\n",
    "                html.Div(id='prop_id'),\n",
    "                html.Div(id='prop_type'),\n",
    "                html.Div(id='prop_value'),\n",
    "\n",
    "                html.Div(\n",
    "                id=\"patient_volume_card\",\n",
    "                className='mini_container',\n",
    "                children=[\n",
    "                    dcc.Loading(dcc.Graph(id='hist')),\n",
    "\n",
    "                    html.Hr(), html.B(\"Email Heatmap\"),\n",
    "                    html.Div(\n",
    "                        'The traffic of rejection emails! Click on the cells to see the actual entry.'),\n",
    "                    dcc.Graph(id=\"patient_volume_hm\"),\n",
    "                    dcc.RangeSlider(\n",
    "                        id='datetime_RangeSlider',\n",
    "                        updatemode='mouseup',  # don't let it update till mouse released\n",
    "                        min=0,\n",
    "                        #                 disabled=True,\n",
    "                        max=(email_df['date_es'].max().date() - email_df['date_es'].min().date()).days),\n",
    "                    html.Div(id='table_div', style={'display': 'none'},\n",
    "                             children=[dash_table.DataTable(\n",
    "                                 id='table',\n",
    "                                 style_cell={'textAlign': 'left', 'padding': '5px',\n",
    "                                             'overflow': 'hidden',\n",
    "                                             'textOverflow': 'ellipsis'},\n",
    "                                 style_data={'whiteSpace': 'normal'},\n",
    "                                 css=[{\n",
    "                                     'selector': '.dash-cell div.dash-cell-value',\n",
    "                                     'rule': 'display: inline; white-space: inherit; overflow: inherit; text-overflow: inherit;'\n",
    "                                 }],\n",
    "                                 columns=[\n",
    "                                     {'name': i, 'id': i, 'deletable': True} for i in ['date_es', 'subject']\n",
    "                                 ],\n",
    "                                 page_current=0,\n",
    "#                                  page_size=1,\n",
    "#                                  page_action='custom',\n",
    "\n",
    "#                                  sort_action='custom',\n",
    "#                                  sort_mode='single',\n",
    "                                 sort_by=[])],\n",
    "\n",
    "                             )\n",
    "                ],\n",
    "            )]),\n",
    "        html.Div(id='para', className=\"columns pretty_container\", children=[\n",
    "            html.H5('Parallel Coordinates of the Flow of Rej. Emails.'),\n",
    "            html.Div(\"Let's highlight the most prominent streamline of the email flow. \\\n",
    "                     It could come in handy when we observed some trends in the data. \\\n",
    "            This plot will update automatically with the Date/Day/Hour you selected in the Control widgets.\",\n",
    "                    style={'margin':\"10px\"}),\n",
    "            dcc.Graph(id='paco')]\n",
    "        ),\n",
    "        html.Div(id='temperal pro', className='columns pretty_container', children=[\n",
    "\n",
    "            html.H5(\"Temporal Process Analytics\", style={'margin-left':'10px'}),\n",
    "            html.P(\n",
    "                children=\"A Temporal Process is a kind of random process whose realization consists of discrete events \\\n",
    "                localized in time. Compared with \\\n",
    "                traditional Time-Seris, each data entry was allocated in different time interval. The scattering nature of receiving\\\n",
    "                an email fits better with a Temporal Process Analysis. \\n \\\n",
    "                A very popular kind of termporal process is the Haweks process, which could be consider\\\n",
    "                as an 'auto-regression' type of process. Here I used the Haweks Process to simulate the events.\\\n",
    "                You can select the Kernal and the days to forecast below.\", \n",
    "                style={'margin':'10px'}\n",
    "            ),\n",
    "            html.Div(\n",
    "                id=\"select model\",\n",
    "                style={\"text-align\": \"center\"},\n",
    "                className=\"six columns\", children=[html.Strong('Select the Kernal'),\n",
    "                                                   dcc.RadioItems(\n",
    "                                                       id='modelpicker',\n",
    "                                                       options=[\n",
    "                                                           {'label': 'Exponential Kernel',\n",
    "                                                               'value': 'Exponential'},\n",
    "                                                           {'label': 'ExponentialSum Kernel',\n",
    "                                                            'value': 'ExponentialSum'}\n",
    "                                                       ],\n",
    "                    value='Exponential'\n",
    "                )\n",
    "\n",
    "                ]),\n",
    "            html.Div(\n",
    "                id=\"select days\",\n",
    "                style={\"text-align\": \"center\"},\n",
    "                className=\"six columns\", children=[html.Strong('Select the # of Days in the future to Predict'),\n",
    "                                                   dcc.Slider(\n",
    "                    id='daysslider',\n",
    "                    min=1,\n",
    "                    max=100,\n",
    "                    step=1,\n",
    "                    value=10,\n",
    "                    updatemode='drag'\n",
    "                )\n",
    "                ]),\n",
    "            html.Br(),\n",
    "            html.Br(),\n",
    "            html.Br(),\n",
    "            html.Br(),\n",
    "            html.Div(className='nine columns', children=dcc.Loading(\n",
    "                dcc.Graph(id='processline'))),\n",
    "            html.Div(className='three columns container', children=[\n",
    "                html.Div(className='container', children=[html.Div(\n",
    "                    [html.H6(), html.P(\n",
    "                        \"  No. Days After the Last Rej Letter that I Received\"),\n",
    "                        html.Strong(id=\"days_after\")],\n",
    "\n",
    "                    className=\"mini_container\",\n",
    "                ), html.Br(), html.Br(),\n",
    "                    html.Div(\n",
    "                        [html.H6(), html.P(\n",
    "                            'Exact Time of the Email (Received/To be Received)'),\n",
    "                         html.Strong(id=\"exact_time\")],\n",
    "\n",
    "                        className=\"mini_container\",\n",
    "                )\n",
    "\n",
    "\n",
    "                ]\n",
    "                )\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "            ]),\n",
    "\n",
    "\n",
    "\n",
    "        ]),\n",
    "\n",
    "        html.Div(className='columns mini_container', children=[\n",
    "            html.H5(\"Email Content Analysis\",style={'margin':'10px'}),\n",
    "            html.Div(\"After cleaning the text of the emails, we can find out what words or phrases are the important or interesting .\\\n",
    "            I provided two commonly used metrics for you to rank the words/phrases. On the right panel, I have tried to present you with\\\n",
    "            the interesting bigrams, a.k.a. word collocations. Feel free to change the metric to see \\\n",
    "            which words are connected.\",style={'margin':'10px'}),\n",
    "            html.P('P.S. It might take a long time for the left graph to show up.',style={'margin':'10px'}),\n",
    "            html.Div(id='phrase', className='six columns',\n",
    "                     style={'text-align': 'center'},\n",
    "                     children=[\n",
    "                         html.Strong(\n",
    "                             'Select the Metric and the # of Words to Show'),\n",
    "                         html.Div(children=[dcc.Dropdown(\n",
    "                             id='tf_selector',\n",
    "                             options=[\n",
    "                                 {'label': 'Word Count(Term Frequency)',\n",
    "                                  'value': 'tf'},\n",
    "                                 {'label': 'Term Frequency-Inverse Document Frequency(TF-IDF)', 'value': 'tfidf'}],\n",
    "                             value='tf'\n",
    "                         ),\n",
    "\n",
    "                             dcc.Dropdown(\n",
    "                             id='rank_selector',\n",
    "                             options=[\n",
    "                                 {'label': '10', 'value': 10},\n",
    "                                 {'label': '15', 'value': 15},\n",
    "                                 {'label': '20', 'value': 20}],\n",
    "                             value=10\n",
    "                         )]),\n",
    "                         html.Br(),\n",
    "                         dcc.Loading(dcc.Graph(id='barchart')),\n",
    "                         dcc.Loading(dcc.Graph(id='treechart'))\n",
    "\n",
    "\n",
    "\n",
    "                     ]),\n",
    "\n",
    "            html.Div(style={'margin-left': '7%', 'text-align': 'center'}, className='five columns', children=[\n",
    "                html.Strong('Select the Interestingness Metric'),\n",
    "                dcc.RadioItems(\n",
    "                    id='colmetric',\n",
    "                    options=[\n",
    "                        {'label': 'Point-Wise Mutual Infomation', 'value': 'pmi'},\n",
    "                        {'label': 'Chi-Square', 'value': 'chisquare'},\n",
    "                        {'label': 'Likelihood Ratio', 'value': 'likelihood_ratio'}\n",
    "                    ],\n",
    "                    value='pmi'),\n",
    "                html.Div(id='cos'),\n",
    "                html.Br(),\n",
    "                html.Br(),\n",
    "                dash_table.DataTable(\n",
    "                    id='co_table',\n",
    "                    style_cell={'textAlign': 'left', 'padding': '5px',\n",
    "                                'overflow': 'hidden',\n",
    "                                'textOverflow': 'ellipsis'},\n",
    "                    style_data={'whiteSpace': 'normal'},\n",
    "                    css=[{\n",
    "                        'selector': '.dash-cell div.dash-cell-value',\n",
    "                        'rule': 'display: inline; white-space: inherit; overflow: inherit; text-overflow: inherit;'\n",
    "                    }],\n",
    "                    columns=[\n",
    "                        {'name': i, 'id': i, 'deletable': True} for i in ['collacation', 'metric']\n",
    "                    ],\n",
    "                    page_current=0,\n",
    "                    page_size=1,\n",
    "                    page_action='custom',\n",
    "\n",
    "                    sort_action='custom',\n",
    "                    sort_mode='single',\n",
    "                    sort_by=[])\n",
    "\n",
    "\n",
    "            ])\n",
    "\n",
    "        ]),\n",
    "        html.Div(className='mini_container columns', children=[\n",
    "            #             html.H5('Topic Modelling'),\n",
    "\n",
    "            html.Div(id='clickdata'),\n",
    "            html.Div(className='columns', children=[\n",
    "                html.Div(className='six columns',\n",
    "                         children=[html.Div(className='six columns container',\n",
    "                                            style={\n",
    "                                                'width': \"70%\",\n",
    "                                            'margin':\"30px\"},\n",
    "                                            children=[html.H5('Topic Modelling'),html.Br(), html.Blockquote('We can further \\\n",
    "                                            explore what these emails are mainly talking about by applying topic modeling techniques to \\\n",
    "            the texts. To achieve the best results, the texts were cleaned by removing extra elements(like HTML tags), punctuations, and numbers.\\\n",
    "            The stopwords are removed as well. I used the NLTK stopword collection and extended it with other self-defined words, like my name😂. \\\n",
    "            Then, the n-grams(I only used the uni-, bi- and, tri-grams here.) are generated. As an unsupervised learning method, \\\n",
    "            the number of topics should be specified, here the number is automatically selected by maximizing the coherence values; however, \\\n",
    "            you can change the number of topics by clicking the dots in the line chart.')])]),\n",
    "\n",
    "                html.Div(className='six columns',\n",
    "                         style={'text-align':'center'},\n",
    "                         children=[\n",
    "                             html.Div(style={'display': 'inline-flex'}, children=[html.Div('Number of Topics Selected', style={'margin-right': '20px'})\n",
    "                                                                                  , dcc.Input(id=\"current_k\", value=4, disabled=True)]),\n",
    "                             #                              style={'display':'inline-flex'},\n",
    "                             dcc.Graph(id='coherence',\n",
    "                                       figure=px.line(choose_k, x='# of Topics', y='coherence').update_traces(\n",
    "                                           mode='lines+markers')\n",
    "                                       )]\n",
    "                         ),\n",
    "                #                 html.Hr(),\n",
    "                html.Br(),\n",
    "                 html.H6(\"Let's see the temporal distribution and the linguistic features of these topics.\",\n",
    "                        style={'text-align': 'center', 'width': \"100%\",'margin-top':'20px'}, className='columns'),\n",
    "                html.Div(className='six columns', children=[\n",
    "                     dcc.Loading(dcc.Graph(id='sunburst'))]),\n",
    "                #                 html.Div(style={\"border-left\":\"1px solid #000\",\"height\":\"500px\"}),\n",
    "                html.Div(className='six columns', children=[\n",
    "                    dcc.Loading(dcc.Graph(id='polar'))]),\n",
    "                #                 html.Hr(),\n",
    "                html.Br(),\n",
    "                html.Br(),\n",
    "                html.H6('Visualizing the Topic Modeling Results with t-SNE',\n",
    "                        style={'text-align': 'center', 'width': \"100%\",'margin-top':'20px'}, className='columns'),\n",
    "                html.Hr(),\n",
    "                html.Div(className='columns',\n",
    "                         children=[dcc.Loading(dcc.Graph(id='tsne'))])\n",
    "            ])]),\n",
    "        html.Div(id='contact_info',style={'text-align': 'center'}, className='pretty_container twelve columns', children=[\n",
    "            'Thanks for playing with it! You can contact me via my ',\n",
    "            html.A(\n",
    "                'LinkedIn', href='https://www.linkedin.com/in/yukun-yang-1044ab157/', target=\"_blank\"),\n",
    "            ', or ',\n",
    "            html.A('Personal Website',\n",
    "                   href='http://www.yukunyang.info', target=\"_blank\"),\n",
    "            '. You can also Email me at ',\n",
    "            html.A('contact@yukunyang.info',\n",
    "                   href='mailto:contact@yukunyang.info', target=\"_blank\")\n",
    "        ])\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    ])\n",
    "\n",
    "\n",
    "@app.callback(\n",
    "    Output('current_k', 'value'),\n",
    "    [Input('coherence', 'clickData')])\n",
    "def click_to_change_k(clickData):\n",
    "\n",
    "    if clickData == None:\n",
    "        return 4\n",
    "    else:\n",
    "        return clickData['points'][0]['x']\n",
    "\n",
    "\n",
    "@app.callback(\n",
    "    [Output('sunburst', 'figure'),\n",
    "     Output('polar', 'figure'),\n",
    "     Output('tsne', 'figure')],\n",
    "    [Input('current_k', 'value')])\n",
    "def change_of_k(k):\n",
    "    lda_model = model_list[int((k-2)/2)]\n",
    "\n",
    "    df_topic_sents_keywords = format_topics_sentences(\n",
    "        ldamodel=lda_model, corpus=corpus, texts=data_lemmatized)\n",
    "\n",
    "    # Format\n",
    "    df_dominant_topic = df_topic_sents_keywords.reset_index()\n",
    "    df_dominant_topic.columns = [\n",
    "        'Document_No', 'Dominant_Topic', 'Topic_Perc_Contrib', 'Keywords', 'Text']\n",
    "\n",
    "    topic_email = pd.merge(how='left', left=email_df.dropna().reset_index(\n",
    "    ), right=df_dominant_topic, left_index=True, right_index=True)\n",
    "    topic_email['Month'] = topic_email['date_es'].dt.month.map(\n",
    "        {3: \"March\", 4: \"April\", 5: 'May', 6: 'June'})\n",
    "\n",
    "    topic_email['lexicon_count'] = topic_email.cleaned.apply(\n",
    "        textstat.lexicon_count, removepunct=True)\n",
    "    topic_email['reading_ease'] = topic_email.extracted.apply(\n",
    "        textstat.flesch_reading_ease)\n",
    "    topic_email['unique_term_count'] = topic_email.cleaned.apply(\n",
    "        lexicalrichness.LexicalRichness).apply(lambda x: x.terms)\n",
    "    topic_email['lexical_diversity'] = topic_email.cleaned.apply(\n",
    "        lexicalrichness.LexicalRichness).apply(lambda x: x.mtld(threshold=0.72))\n",
    "\n",
    "    month_topic_dis = topic_email.groupby(['Month', 'Dominant_Topic'])[\n",
    "        'date'].count().reset_index().rename(columns={'date': 'Count'})\n",
    "\n",
    "    categories = ['lexicon_count', 'reading_ease',\n",
    "                  'unique_term_count', 'lexical_diversity']\n",
    "    melted_df = pd.melt(topic_email.groupby('Dominant_Topic')[categories].mean().reset_index(), id_vars=['Dominant_Topic'], value_vars=categories\n",
    "                        )\n",
    "    polar = px.line_polar(melted_df, r=\"value\", theta=\"variable\",\n",
    "                          color=\"Dominant_Topic\", line_close=True, title='Linguistic Features of Topics')\n",
    "    sun = px.sunburst(month_topic_dis, path=['Month', 'Dominant_Topic'], values='Count',\n",
    "                      color='Count', title='Monthly Topic Distribution', color_continuous_scale=px.colors.sequential.Blues)\n",
    "    polar.update(layout=dict(title=dict(x=0.5)))\n",
    "    sun.update(layout=dict(title=dict(x=0.5)))\n",
    "    from sklearn.manifold import TSNE\n",
    "    topic_weights = []\n",
    "\n",
    "    topic_weights = pd.DataFrame()\n",
    "\n",
    "    for i, row_list in enumerate(lda_model[corpus]):\n",
    "        #     print(i,row_list)\n",
    "        for j in row_list:\n",
    "            topic_weights.loc[i, j[0]] = j[-1]\n",
    "    #     topic_weights.append([row_list[0][-1]])\n",
    "\n",
    "    arr = topic_weights.fillna(0).values\n",
    "    topic_num = np.argmax(arr, axis=1)\n",
    "\n",
    "    # tSNE Dimension Reduction\n",
    "    tsne_model = TSNE(n_components=2, verbose=1,\n",
    "                      random_state=0, angle=.99, init='pca')\n",
    "    tsne_lda = tsne_model.fit_transform(arr)\n",
    "\n",
    "    tsne_df = pd.DataFrame(tsne_lda)\n",
    "    tsne_df = tsne_df.rename(columns={0: 'x', 1: 'y'})\n",
    "    tsne_df['Dominant_Topic'] = topic_num\n",
    "    tsne_df = pd.merge(left=tsne_df, right=df_dominant_topic,\n",
    "                       left_on='Dominant_Topic', right_on='Dominant_Topic')\n",
    "    tsne_df['Dominant_Topic'] = tsne_df['Dominant_Topic'].apply(\n",
    "        lambda x: 'Topic'+str(x))\n",
    "    tsne = px.scatter(tsne_df, x='x', y='y', color='Dominant_Topic',\n",
    "                      color_discrete_sequence=px.colors.qualitative.Pastel,\n",
    "                      hover_data=['x', 'y', 'Keywords'])\n",
    "#     tsne.update_traces(hovertemplate= \"x:%{x}: <br>y: %{y} </br> Topic Keywords:%{hover_data}\")\n",
    "    return sun, polar, tsne\n",
    "\n",
    "\n",
    "@app.callback([dash.dependencies.Output('cos', 'children'),\n",
    "               dash.dependencies.Output('co_table', 'data')],\n",
    "              [dash.dependencies.Input('colmetric', 'value')]\n",
    "              )\n",
    "def update_cos(metric):\n",
    "\n",
    "    co_list = collo(metric)\n",
    "    table = pd.DataFrame(co_list, columns=['collacation', 'metric'])\n",
    "\n",
    "    top_10 = table.head(10)\n",
    "    top_10['collacation'] = top_10.apply(\n",
    "        lambda x: ' '.join(word for word in x['collacation']), axis=1)\n",
    "\n",
    "    top_10 = top_10.to_dict('record')\n",
    "    return[make_circos(co_list)], top_10\n",
    "\n",
    "\n",
    "@app.callback([dash.dependencies.Output('barchart', 'figure'),\n",
    "               dash.dependencies.Output('treechart', 'figure')],\n",
    "              [dash.dependencies.Input('tf_selector', 'value'),\n",
    "               dash.dependencies.Input('rank_selector', 'value')]\n",
    "              )\n",
    "def update_import_words(metric, rank):\n",
    "\n",
    "    return make_important_graphs(important_words(metric, rank))\n",
    "\n",
    "\n",
    "@app.callback([dash.dependencies.Output('daysslider', 'marks')],\n",
    "              [dash.dependencies.Input('daysslider', 'value')])\n",
    "def add_marks(value):\n",
    "    return [{value: {'label': str(value)+' days'}}]\n",
    "\n",
    "\n",
    "@app.callback([dash.dependencies.Output('days_after', 'children'),\n",
    "               dash.dependencies.Output('exact_time', 'children')],\n",
    "              [dash.dependencies.Input('processline', 'hoverData')])\n",
    "def conver_point(hoverData):\n",
    "\n",
    "    if hoverData == None:\n",
    "        return ('Please Hover over the Dots in the Graph', 'Please Hover over the Dots in the Graph')\n",
    "\n",
    "    if (hoverData[\"points\"][0][\"curveNumber\"] == 1) or (hoverData[\"points\"][0][\"curveNumber\"] == 3):\n",
    "        days_passed = hoverData[\"points\"][0]['x']\n",
    "        hours_passed = days_passed*24\n",
    "        exact_date = email_df.date_es.min()+datetime.timedelta(hours=hours_passed)\n",
    "\n",
    "        return round(days_passed, 2), exact_date.strftime(\"%m/%d/%Y, %H:%M:%S\")\n",
    "    else:\n",
    "        return no_update, no_update\n",
    "\n",
    "\n",
    "@app.callback([dash.dependencies.Output('processline', 'figure')],\n",
    "              [dash.dependencies.Input('modelpicker', 'value'),\n",
    "               dash.dependencies.Input('daysslider', 'value')])\n",
    "def update_process(model, days):\n",
    "\n",
    "    return [haweks(model, days)]\n",
    "\n",
    "\n",
    "@app.callback([dash.dependencies.Output('table_div', 'style'),\n",
    "               dash.dependencies.Output('table', 'data')],\n",
    "              [dash.dependencies.Input('patient_volume_hm', 'clickData')],\n",
    "              [dash.dependencies.State('my-date-picker-range', 'start_date'),\n",
    "               dash.dependencies.State('my-date-picker-range', 'end_date'),\n",
    "               dash.dependencies.State('weekdays', 'value'),\n",
    "               dash.dependencies.State('time', 'value')])\n",
    "def test(heatmap_click, start_date, end_date, weekdays, time):\n",
    "\n",
    "    if heatmap_click == None:\n",
    "        return no_update\n",
    "    else:\n",
    "        ctx = dash.callback_context\n",
    "\n",
    "        prop_id = \"\"\n",
    "        prop_type = \"\"\n",
    "        triggered_value = None\n",
    "        if ctx.triggered:\n",
    "            prop_id = ctx.triggered[0][\"prop_id\"].split(\".\")[0]\n",
    "            prop_type = ctx.triggered[0][\"prop_id\"].split(\".\")[1]\n",
    "            triggered_value = ctx.triggered[0][\"value\"]\n",
    "\n",
    "            filtered = filter_df(dt.strptime(\n",
    "                start_date, \"%Y-%m-%d\"), dt.strptime(end_date, \"%Y-%m-%d\"), weekdays, time)\n",
    "            hour_of_day = heatmap_click[\"points\"][0][\"x\"]\n",
    "            weekday = heatmap_click[\"points\"][0][\"y\"]\n",
    "\n",
    "            click_df = filtered[filtered.hour == hour_of_day]\n",
    "            click_df = click_df[click_df.weekdays == weekday]\n",
    "\n",
    "        return ({'display': 'block'}, click_df.to_dict('record'))\n",
    "\n",
    "\n",
    "# @dcb.callback([Output(\"input\", \"value\"), Output(\"slider\", \"value\")], [Input(\"sync\", \"data\")],\n",
    "#               [State(\"input\", \"value\"), State(\"slider\", \"value\")])\n",
    "# def update_components(current_value, input_prev, slider_prev):\n",
    "#     # Update only inputs that are out of sync (this step \"breaks\" the circular dependency).\n",
    "#     input_value = current_value if current_value != input_prev else dash.no_update\n",
    "#     slider_value = current_value if current_value != slider_prev else dash.no_update\n",
    "#     return [input_value, slider_value]\n",
    "\n",
    "\n",
    "@app.callback(\n",
    "    [dash.dependencies.Output('patient_volume_hm', 'figure'),\n",
    "     dash.dependencies.Output('datetime_RangeSlider', 'value'),\n",
    "     dash.dependencies.Output('datetime_RangeSlider', 'marks'),\n",
    "     dash.dependencies.Output('hist', 'figure'),\n",
    "     dash.dependencies.Output('days_selected', 'children'),\n",
    "     dash.dependencies.Output('total', 'children'),\n",
    "     dash.dependencies.Output('peak_date', 'children'),\n",
    "     dash.dependencies.Output('peak_num', 'children'),\n",
    "     dash.dependencies.Output('paco', 'figure')\n",
    "     ],\n",
    "    [dash.dependencies.Input('my-date-picker-range', 'start_date'),\n",
    "     dash.dependencies.Input('my-date-picker-range', 'end_date'),\n",
    "     dash.dependencies.Input('weekdays', 'value'),\n",
    "     dash.dependencies.Input('time', 'value'),\n",
    "     dash.dependencies.Input('patient_volume_hm', 'clickData')])\n",
    "def update_output_from_picker(start_date, end_date, weekdays, time, click):\n",
    "    heatmapdata = generate_patient_volume_heatmap(dt.strptime(\n",
    "        start_date, \"%Y-%m-%d\"), dt.strptime(end_date, \"%Y-%m-%d\"), click, None, weekdays, time)\n",
    "    hist_plot = generate_hist(start_date, end_date, weekdays, time)\n",
    "\n",
    "    ind = np.unravel_index(np.argmax(\n",
    "        heatmapdata['data'][0]['z'], axis=None), heatmapdata['data'][0]['z'].shape)\n",
    "    day = heatmapdata['data'][0]['y'][ind[0]]\n",
    "    hour = heatmapdata['data'][0]['x'][ind[-1]]\n",
    "\n",
    "    filtered = filter_df(dt.strptime(\n",
    "        start_date, \"%Y-%m-%d\"), dt.strptime(end_date, \"%Y-%m-%d\"), weekdays, time)\n",
    "\n",
    "    return (heatmapdata,\n",
    "            cal_slider(start_date, end_date),\n",
    "            {0: email_df['date_es'].min().date(), cal_slider(start_date, end_date)[0]: start_date, cal_slider(\n",
    "                start_date, end_date)[-1]: end_date, 100: email_df['date_es'].max().date()},\n",
    "            hist_plot,\n",
    "            (dt.strptime(end_date, \"%Y-%m-%d\") -\n",
    "             dt.strptime(start_date, \"%Y-%m-%d\")).days,\n",
    "            heatmapdata['data'][0]['z'].sum(),\n",
    "            day+' '+hour,\n",
    "            heatmapdata['data'][0]['z'].max(),\n",
    "            generate_paco(filtered))\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    app.run_server(port=8070)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.6 64-bit",
   "language": "python",
   "name": "python37664bitaad11810a68246cf96b4e903e9fb7d27"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "404.4px",
    "left": "1192.6px",
    "right": "20px",
    "top": "120px",
    "width": "323.4px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}