Prob_citing_ref_by_successful_plos.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import sys\n",
    "import pickle\n",
    "import gzip\n",
    "import os,glob\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import operator\n",
    "import random\n",
    "from  scipy import stats\n",
    "#sys.path\n",
    "\n",
    "import datetime\n",
    "import math\n",
    "import time\n",
    "import itertools\n",
    "\n",
    "\n",
    "from pymongo import MongoClient\n",
    "\n",
    "import json\n",
    "\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "from plotly import tools\n",
    "import plotly.figure_factory as ff\n",
    "\n",
    "\n",
    "########## to be able to plot offline (without sending the plots to the plotly server every time)\n",
    "import plotly.offline as offline\n",
    "from plotly.graph_objs import *\n",
    "from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\n",
    "\n",
    "init_notebook_mode(connected=True)\n",
    "################\n",
    "\n",
    "from IPython.core.display import display,HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))  # to make the notebook use the entire width of the browser\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>occurence</th>\n",
       "      <th>reference_UT</th>\n",
       "      <th>reference_rank</th>\n",
       "      <th>regex_sect_index</th>\n",
       "      <th>cite_count</th>\n",
       "      <th>ref_pub_year</th>\n",
       "      <th>paper_cite_count</th>\n",
       "      <th>plos_pub_year</th>\n",
       "      <th>sect_char_pos</th>\n",
       "      <th>sect_char_total</th>\n",
       "      <th>...</th>\n",
       "      <th>log2_num_cit_ref</th>\n",
       "      <th>log2_num_cit_paper</th>\n",
       "      <th>diff_year_plos_ref</th>\n",
       "      <th>rel_loc_in_sect</th>\n",
       "      <th>plos_field</th>\n",
       "      <th>isolated_citation</th>\n",
       "      <th>paper_UT</th>\n",
       "      <th>total_refs</th>\n",
       "      <th>categ_codes</th>\n",
       "      <th>self_citation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>A1995QY75100004</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>1995.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>139</td>\n",
       "      <td>4029</td>\n",
       "      <td>...</td>\n",
       "      <td>5.906891</td>\n",
       "      <td>1.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.034500</td>\n",
       "      <td>['D RO MULTIDISCIPLINARY SCIENCES']</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "      <td>37</td>\n",
       "      <td>0 5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>A1995QY75100004</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>60.0</td>\n",
       "      <td>1995.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>494</td>\n",
       "      <td>5398</td>\n",
       "      <td>...</td>\n",
       "      <td>5.906891</td>\n",
       "      <td>1.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.091515</td>\n",
       "      <td>['D RO MULTIDISCIPLINARY SCIENCES']</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "      <td>37</td>\n",
       "      <td>0 5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>000222499800017</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>266</td>\n",
       "      <td>4029</td>\n",
       "      <td>...</td>\n",
       "      <td>6.022368</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.066021</td>\n",
       "      <td>['D RO MULTIDISCIPLINARY SCIENCES']</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "      <td>37</td>\n",
       "      <td>0 5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>000222499800017</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>65.0</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>495</td>\n",
       "      <td>5398</td>\n",
       "      <td>...</td>\n",
       "      <td>6.022368</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.091701</td>\n",
       "      <td>['D RO MULTIDISCIPLINARY SCIENCES']</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "      <td>37</td>\n",
       "      <td>0 5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>000263911400006</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2009.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>142</td>\n",
       "      <td>4029</td>\n",
       "      <td>...</td>\n",
       "      <td>2.321928</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.035244</td>\n",
       "      <td>['D RO MULTIDISCIPLINARY SCIENCES']</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "      <td>37</td>\n",
       "      <td>0 5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   occurence     reference_UT  reference_rank  regex_sect_index  cite_count  \\\n",
       "0          1  A1995QY75100004               1                 0        60.0   \n",
       "1          2  A1995QY75100004               1                 3        60.0   \n",
       "2          1  000222499800017               2                 0        65.0   \n",
       "3          3  000222499800017               2                 3        65.0   \n",
       "4          1  000263911400006               3                 0         5.0   \n",
       "\n",
       "   ref_pub_year  paper_cite_count  plos_pub_year  sect_char_pos  \\\n",
       "0        1995.0                 2         2013.0            139   \n",
       "1        1995.0                 2         2013.0            494   \n",
       "2        2004.0                 2         2013.0            266   \n",
       "3        2004.0                 2         2013.0            495   \n",
       "4        2009.0                 2         2013.0            142   \n",
       "\n",
       "   sect_char_total      ...      log2_num_cit_ref log2_num_cit_paper  \\\n",
       "0             4029      ...              5.906891                1.0   \n",
       "1             5398      ...              5.906891                1.0   \n",
       "2             4029      ...              6.022368                1.0   \n",
       "3             5398      ...              6.022368                1.0   \n",
       "4             4029      ...              2.321928                1.0   \n",
       "\n",
       "  diff_year_plos_ref  rel_loc_in_sect                           plos_field  \\\n",
       "0               18.0         0.034500  ['D RO MULTIDISCIPLINARY SCIENCES']   \n",
       "1               18.0         0.091515  ['D RO MULTIDISCIPLINARY SCIENCES']   \n",
       "2                9.0         0.066021  ['D RO MULTIDISCIPLINARY SCIENCES']   \n",
       "3                9.0         0.091701  ['D RO MULTIDISCIPLINARY SCIENCES']   \n",
       "4                4.0         0.035244  ['D RO MULTIDISCIPLINARY SCIENCES']   \n",
       "\n",
       "   isolated_citation         paper_UT total_refs  categ_codes self_citation  \n",
       "0                  0  000324515600133         37          0 5             0  \n",
       "1                  0  000324515600133         37          0 5             0  \n",
       "2                  0  000324515600133         37          0 5             1  \n",
       "3                  0  000324515600133         37          0 5             1  \n",
       "4                  0  000324515600133         37          0 5             0  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "                                          # df_reference_cite_plos_merged     (original with ALL columns and all entries, without removing any references)\n",
    "\n",
    "%time df_merged = pickle.load(open('../data/df_reference_cite_plos_merged_simplified_added_more_columns.pkl', 'rb'))\n",
    "print (\"done loading pickles\", df_merged.shape)\n",
    "\n",
    "df_merged = df_merged[df_merged['cite_count'] != -1]\n",
    "\n",
    "\n",
    "df_merged.head()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6924262, 23)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged.shape\n",
    "# #### i add a new column with shuffled paper_UTs  for the null model to compare usage of references by top/non-top plos papers\n",
    "# df_merged.shape\n",
    "# lista_values = list(df_merged.paper_UT)   #[i for i in range (len(df_merged))]\n",
    "# random.shuffle(lista_values)\n",
    "# df_merged['randomized_paper_UT'] = lista_values\n",
    "# df_merged.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>occurence</th>\n",
       "      <th>reference_UT</th>\n",
       "      <th>reference_rank</th>\n",
       "      <th>regex_sect_index</th>\n",
       "      <th>cite_count</th>\n",
       "      <th>ref_pub_year</th>\n",
       "      <th>paper_cite_count</th>\n",
       "      <th>plos_pub_year</th>\n",
       "      <th>sect_char_pos</th>\n",
       "      <th>sect_char_total</th>\n",
       "      <th>...</th>\n",
       "      <th>log2_num_cit_ref</th>\n",
       "      <th>log2_num_cit_paper</th>\n",
       "      <th>diff_year_plos_ref</th>\n",
       "      <th>rel_loc_in_sect</th>\n",
       "      <th>plos_field</th>\n",
       "      <th>isolated_citation</th>\n",
       "      <th>paper_UT</th>\n",
       "      <th>total_refs</th>\n",
       "      <th>categ_codes</th>\n",
       "      <th>self_citation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [occurence, reference_UT, reference_rank, regex_sect_index, cite_count, ref_pub_year, paper_cite_count, plos_pub_year, sect_char_pos, sect_char_total, plos_j1, ref_field, ref_j1, log2_num_cit_ref, log2_num_cit_paper, diff_year_plos_ref, rel_loc_in_sect, plos_field, isolated_citation, paper_UT, total_refs, categ_codes, self_citation]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 23 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged[df_merged['paper_cite_count'] == -1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class MongoConnection(object):\n",
    "    def __init__(self, cxnSettings, **kwargs):\n",
    "        self.settings = cxnSettings\n",
    "        self.mongoURI = self._constructURI()\n",
    "        self.connect(**kwargs)\n",
    "        self.ensure_index()\n",
    "\n",
    "    def _constructURI(self):\n",
    "        '''\n",
    "        Construct the mongo URI\n",
    "        '''\n",
    "        mongoURI = 'mongodb://'\n",
    "        #User/password handling\n",
    "        if 'user'in self.settings and 'password' in self.settings:\n",
    "            mongoURI += self.settings['user'] + ':' + self.settings['password']\n",
    "            mongoURI += '@'\n",
    "        elif 'user' in self.settings:\n",
    "            print('Missing password for given user, proceeding without either')\n",
    "        elif 'password' in self.settings:\n",
    "            print('Missing user for given passord, proceeding without either')\n",
    "        #Host and port\n",
    "        try:\n",
    "            mongoURI += self.settings['host'] + ':'\n",
    "        except KeyError:\n",
    "            print('Missing the hostname. Cannot connect without host')\n",
    "            sys.exit()\n",
    "        try:\n",
    "            mongoURI += str(self.settings['port'])\n",
    "        except KeyError:\n",
    "            print('Missing the port. Substituting default port of 27017')\n",
    "            mongoURI += str('27017')\n",
    "        return mongoURI\n",
    "\n",
    "    def connect(self, **kwargs):\n",
    "        '''\n",
    "        Establish the connection, database, and collection\n",
    "        '''\n",
    "        self.connection = MongoClient(self.mongoURI, **kwargs)\n",
    "        #########\n",
    "        try:\n",
    "            self.db = self.connection[self.settings['db']]\n",
    "        except KeyError:\n",
    "            print(\"Must specify a database as a 'db' key in the settings file\")\n",
    "            sys.exit()\n",
    "        #########\n",
    "        try:\n",
    "            self.collection = self.db[self.settings['collection']]\n",
    "        except KeyError:\n",
    "            print('Should have a collection.', end='')\n",
    "            print('Starting a collection in database', end='')\n",
    "            print(' for current connection as test.')\n",
    "            self.collection = self.db['test']\n",
    "\n",
    "    def tearDown(self):\n",
    "        '''\n",
    "        Closes the connection\n",
    "        '''\n",
    "        self.connection.close()\n",
    "\n",
    "    def ensure_index(self):\n",
    "        '''\n",
    "        Ensures the connection has all given indexes.\n",
    "        indexes: list of (`key`, `direction`) pairs.\n",
    "            See docs.mongodb.org/manual/core/indexes/ for possible `direction`\n",
    "            values.\n",
    "        '''\n",
    "        if 'indexes' in self.settings:\n",
    "            for index in self.settings['indexes']:\n",
    "                self.collection.ensure_index(index[0], **index[1])\n",
    "\n",
    "\n",
    "# In[3]:\n",
    "\n",
    "\n",
    "merged_papers_settings = {\n",
    "    \"host\": \"chicago.chem-eng.northwestern.edu\",\n",
    "    \"port\": \"27017\",\n",
    "    \"db\": \"web_of_science_aux\",\n",
    "    \"collection\": \"merged_papers\",\n",
    "    \"user\": \"mongoreader\",\n",
    "    \"password\": \"emptycoffeecup\"\n",
    "}\n",
    "\n",
    "papers_con = MongoConnection(merged_papers_settings)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2607453, 5)\n",
      "2607453\n",
      "\n",
      " (158813, 6)\n",
      "158813\n"
     ]
    }
   ],
   "source": [
    "df_ref=df_merged.drop_duplicates(subset=['reference_UT','cite_count','ref_pub_year'])\n",
    "df_ref = df_ref[['reference_UT','cite_count','ref_pub_year','ref_field','ref_j1']]\n",
    "print (df_ref.shape)\n",
    "\n",
    "\n",
    "lista_all_ref = list(df_merged.reference_UT.unique())\n",
    "print (len(lista_all_ref))\n",
    "\n",
    "\n",
    "\n",
    "df_plos=df_merged.drop_duplicates(subset=['paper_UT','paper_cite_count','plos_pub_year'])\n",
    "df_plos = df_plos[['paper_UT','paper_cite_count','plos_pub_year','plos_field','plos_j1','total_refs']]\n",
    "print (\"\\n\",df_plos.shape)\n",
    "\n",
    "\n",
    "\n",
    "lista_all_plos = list(df_merged.paper_UT.unique())\n",
    "print (len(lista_all_plos))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_plos.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# lista_all_plos_UTs = pickle.load(open('/home/staff/julia/at_Northwestern/In_Text_Citations/In-Text-Citations-New/data/lista_all_plos_UTs.pkl', 'rb'))\n",
    "# new_lista_all_plos_UTs = ['000'+str(item) for item in lista_all_plos_UTs]   # i need to add the 000 so it matches the db UTs\n",
    "\n",
    "# #print (len(new_lista_all_plos_UTs))\n",
    "\n",
    "\n",
    "# # In[15]:\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# #lista_UTs=['000322590800016','000254928800015','000305349100026','000321341000076']\n",
    "\n",
    "# start=0\n",
    "# stop=1000\n",
    "# while stop <=159000:\n",
    "\n",
    "\n",
    "#     lista=new_lista_all_plos_UTs[start:stop]\n",
    "#     query = papers_con.collection.find({\"UT\":{\"$in\":lista}},{\"UT\":1,\"AR\":1}, no_cursor_timeout=True)  \n",
    "\n",
    "\n",
    "#     #print (start, stop)\n",
    "\n",
    "#     for item in query:  # query (cursor) is an iterator (once i iterate over it once, it is empty), and every item is a dict\n",
    "\n",
    "#         UT=item[\"UT\"]\n",
    "#         doi=item['AR'][-1]\n",
    "#         #string =str(UT)+\" \"+str(doi)\n",
    "#         if \".pone.\" in doi:\n",
    "#             print ( doi)\n",
    "\n",
    "#     stop +=1000\n",
    "#     start +=1000\n",
    "        \n",
    "     \n",
    "#     query.close()  # because i am using the no_cursor_timeout=True, i need also this, or cursor keeps waiting so ur resources are used up\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merged.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(preselection_df3.paper_UT.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2013]\n",
      "size of preselection1 (by plos years): (1533850, 23)\n",
      "    size of preselection3 (only young references): (145356, 23) young\n",
      "     N plos: 29061  N records: 145356\n",
      "\n",
      "\n",
      "citation bins for the selected plos: [0.2, 0.95, 1]\n",
      "0-3 13632\n",
      "3-28 72759\n",
      "28-328 7337\n",
      "\n",
      "\n",
      "citation bins for the references in the selected plos: [0.5, 0.95, 1] [(0.5, 18.0), (0.95, 104.0), (1.0, 6453.0)]\n",
      "0-18 42804\n",
      "18-104 40015\n",
      "104-6453 4391\n",
      "\n",
      "\n",
      "# UTs top 5.0 % plos: 1494\n",
      "# UTs top 5.0 % ref: 4391\n",
      "# UTs bottom  20.0 % plos: 4639\n",
      "# UTs bottom  50.0 % ref: 42804\n",
      "Tot # records: 145356 , # plos: 29061\n",
      "fraction of usage of top ref by \n",
      "  top 5.0 % plos: 0.14983365764860962\n",
      "  bottom 20.0 % plos: 0.09723181219006967\n",
      "fraction of usage of non-top ref by \n",
      "  top 5.0 % plos: 0.03384774348376775\n",
      "  bottom 20.0 % plos: 0.16220585294313714\n",
      "0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/staff/julia/.pyenv/versions/3.6.4/envs/psy/lib/python3.6/site-packages/ipykernel_launcher.py:260: SettingWithCopyWarning:\n",
      "\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "avg randomized!!\n",
      "fraction of usage of top ref by\n",
      "  top 5.0 % plos: 0.0744278450819\n",
      "  bottom 20.0 % plos: 0.124877283284\n",
      "\n",
      "\n",
      "avg randomized\n",
      "fraction of usage of non-top ref by \n",
      "  top 5.0 % plos: 0.0741880541297\n",
      "  bottom 20.0 % plos: 0.124960835944 \n",
      "\n",
      "\n",
      "\n",
      "zscore top ref by top plos: 36.3976155882\n",
      "zscore nontop ref by top plos: -56.137270554\n",
      "zscore top ref by bottom plos: -9.87264576258\n",
      "zscore nontop ref by bottom plos: 32.6140201509\n",
      "\n",
      "\n",
      "Total number of references in different types of papers\n",
      "\n",
      " (29061, 24)\n",
      "All: 29061 47.0\n",
      "Top plos: 1494 52.0\n",
      "Bottom plos 4639 42.0\n",
      "Random: 1494 46.0\n",
      "comparison   ALL- top plos: Ks_2sampResult(statistic=0.11357953751622574, pvalue=1.8844313067244635e-16) \n",
      "\n",
      "comparison   ALL- bottom plos: Ks_2sampResult(statistic=0.11266635042349721, pvalue=1.0569347264871222e-44) \n",
      "\n",
      "comparison   top plos - bottom plos: Ks_2sampResult(statistic=0.22329888642736495, pvalue=9.9667324220997036e-50) \n",
      "\n",
      "comparison   top plos - random: Ks_2sampResult(statistic=0.12784471218206156, pvalue=3.9793878331368452e-11) \n",
      "\n",
      "(1533850, 23) (145356, 24)\n",
      "(870897, 23) (87211, 24) (34593, 23) (1494, 23)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'file:///home/staff/julia/at_Northwestern/In_Text_Citations/In-Text-Citations-New/notebooks/histogram_tot_num_citations_received_by_top_plos_and_by_top_young_ref.html'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "########### i compare the usafe of top/nontop references by top/nontop plos papers with a null model that comes from randomizing the data\n",
    "\n",
    "Niter=100\n",
    "\n",
    "years=[2013]\n",
    "\n",
    "string_references_age=\"young\"   #young\"#old\"  # young # all   for the selection of what references i include\n",
    "string_isolated_ref=\"\"  #\"\"   # 0  or 1 (or empty string, to include all ref)\n",
    "string_self_ref=\"\"#1   # 0  or 1 (or empty string, to include all ref)\n",
    "string_journal=\"\"\n",
    "string_plos_field=\"\"#['D CU BIOLOGY']\"\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "##### preselection by plos year\n",
    "print (years)\n",
    "preselection_df = df_merged[df_merged['plos_pub_year'].isin(years)]  \n",
    "print (\"size of preselection1 (by plos years):\",preselection_df.shape)\n",
    "\n",
    "\n",
    "\n",
    "#### i remove self-citations\n",
    "if (string_self_ref==0) or  ( string_self_ref == 1 ): \n",
    "    preselection_df = preselection_df[preselection_df['self_citation']== string_self_ref ]  \n",
    "    print (\" size of preselection1 (self-cit):\",preselection_df.shape)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "######### preselection by isolated or group references:\n",
    "preselection_df0 = preselection_df   \n",
    "if (string_isolated_ref==0) or  ( string_isolated_ref == 1 ): \n",
    "    preselection_df0 = preselection_df[preselection_df['isolated_citation']== string_isolated_ref ]        \n",
    "    print (\"  size of preselection1 (by isolated/group ref):\",preselection_df0.shape, string_isolated_ref)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "######### preselection by plos journal:\n",
    "if string_journal==\"\": \n",
    "    preselection_df1 = preselection_df0\n",
    "else:    \n",
    "    preselection_df1 = preselection_df0[preselection_df0['plos_j1']== string_journal ]  \n",
    "    print (\"   size of preselection2 (by plos journal):\",preselection_df1.shape, string_journal)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "######### preselection by plos field:\n",
    "if string_plos_field==\"\": \n",
    "    preselection_df2 = preselection_df1\n",
    "else:    \n",
    "    preselection_df2 = preselection_df1[preselection_df1['plos_field']== string_plos_field ]  \n",
    "    print (\"    size of preselection2 (by plos field):\",preselection_df2.shape, string_plos_field)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    " \n",
    "\n",
    "\n",
    "\n",
    "##### preselection only young/old references:       \n",
    "preselection_df3 = preselection_df2\n",
    "if string_references_age == \"young\":\n",
    "    time_window_age = 1   \n",
    "    preselection_df3 = preselection_df2[preselection_df2['ref_pub_year'] >= (min(years)-time_window_age) ]   \n",
    "    \n",
    "    print (\"    size of preselection3 (only young references):\",preselection_df3.shape, string_references_age)\n",
    "\n",
    "elif string_references_age == \"old\":\n",
    "    time_window_age = 10    \n",
    "    preselection_df3 = preselection_df2[preselection_df2['ref_pub_year'] <= (min(years)-time_window_age) ]   \n",
    "    \n",
    "    print (\"    size of preselection3 (only old references):\",preselection_df3.shape,string_references_age )\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "N_plos=len(preselection_df3.paper_UT.unique())        ## this values are overall, for the title   \n",
    "N_all=len(preselection_df3)\n",
    "\n",
    "    \n",
    "    \n",
    "         \n",
    "print (\"     N plos:\", N_plos, \" N records:\", N_all)        \n",
    " \n",
    "    \n",
    "    \n",
    "    \n",
    "############## i define quantiles for plos papers (for that subselection, and based on their FINAL number of citations):\n",
    "list_q_plos=[.2,.95,1]\n",
    "df_for_quantiles_plos = preselection_df3.drop_duplicates(subset=['paper_UT'])   # ojo!!! dont use preselection_df3 directly because there are REPETITIONS!!!!\n",
    "\n",
    "quantiles=sorted(list(df_for_quantiles_plos['paper_cite_count'].quantile(list_q_plos).to_dict().items())) #mean 10.68 \n",
    " \n",
    "print (\"\\n\\ncitation bins for the selected plos:\", list_q_plos)#,quantiles, df_for_quantiles_plos.shape)   \n",
    "\n",
    "lista_bins_plos=[]\n",
    "old_value=0\n",
    "for item in quantiles:\n",
    "    pair=[old_value, int(item[1])]\n",
    "    lista_bins_plos.append(pair)\n",
    "    old_value = int(item[1])\n",
    "                           \n",
    "#print (lista_bins_plos, min(preselection_df3['paper_cite_count']), max(preselection_df3['paper_cite_count']))\n",
    "\n",
    "\n",
    "\n",
    "cont = 0\n",
    "dict_bin_list_plos_UT={}\n",
    "for item in lista_bins_plos:\n",
    "    \n",
    "    minimo = item[0]\n",
    "    maximo = item[1]   \n",
    "\n",
    "    df_select = preselection_df3[(preselection_df3['paper_cite_count'] >= minimo)  &  (preselection_df3['paper_cite_count'] < maximo)]\n",
    "    llave=str(minimo)+\"-\"+str(maximo)\n",
    "    dict_bin_list_plos_UT[llave]= list(df_select.paper_UT.unique())\n",
    "    print (llave, len(list(df_select.reference_UT.unique())))\n",
    "    max_key_plos=llave\n",
    "\n",
    "    \n",
    "    if cont ==0:\n",
    "        min_key_plos = llave\n",
    "    cont  +=1\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "########## i define quantiles for references (based on their FINAL number of citations)\n",
    "list_q_ref=[0.5,.95,1]\n",
    "df_for_quantiles_ref = preselection_df3.drop_duplicates(subset=['reference_UT'])   # ojo!!! remember to remove REPETITIONS!!!!\n",
    "quantiles=sorted(list(df_for_quantiles_ref['cite_count'].quantile(list_q_ref).to_dict().items())) #mean 10.68 \n",
    " \n",
    "print (\"\\n\\ncitation bins for the references in the selected plos:\", list_q_ref,quantiles)    \n",
    "\n",
    "lista_bins=[]\n",
    "old_value=0\n",
    "for item in quantiles:\n",
    "    pair=[old_value, int(item[1])]\n",
    "    lista_bins.append(pair)\n",
    "    old_value = int(item[1])\n",
    "                           \n",
    "\n",
    "\n",
    "\n",
    "cont = 0\n",
    "dict_bin_list_ref_UT={}\n",
    "for item in lista_bins:\n",
    "    \n",
    "    minimo = item[0]\n",
    "    maximo = item[1]    \n",
    "     \n",
    "    df_select = preselection_df3[(preselection_df3['cite_count'] >= minimo)  &  (preselection_df3['cite_count'] < maximo)]\n",
    "    llave=str(minimo)+\"-\"+str(maximo)\n",
    "    dict_bin_list_ref_UT[llave]=list(df_select.reference_UT.unique())\n",
    "    print (llave, len(list(df_select.reference_UT.unique())))\n",
    "    max_key_ref=llave\n",
    "\n",
    "    if cont ==0:\n",
    "        min_key_ref = llave\n",
    "    cont  +=1\n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "############### i create the list of top plos, top ref, bottom plos and bottom ref:\n",
    "#########################\n",
    "\n",
    "lista_top_plos = dict_bin_list_plos_UT[max_key_plos]\n",
    "print (\"\\n\\n# UTs top\",(100-100*list_q_plos[-2]),\"% plos:\",len(lista_top_plos))\n",
    "\n",
    "lista_top_ref=dict_bin_list_ref_UT[max_key_ref]\n",
    "print (\"# UTs top\",(100-100*list_q_ref[-2]),\"% ref:\", len(lista_top_ref))\n",
    "\n",
    "\n",
    "lista_bottom_plos = dict_bin_list_plos_UT[min_key_plos]\n",
    "print (\"# UTs bottom \",(100*list_q_plos[0]),\"% plos:\",len(lista_bottom_plos))\n",
    "\n",
    "lista_bottom_ref=dict_bin_list_ref_UT[min_key_ref]\n",
    "print (\"# UTs bottom \",(100*list_q_ref[0]),\"% ref:\", len(lista_bottom_ref))\n",
    "\n",
    "list_plos_in_year= list(preselection_df3.paper_UT.unique())\n",
    "print (\"Tot # records:\",len(preselection_df3),\", # plos:\",len(list_plos_in_year))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "######  i look at the usage of the top ref\n",
    "################################################  \n",
    "\n",
    "df_top_ref = preselection_df3[preselection_df3['reference_UT'].isin(lista_top_ref)]\n",
    "\n",
    "\n",
    "df_top_ref_top_plos = df_top_ref[df_top_ref['paper_UT'].isin(lista_top_plos)]\n",
    "df_top_ref_bottom_plos = df_top_ref[df_top_ref['paper_UT'].isin(lista_bottom_plos)]\n",
    "\n",
    "\n",
    "usage_top_ref_top_plos = len(df_top_ref_top_plos)/float(len(df_top_ref))\n",
    "usage_top_ref_bottom_plos = len(df_top_ref_bottom_plos)/float(len(df_top_ref))\n",
    "\n",
    "\n",
    "print (\"fraction of usage of top ref by \")\n",
    "print (\"  top\",(100-100*list_q_plos[-2]),\"% plos:\",  usage_top_ref_top_plos)\n",
    "print (\"  bottom\",(100*list_q_plos[0]),\"% plos:\",usage_top_ref_bottom_plos  )\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "######  i look at the usage of the non-top ref\n",
    "################################################      \n",
    "\n",
    "df_non_top_ref = preselection_df3[preselection_df3['reference_UT'].isin(lista_bottom_ref)]\n",
    "\n",
    "\n",
    "df_non_top_ref_top_plos = df_non_top_ref[df_non_top_ref['paper_UT'].isin(lista_top_plos)]\n",
    "df_non_top_ref_bottom_plos = df_non_top_ref[df_non_top_ref['paper_UT'].isin(lista_bottom_plos)]\n",
    "\n",
    "usage_non_top_ref_top_plos = len(df_non_top_ref_top_plos)/float(len(df_non_top_ref))\n",
    "usage_non_top_ref_bottom_plos = len(df_non_top_ref_bottom_plos)/float(len(df_non_top_ref))\n",
    "\n",
    "\n",
    "print (\"fraction of usage of non-top ref by \")\n",
    "print (\"  top\",(100-100*list_q_plos[-2]),\"% plos:\", usage_non_top_ref_top_plos )\n",
    "print (\"  bottom\",(100*list_q_plos[0]),\"% plos:\", usage_non_top_ref_bottom_plos )\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "####################\n",
    "# I canculate the null model (usage of references by top and non top plos papers, from the randomized data)\n",
    "#################################################  \n",
    "\n",
    "lista_usage_top_ref_by_top_plos_rand = []\n",
    "lista_usage_top_ref_by_bottom_plos_rand = []\n",
    "\n",
    "lista_usage_nontop_ref_by_top_plos_rand = []\n",
    "lista_usage_nontop_ref_by_bottom_plos_rand = []\n",
    "\n",
    "\n",
    "\n",
    "for i in range(Niter):\n",
    "    \n",
    "    print (i)\n",
    "    \n",
    "    lista_values = list(preselection_df3.paper_UT)   #[i for i in range (len(df_merged))]\n",
    "    random.shuffle(lista_values)\n",
    "    preselection_df3['randomized_paper_UT'] = lista_values\n",
    "   \n",
    "\n",
    "\n",
    "\n",
    "    ####### (RANDOMIZED)  i look at the usage of the top ref\n",
    "    df_top_ref_rand = preselection_df3[preselection_df3['reference_UT'].isin(lista_top_ref)]\n",
    "    \n",
    "    df_top_ref_top_plos_rand = df_top_ref_rand[df_top_ref_rand['randomized_paper_UT'].isin(lista_top_plos)]\n",
    "    df_top_ref_bottom_plos_rand = df_top_ref_rand[df_top_ref_rand['randomized_paper_UT'].isin(lista_bottom_plos)]\n",
    "    \n",
    "    \n",
    "    usage_top_ref_top_plos_rand = len(df_top_ref_top_plos_rand)/float(len(df_top_ref_rand))\n",
    "    usage_top_ref_bottom_plos_rand = len(df_top_ref_bottom_plos_rand)/float(len(df_top_ref_rand))\n",
    "    \n",
    "    \n",
    "    lista_usage_top_ref_by_top_plos_rand.append(usage_top_ref_top_plos_rand)\n",
    "    lista_usage_top_ref_by_bottom_plos_rand.append(usage_top_ref_bottom_plos_rand)\n",
    "    \n",
    "\n",
    "    \n",
    "    \n",
    "\n",
    "\n",
    "    #######  (RANDOMIZED) i look at the usage of the non-top ref            \n",
    "    df_non_top_ref_rand = preselection_df3[preselection_df3['reference_UT'].isin(lista_bottom_ref)]        \n",
    "    \n",
    "    df_non_top_ref_top_plos_rand = df_non_top_ref_rand[df_non_top_ref_rand['randomized_paper_UT'].isin(lista_top_plos)]\n",
    "    df_non_top_ref_bottom_plos_rand = df_non_top_ref_rand[df_non_top_ref_rand['randomized_paper_UT'].isin(lista_bottom_plos)]\n",
    "    \n",
    "    \n",
    "    usage_non_top_ref_top_plos_rand = len(df_non_top_ref_top_plos_rand)/float(len(df_non_top_ref_rand))\n",
    "    usage_non_top_ref_bottom_plos_rand = len(df_non_top_ref_bottom_plos_rand)/float(len(df_non_top_ref_rand))\n",
    "    \n",
    "    \n",
    "    lista_usage_nontop_ref_by_top_plos_rand.append(usage_non_top_ref_top_plos_rand)\n",
    "    lista_usage_nontop_ref_by_bottom_plos_rand.append(usage_non_top_ref_bottom_plos_rand)\n",
    "\n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "print (\"\\n\\n\\n\\navg randomized!!\")\n",
    "print(\"fraction of usage of top ref by\")\n",
    "print (\"  top\",(100-100*list_q_plos[-2]),\"% plos:\",  np.mean(lista_usage_top_ref_by_top_plos_rand) )   \n",
    "print (\"  bottom\",(100*list_q_plos[0]),\"% plos:\",np.mean(lista_usage_top_ref_by_bottom_plos_rand)  )\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "print (\"\\n\\navg randomized\")\n",
    "print (\"fraction of usage of non-top ref by \")\n",
    "print (\"  top\",(100-100*list_q_plos[-2]),\"% plos:\", np.mean(lista_usage_nontop_ref_by_top_plos_rand) )   \n",
    "print (\"  bottom\",(100*list_q_plos[0]),\"% plos:\", np.mean(lista_usage_nontop_ref_by_bottom_plos_rand) ,\"\\n\\n\\n\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "################ i plot it\n",
    "##########################################\n",
    "lista_bin_names=[\" Usage of top \"+str(int(100-100*list_q_ref[-2]))+\"% ref\", \" Usage of bottom \"+str(int(100*list_q_ref[0]))+\" ref\"]\n",
    "\n",
    "\n",
    "\n",
    "lista_for_top_plos = [ usage_top_ref_top_plos, usage_non_top_ref_top_plos]\n",
    "lista_for_bottom_plos = [usage_top_ref_bottom_plos, usage_non_top_ref_bottom_plos]\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "###### this is the null model \n",
    "lista_expectations_top = [np.mean(lista_usage_top_ref_by_top_plos_rand),np.mean(lista_usage_nontop_ref_by_top_plos_rand)]  \n",
    "lista_expectations_bottom = [np.mean(lista_usage_top_ref_by_bottom_plos_rand), np.mean(lista_usage_nontop_ref_by_bottom_plos_rand)] \n",
    "\n",
    "\n",
    "list_errors_top = [2.*np.std(lista_usage_top_ref_by_top_plos_rand), 2.*np.std(lista_usage_nontop_ref_by_top_plos_rand)] \n",
    "list_errors_bottom = [ 2.*np.std(lista_usage_top_ref_by_bottom_plos_rand), 2.*np.std(lista_usage_nontop_ref_by_bottom_plos_rand)] \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "z_score_top_ref_by_top_plos = (usage_top_ref_top_plos - np.mean(lista_usage_top_ref_by_top_plos_rand))/np.std(lista_usage_top_ref_by_top_plos_rand)\n",
    "z_score_nontop_ref_by_top_plos = (usage_non_top_ref_top_plos - np.mean(lista_usage_nontop_ref_by_top_plos_rand))/np.std(lista_usage_nontop_ref_by_top_plos_rand)\n",
    "\n",
    "z_score_top_ref_by_bottom_plos = (usage_top_ref_bottom_plos - np.mean(lista_usage_top_ref_by_bottom_plos_rand))/np.std(lista_usage_top_ref_by_bottom_plos_rand)\n",
    "z_score_nontop_ref_by_bottom_plos = (usage_non_top_ref_bottom_plos - np.mean(lista_usage_nontop_ref_by_bottom_plos_rand))/np.std(lista_usage_nontop_ref_by_bottom_plos_rand)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "print ('zscore top ref by top plos:', z_score_top_ref_by_top_plos)\n",
    "print ('zscore nontop ref by top plos:', z_score_nontop_ref_by_top_plos)\n",
    "\n",
    "print ('zscore top ref by bottom plos:', z_score_top_ref_by_bottom_plos)\n",
    "print ('zscore nontop ref by bottom plos:', z_score_nontop_ref_by_bottom_plos)\n",
    "\n",
    "\n",
    "\n",
    "title_string='zs top-ref by top plos: '+str(z_score_top_ref_by_top_plos)+';  zs top-ref by bottom plos: '+str(z_score_top_ref_by_bottom_plos)+\\\n",
    "'<br>zs nontop-ref by top plos: '+str(z_score_nontop_ref_by_top_plos)+';  zs nontop-ref by bottom plos: '+str(z_score_nontop_ref_by_bottom_plos)\n",
    "\n",
    "\n",
    "trace1 = go.Bar(\n",
    "    x=lista_bin_names,\n",
    "    y=lista_for_top_plos,\n",
    "    name='Top '+str(int(100-100*list_q_plos[-2]))+'% Plos',\n",
    "    #text= lista_text_young,\n",
    "    marker=dict(\n",
    "        color='#00e673',#rgb(158,202,225)',\n",
    "#         line=dict(\n",
    "#             color='rgb(8,48,107)',\n",
    "#             width=1.5,\n",
    "#         )\n",
    "    ),\n",
    "    \n",
    "    \n",
    "    \n",
    ")\n",
    "\n",
    "\n",
    "trace2 = go.Bar(\n",
    "    x=lista_bin_names,\n",
    "    y=lista_expectations_top,\n",
    "    name='expectations top '+str(int(100-100*list_q_plos[-2]))+'% Plos ('+str(Niter)+' iter)',\n",
    "  #  text= lista_text_old,\n",
    "    error_y=dict(\n",
    "       # type='data',\n",
    "        array=list_errors_top,#[0.5, 1, 2],\n",
    "        visible=True\n",
    "    ),\n",
    "    marker=dict(\n",
    "        color='#e6fff2',#'rgb(158,202,225)',\n",
    "#         line=dict(\n",
    "#             color='rgb(8,48,107)',\n",
    "#             width=1.5,\n",
    "#         )\n",
    "    ),\n",
    ")\n",
    "\n",
    "\n",
    "trace3 = go.Bar(\n",
    "    x=lista_bin_names,\n",
    "    y=lista_for_bottom_plos,\n",
    "    name='Bottom '+str(int(list_q_plos[0]*100))+'% Plos',\n",
    "   # text= lista_text_old,    \n",
    "    marker=dict(\n",
    "        color='#ff6666',#'rgb(158,202,225)',\n",
    "#         line=dict(\n",
    "#             color='rgb(8,48,107)',\n",
    "#             width=1.5,\n",
    "#         )\n",
    "    ),\n",
    ")\n",
    "\n",
    "\n",
    "trace4 = go.Bar(\n",
    "    x=lista_bin_names,\n",
    "    y=lista_expectations_bottom,\n",
    "    name='expectations bottom '+str(int(list_q_plos[0]*100))+'% Plos ('+str(Niter)+' iter)',\n",
    "  #  text= lista_text_old,\n",
    "    error_y=dict(\n",
    "       # type='data',\n",
    "        array=list_errors_bottom,#[0.5, 1, 2],\n",
    "        visible=True\n",
    "    ),\n",
    "    marker=dict(\n",
    "        color='#ffe6e6',#'rgb(158,202,225)',\n",
    "#         line=dict(\n",
    "#             color='rgb(8,48,107)',\n",
    "#             width=1.5,\n",
    "#         )\n",
    "    ),\n",
    ")\n",
    "\n",
    "\n",
    "data = [trace1, trace2, trace3, trace4]\n",
    "layout = go.Layout(\n",
    "    barmode='group',\n",
    "    title=title_string,\n",
    " #   xaxis = dict(title= 'Plos Citation percentile'),\n",
    "    yaxis = dict(title= 'Fraction of reference usage'),\n",
    "    \n",
    ")\n",
    "\n",
    "\n",
    "# layout = {\n",
    "#   'xaxis': {'title': 'Plos Citation percentile'},\n",
    "#   'yaxis': {'title': 'Fraction of references in paper'},\n",
    " \n",
    "# }\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "fig = go.Figure(data=data, layout=layout)\n",
    "#py.iplot(fig, filename='grouped-bar')\n",
    "\n",
    "\n",
    "fig_filename='fract_young_old_ref_vs_plos_cit_bin'\n",
    "offline.plot(fig, auto_open=True, image = 'png', image_filename=fig_filename ,image_width=2000, image_height=2000,\n",
    "              filename=fig_filename+'.html', validate=True)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#################### i plot the total number of references in papers by top/bottom plos category\n",
    "################################\n",
    "\n",
    "print(\"\\n\\nTotal number of references in different types of papers\")\n",
    "df_plos_select = preselection_df3.drop_duplicates(subset=['paper_UT','paper_cite_count','plos_pub_year'])\n",
    "\n",
    "print (\"\\n\",df_plos_select.shape)\n",
    "\n",
    "\n",
    "x_all = df_plos_select['total_refs']\n",
    "print (\"All:\",len(x_all), x_all.median())\n",
    "trace1 = go.Histogram(x=x_all,\n",
    "                     name='All plos '+str(int(np.nanmedian(x_all))),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "\n",
    "df_selected_top = df_plos_select[df_plos_select['paper_UT'].isin(lista_top_plos)]  # occurrences of not top references\n",
    "x_top = df_selected_top['total_refs']\n",
    "print (\"Top plos:\",len(x_top), x_top.median())\n",
    "trace2 = go.Histogram(x=x_top,\n",
    "                     name='top plos '+str(int(np.nanmedian(x_top))),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "\n",
    "df_selected_bottom = df_plos_select[df_plos_select['paper_UT'].isin(lista_bottom_plos)]  # occurrences of not top references\n",
    "x_bottom = df_selected_bottom['total_refs']\n",
    "print (\"Bottom plos\",len(x_bottom), x_bottom.median())\n",
    "trace3 = go.Histogram(x=x_bottom,\n",
    "                     name='bottom plos '+str(int(np.nanmedian(x_bottom))),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "\n",
    "random_lista = random.sample(list(x_all), len(x_top))\n",
    "print (\"Random:\",len(random_lista), np.median(random_lista))\n",
    "trace4 = go.Histogram(x=random_lista,\n",
    "                     name='random sample '+str(int(np.nanmedian(random_lista))),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "data = [trace1, trace2, trace3]#, trace4]\n",
    "\n",
    "\n",
    "\n",
    "fig = go.Figure(data=data)#, layout=layout)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#layout = go.Layout(barmode='overlay')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "fig_filename='histogram_tot_num_references_per_paper'\n",
    "offline.plot(fig, auto_open=True, image = 'png', image_filename=fig_filename ,image_width=2000, image_height=2000,\n",
    "              filename=fig_filename+'.html', validate=True)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "print (\"comparison   ALL- top plos:\", stats.ks_2samp(x_all, x_top),\"\\n\"  )\n",
    "print (\"comparison   ALL- bottom plos:\", stats.ks_2samp(x_all, x_bottom),\"\\n\"  )\n",
    "print (\"comparison   top plos - bottom plos:\", stats.ks_2samp(x_top, x_bottom),\"\\n\"  )\n",
    "print (\"comparison   top plos - random:\", stats.ks_2samp(x_top, random_lista),\"\\n\"  )\n",
    "\n",
    "     \n",
    "\n",
    "    \n",
    "    \n",
    "##################################\n",
    "    \n",
    "    \n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "print (preselection_df.shape, preselection_df3.shape)\n",
    "\n",
    "\n",
    "\n",
    "df_ref = preselection_df.drop_duplicates(subset=['reference_UT'])\n",
    "df_ref_young = preselection_df3.drop_duplicates(subset=['reference_UT'])\n",
    "\n",
    "df_plos = preselection_df.drop_duplicates(subset=['paper_UT'])\n",
    "df_plos_top = df_plos[df_plos['paper_UT'].isin(lista_top_plos)]\n",
    "\n",
    "\n",
    "print (df_ref.shape, df_ref_young.shape, df_plos.shape,df_plos_top.shape )\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#################### i plot the total number of citations received by the young (TOP) references and by the focus plos (TOP) papers\n",
    "################################\n",
    "\n",
    "\n",
    "\n",
    "lista_plos = list(df_plos['paper_cite_count'].dropna())\n",
    "trace1 = go.Histogram(x=lista_plos,\n",
    "                     name=\"all plos; median: \"+str(df_plos['paper_cite_count'].median()),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "\n",
    "lista_plos_top = list(df_plos_top['paper_cite_count'].dropna())\n",
    "trace2 = go.Histogram(x=lista_plos_top,\n",
    "                     name=\"top plos; median: \"+str(df_plos_top['paper_cite_count'].median()),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "lista_ref = list(df_ref['cite_count'].dropna())\n",
    "trace3 = go.Histogram(x=lista_ref,\n",
    "                     name=\"all ref; median: \"+str(df_ref['cite_count'].median()),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "\n",
    "lista_ref_young = list(df_ref_young['cite_count'].dropna())\n",
    "trace4 = go.Histogram(x=lista_ref_young,\n",
    "                     name=\"all (young) ref; median: \"+str(df_ref_young['cite_count'].median()),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "data =[trace1, trace2,trace3, trace4 ]\n",
    "\n",
    "# layout = go.Layout(\n",
    "#             title=str(years),\n",
    "#             xaxis = dict(title= 'Final number of citations received'),\n",
    "#                 # type='log'),#, autorange=True),\n",
    "#             yaxis = dict(title= 'PDF'),\n",
    "#     )\n",
    "#                 # type='log'),#, autorange=True),        \n",
    "                  \n",
    "   \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "fig = go.Figure(data=data)#, layout=layout)\n",
    "\n",
    "\n",
    "fig_filename='histogram_tot_num_citations_received_by_top_plos_and_by_top_young_ref'\n",
    "offline.plot(fig, auto_open=True, image = 'png', image_filename=fig_filename ,image_width=2000, image_height=2000,\n",
    "              filename=fig_filename+'.html', validate=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f848bb23ba8>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEACAYAAACpoOGTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAD7hJREFUeJzt3XuwH2V9x/H3x0QEbAtqTilNwIOVwVKrU3qKzjC2VuwUxQLtWMWxNio1tcVb7YwG6xT/YUanVoRObU2FitaieAWLtgL1Mp2pYKKO3KRmUCQR5FgveBsp+O0fvw05xiecX07O/vaXc96vmTPZffbZ3W+A8Mnus/tsqgpJkvb0oKELkCRNJwNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUtHboAvbHunXranZ2dugyJOmAsm3btm9U1cxi/Q7ogJidnWXr1q1DlyFJB5Qkt43Tz1tMkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU29BUSSi5PcleSGBW1/k+SLSb6Q5INJDl+w7Zwk25PckuR3+6pLkjSePq8g3g6cskfbVcBjq+pxwP8A5wAkOR44E/iVbp+3JFnTY22SpEX0FhBV9Sngm3u0fayq7u1WPw1s6JZPB95dVT+qqi8D24ET+6oNYHbzlX0eXpIOeEOOQbwQ+Gi3vB64fcG2HV2bJGkggwREkr8C7gXetYR9NyXZmmTr/Pz88hcnSQIGCIgkzweeATy3qqpr3gkctaDbhq7tp1TVlqqaq6q5mZlF55qSJC3RRAMiySnAq4DTquoHCzZdAZyZ5CFJjgGOBa6bZG2SpJ/U22yuSS4FngysS7IDOJfRU0sPAa5KAvDpqnpxVd2Y5DLgJka3ns6uqvv6qk2StLjeAqKqntNovugB+p8HnNdXPZKkfeOb1JKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBsQ9mN185dAmSNDEGhCSpyYCQJDUZEJKkJgNCktTUW0AkuTjJXUluWND28CRXJflS9+vDuvYkuTDJ9iRfSHJCX3VJksbT5xXE24FT9mjbDFxTVccC13TrAE8Dju1+NgH/0GNdkqQx9BYQVfUp4Jt7NJ8OXNItXwKcsaD9HTXyaeDwJEf2VZskaXGTHoM4oqru6JbvBI7oltcDty/ot6NrkyQNZLBB6qoqoPZ1vySbkmxNsnV+fr6Hysbni3OSVrJJB8TXd9066n69q2vfCRy1oN+Gru2nVNWWqpqrqrmZmZlei5Wk1WzSAXEFsLFb3ghcvqD9j7unmZ4IfGfBrShJ0gDW9nXgJJcCTwbWJdkBnAu8HrgsyVnAbcCzuu4fAZ4ObAd+ALygr7okSePpLSCq6jl72XRyo28BZ/dViyRp3/kmtSSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpaZCASPIXSW5MckOSS5McnOSYJNcm2Z7kPUkOGqI2SdLIxAMiyXrgZcBcVT0WWAOcCbwBOL+qHg18Czhr0rVJknYb6hbTWuCQJGuBQ4E7gKcA7+u2XwKcMVBtkiQGCIiq2gm8Efgqo2D4DrAN+HZV3dt12wGsn3RtkqTdhrjF9DDgdOAY4BeBhwKn7MP+m5JsTbJ1fn6+pyolSUPcYnoq8OWqmq+q/wM+AJwEHN7dcgLYAOxs7VxVW6pqrqrmZmZmJlOxJK1CQwTEV4EnJjk0SYCTgZuAjwPP7PpsBC4foDZJUmeIMYhrGQ1Gfxa4vqthC/Bq4JVJtgOPAC6adG2SpN3WLt4FkvxqVV2/XCetqnOBc/dovhU4cbnOIUnaP+NeQbwlyXVJ/jzJYb1WJEmaCmMFRFU9CXgucBSwLcm/JvmdXiuTJA1q7DGIqvoS8FpGYwW/BVyY5ItJ/qCv4iRJwxkrIJI8Lsn5wM2M3nj+var65W75/B7rkyQNZKxBauDvgLcBr6mqH+5qrKqvJXltL5VJkgY1bkCcCvywqu4DSPIg4OCq+kFVvbO36iRJgxl3DOJq4JAF64d2bZKkFWrcgDi4qr63a6VbPrSfklaO2c1XDl2CJC3ZuAHx/SQn7FpJ8uvADx+gvyTpADfuGMQrgPcm+RoQ4BeAZ/dWlSRpcGMFRFV9JsljgOO6plu6mVglSSvUuFcQAL8BzHb7nJCEqnpHL1VJkgY37mR97wR+Cfg8cF/XXIABIUkr1LhXEHPA8VVVfRYjSZoe4z7FdAOjgWlJ0iox7hXEOuCmJNcBP9rVWFWn9VKVJGlw4wbE6/osQpI0fcZ9zPWTSR4JHFtVVyc5FFjTb2mSpCGNO933ixh9R/qtXdN64EN9FSVJGt64g9RnAycBd8P9Hw/6+b6KkiQNb9yA+FFV3bNrJclaRu9BSJJWqHED4pNJXgMc0n2L+r3Ah/srS5I0tHEDYjMwD1wP/CnwEUbfp5YkrVDjPsX0Y+Cfuh9J0iow7lxMX6Yx5lBVj1r2iiRJU2Ff5mLa5WDgD4GHL/WkSQ4H3gY8llHwvBC4BXgPoxljvwI8q6q+tdRzSJL2z1hjEFX1vwt+dlbVm4FT9+O8FwD/XlWPAR4P3MxonOOaqjoWuKZblyQNZNxbTCcsWH0QoyuKffmWxMJjHQb8JvB8gO7x2XuSnA48uet2CfAJ4NVLOYckaf+N+z/5v12wfC/dLaAlnvMYRk9E/XOSxwPbgJcDR1TVHV2fO4Ejlnj8qTW7+Uq+8vr9ufCSpMkZ9ymm317mc54AvLSqrk1yAXvcTqqqStJ8ES/JJmATwNFHH72MZUmSFhr3FtMrH2h7Vb1pH865A9hRVdd26+9jFBBfT3JkVd2R5Ejgrr2cawuwBWBubs63uSWpJ+O+KDcH/BmjSfrWAy9mdBXws93P2KrqTuD2JMd1TScDNwFXABu7to3A5ftyXEnS8hp3DGIDcEJVfRcgyeuAK6vqj5Z43pcC70pyEHAr8AJGYXVZkrOA21j6GIckaRmMGxBHAPcsWL+H/RhErqrP85PvVuxy8lKPKUlaXuMGxDuA65J8sFs/g9GjqJKkFWrcp5jOS/JR4Eld0wuq6nP9lSVJGtq4g9QAhwJ3V9UFwI4kx/RUkyRpCoz7ydFzGb3VfE7X9GDgX/oqSpI0vHGvIH4fOA34PkBVfY19fLxVknRgGTcg7qmqopvyO8lD+ytJkjQNxg2Iy5K8FTg8yYuAq/HjQZK0oo37FNMbu29R3w0cB/x1VV3Va2WSpEEtGhBJ1gBXdxP2GQqStEoseoupqu4Dftx9x0GStEqM+yb194Drk1xF9yQTQFW9rJeqJEmDGzcgPtD9aGB+dEjSpDxgQCQ5uqq+WlXOuyRJq8xiYxAf2rWQ5P091yJJmiKLBUQWLD+qz0IkSdNlsYCovSxLkla4xQapH5/kbkZXEod0y3TrVVU/12t1kqTBPGBAVNWaSRUiSZou+/I9CEnSKmJASJKaDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDUZEJKkpsECIsmaJJ9L8m/d+jFJrk2yPcl7khw0VG2SpGGvIF4O3Lxg/Q3A+VX1aOBbwFmDVCVJAgYKiCQbgFOBt3XrAZ4CvK/rcglwxhC1SZJGhrqCeDPwKuDH3fojgG9X1b3d+g5g/RCFSZJGJh4QSZ4B3FVV25a4/6YkW5NsnZ+fX+bqDiyzm68cugRJK9gQVxAnAacl+Qrwbka3li4ADk+ya3bZDcDO1s5VtaWq5qpqbmZmZhL1StKqNPGAqKpzqmpDVc0CZwL/WVXPBT4OPLPrthG4fNK1SZJ2m6b3IF4NvDLJdkZjEhcNXI8krWqLfVGuV1X1CeAT3fKtwIlD1iNJ2m2ariAkSVPEgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDYoWb3Xxlc1mSFmNArEAGgaTlYEBIkpoMCElSkwEhSWoyICRJTRMPiCRHJfl4kpuS3Jjk5V37w5NcleRL3a8Pm3RtkqTdhriCuBf4y6o6HngicHaS44HNwDVVdSxwTbcuSRrIxAOiqu6oqs92y98FbgbWA6cDl3TdLgHOmHRt2s1HZSUNOgaRZBb4NeBa4IiquqPbdCdwxEBlSZIYMCCS/AzwfuAVVXX3wm1VVUDtZb9NSbYm2To/Pz+BSiVpdRokIJI8mFE4vKuqPtA1fz3Jkd32I4G7WvtW1ZaqmququZmZmckULEmr0BBPMQW4CLi5qt60YNMVwMZueSNw+aRrkyTttnaAc54EPA+4Psnnu7bXAK8HLktyFnAb8KwBapMkdSYeEFX1X0D2svnkSdYiSdo736SWJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoT2y77O+uossdKBw4DQsjMEpJXBgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQGgQfc3X5DxQ0vIxICRJTQaEBuXf+KXpZUBIkpoMCImlX8l4BaSVbOoCIskpSW5Jsj3J5qHrkaTVaqoCIska4O+BpwHHA89JcvywVelAsC9/kx+379767Wv7ctYkTdJUBQRwIrC9qm6tqnuAdwOnD1yTdMAYImgMt+Uxjf8cpy0g1gO3L1jf0bVJkiYsVTV0DfdL8kzglKr6k279ecATquolC/psAjZ1q8cBt/RY0mHAd3o8/lIMUVPf51zu4y/H8fbnGEvZdx3wjSWeT+Obxj/T+2Opv59HVtXMYp3WLuHAfdoJHLVgfUPXdr+q2gJsmUQxSbZU1abFe07OEDX1fc7lPv5yHG9/jrGUfZNsraq5pZxP45vGP9P7o+/fz7TdYvoMcGySY5IcBJwJXDFgPR8e8Nx7M0RNfZ9zuY+/HMfbn2NM4383Gllp/256/f1M1S0mgCRPB94MrAEurqrzBi5J6p1XEJpGUxcQ0mqUZFN3+1SaGgaEJKlp2sYgJElTwoCQJDUZEJKkpml7D0ISkOQM4FTg54CLqupjA5ekVcgrCGlCklyc5K4kN+zR/lMzGFfVh6rqRcCLgWcPUa9kQEiT83bglIUNY8xg/NpuuzRxBoQ0IVX1KeCbezQ3ZzDOyBuAj1bVZyddqwSOQUhDa81g/ATgpcBTgcOSPLqq/nGI4rS6GRDSFKqqC4ELh65Dq5u3mKRhLTqDsTQUA0Ia1rTNYCzdz4CQJiTJpcB/A8cl2ZHkrKq6F3gJ8B/AzcBlVXXjkHVKuzhZnySpySsISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElS0/8DbmjvDRcKvDYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f848bb234e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f848bb23ba8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "df_plos_top['paper_cite_count'].plot.hist(bins=2000,logx=True)\n",
    "plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "328"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(lista_plos)[-1]\n",
    "#sorted(lista_plos_top)[-1]\n",
    "#sorted(lista_ref)[-1]\n",
    "#sorted(lista_ref_young)[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2013]\n",
      "size of preselection1 (by plos years): (1533850, 23)\n",
      "    size of preselection3 (only young references): (145356, 23) young\n",
      "     N plos: 29061  N records: 145356\n",
      "\n",
      "\n",
      "citation bins for the selected plos: [0.2, 0.95, 1]\n",
      "0-3 13632\n",
      "3-28 72759\n",
      "28-328 7337\n",
      "\n",
      "\n",
      "# UTs top 5.0 % plos: 1494\n",
      "\n",
      "\n",
      "citation bins for the references in the selected plos: [0.5, 0.95, 1] [(0.5, 18.0), (0.95, 104.0), (1.0, 6453.0)]\n",
      "0-18 42804\n",
      "18-104 40015\n",
      "104-6453 4391\n",
      "# UTs top 5.0 % ref: 4391\n",
      "(1533850, 23) (145356, 23)\n",
      "(870897, 23) (87211, 23) (34593, 23) (1494, 23)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'file:///home/staff/julia/at_Northwestern/In_Text_Citations/In-Text-Citations-New/notebooks/histogram_tot_num_citations_received_by_top_plos_and_by_top_young_ref.html'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "###### check: if the \"endorsement effect\" is real (as opposed to just better plos papers being better at spotting quality in young references), \n",
    "####          succesful plos would get more citations than the young references they cite --->> i get the corresponding distributions:\n",
    "\n",
    "\n",
    "\n",
    "years=[2013]\n",
    "\n",
    "string_references_age=\"young\"   #young\"#old\"  # young # all   for the selection of what references i include\n",
    "string_isolated_ref=\"\"  #\"\"   # 0  or 1 (or empty string, to include all ref)\n",
    "string_self_ref=\"\"#1   # 0  or 1 (or empty string, to include all ref)\n",
    "string_journal=\"\"\n",
    "string_plos_field=\"\"#['D CU BIOLOGY']\"\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "##### preselection by plos year\n",
    "print (years)\n",
    "preselection_df = df_merged[df_merged['plos_pub_year'].isin(years)]  \n",
    "print (\"size of preselection1 (by plos years):\",preselection_df.shape)\n",
    "\n",
    "\n",
    "\n",
    "#### i remove self-citations\n",
    "if (string_self_ref==0) or  ( string_self_ref == 1 ): \n",
    "    preselection_df = preselection_df[preselection_df['self_citation']== string_self_ref ]  \n",
    "    print (\" size of preselection1 (self-cit):\",preselection_df.shape)\n",
    "\n",
    "\n",
    "\n",
    "######### preselection by isolated or group references:\n",
    "preselection_df0 = preselection_df   \n",
    "if (string_isolated_ref==0) or  ( string_isolated_ref == 1 ): \n",
    "    preselection_df0 = preselection_df[preselection_df['isolated_citation']== string_isolated_ref ]        \n",
    "    print (\"  size of preselection1 (by isolated/group ref):\",preselection_df0.shape, string_isolated_ref)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "######### preselection by plos journal:\n",
    "if string_journal==\"\": \n",
    "    preselection_df1 = preselection_df0\n",
    "else:    \n",
    "    preselection_df1 = preselection_df0[preselection_df0['plos_j1']== string_journal ]  \n",
    "    print (\"   size of preselection2 (by plos journal):\",preselection_df1.shape, string_journal)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "######### preselection by plos field:\n",
    "if string_plos_field==\"\": \n",
    "    preselection_df2 = preselection_df1\n",
    "else:    \n",
    "    preselection_df2 = preselection_df1[preselection_df1['plos_field']== string_plos_field ]  \n",
    "    print (\"    size of preselection2 (by plos field):\",preselection_df2.shape, string_plos_field)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    " \n",
    "\n",
    "\n",
    "\n",
    "##### preselection only young/old references:       \n",
    "preselection_df3 = preselection_df2\n",
    "if string_references_age == \"young\":\n",
    "    time_window_age = 1   \n",
    "    preselection_df3 = preselection_df2[preselection_df2['ref_pub_year'] >= (min(years)-time_window_age) ]   \n",
    "    \n",
    "    print (\"    size of preselection3 (only young references):\",preselection_df3.shape, string_references_age)\n",
    "\n",
    "elif string_references_age == \"old\":\n",
    "    time_window_age = 10    \n",
    "    preselection_df3 = preselection_df2[preselection_df2['ref_pub_year'] <= (min(years)-time_window_age) ]   \n",
    "    \n",
    "    print (\"    size of preselection3 (only old references):\",preselection_df3.shape,string_references_age )\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "N_plos=len(preselection_df3.paper_UT.unique())        ## this values are overall, for the title   \n",
    "N_all=len(preselection_df3)\n",
    "\n",
    "    \n",
    "    \n",
    "         \n",
    "print (\"     N plos:\", N_plos, \" N records:\", N_all)        \n",
    " \n",
    "    \n",
    "    \n",
    "    \n",
    "############## i define quantiles for plos papers (for that subselection, and based on their FINAL number of citations):\n",
    "list_q_plos=[.2,.95,1]\n",
    "df_for_quantiles_plos = preselection_df3.drop_duplicates(subset=['paper_UT'])   # ojo!!! dont use preselection_df3 directly because there are REPETITIONS!!!!\n",
    "\n",
    "quantiles=sorted(list(df_for_quantiles_plos['paper_cite_count'].quantile(list_q_plos).to_dict().items())) #mean 10.68 \n",
    " \n",
    "print (\"\\n\\ncitation bins for the selected plos:\", list_q_plos)#,quantiles, df_for_quantiles_plos.shape)   \n",
    "\n",
    "lista_bins_plos=[]\n",
    "old_value=0\n",
    "for item in quantiles:\n",
    "    pair=[old_value, int(item[1])]\n",
    "    lista_bins_plos.append(pair)\n",
    "    old_value = int(item[1])\n",
    "                           \n",
    "#print (lista_bins_plos, min(preselection_df3['paper_cite_count']), max(preselection_df3['paper_cite_count']))\n",
    "\n",
    "\n",
    "\n",
    "cont = 0\n",
    "dict_bin_list_plos_UT={}\n",
    "for item in lista_bins_plos:\n",
    "    \n",
    "    minimo = item[0]\n",
    "    maximo = item[1]   \n",
    "\n",
    "    df_select = preselection_df3[(preselection_df3['paper_cite_count'] >= minimo)  &  (preselection_df3['paper_cite_count'] < maximo)]\n",
    "    llave=str(minimo)+\"-\"+str(maximo)\n",
    "    dict_bin_list_plos_UT[llave]= list(df_select.paper_UT.unique())\n",
    "    print (llave, len(list(df_select.reference_UT.unique())))\n",
    "    max_key_plos=llave\n",
    "\n",
    "    \n",
    "    if cont ==0:\n",
    "        min_key_plos = llave\n",
    "    cont  +=1\n",
    "    \n",
    "    \n",
    "lista_top_plos = dict_bin_list_plos_UT[max_key_plos]    # i create the list of top plos\n",
    "print (\"\\n\\n# UTs top\",(100-100*list_q_plos[-2]),\"% plos:\",len(lista_top_plos))\n",
    "    \n",
    "   \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "########## i define quantiles for references (based on their FINAL number of citations)\n",
    "list_q_ref=[0.5,.95,1]\n",
    "df_for_quantiles_ref = preselection_df3.drop_duplicates(subset=['reference_UT'])   # ojo!!! remember to remove REPETITIONS!!!!\n",
    "quantiles=sorted(list(df_for_quantiles_ref['cite_count'].quantile(list_q_ref).to_dict().items())) #mean 10.68 \n",
    " \n",
    "print (\"\\n\\ncitation bins for the references in the selected plos:\", list_q_ref,quantiles)    \n",
    "\n",
    "lista_bins=[]\n",
    "old_value=0\n",
    "for item in quantiles:\n",
    "    pair=[old_value, int(item[1])]\n",
    "    lista_bins.append(pair)\n",
    "    old_value = int(item[1])\n",
    "                           \n",
    "\n",
    "\n",
    "\n",
    "cont = 0\n",
    "dict_bin_list_ref_UT={}\n",
    "for item in lista_bins:\n",
    "    \n",
    "    minimo = item[0]\n",
    "    maximo = item[1]    \n",
    "     \n",
    "    df_select = preselection_df3[(preselection_df3['cite_count'] >= minimo)  &  (preselection_df3['cite_count'] < maximo)]\n",
    "    llave=str(minimo)+\"-\"+str(maximo)\n",
    "    dict_bin_list_ref_UT[llave]=list(df_select.reference_UT.unique())\n",
    "    print (llave, len(list(df_select.reference_UT.unique())))\n",
    "    max_key_ref=llave\n",
    "\n",
    "    if cont ==0:\n",
    "        min_key_ref = llave\n",
    "    cont  +=1\n",
    "\n",
    "\n",
    "\n",
    "lista_top_ref=dict_bin_list_ref_UT[max_key_ref]\n",
    "print (\"# UTs top\",(100-100*list_q_ref[-2]),\"% ref:\", len(lista_top_ref))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "print (preselection_df.shape, preselection_df3.shape)\n",
    "\n",
    "\n",
    "\n",
    "df_ref = preselection_df.drop_duplicates(subset=['reference_UT'])\n",
    "df_ref_young = preselection_df3.drop_duplicates(subset=['reference_UT'])\n",
    "\n",
    "df_plos = preselection_df.drop_duplicates(subset=['paper_UT'])\n",
    "df_plos_top = df_plos[df_plos['paper_UT'].isin(lista_top_plos)]\n",
    "\n",
    "\n",
    "print (df_ref.shape, df_ref_young.shape, df_plos.shape,df_plos_top.shape )\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#################### i plot the total number of citations received by the young (TOP) references and by the focus plos (TOP) papers\n",
    "################################\n",
    "\n",
    "\n",
    "\n",
    "lista= list(df_plos['paper_cite_count'].dropna())\n",
    "trace1 = go.Histogram(x=lista,\n",
    "                     name=\"all plos; median: \"+str(df_plos['paper_cite_count'].median()),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "\n",
    "lista= list(df_plos_top['paper_cite_count'].dropna())\n",
    "trace2 = go.Histogram(x=lista,\n",
    "                     name=\"top plos; median: \"+str(df_plos_top['paper_cite_count'].median()),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "lista= list(df_ref['cite_count'].dropna())\n",
    "trace3 = go.Histogram(x=lista,\n",
    "                     name=\"all ref; median: \"+str(df_ref['cite_count'].median()),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "\n",
    "lista= list(df_ref_young['cite_count'].dropna())\n",
    "trace4 = go.Histogram(x=lista,\n",
    "                     name=\"all (young) ref; median: \"+str(df_ref_young['cite_count'].median()),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "data =[trace1, trace2,trace3, trace4 ]\n",
    "\n",
    "# layout = go.Layout(\n",
    "#             title=str(years),\n",
    "#             xaxis = dict(title= 'Final number of citations received'),\n",
    "#                 # type='log'),#, autorange=True),\n",
    "#             yaxis = dict(title= 'PDF'),\n",
    "#     )\n",
    "#                 # type='log'),#, autorange=True),        \n",
    "                  \n",
    "   \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "fig = go.Figure(data=data)#, layout=layout)\n",
    "\n",
    "\n",
    "fig_filename='histogram_tot_num_citations_received_by_top_plos_and_by_top_young_ref'\n",
    "offline.plot(fig, auto_open=True, image = 'png', image_filename=fig_filename ,image_width=2000, image_height=2000,\n",
    "              filename=fig_filename+'.html', validate=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "158813 158813 5617055 2607453 43.60009571 0.294875095546\n",
      "34593 34593 1244069 870897 44.3398953546 0.29303700373\n",
      "29061 29061 112851 87211 5.00175492929 0.688195755498\n",
      "1494 1494 8266 7337 7.21084337349 1.78019850253\n",
      "811 811 1723 1162 2.94327990136 0.246966113346\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'file:///home/staff/julia/at_Northwestern/In_Text_Citations/In-Text-Citations-New/notebooks/histogram_rations_num_cit_plos_num_cit_young_ref.html'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "###### more checks: if the \"endorsement effect\" is real (as opposed to just better plos papers being better at spotting quality in young references), \n",
    "#                   succesful plos would get more citations than the young references they cite --->> i get the (distributions of) ratios:\n",
    "lista_paper_UT=[]\n",
    "lista_ref_UT=[]\n",
    "lista_num_ref=[]\n",
    "lista_ratios_all=[]\n",
    "\n",
    "\n",
    "for paper_UT, group in df_merged.groupby(['paper_UT']):  ### for df_merged: 158813 158813 5617059 2607457 43.600139787 0.0542472340106   # for preselection_df3: 6211 6211 30225 25508 6.34068587989 0.472185840259  # selected year: 7005 7005 260294 214744 46.4395431834 0.110949088539\n",
    "        \n",
    "        lista_paper_UT.append(paper_UT)\n",
    "        lista_ref_UT += list(group.reference_UT.unique())\n",
    "        lista_num_ref.append(len(group))\n",
    "        num_cit_paper = float(group.paper_cite_count.unique()) # the group refers to one single plos\n",
    "        lista_num_cit_ref = list(group.cite_count.values)\n",
    "        list_ratios = [num_cit_paper/item   if item !=0  else 0 for item in lista_num_cit_ref]  #num_cit_paper / avg_num_cit_ref\n",
    "        lista_ratios_all += list_ratios #.append(ratio)        \n",
    "        \n",
    "print (len(lista_paper_UT), len(set(lista_paper_UT)), len(lista_ref_UT), len(set(lista_ref_UT)), np.mean(lista_num_ref), np.mean(lista_ratios_all) )\n",
    "\n",
    "\n",
    "\n",
    "lista_paper_UT=[]\n",
    "lista_ref_UT=[]\n",
    "lista_num_ref=[]\n",
    "\n",
    "lista_ratios_selected_year=[]\n",
    "\n",
    "for paper_UT, group in preselection_df.groupby(['paper_UT']):  ### for df_merged: 158813 158813 5617059 2607457 43.600139787 0.0542472340106   # for preselection_df3: 6211 6211 30225 25508 6.34068587989 0.472185840259  # selected year: 7005 7005 260294 214744 46.4395431834 0.110949088539\n",
    "        \n",
    "        lista_paper_UT.append(paper_UT)\n",
    "        lista_ref_UT += list(group.reference_UT.unique())\n",
    "        lista_num_ref.append(len(group))\n",
    "        num_cit_paper = float(group.paper_cite_count.unique()) # the group refers to one single plos\n",
    "        lista_num_cit_ref = list(group.cite_count.values)\n",
    "        list_ratios = [num_cit_paper/item   if item !=0  else 0 for item in lista_num_cit_ref] #num_cit_paper / avg_num_cit_ref\n",
    "        lista_ratios_selected_year += list_ratios #.append(ratio)   \n",
    "        \n",
    "        \n",
    "print (len(lista_paper_UT), len(set(lista_paper_UT)), len(lista_ref_UT), len(set(lista_ref_UT)), np.mean(lista_num_ref), np.mean(lista_ratios_selected_year) )\n",
    "\n",
    "\n",
    "\n",
    "lista_paper_UT=[]\n",
    "lista_ref_UT=[]\n",
    "lista_num_ref=[]\n",
    "\n",
    "lista_ratios_selected_year_young_ref=[]\n",
    "\n",
    "\n",
    "for paper_UT, group in preselection_df3.groupby(['paper_UT']):  ### for df_merged: 158813 158813 5617059 2607457 43.600139787 0.0542472340106   # for preselection_df3: 6211 6211 30225 25508 6.34068587989 0.472185840259  # selected year: 7005 7005 260294 214744 46.4395431834 0.110949088539\n",
    "        \n",
    "        lista_paper_UT.append(paper_UT)\n",
    "        lista_ref_UT += list(group.reference_UT.unique())\n",
    "        lista_num_ref.append(len(group))\n",
    "        num_cit_paper = float(group.paper_cite_count.unique()) # the group refers to one single plos\n",
    "        lista_num_cit_ref = list(group.cite_count.values)\n",
    "        list_ratios = [num_cit_paper/item   if item !=0  else 0 for item in lista_num_cit_ref]  #num_cit_paper / avg_num_cit_ref\n",
    "        lista_ratios_selected_year_young_ref += list_ratios #.append(ratio)   \n",
    "        \n",
    "        \n",
    "print (len(lista_paper_UT), len(set(lista_paper_UT)), len(lista_ref_UT), len(set(lista_ref_UT)), np.mean(lista_num_ref), np.mean(lista_ratios_selected_year_young_ref) )\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "lista_paper_UT=[]\n",
    "lista_ref_UT=[]\n",
    "lista_num_ref=[]\n",
    "\n",
    "lista_ratios_selected_year_young_ref_top_plos=[]\n",
    "df_select = preselection_df3[preselection_df3['paper_UT'].isin(lista_top_plos)]\n",
    "\n",
    "for paper_UT, group in df_select.groupby(['paper_UT']):  ### for df_merged: 158813 158813 5617059 2607457 43.600139787 0.0542472340106   # for preselection_df3: 6211 6211 30225 25508 6.34068587989 0.472185840259  # selected year: 7005 7005 260294 214744 46.4395431834 0.110949088539\n",
    "        \n",
    "        lista_paper_UT.append(paper_UT)\n",
    "        lista_ref_UT += list(group.reference_UT.unique())\n",
    "        lista_num_ref.append(len(group))\n",
    "        num_cit_paper = float(group.paper_cite_count.unique()) # the group refers to one single plos\n",
    "        \n",
    "        lista_num_cit_ref = list(group.cite_count.values)\n",
    "        list_ratios = [num_cit_paper/item   if item !=0  else 0 for item in lista_num_cit_ref]  #num_cit_paper / avg_num_cit_ref\n",
    "        lista_ratios_selected_year_young_ref_top_plos += list_ratios #.append(ratio)   \n",
    "        \n",
    "        \n",
    "print (len(lista_paper_UT), len(set(lista_paper_UT)), len(lista_ref_UT), len(set(lista_ref_UT)), np.mean(lista_num_ref), np.mean(lista_ratios_selected_year_young_ref_top_plos) )\n",
    "\n",
    "\n",
    "\n",
    "lista_paper_UT=[]\n",
    "lista_ref_UT=[]\n",
    "lista_num_ref=[]\n",
    "\n",
    "lista_ratios_selected_year_young_ref_top_plos_top_ref=[]\n",
    "\n",
    "df_select = preselection_df3[preselection_df3['paper_UT'].isin(lista_top_plos)]\n",
    "df_select = df_select[df_select['reference_UT'].isin(lista_top_ref)]\n",
    "\n",
    "\n",
    "for paper_UT, group in df_select.groupby(['paper_UT']):  ### for df_merged: 158813 158813 5617059 2607457 43.600139787 0.0542472340106   # for preselection_df3: 6211 6211 30225 25508 6.34068587989 0.472185840259  # selected year: 7005 7005 260294 214744 46.4395431834 0.110949088539\n",
    "        \n",
    "        lista_paper_UT.append(paper_UT)\n",
    "        lista_ref_UT += list(group.reference_UT.unique())\n",
    "        lista_num_ref.append(len(group))\n",
    "        num_cit_paper = float(group.paper_cite_count.unique()) # the group refers to one single plos\n",
    "        lista_num_cit_ref = list(group.cite_count.values)\n",
    "        list_ratios = [num_cit_paper/item   if item !=0  else 0 for item in lista_num_cit_ref]  #num_cit_paper / avg_num_cit_ref\n",
    "        lista_ratios_selected_year_young_ref_top_plos_top_ref += list_ratios #.append(ratio)   \n",
    "        \n",
    "print (len(lista_paper_UT), len(set(lista_paper_UT)), len(lista_ref_UT), len(set(lista_ref_UT)), np.mean(lista_num_ref), np.mean(lista_ratios_selected_year_young_ref_top_plos_top_ref) )\n",
    "\n",
    "\n",
    "# 158813 158813 5617059 2607457 43.600139787 0.0542472340106\n",
    "# 7005 7005 260294 214744 46.4395431834 0.110949088539\n",
    "# 6211 6211 30225 25508 6.34068587989 0.472185840259\n",
    "# 312 312 2266 2092 9.46474358974 1.36087421586\n",
    "# 169 169 379 293 3.01775147929 0.209680689138\n",
    "\n",
    "\n",
    "\n",
    "\n",
    " \n",
    "\n",
    "trace1 = go.Histogram(x=lista_ratios_all,\n",
    "                     name=\"all plos; mean: \"+str(np.nanmean(lista_ratios_all)),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "trace2 = go.Histogram(x=lista_ratios_selected_year,\n",
    "                     name=\"plos in selected year, all ref; mean: \"+str(np.nanmean(lista_ratios_selected_year)),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "trace3 = go.Histogram(x=lista_ratios_selected_year_young_ref,\n",
    "                     name=\"plos in selected year, young ref only; mean: \"+str(np.nanmean(lista_ratios_selected_year_young_ref)),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "trace4 = go.Histogram(x=lista_ratios_selected_year_young_ref_top_plos,\n",
    "                     name=\"TOP plos in selected year, young ref only; mean: \"+str(np.nanmean(lista_ratios_selected_year_young_ref_top_plos)),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "trace5 = go.Histogram(x=lista_ratios_selected_year_young_ref_top_plos_top_ref,\n",
    "                     name=\"TOP plos in selected year, TOP young ref only; mean: \"+str(np.nanmean(lista_ratios_selected_year_young_ref_top_plos_top_ref)),\n",
    "                      histnorm='probability',\n",
    "                     opacity = 0.75)\n",
    "\n",
    "\n",
    "data =[trace1,  trace2, trace3, trace4, trace5 ]\n",
    "\n",
    "layout = go.Layout(\n",
    "            title=str(years),\n",
    "            xaxis = dict(title= 'Ratio final number of citations plos / final number citations ref'),\n",
    "                # type='log'),#, autorange=True),\n",
    "            yaxis = dict(title= 'PDF'),\n",
    "    )\n",
    "                # type='log'),#, autorange=True),        \n",
    "                  \n",
    "   \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "fig = go.Figure(data=data, layout=layout)\n",
    "\n",
    "\n",
    "fig_filename='histogram_rations_num_cit_plos_num_cit_young_ref'\n",
    "offline.plot(fig, auto_open=True, image = 'png', image_filename=fig_filename ,image_width=2000, image_height=2000,\n",
    "              filename=fig_filename+'.html', validate=True)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "PLOS FROM YEAR: [2008]\n",
      "size of preselection1 (by plos years): (127714, 23)\n",
      "    size of preselection3 (only young references): (17779, 23) young\n",
      "     N plos: 2707  N records: 17779\n",
      "citation bins for the selected plos: [0.2, 0.95, 1]\n",
      "\n",
      "\n",
      "citation bins for the references in the selected plos: [0.5, 0.95, 1] [(0.5, 58.0), (0.95, 467.10000000000036), (1.0, 20654.0)]\n",
      "11879 2707 1006 1901\n",
      "entering query for all citations of young references ..............\n",
      "1833140\n",
      "1123339\n",
      "done collecting the list of citing papers of the young references.\n",
      "done loading dict_UT_year\n",
      "full dict all citing papers: 1119422   only early citing papers: 86670\n",
      "new dict created\n",
      "[2008]\n",
      "avg # early citations of young ref 13.6147823891 60.3934214023 11879\n",
      "  of young ref. cited by top plos 24.7783300199 83.9423158842 1006\n",
      "  of young ref. cited by bottom plos 15.3371909521 69.5331219346 1901\n",
      "\n",
      "comparison   ALL- top plos: Ks_2sampResult(statistic=0.19824666781698902, pvalue=2.4146900525414032e-32)\n",
      "comparison   ALL- bottom plos: Ks_2sampResult(statistic=0.032868244187101603, pvalue=0.056748295649428307)\n",
      "comparison   top plos- bottom plos: Ks_2sampResult(statistic=0.22656172381805956, pvalue=4.8459508980822651e-30)\n",
      "\n",
      "\n",
      "\n",
      "PLOS FROM YEAR: [2009]\n",
      "size of preselection1 (by plos years): (208881, 23)\n",
      "    size of preselection3 (only young references): (25965, 23) young\n",
      "     N plos: 4190  N records: 25965\n",
      "citation bins for the selected plos: [0.2, 0.95, 1]\n",
      "\n",
      "\n",
      "citation bins for the references in the selected plos: [0.5, 0.95, 1] [(0.5, 49.0), (0.95, 384.0), (1.0, 49346.0)]\n",
      "17240 4190 1495 2765\n",
      "entering query for all citations of young references ..............\n",
      "2273489\n",
      "1341757\n",
      "done collecting the list of citing papers of the young references.\n",
      "done loading dict_UT_year\n",
      "full dict all citing papers: 1339317   only early citing papers: 122099\n",
      "new dict created\n",
      "[2009]\n",
      "avg # early citations of young ref 13.2332366589 153.685822556 17240\n",
      "  of young ref. cited by top plos 24.2474916388 138.323864342 1495\n",
      "  of young ref. cited by bottom plos 14.8216998192 103.254534409 2765\n",
      "\n",
      "comparison   ALL- top plos: Ks_2sampResult(statistic=0.17956296704405245, pvalue=3.2922118338862529e-39)\n",
      "comparison   ALL- bottom plos: Ks_2sampResult(statistic=0.047400280268352746, pvalue=4.2415662484295279e-05)\n",
      "comparison   top plos- bottom plos: Ks_2sampResult(statistic=0.21132769267056556, pvalue=2.2989725162264001e-38)\n",
      "\n",
      "\n",
      "\n",
      "PLOS FROM YEAR: [2010]\n",
      "size of preselection1 (by plos years): (325309, 23)\n",
      "    size of preselection3 (only young references): (39382, 23) young\n",
      "     N plos: 6211  N records: 39382\n",
      "citation bins for the selected plos: [0.2, 0.95, 1]\n",
      "\n",
      "\n",
      "citation bins for the references in the selected plos: [0.5, 0.95, 1] [(0.5, 41.0), (0.95, 296.6499999999978), (1.0, 7858.0)]\n",
      "25508 6211 2092 4052\n",
      "entering query for all citations of young references ..............\n",
      "2809570\n",
      "1477026\n",
      "done collecting the list of citing papers of the young references.\n",
      "done loading dict_UT_year\n",
      "full dict all citing papers: 1469091   only early citing papers: 144923\n",
      "new dict created\n",
      "[2010]\n",
      "avg # early citations of young ref 11.784930218 46.5120037539 25508\n",
      "  of young ref. cited by top plos 23.1682600382 112.957518982 2092\n",
      "  of young ref. cited by bottom plos 14.1201875617 80.3866426009 4052\n",
      "\n",
      "comparison   ALL- top plos: Ks_2sampResult(statistic=0.19538083654481281, pvalue=6.8559184128914402e-65)\n",
      "comparison   ALL- bottom plos: Ks_2sampResult(statistic=0.035063888749997857, pvalue=0.00035613146762742532)\n",
      "comparison   top plos- bottom plos: Ks_2sampResult(statistic=0.19909814854312674, pvalue=3.0350331167612209e-48)\n",
      "\n",
      "\n",
      "\n",
      "PLOS FROM YEAR: [2011]\n",
      "size of preselection1 (by plos years): (677899, 23)\n",
      "    size of preselection3 (only young references): (73048, 23) young\n",
      "     N plos: 12588  N records: 73048\n",
      "citation bins for the selected plos: [0.2, 0.95, 1]\n",
      "\n",
      "\n",
      "citation bins for the references in the selected plos: [0.5, 0.95, 1] [(0.5, 31.0), (0.95, 206.0), (1.0, 21063.0)]\n",
      "45372 12588 3822 8045\n",
      "entering query for all citations of young references ..............\n",
      "3652922\n",
      "1778789\n",
      "done collecting the list of citing papers of the young references.\n",
      "done loading dict_UT_year\n",
      "full dict all citing papers: 1768646   only early citing papers: 208159\n",
      "new dict created\n",
      "[2011]\n",
      "avg # early citations of young ref 9.92548267654 39.471293241 45372\n",
      "  of young ref. cited by top plos 18.746729461 81.0825295146 3822\n",
      "  of young ref. cited by bottom plos 12.3558732132 71.5666613412 8045\n",
      "\n",
      "comparison   ALL- top plos: Ks_2sampResult(statistic=0.21018000714415108, pvalue=3.0695323084936781e-136)\n",
      "comparison   ALL- bottom plos: Ks_2sampResult(statistic=0.034162753295223391, pvalue=2.2570001966727035e-07)\n",
      "comparison   top plos- bottom plos: Ks_2sampResult(statistic=0.21914502378854689, pvalue=5.0077904593095673e-109)\n",
      "\n",
      "\n",
      "\n",
      "PLOS FROM YEAR: [2012]\n",
      "size of preselection1 (by plos years): (1213058, 23)\n",
      "    size of preselection3 (only young references): (123026, 23) young\n",
      "     N plos: 22530  N records: 123026\n",
      "citation bins for the selected plos: [0.2, 0.95, 1]\n",
      "\n",
      "\n",
      "citation bins for the references in the selected plos: [0.5, 0.95, 1] [(0.5, 24.0), (0.95, 142.0), (1.0, 21063.0)]\n",
      "72922 22530 6303 13337\n",
      "entering query for all citations of young references ..............\n",
      "4393410\n",
      "1973336\n",
      "done collecting the list of citing papers of the young references.\n",
      "done loading dict_UT_year\n",
      "full dict all citing papers: 1968312   only early citing papers: 281707\n",
      "new dict created\n",
      "[2012]\n",
      "avg # early citations of young ref 8.80694440635 38.4419512169 72922\n",
      "  of young ref. cited by top plos 17.878311915 101.291021122 6303\n",
      "  of young ref. cited by bottom plos 11.0258678863 72.3948999706 13337\n",
      "\n",
      "comparison   ALL- top plos: Ks_2sampResult(statistic=0.22161914963087731, pvalue=1.0296772917496555e-248)\n",
      "comparison   ALL- bottom plos: Ks_2sampResult(statistic=0.027790918973928491, pvalue=5.2493967921373998e-08)\n",
      "comparison   top plos- bottom plos: Ks_2sampResult(statistic=0.21641761509397389, pvalue=3.3632690766332916e-175)\n",
      "\n",
      "\n",
      "\n",
      "PLOS FROM YEAR: [2013]\n",
      "size of preselection1 (by plos years): (1533850, 23)\n",
      "    size of preselection3 (only young references): (145356, 23) young\n",
      "     N plos: 29061  N records: 145356\n",
      "citation bins for the selected plos: [0.2, 0.95, 1]\n",
      "\n",
      "\n",
      "citation bins for the references in the selected plos: [0.5, 0.95, 1] [(0.5, 18.0), (0.95, 104.0), (1.0, 6453.0)]\n",
      "87211 29061 7337 13632\n",
      "entering query for all citations of young references ..............\n",
      "4151712\n",
      "1830462\n",
      "done collecting the list of citing papers of the young references.\n",
      "done loading dict_UT_year\n",
      "full dict all citing papers: 1828135   only early citing papers: 334385\n",
      "new dict created\n",
      "[2013]\n",
      "avg # early citations of young ref 8.64694820607 37.7427057949 87211\n",
      "  of young ref. cited by top plos 19.2522829494 103.724475471 7337\n",
      "  of young ref. cited by bottom plos 11.0004401408 74.7585471689 13632\n",
      "\n",
      "comparison   ALL- top plos: Ks_2sampResult(statistic=0.25206913784990048, pvalue=0.0)\n",
      "comparison   ALL- bottom plos: Ks_2sampResult(statistic=0.030569644230174498, pvalue=5.1331957679698765e-10)\n",
      "comparison   top plos- bottom plos: Ks_2sampResult(statistic=0.26356843985177703, pvalue=3.0173416641780143e-289)\n",
      "\n",
      "\n",
      "\n",
      "PLOS FROM YEAR: [2014]\n",
      "size of preselection1 (by plos years): (1276963, 23)\n",
      "    size of preselection3 (only young references): (108885, 23) young\n",
      "     N plos: 24533  N records: 108885\n",
      "citation bins for the selected plos: [0.2, 0.95, 1]\n",
      "\n",
      "\n",
      "citation bins for the references in the selected plos: [0.5, 0.95, 1] [(0.5, 13.0), (0.95, 76.0), (1.0, 7753.0)]\n",
      "71424 24533 5546 13843\n",
      "entering query for all citations of young references ..............\n",
      "2778303\n",
      "1392448\n",
      "done collecting the list of citing papers of the young references.\n",
      "done loading dict_UT_year\n",
      "full dict all citing papers: 1388278   only early citing papers: 312452\n",
      "new dict created\n",
      "[2014]\n",
      "avg # early citations of young ref 8.72236223118 41.281711257 71424\n",
      "  of young ref. cited by top plos 18.1754417598 115.172979534 5546\n",
      "  of young ref. cited by bottom plos 10.3344650726 75.6977481795 13843\n",
      "\n",
      "comparison   ALL- top plos: Ks_2sampResult(statistic=0.22130695340340223, pvalue=4.2438128477523467e-220)\n",
      "comparison   ALL- bottom plos: Ks_2sampResult(statistic=0.03815890969893565, pvalue=4.0036318039170119e-15)\n",
      "comparison   top plos- bottom plos: Ks_2sampResult(statistic=0.24540920344706391, pvalue=2.3177994251991694e-208)\n",
      "\n",
      "\n",
      "\n",
      "PLOS FROM YEAR: [2015]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size of preselection1 (by plos years): (1190256, 23)\n",
      "    size of preselection3 (only young references): (97191, 23) young\n",
      "     N plos: 22937  N records: 97191\n",
      "citation bins for the selected plos: [0.2, 0.95, 1]\n",
      "\n",
      "\n",
      "citation bins for the references in the selected plos: [0.5, 0.95, 1] [(0.5, 7.0), (0.95, 45.0), (1.0, 4663.0)]\n",
      "65776 22937 6060 0\n",
      "entering query for all citations of young references ..............\n",
      "1903021\n",
      "1013627\n",
      "done collecting the list of citing papers of the young references.\n",
      "entering long query for ONLY early citations of young references ..............\n",
      "1008515 ../data/dict_UT_year_for_citing_papers_of_young_ref_2015.pkl\n",
      "full dict all citing papers: 1008515   only early citing papers: 303848\n",
      "new dict created\n",
      "[2015]\n",
      "avg # early citations of young ref 8.56911755762 43.7013040637 65772\n",
      "  of young ref. cited by top plos 17.2087458746 113.521992858 6060\n",
      "  of young ref. cited by bottom plos nan nan 0\n",
      "\n",
      "comparison   ALL- top plos: Ks_2sampResult(statistic=0.21439316619127702, pvalue=1.1174827371683936e-222)\n",
      "comparison   ALL- bottom plos: Ks_2sampResult(statistic=nan, pvalue=nan)\n",
      "comparison   top plos- bottom plos: Ks_2sampResult(statistic=nan, pvalue=nan)\n",
      "\n",
      "\n",
      "\n",
      "PLOS FROM YEAR: [2016]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/staff/julia/.pyenv/versions/3.6.4/envs/psy/lib/python3.6/site-packages/numpy/core/fromnumeric.py:2909: RuntimeWarning:\n",
      "\n",
      "Mean of empty slice.\n",
      "\n",
      "/home/staff/julia/.pyenv/versions/3.6.4/envs/psy/lib/python3.6/site-packages/numpy/core/_methods.py:80: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in double_scalars\n",
      "\n",
      "/home/staff/julia/.pyenv/versions/3.6.4/envs/psy/lib/python3.6/site-packages/numpy/core/_methods.py:135: RuntimeWarning:\n",
      "\n",
      "Degrees of freedom <= 0 for slice\n",
      "\n",
      "/home/staff/julia/.pyenv/versions/3.6.4/envs/psy/lib/python3.6/site-packages/numpy/core/_methods.py:105: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in true_divide\n",
      "\n",
      "/home/staff/julia/.pyenv/versions/3.6.4/envs/psy/lib/python3.6/site-packages/numpy/core/_methods.py:127: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in double_scalars\n",
      "\n",
      "/home/staff/julia/.pyenv/versions/3.6.4/envs/psy/lib/python3.6/site-packages/scipy/stats/stats.py:4761: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in true_divide\n",
      "\n",
      "/home/staff/julia/.pyenv/versions/3.6.4/envs/psy/lib/python3.6/site-packages/scipy/stats/stats.py:4766: RuntimeWarning:\n",
      "\n",
      "divide by zero encountered in double_scalars\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size of preselection1 (by plos years): (311116, 23)\n",
      "    size of preselection3 (only young references): (16481, 23) young\n",
      "     N plos: 5129  N records: 16481\n",
      "citation bins for the selected plos: [0.2, 0.95, 1]\n",
      "\n",
      "\n",
      "citation bins for the references in the selected plos: [0.5, 0.95, 1] [(0.5, 4.0), (0.95, 28.0), (1.0, 2711.0)]\n",
      "12409 5129 1652 0\n",
      "entering query for all citations of young references ..............\n",
      "337811\n",
      "253814\n",
      "done collecting the list of citing papers of the young references.\n",
      "entering long query for ONLY early citations of young references ..............\n",
      "249623 ../data/dict_UT_year_for_citing_papers_of_young_ref_2016.pkl\n",
      "full dict all citing papers: 249623   only early citing papers: 118557\n",
      "new dict created\n",
      "[2016]\n",
      "avg # early citations of young ref 12.9108639587 106.543930357 12408\n",
      "  of young ref. cited by top plos 19.4491525424 192.385110703 1652\n",
      "  of young ref. cited by bottom plos nan nan 0\n",
      "\n",
      "comparison   ALL- top plos: Ks_2sampResult(statistic=0.12772728833853969, pvalue=3.2277246677121574e-21)\n",
      "comparison   ALL- bottom plos: Ks_2sampResult(statistic=nan, pvalue=nan)\n",
      "comparison   top plos- bottom plos: Ks_2sampResult(statistic=nan, pvalue=nan)\n"
     ]
    }
   ],
   "source": [
    "########  i study the number of citations that young references had at the time of the plos' publication\n",
    "\n",
    "\n",
    "#############################3#############\n",
    "#############################################     [[2006],[2007],[2008],[2009],[2010],[2011],[2012],[2013],[2014],[2015],[2016]]\n",
    "for years in [[2008],[2009],[2010],[2011],[2012],[2013],[2014],[2015],[2016]]:\n",
    "\n",
    "\n",
    "\n",
    "    print (\"\\n\\n\\nPLOS FROM YEAR:\", years)\n",
    "\n",
    "    string_references_age=\"young\"   #young\"#old\"  # young # all   for the selection of what references i include\n",
    "    string_isolated_ref=\"\"  #\"\"   # 0  or 1 (or empty string, to include all ref)\n",
    "    string_self_ref=\"\"#1   # 0  or 1 (or empty string, to include all ref)\n",
    "    string_journal=\"\"\n",
    "    string_plos_field=\"\"#['D CU BIOLOGY']\"\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    ##### preselection by plos year   \n",
    "    preselection_df = df_merged[df_merged['plos_pub_year'].isin(years)]  \n",
    "    print (\"size of preselection1 (by plos years):\",preselection_df.shape)\n",
    "\n",
    "\n",
    "    #### i remove self-citations\n",
    "    if (string_self_ref==0) or  ( string_self_ref == 1 ): \n",
    "        preselection_df = preselection_df[preselection_df['self_citation']== string_self_ref ]  \n",
    "        print (\" size of preselection1 (self-cit):\",preselection_df.shape)\n",
    "\n",
    "\n",
    "    ######### preselection by isolated or group references:\n",
    "    preselection_df0 = preselection_df   \n",
    "    if (string_isolated_ref==0) or  ( string_isolated_ref == 1 ): \n",
    "        preselection_df0 = preselection_df[preselection_df['isolated_citation']== string_isolated_ref ]        \n",
    "        print (\"  size of preselection1 (by isolated/group ref):\",preselection_df0.shape, string_isolated_ref)\n",
    "\n",
    "\n",
    "    ######### preselection by plos journal:\n",
    "    if string_journal==\"\": \n",
    "        preselection_df1 = preselection_df0\n",
    "    else:    \n",
    "        preselection_df1 = preselection_df0[preselection_df0['plos_j1']== string_journal ]  \n",
    "        print (\"   size of preselection2 (by plos journal):\",preselection_df1.shape, string_journal)\n",
    "\n",
    "\n",
    "    ######### preselection by plos field:\n",
    "    if string_plos_field==\"\": \n",
    "        preselection_df2 = preselection_df1\n",
    "    else:    \n",
    "        preselection_df2 = preselection_df1[preselection_df1['plos_field']== string_plos_field ]  \n",
    "        print (\"    size of preselection2 (by plos field):\",preselection_df2.shape, string_plos_field)\n",
    "\n",
    "\n",
    "    ##### preselection only young/old references:       \n",
    "    preselection_df3 = preselection_df2\n",
    "    if string_references_age == \"young\":\n",
    "        time_window_age = 1   \n",
    "        preselection_df3 = preselection_df2[preselection_df2['ref_pub_year'] >= (min(years)-time_window_age) ]   \n",
    "\n",
    "        print (\"    size of preselection3 (only young references):\",preselection_df3.shape, string_references_age)\n",
    "\n",
    "    elif string_references_age == \"old\":\n",
    "        time_window_age = 10    \n",
    "        preselection_df3 = preselection_df2[preselection_df2['ref_pub_year'] <= (min(years)-time_window_age) ]   \n",
    "\n",
    "        print (\"    size of preselection3 (only old references):\",preselection_df3.shape,string_references_age )\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    N_plos=len(preselection_df3.paper_UT.unique())        ## this values are overall, for the title   \n",
    "    N_all=len(preselection_df3)\n",
    "\n",
    "\n",
    "    print (\"     N plos:\", N_plos, \" N records:\", N_all)        \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    ############## i define quantiles for plos papers (for that subselection, and based on their FINAL number of citations):\n",
    "    list_q_plos=[.2,.95,1]\n",
    "    df_for_quantiles_plos = preselection_df3.drop_duplicates(subset=['paper_UT'])   # ojo!!! dont use preselection_df3 directly because there are REPETITIONS!!!!\n",
    "\n",
    "    quantiles=sorted(list(df_for_quantiles_plos['paper_cite_count'].quantile(list_q_plos).to_dict().items())) #mean 10.68 \n",
    "\n",
    "    print (\"citation bins for the selected plos:\", list_q_plos)#,quantiles, df_for_quantiles_plos.shape)   \n",
    "\n",
    "    lista_bins_plos=[]\n",
    "    old_value=0\n",
    "    for item in quantiles:\n",
    "        pair=[old_value, int(item[1])]\n",
    "        lista_bins_plos.append(pair)\n",
    "        old_value = int(item[1])\n",
    "\n",
    "    #print (lista_bins_plos, min(preselection_df3['paper_cite_count']), max(preselection_df3['paper_cite_count']))\n",
    "\n",
    "\n",
    "\n",
    "    cont = 0\n",
    "    dict_bin_list_plos_UT={}\n",
    "    for item in lista_bins_plos:\n",
    "\n",
    "        minimo = item[0]\n",
    "        maximo = item[1]   \n",
    "\n",
    "        df_select = preselection_df3[(preselection_df3['paper_cite_count'] >= minimo)  &  (preselection_df3['paper_cite_count'] < maximo)]\n",
    "        llave=str(minimo)+\"-\"+str(maximo)\n",
    "        dict_bin_list_plos_UT[llave]= list(df_select.paper_UT.unique())\n",
    "        #print (llave, len(list(df_select.reference_UT.unique())))\n",
    "        max_key_plos=llave\n",
    "\n",
    "\n",
    "        if cont ==0:\n",
    "            min_key_plos = llave\n",
    "        cont  +=1\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    ########## i define quantiles for references (based on their FINAL number of citations)\n",
    "    list_q_ref=[0.5,.95,1]\n",
    "    df_for_quantiles_ref = preselection_df3.drop_duplicates(subset=['reference_UT'])   # ojo!!! remember to remove REPETITIONS!!!!\n",
    "    quantiles=sorted(list(df_for_quantiles_ref['cite_count'].quantile(list_q_ref).to_dict().items())) #mean 10.68 \n",
    "\n",
    "    print (\"\\n\\ncitation bins for the references in the selected plos:\", list_q_ref,quantiles)    \n",
    "\n",
    "    lista_bins=[]\n",
    "    old_value=0\n",
    "    for item in quantiles:\n",
    "        pair=[old_value, int(item[1])]\n",
    "        lista_bins.append(pair)\n",
    "        old_value = int(item[1])\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    cont = 0\n",
    "    dict_bin_list_ref_UT={}\n",
    "    for item in lista_bins:\n",
    "\n",
    "        minimo = item[0]\n",
    "        maximo = item[1]    \n",
    "\n",
    "        df_select = preselection_df3[(preselection_df3['cite_count'] >= minimo)  &  (preselection_df3['cite_count'] < maximo)]\n",
    "        llave=str(minimo)+\"-\"+str(maximo)\n",
    "        dict_bin_list_ref_UT[llave]=list(df_select.reference_UT.unique())\n",
    "        #print (llave, len(list(df_select.reference_UT.unique())))\n",
    "        max_key_ref=llave\n",
    "\n",
    "        if cont ==0:\n",
    "            min_key_ref = llave\n",
    "        cont  +=1\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    ############### i create the list of top plos, top ref, bottom plos and bottom ref:\n",
    "    #########################\n",
    "\n",
    "    lista_top_plos = dict_bin_list_plos_UT[max_key_plos]\n",
    "    #print (\"\\n\\n# UTs top\",(100-100*list_q_plos[-2]),\"% plos:\",len(lista_top_plos))\n",
    "\n",
    "#     lista_top_ref=dict_bin_list_ref_UT[max_key_ref]\n",
    "#     #print (\"# UTs top\",(100-100*list_q_ref[-2]),\"% ref:\", len(lista_top_ref))\n",
    "\n",
    "    lista_bottom_plos = dict_bin_list_plos_UT[min_key_plos]\n",
    "    #print (\"# UTs bottom \",(100*list_q_plos[0]),\"% plos:\",len(lista_bottom_plos))\n",
    "\n",
    "#     lista_bottom_ref=dict_bin_list_ref_UT[min_key_ref]\n",
    "    #print (\"# UTs bottom \",(100*list_q_ref[0]),\"% ref:\", len(lista_bottom_ref))\n",
    "\n",
    "    list_plos_in_year= list(preselection_df3.paper_UT.unique())\n",
    "    #print (\"Tot # records:\",len(preselection_df3),\", # plos:\",len(list_plos_in_year))\n",
    "\n",
    "    #############################################\n",
    "    ###############################################\n",
    "    #############################################\n",
    "    ###############################################\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "   \n",
    "    lista_young_ref = list(preselection_df3.reference_UT.unique())\n",
    "    \n",
    "    lista_plos_cite_young_ref = list(preselection_df3.paper_UT.unique())\n",
    "\n",
    "    lista_young_ref_cited_by_top_plos =  list(preselection_df3[preselection_df3['paper_UT'].isin(lista_top_plos)].reference_UT.unique())    \n",
    "\n",
    "    lista_young_ref_cited_by_bottom_plos =  list(preselection_df3[preselection_df3['paper_UT'].isin(lista_bottom_plos)].reference_UT.unique())    \n",
    "\n",
    "    print(len(lista_young_ref), len(lista_plos_cite_young_ref), len(lista_young_ref_cited_by_top_plos), len(lista_young_ref_cited_by_bottom_plos))\n",
    "\n",
    "                                          \n",
    "    \n",
    "    print (\"entering query for all citations of young references ..............\")\n",
    "    ############ i look for all citations of the young references used by plos of the focus year:\n",
    "    cursor = papers_con.collection.find({\"UT\":{\"$in\":lista_young_ref}},{\"UT\":1,'citations':1}, no_cursor_timeout=True)  # it returns an iterator (it gets empty after iterating over it once) # second {} to select the fields it returns, if i dont want them all\n",
    "\n",
    "    list_all_citing_papers_of_the_ref=[]\n",
    "    dict_UT_ref_list_citing_papers={}\n",
    "    for item in cursor:  # cursor is an iterator (once i iterate over it once, it is empty), and every item is a dict\n",
    "\n",
    "        ref = item[\"UT\"]\n",
    "\n",
    "        list_citing_papers=item['citations']\n",
    "        dict_UT_ref_list_citing_papers[ref]= list_citing_papers\n",
    "\n",
    "        list_all_citing_papers_of_the_ref +=list_citing_papers\n",
    "\n",
    "\n",
    "    cursor.close()  # because i am using the no_cursor_timeout=True, i need also this, or cursor keeps waiting so ur resources are used up\n",
    "    print (len(list_all_citing_papers_of_the_ref))                                     \n",
    "    list_all_citing_papers_of_the_ref = list(set(list_all_citing_papers_of_the_ref))                                     \n",
    "    print (len(list_all_citing_papers_of_the_ref))\n",
    "    print (\"done collecting the list of citing papers of the young references.\")  \n",
    "\n",
    "    #################################\n",
    "\n",
    "\n",
    "   \n",
    "\n",
    "\n",
    "\n",
    "    lista_early_years_for_ref = sorted(years + [min(years)-1])   # i only care about the citations that the young references get early on (around the plos' publication year)\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    try: \n",
    "        filename = '../data/dict_UT_year_for_citing_papers_of_young_ref_'+str(years).strip(']').strip('[')+'.pkl'\n",
    "        dict_UT_year = pickle.load(open(filename, 'rb'))\n",
    "        print (\"done loading dict_UT_year\")\n",
    "    \n",
    "    except:\n",
    "    \n",
    "        ############ i look for ONLY the early citations of the young references for the focus year:\n",
    "        print (\"entering long query for ONLY early citations of young references ..............\")\n",
    "        dict_UT_year={}\n",
    "        start=0\n",
    "        stop=10000\n",
    "        while stop <=len(list_all_citing_papers_of_the_ref):\n",
    "\n",
    "            #print (\"  \",start, stop, \"...\")\n",
    "\n",
    "\n",
    "            lista=list_all_citing_papers_of_the_ref[start:stop]       \n",
    "\n",
    "            cursor = papers_con.collection.find({\"UT\":{\"$in\":lista}},{\"UT\":1, 'issue':1}, no_cursor_timeout=True)  # it returns an iterator (it gets empty after iterating over it once) # second {} to select the fields it returns, if i dont want them all\n",
    "\n",
    "            #cursor = papers_con.collection.find({\"UT\":{\"$in\":lista},\"issue.PY\":{\"$in\":lista_early_years_for_ref}},{\"UT\":1, 'issue':1}, no_cursor_timeout=True)  # it returns an iterator (it gets empty after iterating over it once) # second {} to select the fields it returns, if i dont want them all\n",
    "           ### is it faster to retrieve a cursor without the PY condition??!\n",
    "\n",
    "\n",
    "            for item in cursor:  # cursor is an iterator (once i iterate over it once, it is empty), and every item is a dict\n",
    "\n",
    "                cit = item[\"UT\"]           \n",
    "                year_ref=int(item['issue']['PY'])\n",
    "                #if year_ref in lista_early_years_for_ref:\n",
    "\n",
    "                dict_UT_year[cit]=year_ref\n",
    "\n",
    "            stop  += 10000\n",
    "            start += 10000\n",
    "            cursor.close()  # because i am using the no_cursor_timeout=True, i need also this, or cursor keeps waiting so ur resources are used up\n",
    "\n",
    "\n",
    "\n",
    "        with open('../data/dict_UT_year_for_citing_papers_of_young_ref_'+str(years).strip(']').strip('[')+'.pkl', 'wb') as handle:\n",
    "            pickle.dump(dict_UT_year, handle, protocol = 2)   \n",
    "        print ( len(dict_UT_year), '../data/dict_UT_year_for_citing_papers_of_young_ref_'+str(years).strip(']').strip('[')+'.pkl')\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    dict_only_early_cit_UT_year={}\n",
    "    for UT in dict_UT_year:\n",
    "        year_cit = dict_UT_year[UT]\n",
    "        if year_cit in lista_early_years_for_ref:\n",
    "            \n",
    "            dict_only_early_cit_UT_year[UT] = year_cit\n",
    "    \n",
    "    \n",
    "    print (\"full dict all citing papers:\",len(dict_UT_year), \"  only early citing papers:\",len(dict_only_early_cit_UT_year))\n",
    "    \n",
    "\n",
    "    ###################  i get a dict with only the early citations of the young ref:\n",
    "\n",
    "    new_dict_UT_early_cit={}\n",
    "    for ref in dict_UT_ref_list_citing_papers:\n",
    "        lista_citing=dict_UT_ref_list_citing_papers[ref]\n",
    "        new_dict_UT_early_cit[ref] =0.\n",
    "        for citing in lista_citing:\n",
    "            try:\n",
    "                dict_only_early_cit_UT_year[citing]\n",
    "                new_dict_UT_early_cit[ref] +=1.\n",
    "            except KeyError: pass\n",
    "\n",
    "    print ('new dict created')       \n",
    "    ################\n",
    "\n",
    "\n",
    "    \n",
    "#       lista_young_ref     \n",
    "#     lista_plos_cite_young_ref \n",
    "#     lista_young_ref_cited_by_top_plos \n",
    "#     lista_young_ref_cited_by_bottom_plos   \n",
    "\n",
    "    \n",
    "    \n",
    "    ####### i get the avg number of early citations for different subsets of young ref:\n",
    "    list_values_all = []\n",
    "    list_values_ref_by_top_plos = []\n",
    "    list_values_ref_by_bottom_plos = []\n",
    "    for ref in new_dict_UT_early_cit:\n",
    "        if ref in lista_young_ref:\n",
    "            list_values_all.append(new_dict_UT_early_cit[ref])\n",
    "            \n",
    "        if ref in lista_young_ref_cited_by_top_plos:\n",
    "            list_values_ref_by_top_plos.append(new_dict_UT_early_cit[ref])\n",
    "            \n",
    "        if ref in lista_young_ref_cited_by_bottom_plos:\n",
    "            list_values_ref_by_bottom_plos.append(new_dict_UT_early_cit[ref])\n",
    "\n",
    "\n",
    "    print (years)\n",
    "    print (\"avg # early citations of young ref\",np.mean(list_values_all), 2.*np.std(list_values_all),len(list_values_all))\n",
    "    print (\"  of young ref. cited by top plos\",np.mean(list_values_ref_by_top_plos), 2.*np.std(list_values_ref_by_top_plos), len(list_values_ref_by_top_plos))\n",
    "    print (\"  of young ref. cited by bottom plos\",np.mean(list_values_ref_by_bottom_plos), 2.*np.std(list_values_ref_by_bottom_plos), len(list_values_ref_by_bottom_plos))\n",
    "\n",
    "    print (\"\\ncomparison   ALL- top plos:\", stats.ks_2samp(list_values_all, list_values_ref_by_top_plos) )\n",
    "    print (\"comparison   ALL- bottom plos:\", stats.ks_2samp(list_values_all, list_values_ref_by_bottom_plos)  )\n",
    "    print (\"comparison   top plos- bottom plos:\", stats.ks_2samp(list_values_ref_by_top_plos, list_values_ref_by_bottom_plos)  )\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['000367805100045',\n",
       " '000367888100170',\n",
       " '000373039400002',\n",
       " '000372565700072',\n",
       " '000373038200006',\n",
       " '000373038600002',\n",
       " '000369552800029',\n",
       " '000369554000046',\n",
       " '000369368200049',\n",
       " '000371223400050',\n",
       " '000368529100016',\n",
       " '000371276100203',\n",
       " '000369527800125',\n",
       " '000373038200004',\n",
       " '000373272500001']"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(set(lista_young_ref)&set(lista_young_ref_cited_by_top_plos))\n",
    "#list(set(lista_young_ref_cited_by_top_plos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}