scopus.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Annotating dataset with Scopus/WOS.."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This document allow annotate the current dataset extracted from ISTEX database by using SCOPUS categories as annotation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "ALL_LABELS = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "51805it [00:36, 1435.97it/s]\n",
      "96642it [00:55, 1729.12it/s]\n"
     ]
    }
   ],
   "source": [
    "import pymongo\n",
    "from pymongo import MongoClient\n",
    "import tqdm\n",
    "from collections import defaultdict, Counter\n",
    "import json\n",
    "import re \n",
    "\n",
    "client = MongoClient('localhost', 27017)\n",
    "db = client.istex\n",
    "cambridge = db.cambridge\n",
    "springer = db.springer\n",
    "total = springer.find({'categories': {'$exists': True}, 'abstract': {'$exists': True}, 'title': {'$exists': True}}).count() + cambridge.find({'categories': {'$exists': True}, 'abstract': {'$exists': True}, 'title': {'$exists': True}}).count()\n",
    "categories = defaultdict(set)\n",
    "\n",
    "cnt = Counter()\n",
    "\n",
    "with open('springer_cambridge.txt', 'w') as outfile:\n",
    "    for col in [springer, cambridge]:\n",
    "        for i in tqdm.tqdm(col.find({'categories': {'$exists': True}, 'abstract': {'$exists': True}, 'title': {'$exists': True}, 'language': 'eng'})):\n",
    "            for ar, v in i['categories'].iteritems():\n",
    "                if ar == 'scopus':\n",
    "                    cat = set()\n",
    "                    for el in v:\n",
    "                        if ALL_LABELS:\n",
    "                                cat.add(el)\n",
    "                        else:\n",
    "                            if '1 -' in el:\n",
    "                                cat.add(el)\n",
    "                    cat = list(cat)\n",
    "                    cat = map(lambda x: ' '.join(re.findall(r'\\b[^\\d\\W]+\\b', x)), cat)\n",
    "                    json.dump({'abstract': i['abstract'], 'label': ','.join(cat), 'id': i['id'], 'title': i['title']}, outfile)\n",
    "                    outfile.write('\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "d = pd.read_json(\"springer_cambridge.txt\", orient='records', lines=True)\n",
    "d = d[d.label != '']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    Life Sciences,Environmental Science,Aquatic Sc...\n",
       "1    Life Sciences,Pharmacology medical,Pharmacolog...\n",
       "2    Biochemistry Genetics and Molecular Biology,Li...\n",
       "3    Life Sciences,Environmental Science,Aquatic Sc...\n",
       "4    Life Sciences,Microbiology,Genetics,Biochemist...\n",
       "Name: label, dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d.label.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'life_sciences,environmental_science,aquatic_science,ecology,agricultural_and_biological_sciences,physical_sciences,ecology_evolution_behavior_and_systematics'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d.label = d.label.apply(lambda x: x.lower().strip().replace(' ', '_'))\n",
    "d.label[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 83933/83933 [00:00<00:00, 286187.37it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{u'accounting',\n",
       " u'agricultural_and_biological_sciences',\n",
       " u'agricultural_and_biological_sciences_miscellaneous',\n",
       " u'agronomy_and_crop_science',\n",
       " u'algebra_and_number_theory',\n",
       " u'analysis',\n",
       " u'analytical_chemistry',\n",
       " u'anatomy',\n",
       " u'anesthesiology_and_pain_medicine',\n",
       " u'animal_science_and_zoology',\n",
       " u'anthropology',\n",
       " u'applied_mathematics',\n",
       " u'applied_microbiology_and_biotechnology',\n",
       " u'applied_psychology',\n",
       " u'aquatic_science',\n",
       " u'archaeology',\n",
       " u'architecture',\n",
       " u'artificial_intelligence',\n",
       " u'arts_and_humanities',\n",
       " u'arts_and_humanities_miscellaneous',\n",
       " u'astronomy_and_astrophysics',\n",
       " u'atmospheric_science',\n",
       " u'atomic_and_molecular_physics_and_optics',\n",
       " u'behavioral_neuroscience',\n",
       " u'biochemistry',\n",
       " u'biochemistry_genetics_and_molecular_biology',\n",
       " u'biochemistry_medical',\n",
       " u'bioengineering',\n",
       " u'biological_psychiatry',\n",
       " u'biomaterials',\n",
       " u'biomedical_engineering',\n",
       " u'biophysics',\n",
       " u'biotechnology',\n",
       " u'business_and_international_management',\n",
       " u'business_management_and_accounting',\n",
       " u'business_management_and_accounting_miscellaneous',\n",
       " u'cancer_research',\n",
       " u'cardiology_and_cardiovascular_medicine',\n",
       " u'cell_biology',\n",
       " u'cellular_and_molecular_neuroscience',\n",
       " u'chemical_engineering',\n",
       " u'chemistry',\n",
       " u'chiropractics',\n",
       " u'civil_and_structural_engineering',\n",
       " u'classics',\n",
       " u'clinical_biochemistry',\n",
       " u'clinical_neurology',\n",
       " u'clinical_psychology',\n",
       " u'cognitive_neuroscience',\n",
       " u'community_and_home_care',\n",
       " u'computational_mathematics',\n",
       " u'computational_mechanics',\n",
       " u'computational_theory_and_mathematics',\n",
       " u'computer_graphics_and_computer_aided_design',\n",
       " u'computer_networks_and_communications',\n",
       " u'computer_science',\n",
       " u'computer_science_applications',\n",
       " u'computer_vision_and_pattern_recognition',\n",
       " u'condensed_matter_physics',\n",
       " u'conservation',\n",
       " u'control_and_optimization',\n",
       " u'control_and_systems_engineering',\n",
       " u'cultural_studies',\n",
       " u'decision_sciences',\n",
       " u'demography',\n",
       " u'dermatology',\n",
       " u'development',\n",
       " u'developmental_and_educational_psychology',\n",
       " u'developmental_biology',\n",
       " u'earth_and_planetary_sciences',\n",
       " u'earth_and_planetary_sciences_miscellaneous',\n",
       " u'earth_surface_processes',\n",
       " u'ecology',\n",
       " u'ecology_evolution_behavior_and_systematics',\n",
       " u'economics_and_econometrics',\n",
       " u'economics_econometrics_and_finance',\n",
       " u'economics_econometrics_and_finance_miscellaneous',\n",
       " u'education',\n",
       " u'electrical_and_electronic_engineering',\n",
       " u'emergency',\n",
       " u'emergency_medicine',\n",
       " u'endocrinology',\n",
       " u'endocrinology_diabetes_and_metabolism',\n",
       " u'engineering',\n",
       " u'environmental_chemistry',\n",
       " u'environmental_engineering',\n",
       " u'environmental_science',\n",
       " u'environmental_science_miscellaneous',\n",
       " u'epidemiology',\n",
       " u'experimental_and_cognitive_psychology',\n",
       " u'finance',\n",
       " u'fluid_flow_and_transfer_processes',\n",
       " u'food_science',\n",
       " u'forestry',\n",
       " u'gastroenterology',\n",
       " u'general_agricultural_and_biological_sciences',\n",
       " u'general_arts_and_humanities',\n",
       " u'general_biochemistry_genetics_and_molecular_biology',\n",
       " u'general_business_management_and_accounting',\n",
       " u'general_chemical_engineering',\n",
       " u'general_chemistry',\n",
       " u'general_computer_science',\n",
       " u'general_earth_and_planetary_sciences',\n",
       " u'general_economics_econometrics_and_finance',\n",
       " u'general_engineering',\n",
       " u'general_environmental_science',\n",
       " u'general_materials_science',\n",
       " u'general_mathematics',\n",
       " u'general_medicine',\n",
       " u'general_neuroscience',\n",
       " u'general_nursing',\n",
       " u'general_physics_and_astronomy',\n",
       " u'general_psychology',\n",
       " u'general_social_sciences',\n",
       " u'genetics',\n",
       " u'genetics_clinical',\n",
       " u'geochemistry_and_petrology',\n",
       " u'geography_planning_and_development',\n",
       " u'geology',\n",
       " u'geophysics',\n",
       " u'geriatrics_and_gerontology',\n",
       " u'gerontology',\n",
       " u'hardware_and_architecture',\n",
       " u'health_policy',\n",
       " u'health_professions',\n",
       " u'health_sciences',\n",
       " u'health_social_science',\n",
       " u'health_toxicology_and_mutagenesis',\n",
       " u'histology',\n",
       " u'history',\n",
       " u'history_and_philosophy_of_science',\n",
       " u'horticulture',\n",
       " u'human_computer_interaction',\n",
       " u'immunology',\n",
       " u'immunology_and_microbiology',\n",
       " u'industrial_and_manufacturing_engineering',\n",
       " u'infectious_diseases',\n",
       " u'information_systems',\n",
       " u'information_systems_and_management',\n",
       " u'inorganic_chemistry',\n",
       " u'insect_science',\n",
       " u'instrumentation',\n",
       " u'internal_medicine',\n",
       " u'issues_ethics_and_legal_aspects',\n",
       " u'language_and_linguistics',\n",
       " u'law',\n",
       " u'leadership_and_management',\n",
       " u'life_sciences',\n",
       " u'linguistics_and_language',\n",
       " u'literature_and_literary_theory',\n",
       " u'logic',\n",
       " u'management_monitoring_policy_and_law',\n",
       " u'management_science_and_operations_research',\n",
       " u'materials_science',\n",
       " u'mathematical_physics',\n",
       " u'mathematics',\n",
       " u'mathematics_miscellaneous',\n",
       " u'mechanical_engineering',\n",
       " u'mechanics_of_materials',\n",
       " u'medical_laboratory_technology',\n",
       " u'medical_surgical',\n",
       " u'medicine',\n",
       " u'medicine_miscellaneous',\n",
       " u'metals_and_alloys',\n",
       " u'microbiology',\n",
       " u'molecular_biology',\n",
       " u'museology',\n",
       " u'music',\n",
       " u'nature_and_landscape_conservation',\n",
       " u'neurology',\n",
       " u'neuropsychology_and_physiological_psychology',\n",
       " u'neuroscience',\n",
       " u'nuclear_and_high_energy_physics',\n",
       " u'numerical_analysis',\n",
       " u'nursing',\n",
       " u'nutrition_and_dietetics',\n",
       " u'obstetrics_and_gynaecology',\n",
       " u'ocean_engineering',\n",
       " u'oceanography',\n",
       " u'oncology',\n",
       " u'ophthalmology',\n",
       " u'organic_chemistry',\n",
       " u'organizational_behavior_and_human_resource_management',\n",
       " u'orthopedics_and_sports_medicine',\n",
       " u'otorhinolaryngology',\n",
       " u'palaeontology',\n",
       " u'parasitology',\n",
       " u'pathology_and_forensic_medicine',\n",
       " u'pediatrics_perinatology_and_child_health',\n",
       " u'pharmaceutical_science',\n",
       " u'pharmacology',\n",
       " u'pharmacology_medical',\n",
       " u'pharmacology_toxicology_and_pharmaceutics',\n",
       " u'philosophy',\n",
       " u'phychiatric_mental_health',\n",
       " u'physical_and_theoretical_chemistry',\n",
       " u'physical_sciences',\n",
       " u'physical_therapy_sports_therapy_and_rehabilitation',\n",
       " u'physics_and_astronomy',\n",
       " u'physics_and_astronomy_miscellaneous',\n",
       " u'physiology',\n",
       " u'physiology_medical',\n",
       " u'plant_science',\n",
       " u'political_science_and_international_relations',\n",
       " u'pollution',\n",
       " u'psychiatry_and_mental_health',\n",
       " u'psychology',\n",
       " u'public_administration',\n",
       " u'public_health_environmental_and_occupational_health',\n",
       " u'pulmonary_and_respiratory_medicine',\n",
       " u'radiation',\n",
       " u'radiological_and_ultrasound_technology',\n",
       " u'radiology_nuclear_medicine_and_imaging',\n",
       " u'religious_studies',\n",
       " u'safety_risk_reliability_and_quality',\n",
       " u'sensory_systems',\n",
       " u'social_psychology',\n",
       " u'social_sciences',\n",
       " u'social_sciences_miscellaneous',\n",
       " u'sociology_and_political_science',\n",
       " u'software',\n",
       " u'soil_science',\n",
       " u'space_and_planetary_science',\n",
       " u'speech_and_hearing',\n",
       " u'statistical_and_nonlinear_physics',\n",
       " u'statistics_and_probability',\n",
       " u'statistics_probability_and_uncertainty',\n",
       " u'strategy_and_management',\n",
       " u'structural_biology',\n",
       " u'surgery',\n",
       " u'theoretical_computer_science',\n",
       " u'toxicology',\n",
       " u'transplantation',\n",
       " u'urban_studies',\n",
       " u'urology',\n",
       " u'visual_arts_and_performing_arts',\n",
       " u'water_science_and_technology'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categories = set()\n",
    "for l in tqdm.tqdm(d.label):\n",
    "    l = l.split(',')\n",
    "    for c in l:\n",
    "        categories.add(c)\n",
    "categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sami/.virtualenvs/jupyter/lib/python2.7/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>abstract</th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>title</th>\n",
       "      <th>computational_mathematics</th>\n",
       "      <th>oncology</th>\n",
       "      <th>chemical_engineering</th>\n",
       "      <th>mathematical_physics</th>\n",
       "      <th>health_toxicology_and_mutagenesis</th>\n",
       "      <th>cultural_studies</th>\n",
       "      <th>...</th>\n",
       "      <th>environmental_science_miscellaneous</th>\n",
       "      <th>public_administration</th>\n",
       "      <th>general_engineering</th>\n",
       "      <th>general_computer_science</th>\n",
       "      <th>physics_and_astronomy_miscellaneous</th>\n",
       "      <th>biotechnology</th>\n",
       "      <th>chemistry</th>\n",
       "      <th>inorganic_chemistry</th>\n",
       "      <th>information_systems</th>\n",
       "      <th>anesthesiology_and_pain_medicine</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Abstract: Uptake and metabolic turnover of a v...</td>\n",
       "      <td>91543AB49EF6104156B755A81D4D3E11B1E3CF19</td>\n",
       "      <td>life_sciences,environmental_science,aquatic_sc...</td>\n",
       "      <td>Activity of marine psychrophilic bacteria at e...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Summary: The calcium antagonist falipamil, a c...</td>\n",
       "      <td>EFB951031543D6D1B8479250A5B6BBAB6A7B24C8</td>\n",
       "      <td>life_sciences,pharmacology_medical,pharmacolog...</td>\n",
       "      <td>Atropine and verapamil interactions in healthy...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Summary: Experiments were performed to compare...</td>\n",
       "      <td>9DEA4ED385B3B941293CBADB71A3004AA4BFA390</td>\n",
       "      <td>biochemistry_genetics_and_molecular_biology,li...</td>\n",
       "      <td>Mechanism of asymmetric block of K channels by...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Abstract: Cells of the coccolithophoridCricosp...</td>\n",
       "      <td>152CF0B3C7000B08F84612084EE4CC844F692482</td>\n",
       "      <td>life_sciences,environmental_science,aquatic_sc...</td>\n",
       "      <td>Ultrastructural study and calcium and cadmium ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Abstract: A calcium requirement was shown for ...</td>\n",
       "      <td>0866A00EDFAA77712E45BCCAFEFC042E296D3772</td>\n",
       "      <td>life_sciences,microbiology,genetics,biochemist...</td>\n",
       "      <td>The calcium requirement for functional vesicle...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 241 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            abstract  \\\n",
       "0  Abstract: Uptake and metabolic turnover of a v...   \n",
       "1  Summary: The calcium antagonist falipamil, a c...   \n",
       "2  Summary: Experiments were performed to compare...   \n",
       "3  Abstract: Cells of the coccolithophoridCricosp...   \n",
       "4  Abstract: A calcium requirement was shown for ...   \n",
       "\n",
       "                                         id  \\\n",
       "0  91543AB49EF6104156B755A81D4D3E11B1E3CF19   \n",
       "1  EFB951031543D6D1B8479250A5B6BBAB6A7B24C8   \n",
       "2  9DEA4ED385B3B941293CBADB71A3004AA4BFA390   \n",
       "3  152CF0B3C7000B08F84612084EE4CC844F692482   \n",
       "4  0866A00EDFAA77712E45BCCAFEFC042E296D3772   \n",
       "\n",
       "                                               label  \\\n",
       "0  life_sciences,environmental_science,aquatic_sc...   \n",
       "1  life_sciences,pharmacology_medical,pharmacolog...   \n",
       "2  biochemistry_genetics_and_molecular_biology,li...   \n",
       "3  life_sciences,environmental_science,aquatic_sc...   \n",
       "4  life_sciences,microbiology,genetics,biochemist...   \n",
       "\n",
       "                                               title  \\\n",
       "0  Activity of marine psychrophilic bacteria at e...   \n",
       "1  Atropine and verapamil interactions in healthy...   \n",
       "2  Mechanism of asymmetric block of K channels by...   \n",
       "3  Ultrastructural study and calcium and cadmium ...   \n",
       "4  The calcium requirement for functional vesicle...   \n",
       "\n",
       "   computational_mathematics  oncology  chemical_engineering  \\\n",
       "0                          0         0                     0   \n",
       "1                          0         0                     0   \n",
       "2                          0         0                     0   \n",
       "3                          0         0                     0   \n",
       "4                          0         0                     0   \n",
       "\n",
       "   mathematical_physics  health_toxicology_and_mutagenesis  cultural_studies  \\\n",
       "0                     0                                  0                 0   \n",
       "1                     0                                  0                 0   \n",
       "2                     0                                  0                 0   \n",
       "3                     0                                  0                 0   \n",
       "4                     0                                  0                 0   \n",
       "\n",
       "                 ...                 environmental_science_miscellaneous  \\\n",
       "0                ...                                                   0   \n",
       "1                ...                                                   0   \n",
       "2                ...                                                   0   \n",
       "3                ...                                                   0   \n",
       "4                ...                                                   0   \n",
       "\n",
       "   public_administration  general_engineering  general_computer_science  \\\n",
       "0                      0                    0                         0   \n",
       "1                      0                    0                         0   \n",
       "2                      0                    0                         0   \n",
       "3                      0                    0                         0   \n",
       "4                      0                    0                         0   \n",
       "\n",
       "   physics_and_astronomy_miscellaneous  biotechnology  chemistry  \\\n",
       "0                                    0              0          0   \n",
       "1                                    0              0          0   \n",
       "2                                    0              0          1   \n",
       "3                                    0              0          0   \n",
       "4                                    0              0          1   \n",
       "\n",
       "   inorganic_chemistry  information_systems  anesthesiology_and_pain_medicine  \n",
       "0                    0                    0                                 0  \n",
       "1                    0                    0                                 0  \n",
       "2                    0                    0                                 0  \n",
       "3                    0                    0                                 0  \n",
       "4                    0                    0                                 0  \n",
       "\n",
       "[5 rows x 241 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "for c in categories:\n",
    "    d[c] = np.full((d.shape[0],), 0)\n",
    "    d[c][d.label.str.contains(c)] = 1\n",
    "d.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = d.drop(columns='label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "d.to_csv('scopus.csv', encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "train, test = train_test_split(d, test_size=0.33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(56235, 240)\n",
      "(27698, 240)\n"
     ]
    }
   ],
   "source": [
    "train.to_csv('scopus_train.csv', encoding='utf-8', index=False)\n",
    "test.to_csv('scopus_test.csv', encoding='utf-8', index=False)\n",
    "\n",
    "print(train.shape)\n",
    "print(test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test.to_json('scopus_test.json', orient='records', lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train[train.abstract.str.contains('fiscal policy')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FastText formatting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def labeling(row):\n",
    "    main_part = row['title'] + ' ' + row['abstract']\n",
    "    to_add = []\n",
    "    if row['health_sciences']:\n",
    "        to_add.append('__label__health_sciences ')\n",
    "    if row['life_sciences']:\n",
    "        to_add.append('__label__life_sciences')\n",
    "    if row['physical_sciences']:\n",
    "        to_add.append('__label__physical_sciences') \n",
    "    if row['social_sciences']:\n",
    "        to_add.append('__label__social_sciences')\n",
    "    row['abstract'] = ' '.join(to_add) + ' ' + main_part\n",
    "    return row\n",
    "        \n",
    "ft = d.apply(labeling, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import codecs\n",
    "\n",
    "train, test = train_test_split(ft, test_size=0.33)\n",
    "with codecs.open('fasttext_abstract_train.txt', 'w', encoding='utf8') as f:\n",
    "    for i in train.abstract:\n",
    "        f.write(i)\n",
    "        f.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with codecs.open('fasttext_abstract_test.txt', 'w', encoding='utf8') as f:\n",
    "    for i in test.abstract:\n",
    "        f.write(i)\n",
    "        f.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ft.abstract[0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}