crawl_and_plot.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cdcb8f3c",
   "metadata": {},
   "source": [
    "# Crawl dataset with all submissions info\n",
    "OpenReview Venue Crawling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b15b9499",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import pandas as pd\n",
    "from multiprocessing import Pool"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba524d57",
   "metadata": {},
   "source": [
    "## Crawl list of all submissions\n",
    "Here we scrape the _notes_ , (list of all submissions) using OpenReview's API, way faster than Selenium-based scraping.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37d02c61",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = './data/'\n",
    "venue_id = 'ICLR.cc/2024/Conference'\n",
    "venue_short = 'iclr2024'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc6a63f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openreview\n",
    "\n",
    "# API V2\n",
    "client = openreview.api.OpenReviewClient(\n",
    "    baseurl='https://api2.openreview.net',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2cc64fd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "venue_group = client.get_group(venue_id)\n",
    "submission_name = venue_group.content['submission_name']['value']\n",
    "submissions = client.get_all_notes(invitation=f'{venue_id}/-/{submission_name}', details='directReplies')\n",
    "submissions = [s.to_json() for s in submissions]\n",
    "print(f'Number of submissions: {len(submissions)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d5e5668",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.json_normalize(submissions)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d83b433",
   "metadata": {},
   "source": [
    "## Save filtered dataset \n",
    "We will be saving a smaller version of the dataset in csv format with the data we need for our analysis - this can also be saved directly in Github"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2ab086e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save dataframe as csv\n",
    "# rename title\n",
    "df.rename(columns={'content.title.value': 'title'}, inplace=True)\n",
    "#rename keywords\n",
    "df.rename(columns={'content.keywords.value': 'keywords'}, inplace=True)\n",
    "df.to_csv(f'{DATA_PATH}{venue_short}.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ad0e6cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from imageio import imread\n",
    "from wordcloud import WordCloud\n",
    "\n",
    "gridc = (1., 1., 1)\n",
    "plt.rcParams['grid.color'] = gridc\n",
    "plt.rcParams[\"axes.edgecolor\"] = (0.898, 0.925, 0.965, 1)\n",
    "plt.rc('xtick', labelsize=12)\n",
    "plt.rc('ytick', labelsize=12)\n",
    "sns.set(style='darkgrid', context='talk', palette='colorblind')\n",
    "\n",
    "from IPython.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:95% !important; }</style>\"))\n",
    "display(HTML(\"<style>div#site { height: 100% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe884043",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get list\n",
    "df = pd.read_csv(f'{DATA_PATH}{venue_short}.csv')\n",
    "df['keywords'] = df['keywords'].apply(eval)\n",
    "\n",
    "data = df['keywords']\n",
    "keywords = {}\n",
    "for kw_list in data:\n",
    "    for kw in kw_list:\n",
    "        kw = kw.lower().strip()\n",
    "        keywords[kw] = keywords.get(kw, 0) + 1\n",
    "\n",
    "# Identify and merge singular and plural pairs\n",
    "for kw, count in list(keywords.items()):\n",
    "    if kw.endswith('s') and kw[:-1] in keywords:\n",
    "        singular_kw = kw[:-1]  # Remove 's' to get singular form\n",
    "        plural_count = keywords[kw]\n",
    "        singular_count = keywords[singular_kw]\n",
    "        # Merge counts\n",
    "        keywords[singular_kw] = singular_count + plural_count\n",
    "        del keywords[kw]  # Remove the plural key\n",
    "        \n",
    "# sort values\n",
    "keywords = {k: v for k, v in sorted(keywords.items(), key=lambda item: item[1])[::-1]}\n",
    "keywords = pd.Series(keywords).sort_values(ascending=True)\n",
    "# sort keyword\n",
    "keywords.iloc[-50:].plot.barh(figsize=(8, 12), title='ICLR 2024 Submission Top 50 Keywords')\n",
    "plt.savefig(f'./sources/top50_keywords.png', bbox_inches='tight', dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "793887f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "wc = WordCloud(background_color=\"black\", max_words=300, max_font_size=64, \n",
    "               width=1280, height=640, random_state=0)\n",
    "wc.generate_from_frequencies(keywords.to_dict())\n",
    "fig = plt.figure(figsize=(16, 8))\n",
    "plt.imshow(wc, interpolation=\"bilinear\")\n",
    "plt.axis(\"off\")\n",
    "plt.savefig(f'./sources/wordcloud.png', bbox_inches='tight', dpi=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8fa390e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "logo = imread('./sources/ICLR-mask.png')\n",
    "wc = WordCloud(background_color=\"white\", max_words=300, max_font_size=64, \n",
    "               width=1280, height=640, random_state=0, mask=logo, contour_color='black')\n",
    "wc.generate_from_frequencies(keywords.to_dict())\n",
    "fig = plt.figure(figsize=(16, 8))\n",
    "plt.imshow(logo)\n",
    "plt.imshow(wc, interpolation=\"bilinear\", alpha=.75)\n",
    "plt.axis(\"off\")\n",
    "plt.savefig(f'./sources/wordcloud_nice.png', bbox_inches='tight', dpi=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d24b73e6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2.7.12 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "767d51c1340bd893661ea55ea3124f6de3c7a262a8b4abca0554b478b1e2ff90"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}