Tweetsumm_preprocessing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "#Import repository"
      ],
      "metadata": {
        "id": "r99snyY7rVtS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "!git clone https://github.com/sergiuabed/SICK_Summarization.git\n",
        "ROOT = \"/content/SICK_Summarization/extension_2/Tweetsumm\"\n",
        "%cd $ROOT"
      ],
      "metadata": {
        "id": "GhuE_ExSOIqC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Download Tweetsumm TWCS file"
      ],
      "metadata": {
        "id": "_Xv_v_XQriW4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install kaggle\n",
        "!mkdir ~/.kaggle\n",
        "!cp kaggle.json ~/.kaggle/\n",
        "!chmod 600 ~/.kaggle/kaggle.json\n",
        "!kaggle datasets download thoughtvector/customer-support-on-twitter\n",
        "!unzip customer-support-on-twitter.zip -d data\n"
      ],
      "metadata": {
        "id": "9eyX4zHlgTr7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Install COMET requirements"
      ],
      "metadata": {
        "id": "6F7L3DXEl82r"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!bash Comet/download_model.sh\n",
        "!pip install -U sentence-transformers\n",
        "\n",
        "import torch\n",
        "if torch.cuda.is_available():\n",
        "  DEVICE = \"cuda\"\n",
        "  !pip install -r Comet/requirement-gpu.txt\n",
        "else:\n",
        "  DEVICE = \"cpu\"\n",
        "  !pip install -r Comet/requirement-cpu.txt\n"
      ],
      "metadata": {
        "id": "gLQQRYL_9vh0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Define preprocessing function"
      ],
      "metadata": {
        "id": "R84_9qkrrw4J"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tweet_sum_processor import TweetSumProcessor\n",
        "from Comet.comet import Comet, generate_commonsense\n",
        "from Comet.sbert import select_best_commonsense\n",
        "from sentence_transformers import SentenceTransformer, util\n",
        "from google.colab import files\n",
        "import json\n",
        "import warnings\n",
        "from tqdm.notebook import tqdm\n",
        "\n",
        "TWCS_FILE_PATH = f\"{ROOT}/data/twcs/twcs.csv\"\n",
        "\n",
        "def tweetsumm_preprocessing(input_data, split, summary=False):\n",
        "  warnings.filterwarnings('ignore')\n",
        "  processor = TweetSumProcessor(TWCS_FILE_PATH)\n",
        "  comet = Comet(\"./comet-atomic_2020_BART\")\n",
        "  comet.model.zero_grad()\n",
        "  sbert = SentenceTransformer(\"all-MiniLM-L6-v2\").to(DEVICE)\n",
        "\n",
        "  if not os.path.isdir(\"./dialogue_data\"):\n",
        "    !mkdir ./dialogue_data\n",
        "  if not os.path.isdir(\"./COMET_data\"):\n",
        "    !mkdir ./COMET_data\n",
        "    !mkdir ./COMET_data/tweetsumm\n",
        "    !mkdir ./COMET_data/tweetsumm/comet_inference\n",
        "    !mkdir ./COMET_data/tweetsumm/comet_inference/dialogue\n",
        "    !mkdir ./COMET_data/tweetsumm/comet_inference/summary\n",
        "    !mkdir ./COMET_data/tweetsumm/sbert\n",
        "    !mkdir ./COMET_data/tweetsumm/sbert/dialogue\n",
        "    !mkdir ./COMET_data/tweetsumm/sbert/summary\n",
        "\n",
        "  with open(input_data, \"r\") as f:\n",
        "    dialog_with_summaries = processor.get_dialog_with_summaries(f.readlines())\n",
        "\n",
        "  with open(f\"./dialogue_data/tweetsumm_{split}.json\", \"w\") as f:\n",
        "    print(\"Generating dialogue data...\")\n",
        "    data = {}\n",
        "    for d in dialog_with_summaries:\n",
        "      dialog = json.loads(d.get_json())\n",
        "      data[dialog[\"dialog_id\"]] = {\"turns\":dialog[\"turns\"], \"summaries\":dialog[\"summaries\"][\"abstractive_summaries\"]}\n",
        "    f.write(json.dumps(data, indent=4))\n",
        "\n",
        "  cs_dialog_file_path = f\"./COMET_data/tweetsumm/comet_inference/dialogue/comet_dialogue_{split}.json\"\n",
        "  sbert_cs_dialog_file_path = f\"./COMET_data/tweetsumm/sbert/dialogue/sbert_dialogue_{split}.json\"\n",
        "  print(\"Generating commonsense data for dialogues...\")\n",
        "  with open(cs_dialog_file_path, \"w\") as f, open(sbert_cs_dialog_file_path, \"w\") as g:\n",
        "    comet_data = {}\n",
        "    sbert_data = {}\n",
        "    for d in tqdm(dialog_with_summaries):\n",
        "      dialog = json.loads(d.get_json())\n",
        "      comet_data[dialog[\"dialog_id\"]] = {}\n",
        "      sbert_data[dialog[\"dialog_id\"]] = {}\n",
        "      for i, t in enumerate(dialog[\"turns\"]):\n",
        "        comet_inference = generate_commonsense(comet, t)\n",
        "        sbert_selection = select_best_commonsense(sbert, comet_inference)\n",
        "        comet_data[dialog[\"dialog_id\"]].update({i:comet_inference})\n",
        "        sbert_data[dialog[\"dialog_id\"]].update({i:sbert_selection})\n",
        "    f.write(json.dumps(comet_data, indent=4))\n",
        "    g.write(json.dumps(sbert_data, indent=4))\n",
        "\n",
        "\n",
        "  if summary:\n",
        "    cs_summary_file_path = f\"./COMET_data/tweetsumm/comet_inference/summary/comet_summary_{split}.json\"\n",
        "    sbert_cs_summary_file_path = f\"./COMET_data/tweetsumm/sbert/summary/sbert_summary_{split}.json\"\n",
        "    print(\"Generating commonsense data for summaries...\")\n",
        "    with open(cs_summary_file_path, \"w\") as f, open(sbert_cs_summary_file_path, \"w\") as g:\n",
        "      comet_data = {}\n",
        "      sbert_data = {}\n",
        "      for d in tqdm(dialog_with_summaries):\n",
        "        dialog = json.loads(d.get_json())\n",
        "        comet_data[dialog[\"dialog_id\"]] = {}\n",
        "        sbert_data[dialog[\"dialog_id\"]] = {}\n",
        "        for i, s in enumerate(dialog[\"summaries\"][\"abstractive_summaries\"]):\n",
        "          comet_inference = generate_commonsense(comet, s)\n",
        "          sbert_selection = select_best_commonsense(sbert, comet_inference)\n",
        "          comet_data[dialog[\"dialog_id\"]].update({i:comet_inference})\n",
        "          sbert_data[dialog[\"dialog_id\"]].update({i:sbert_selection})\n",
        "      f.write(json.dumps(comet_data, indent=4))\n",
        "      g.write(json.dumps(sbert_data, indent=4))"
      ],
      "metadata": {
        "id": "0V9RigQSfukF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Apply preprocessing to the data"
      ],
      "metadata": {
        "id": "KF3dPv8Wr2ll"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "TRAIN_FILE_PATH = f\"{ROOT}/tweet_sum_data_files/final_train_tweetsum.jsonl\"\n",
        "VALID_FILE_PATH = f\"{ROOT}/tweet_sum_data_files/final_valid_tweetsum.jsonl\"\n",
        "TEST_FILE_PATH = f\"{ROOT}/tweet_sum_data_files/final_test_tweetsum.jsonl\"\n",
        "\n",
        "print(\"Processing training data\")\n",
        "tweetsumm_preprocessing(TRAIN_FILE_PATH, \"train\", True)\n",
        "\n",
        "print(\"Processing validation data\")\n",
        "tweetsumm_preprocessing(VALID_FILE_PATH, \"valid\")\n",
        "\n",
        "print(\"Processing test data\")\n",
        "tweetsumm_preprocessing(TEST_FILE_PATH, \"test\")\n"
      ],
      "metadata": {
        "id": "OCXkSeUBCs83"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Download data"
      ],
      "metadata": {
        "id": "VFcEhCsDr5ux"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!zip -r tweetsumm_clean.zip dialogue_data\n",
        "files.download(\"tweetsumm_clean.zip\")\n",
        "!zip -r tweetsumm_comet.zip COMET_data\n",
        "files.download(\"tweetsumm_comet.zip\")"
      ],
      "metadata": {
        "id": "xwV2brhcOgF3"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}