Rep_keywords_in_rep_articles.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20f4020a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:03:49.460719Z",
     "iopub.status.busy": "2023-10-31T15:03:49.459843Z",
     "iopub.status.idle": "2023-10-31T15:04:05.207127Z",
     "shell.execute_reply": "2023-10-31T15:04:05.205805Z"
    },
    "papermill": {
     "duration": 15.760851,
     "end_time": "2023-10-31T15:04:05.210040",
     "exception": false,
     "start_time": "2023-10-31T15:03:49.449189",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Installation for IOC defanging - https://github.com/ioc-fang/ioc-fanger\n",
    "!pip install -q ioc-fanger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11f12ba1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:04:05.227249Z",
     "iopub.status.busy": "2023-10-31T15:04:05.226806Z",
     "iopub.status.idle": "2023-10-31T15:04:07.626111Z",
     "shell.execute_reply": "2023-10-31T15:04:07.622375Z"
    },
    "papermill": {
     "duration": 2.411412,
     "end_time": "2023-10-31T15:04:07.628957",
     "exception": false,
     "start_time": "2023-10-31T15:04:05.217545",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "import pandas as pd\n",
    "import ioc_fanger\n",
    "import json\n",
    "import time\n",
    "import re\n",
    "import os\n",
    "import zipfile\n",
    "from collections import defaultdict\n",
    "from nltk.tokenize import word_tokenize\n",
    "from tqdm import tqdm\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "import concurrent.futures\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76e6d80d",
   "metadata": {
    "papermill": {
     "duration": 0.007933,
     "end_time": "2023-10-31T15:04:07.644588",
     "exception": false,
     "start_time": "2023-10-31T15:04:07.636655",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Generate complete list of HF keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "567ad060",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:04:07.663708Z",
     "iopub.status.busy": "2023-10-31T15:04:07.663038Z",
     "iopub.status.idle": "2023-10-31T15:04:09.192267Z",
     "shell.execute_reply": "2023-10-31T15:04:09.189868Z"
    },
    "papermill": {
     "duration": 1.541839,
     "end_time": "2023-10-31T15:04:09.195061",
     "exception": false,
     "start_time": "2023-10-31T15:04:07.653222",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tags = [\"MAL\", \"FILE\", \"OS\", \"PROT\", \"MD5\", \"SHA1\", \"SHA2\",  \"ENCR\", \"TOOL\", \"VULID\", \"VULNAME\",\n",
    "        \"ACT\", \"APT\", \"SECTEAM\", \"IDTY\", \"EMAIL\", \"IP\", \"DOM\", \"URL\",  ]  #\"LOC\", \"TIME\", \n",
    "\n",
    "chunk_size = 500\n",
    "\n",
    "def save_chunk(output_directory, count, chunk):\n",
    "    output_file = os.path.join(output_directory, f\"chunk_{count}.txt\")\n",
    "    with open(output_file, 'w', encoding='utf-8') as file:\n",
    "        file.write('\\n'.join(chunk))\n",
    "\n",
    "output_directory = 'new_lists_folder'\n",
    "if not os.path.exists(output_directory):\n",
    "    os.makedirs(output_directory)\n",
    "\n",
    "output_directory = 'chunks_complete_entity_list'\n",
    "if not os.path.exists(output_directory):\n",
    "    os.makedirs(output_directory)\n",
    "\n",
    "complete_list = set()\n",
    "idx_file = 1\n",
    "count = 0\n",
    "chunk = []\n",
    "tag_list = []\n",
    "for t in tags:\n",
    "    tag = t\n",
    "    print(f'Tag {t} first file index {idx_file}')\n",
    "    countl=0\n",
    "    new_list = []\n",
    "    file_path = f'/home/anon/input/datasets-extracted-entities/REP_extracted_entities/entity_lists/{t}_list.txt'\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        new_file = []\n",
    "        for line in file:\n",
    "            new_file.append(line.strip())\n",
    "        new_file = sorted(new_file, key=lambda x: (-len(x), x))\n",
    "        for line in new_file:\n",
    "            countl += 1\n",
    "            line = line.strip()\n",
    "            if line not in complete_list:\n",
    "                complete_list.add(line)\n",
    "                new_list.append(line)\n",
    "                \n",
    "                chunk.append(line)\n",
    "                count += 1\n",
    "                if count % chunk_size == 0:\n",
    "                    save_chunk('chunks_complete_entity_list', idx_file, chunk)\n",
    "                    tag_list.append(tag)\n",
    "                    chunk = []\n",
    "                    idx_file += 1                    \n",
    "        if chunk:\n",
    "            save_chunk('chunks_complete_entity_list', idx_file, chunk)\n",
    "            tag_list.append(tag)\n",
    "            chunk = []\n",
    "            idx_file += 1\n",
    "            count=0\n",
    "    \n",
    "    print(f'{t} - Before: {countl} - After: {len(new_list)}\\n')\n",
    "    \n",
    "    output_file = os.path.join('new_lists_folder', f\"{t}_list.txt\")\n",
    "    with open(output_file, 'w', encoding='utf-8') as file:\n",
    "        file.write('\\n'.join(new_list))\n",
    "\n",
    "print(f'Chunks: {idx_file}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0abc480",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:04:09.214046Z",
     "iopub.status.busy": "2023-10-31T15:04:09.213615Z",
     "iopub.status.idle": "2023-10-31T15:04:09.222376Z",
     "shell.execute_reply": "2023-10-31T15:04:09.221166Z"
    },
    "papermill": {
     "duration": 0.021765,
     "end_time": "2023-10-31T15:04:09.225240",
     "exception": false,
     "start_time": "2023-10-31T15:04:09.203475",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "len(tag_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05fe795d",
   "metadata": {
    "papermill": {
     "duration": 0.007811,
     "end_time": "2023-10-31T15:04:09.241304",
     "exception": false,
     "start_time": "2023-10-31T15:04:09.233493",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Search REP keywords in REP articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "880105a6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:04:09.260254Z",
     "iopub.status.busy": "2023-10-31T15:04:09.259808Z",
     "iopub.status.idle": "2023-10-31T15:04:09.265815Z",
     "shell.execute_reply": "2023-10-31T15:04:09.264718Z"
    },
    "papermill": {
     "duration": 0.018896,
     "end_time": "2023-10-31T15:04:09.268687",
     "exception": false,
     "start_time": "2023-10-31T15:04:09.249791",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "input_directory_keywords = '/home/anon/working/chunks_complete_entity_list'\n",
    "input_path_REP = '/home/anon/input/datasets-extracted-entities/REP_extracted_entities/REP_extracted_entities.json'\n",
    "output_directory = '/home/anon/working/chunks_REP_keywords_in_REP_articles/'\n",
    "if not os.path.exists(output_directory):\n",
    "    os.makedirs(output_directory)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a18366d0",
   "metadata": {
    "papermill": {
     "duration": 0.009249,
     "end_time": "2023-10-31T15:04:09.286176",
     "exception": false,
     "start_time": "2023-10-31T15:04:09.276927",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Load REP dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42f8893d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:04:09.305024Z",
     "iopub.status.busy": "2023-10-31T15:04:09.304609Z",
     "iopub.status.idle": "2023-10-31T15:04:41.557431Z",
     "shell.execute_reply": "2023-10-31T15:04:41.555896Z"
    },
    "papermill": {
     "duration": 32.269462,
     "end_time": "2023-10-31T15:04:41.563847",
     "exception": false,
     "start_time": "2023-10-31T15:04:09.294385",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load HF posts\n",
    "df_REP = pd.read_json(input_path_REP)\n",
    "\n",
    "print('Number of relevant articles: ', df_REP.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34794c27",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:04:41.582027Z",
     "iopub.status.busy": "2023-10-31T15:04:41.581621Z",
     "iopub.status.idle": "2023-10-31T15:04:41.587933Z",
     "shell.execute_reply": "2023-10-31T15:04:41.586694Z"
    },
    "papermill": {
     "duration": 0.018386,
     "end_time": "2023-10-31T15:04:41.590410",
     "exception": false,
     "start_time": "2023-10-31T15:04:41.572024",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def defang_iocs_in_text(text):\n",
    "    defanged_text = ioc_fanger.defang(text)\n",
    "    return defanged_text\n",
    "\n",
    "def fang_iocs_in_text(text):\n",
    "    fanged_text = ioc_fanger.fang(text)\n",
    "    return fanged_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a32491d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:04:41.608843Z",
     "iopub.status.busy": "2023-10-31T15:04:41.608452Z",
     "iopub.status.idle": "2023-10-31T15:06:15.565550Z",
     "shell.execute_reply": "2023-10-31T15:06:15.564045Z"
    },
    "papermill": {
     "duration": 93.969841,
     "end_time": "2023-10-31T15:06:15.568786",
     "exception": false,
     "start_time": "2023-10-31T15:04:41.598945",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Function to merge sentence lists\n",
    "def merge_sentence_list(article):\n",
    "    merged_list = ' '.join(article).lower()\n",
    "    return merged_list\n",
    "\n",
    "# Function to merge entity lists\n",
    "def merge_entity_lists(entities):\n",
    "    merged_list = []\n",
    "    for entity_type, entity_list in entities.items():\n",
    "        merged_list.extend([fang_iocs_in_text(el).lower() for el in entity_list])\n",
    "    return merged_list\n",
    "\n",
    "\n",
    "#df_REP['merged_content'] = df_REP['flatContent'].apply(merge_sentence_list)\n",
    "df_REP['merged_content'] = df_REP['content'].apply(merge_sentence_list)\n",
    "\n",
    "\n",
    "df_REP['date'] = pd.to_datetime(df_REP['date'])\n",
    "df_REP['date'] = df_REP['date'].dt.strftime('%m-%d-%Y')\n",
    "\n",
    "# Sort the DataFrame by the 'date' column\n",
    "df_REP.sort_values(by='date', inplace=True)\n",
    "\n",
    "# Apply the merge_entity_lists function to each row and create a new column\n",
    "df_REP['merged_entities'] = df_REP['entities'].apply(merge_entity_lists)\n",
    "\n",
    "# Filter the DataFrame to only keep the selected columns\n",
    "df_REP = df_REP[['ID', 'date', 'merged_entities', 'merged_content']]\n",
    "\n",
    "# Convert the filtered DataFrame to a list of dictionaries\n",
    "existing_json_data = df_REP.to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d055bdce",
   "metadata": {
    "papermill": {
     "duration": 0.007977,
     "end_time": "2023-10-31T15:06:15.585445",
     "exception": false,
     "start_time": "2023-10-31T15:06:15.577468",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Search with entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "459a984c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:06:15.604353Z",
     "iopub.status.busy": "2023-10-31T15:06:15.603914Z",
     "iopub.status.idle": "2023-10-31T15:06:15.609890Z",
     "shell.execute_reply": "2023-10-31T15:06:15.609047Z"
    },
    "papermill": {
     "duration": 0.018254,
     "end_time": "2023-10-31T15:06:15.612040",
     "exception": false,
     "start_time": "2023-10-31T15:06:15.593786",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def process_keyword(keyword):\n",
    "    keyword_list = []\n",
    "    for entry in existing_json_data:\n",
    "        if keyword in entry['merged_entities']:\n",
    "            keyword_list.append({\n",
    "                'ID': entry['ID'],\n",
    "                'date': entry['date']\n",
    "            })\n",
    "    return keyword, keyword_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e78b70fd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:06:15.631284Z",
     "iopub.status.busy": "2023-10-31T15:06:15.630227Z",
     "iopub.status.idle": "2023-10-31T15:06:15.639807Z",
     "shell.execute_reply": "2023-10-31T15:06:15.638523Z"
    },
    "papermill": {
     "duration": 0.02177,
     "end_time": "2023-10-31T15:06:15.642279",
     "exception": false,
     "start_time": "2023-10-31T15:06:15.620509",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Define regular expression patterns\n",
    "ip_pattern = re.compile(r'^(\\d+\\.\\d+\\.\\d+\\.\\d+)$')\n",
    "pattern_to_remove = re.compile(r'^[-\\'\\+,/\\\\:=_\\|\\.\\^~0-9]+$')\n",
    "stop_words = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9117ecc6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:06:15.661442Z",
     "iopub.status.busy": "2023-10-31T15:06:15.660983Z",
     "iopub.status.idle": "2023-10-31T15:06:15.668755Z",
     "shell.execute_reply": "2023-10-31T15:06:15.667361Z"
    },
    "papermill": {
     "duration": 0.020229,
     "end_time": "2023-10-31T15:06:15.671022",
     "exception": false,
     "start_time": "2023-10-31T15:06:15.650793",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_REP_keywords_in_REP(keywords_path):\n",
    "    print(f'Working on {keywords_path}')\n",
    "    # Open the set of keywords\n",
    "    with open(keywords_path, 'r', encoding='utf-8') as file:\n",
    "        lines = file.readlines()\n",
    "    # Remove newlines and create a list\n",
    "    keywords_list = [line.strip() for line in lines]\n",
    "    # Filter and keep only the words that match the patterns\n",
    "    keywords_list = [keyword for keyword in keywords_list\n",
    "                     if ip_pattern.match(keyword) or not (pattern_to_remove.match(keyword) or keyword in stop_words or len(keyword) == 1)]\n",
    "\n",
    "    # Create a dictionary to hold the reports by keyword\n",
    "    return {keyword: [] for keyword in keywords_list}, keywords_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca9635f9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:06:15.691035Z",
     "iopub.status.busy": "2023-10-31T15:06:15.690321Z",
     "iopub.status.idle": "2023-10-31T15:06:15.699387Z",
     "shell.execute_reply": "2023-10-31T15:06:15.697863Z"
    },
    "papermill": {
     "duration": 0.022313,
     "end_time": "2023-10-31T15:06:15.702278",
     "exception": false,
     "start_time": "2023-10-31T15:06:15.679965",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def start_searching(REP_keywords_in_REP_dict, tag, keywords_list):\n",
    "    # Use ThreadPoolExecutor to parallelize keyword processing\n",
    "    with concurrent.futures.ThreadPoolExecutor() as executor:\n",
    "        # Submit tasks for each keyword\n",
    "        keyword_futures = {executor.submit(process_keyword, keyword): keyword for keyword in keywords_list}\n",
    "\n",
    "        # Use tqdm to track progress\n",
    "        with tqdm(total=len(keywords_list), desc='Searching in entities') as pbar:\n",
    "            for future in concurrent.futures.as_completed(keyword_futures):\n",
    "                keyword = keyword_futures[future]\n",
    "                keyword_results = future.result()\n",
    "                REP_keywords_in_REP_dict[keyword] = keyword_results\n",
    "                pbar.update(1)  # Update progress bar\n",
    "    \n",
    "    # Save resulting list\n",
    "    final_structure = []\n",
    "\n",
    "    for keyword, value_list in REP_keywords_in_REP_dict.items():\n",
    "            final_structure.append({\n",
    "                'keyword': keyword,\n",
    "                'tag': tag,\n",
    "                'matching_posts': value_list[1]\n",
    "            })\n",
    "            \n",
    "    return final_structure"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dcd33ff",
   "metadata": {
    "papermill": {
     "duration": 0.008334,
     "end_time": "2023-10-31T15:06:15.719628",
     "exception": false,
     "start_time": "2023-10-31T15:06:15.711294",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Search with articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26d00f58",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:06:15.739207Z",
     "iopub.status.busy": "2023-10-31T15:06:15.738756Z",
     "iopub.status.idle": "2023-10-31T15:06:15.755565Z",
     "shell.execute_reply": "2023-10-31T15:06:15.754149Z"
    },
    "papermill": {
     "duration": 0.030154,
     "end_time": "2023-10-31T15:06:15.758468",
     "exception": false,
     "start_time": "2023-10-31T15:06:15.728314",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def read_lines_from_file(file_path):\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        for line in file:\n",
    "            yield line.strip()\n",
    "\n",
    "def create_set_from_file(file_path):\n",
    "    return set(read_lines_from_file(file_path))\n",
    "\n",
    "def create_bulk_search_pattern(keywords_list):\n",
    "    keywords_list = sorted(keywords_list, key=lambda x: (-len(x), x))\n",
    "    #keyword_regex = r\"\\b(?:\" + \"|\".join(re.escape(word) for word in keywords_list) + r\")\\b\"\n",
    "    keyword_regex = r\"(^|\\s)[^a-zA-Z0-9]*(\" + \"|\".join(re.escape(word.lower()) for word in keywords_list) + r\")[^a-zA-Z0-9]*(\\s|$)\"\n",
    "    return re.compile(keyword_regex)\n",
    "\n",
    "def search_matches_in_chunk(chunk, bulk_search_pattern):\n",
    "    word_matching = defaultdict(list)\n",
    "    for post in chunk.itertuples(index=False):\n",
    "        content = post.merged_content.lower()\n",
    "\n",
    "        matches = bulk_search_pattern.findall(content)\n",
    "        for word in matches:\n",
    "            matched_word = word[1].lower()\n",
    "            word_matching[matched_word].append({\n",
    "                'ID': post.ID,\n",
    "                'date': post.date\n",
    "            })\n",
    "    return word_matching\n",
    "\n",
    "def search_matches(input_file_path, output_file_path, df_REP):\n",
    "    # Load keywords of reports\n",
    "    keywords_list = create_set_from_file(input_file_path)\n",
    "\n",
    "    # Convert keywords_list to lowercase for efficient matching\n",
    "    keywords_list_lower = {word.lower() for word in keywords_list}\n",
    "\n",
    "    # Compile the bulk search pattern\n",
    "    bulk_search_pattern = create_bulk_search_pattern(keywords_list_lower)\n",
    "\n",
    "    # Divide the dataframe into smaller chunks for parallel processing\n",
    "    chunk_size = 1000\n",
    "    df_chunks = [df_REP[i:i+chunk_size] for i in range(0, df_REP.shape[0], chunk_size)]\n",
    "                \n",
    "    # Use ThreadPoolExecutor for parallel processing\n",
    "    with ThreadPoolExecutor() as executor:\n",
    "        futures = [executor.submit(search_matches_in_chunk, chunk, bulk_search_pattern) for chunk in df_chunks]\n",
    "        word_matching_dict = {}\n",
    "        for future in tqdm(futures, total=len(futures), desc='Searching in articles'):\n",
    "            chunk_word_matching = future.result()\n",
    "            for word, matches in chunk_word_matching.items():\n",
    "                if word in word_matching_dict:\n",
    "                    word_matching_dict[word].extend(matches)\n",
    "                else:\n",
    "                    word_matching_dict[word] = matches\n",
    "\n",
    "    # Convert word_matching_dict to a list of dictionaries\n",
    "    word_matching = [{'keyword': word, 'matching posts': matches} for word, matches in word_matching_dict.items()]\n",
    "\n",
    "    return word_matching"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "285f2066",
   "metadata": {
    "papermill": {
     "duration": 0.008454,
     "end_time": "2023-10-31T15:06:15.775752",
     "exception": false,
     "start_time": "2023-10-31T15:06:15.767298",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Compare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5362fe6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:06:15.795802Z",
     "iopub.status.busy": "2023-10-31T15:06:15.795052Z",
     "iopub.status.idle": "2023-10-31T15:06:15.805466Z",
     "shell.execute_reply": "2023-10-31T15:06:15.804240Z"
    },
    "papermill": {
     "duration": 0.023624,
     "end_time": "2023-10-31T15:06:15.808048",
     "exception": false,
     "start_time": "2023-10-31T15:06:15.784424",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def compare_struct_and_save(struct_from_entity, struct_from_posts, output_path):\n",
    "    for entry in struct_from_posts:\n",
    "        kw = entry['keyword']\n",
    "        mp = entry['matching posts']\n",
    "        \n",
    "        for idx, item in enumerate(struct_from_entity):\n",
    "            if item['keyword'] == kw:\n",
    "                item['matching_posts'].extend(mp)\n",
    "                break\n",
    "                \n",
    "    \n",
    "    # Create a dictionary to store unique IDs for each keyword using sets\n",
    "    unique_ids = {}\n",
    "    # Iterate through the data and remove duplicates using sets\n",
    "    for item in tqdm(struct_from_entity, 'Removing duplicates'):\n",
    "        keyword = item[\"keyword\"]\n",
    "        #tag = item[\"tag\"]\n",
    "        matching_posts = item[\"matching_posts\"]\n",
    "        # Create a set to store unique IDs for the current keyword\n",
    "        unique_ids[keyword] = set()\n",
    "        # Create a list to store unique posts for the current keyword\n",
    "        unique_posts = []\n",
    "        for post in matching_posts:\n",
    "            post_id = post[\"ID\"]\n",
    "            # Check if the ID is not in the unique_ids set for this keyword\n",
    "            if post_id not in unique_ids[keyword]:\n",
    "                # Add the ID to the unique_ids set\n",
    "                unique_ids[keyword].add(post_id)\n",
    "                # Add the post to the unique_posts list\n",
    "                unique_posts.append(post)\n",
    "        # Update the matching posts for the current keyword\n",
    "        item[\"matching_posts\"] = unique_posts\n",
    "    \n",
    "    with open(output_path, 'w', encoding='utf-8') as outfile:\n",
    "        json.dump(struct_from_entity, outfile, indent=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "318ed1ec",
   "metadata": {
    "papermill": {
     "duration": 0.008895,
     "end_time": "2023-10-31T15:06:15.826299",
     "exception": false,
     "start_time": "2023-10-31T15:06:15.817404",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Execute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b33db75",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:06:15.845974Z",
     "iopub.status.busy": "2023-10-31T15:06:15.845314Z",
     "iopub.status.idle": "2023-10-31T18:55:25.682134Z",
     "shell.execute_reply": "2023-10-31T18:55:25.680836Z"
    },
    "papermill": {
     "duration": 13749.849581,
     "end_time": "2023-10-31T18:55:25.684620",
     "exception": false,
     "start_time": "2023-10-31T15:06:15.835039",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "start = 1\n",
    "end = start + 70  \n",
    "#(1, 71) - (71, 141) - (141, 211) - (211, 281) - (281, 351) - (351, 421) - (421, 491) - (491, 561) - (561, 631) - (631, 698) \n",
    "\n",
    "end = min(end, 698)\n",
    "\n",
    "print('Searching from {0} to {1}'.format(start, end-1))\n",
    "\n",
    "for idx_file in range(start, end):  \n",
    "    iFilename = 'chunk_{0}.txt'.format(idx_file)\n",
    "    input_path = os.path.join(input_directory_keywords, iFilename)    \n",
    "    REP_keywords_in_REP_dict, keywords_list = get_REP_keywords_in_REP(input_path)\n",
    "\n",
    "    oFilename = 'REP_keywords_in_REP_articles_{0}.json'.format(idx_file)\n",
    "    output_path = os.path.join(output_directory, oFilename)\n",
    "    tag = tag_list[idx_file-1]\n",
    "\n",
    "    struct_from_entity = start_searching(REP_keywords_in_REP_dict, tag, keywords_list)\n",
    "    struct_from_posts = search_matches(input_path, output_path, df_REP)\n",
    "    \n",
    "    compare_struct_and_save(struct_from_entity, struct_from_posts, output_path)\n",
    "\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b735da9d",
   "metadata": {
    "papermill": {
     "duration": 1.368574,
     "end_time": "2023-10-31T18:55:28.306341",
     "exception": false,
     "start_time": "2023-10-31T18:55:26.937767",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Save as ZIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08917ef7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T18:55:30.807864Z",
     "iopub.status.busy": "2023-10-31T18:55:30.807456Z",
     "iopub.status.idle": "2023-10-31T18:55:34.337604Z",
     "shell.execute_reply": "2023-10-31T18:55:34.336333Z"
    },
    "papermill": {
     "duration": 4.78803,
     "end_time": "2023-10-31T18:55:34.340381",
     "exception": false,
     "start_time": "2023-10-31T18:55:29.552351",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Directory path containing the output files to zip\n",
    "source_directory = output_directory\n",
    "\n",
    "# Zip file path\n",
    "zip_file_path = '/home/anon/working/HF_keywords_in_HF_posts.zip'\n",
    "\n",
    "# Create a ZIP archive\n",
    "with zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED) as zipf:\n",
    "    # Loop through all files in the directory and add them to the ZIP archive\n",
    "    for root, _, files in os.walk(source_directory):\n",
    "        for file in files:\n",
    "            file_path = os.path.join(root, file)\n",
    "            zipf.write(file_path, os.path.relpath(file_path, source_directory))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 13921.744698,
   "end_time": "2023-10-31T18:55:37.762900",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-10-31T15:03:36.018202",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}