NLP-LLM-TextMining_GeneSearch.ipynb

{"cells":[{"cell_type":"code","source":["# Install Required Libraries\n","!pip install biopython"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"mLkscU7F3Crk","executionInfo":{"status":"ok","timestamp":1741623806308,"user_tz":-60,"elapsed":7788,"user":{"displayName":"Data Analysis","userId":"04405841940174047560"}},"outputId":"e35303e6-7092-4b0a-f12f-36f109aa6a34"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting biopython\n","  Downloading biopython-1.85-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (13 kB)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from biopython) (1.26.4)\n","Downloading biopython-1.85-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m35.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: biopython\n","Successfully installed biopython-1.85\n"]}]},{"cell_type":"code","source":["import requests\n","import time\n","import logging\n","import re\n","import pandas as pd\n","import spacy\n","import nltk\n","from Bio import Entrez\n","from IPython.core.display import display, HTML\n","from nltk.corpus import wordnet as wn\n","from sentence_transformers import SentenceTransformer\n","from sklearn.metrics.pairwise import cosine_similarity\n","\n","# Configure Entrez API Key\n","Entrez.email = \"example@example.com\"  # Replace with your actual email\n","Entrez.api_key = \"your_ENSMEBL_API_key\"  # Replace with your valid API key\n","\n","from huggingface_hub import login\n","login(\"your_HUGGINGFACE_LOGIN_key\")  # Use your token\n","\n","# Set up logging\n","logging.basicConfig(level=logging.ERROR)\n","\n","# Load NLP Models\n","### QC Check for spaCy ###\n","try:\n","    nlp = spacy.load(\"en_core_web_sm\")\n","    logging.info(\"✅ spaCy model loaded successfully.\")\n","except Exception as e:\n","    logging.error(f\"❌ Failed to load spaCy model: {e}\")\n","    nlp = None  # Fallback to None if loading fails\n","\n","### QC Check for nltk (WordNet) ###\n","try:\n","    nltk.data.find(\"corpora/wordnet.zip\")  # Check if WordNet is available\n","    nltk.download('omw-1.4') #Fixed the indentation here\n","    logging.info(\"✅ nltk WordNet found.\")\n","except LookupError:\n","    logging.warning(\"🔴 nltk WordNet not found. Downloading now...\")\n","    nltk.download(\"wordnet\")\n","    print(\"✅ NLTK WordNet downloaded successfully!\")\n","\n","### QC Check for minilm ###\n","try:\n","    minilm_model = SentenceTransformer(\"sentence-transformers/all-MiniLM-L6-v2\")\n","    print(\"✅ MiniLM (BERT) model loaded successfully!\")\n","except Exception as e:\n","    logging.warning(f\"🔴 Failed to load Word2Vec model: {e}. Some functions may not work.\")\n","\n","# Ensembl API Server\n","ENSEMBL_SERVER = \"https://rest.ensembl.org\"\n","\n","# Function to get gene symbol from TAIR ID using Ensembl API\n","def get_gene_symbol(tair_id):\n","    url = f\"{ENSEMBL_SERVER}/xrefs/id/{tair_id}?content-type=application/json\"\n","    response = requests.get(url)\n","    if response.status_code != 200:\n","        logging.error(f\"❌ Error fetching gene symbol for {tair_id}: {response.status_code}\")\n","        return None\n","    data = response.json()\n","    for entry in data:\n","        if entry[\"dbname\"] in [\"TAIR_SYMBOL\", \"Uniprot_gn\"]:\n","            return entry[\"display_id\"]\n","    return None\n","\n","# Function to expand keywords using WordNet, spaCy, and MiniLM\n","def expand_keywords(keyword):\n","    expanded_keywords = set([keyword])\n","\n","    # NLTK WordNet Expansion\n","    nltk_synonyms = set()\n","    for syn in wn.synsets(keyword):\n","        for lemma in syn.lemmas():\n","            nltk_synonyms.add(lemma.name())\n","    if nltk_synonyms:\n","        #print(f\"🔍 NLTK WordNet expanded '{keyword}' to: {list(nltk_synonyms)}\")\n","        expanded_keywords.update(nltk_synonyms)\n","\n","    # spaCy Named Entity Recognition (NER)\n","    doc = nlp(keyword)\n","    spacy_entities = [(ent.text, ent.label_) for ent in doc.ents]\n","    #print(f\"🔍 spaCy NER applied on '{keyword}': {spacy_entities}\")\n","\n","    # MiniLM Semantic Expansion\n","    related_words = [\"ABA\", \"ABI1\", \"abscisic acid\", \"MAPK\", \"GA signaling\"]  # Placeholder examples\n","    embeddings = minilm_model.encode([keyword] + related_words)\n","    similarities = cosine_similarity([embeddings[0]], embeddings[1:])\n","    similar_terms = [related_words[i] for i in similarities.argsort()[0][-3:]]\n","    #print(f\"🔍 MiniLM expanded '{keyword}' to: {similar_terms}\")\n","    expanded_keywords.update(similar_terms)\n","\n","    return list(expanded_keywords)\n","\n","# Define original keywords\n","keywords = [\n","    \"ABA\", \"abscisic acid\", \"ABA signaling pathway\", \"Abscisic acid signaling\", \"ABI1\", \"MAPK cascade\",\n","    \"drought stress\", \"salt stress\", \"cold stress\", \"Seed dormancy\", \"leaf senescence\",\n","    \"stomata\", \"Guard cell signaling\", \"stomatal closure\", \"stomatal regulation\", \"MAPK\"\n","]\n","\n","# Initialize expanded keywords\n","expanded_keywords = set(keywords)\n","for kw in keywords:\n","    expanded_keywords.update(expand_keywords(kw))\n","\n","print(\"✅ Final expanded keyword list:\", expanded_keywords)\n","\n","# Function to safely query NCBI with retries (handles 429 errors)\n","def safe_entrez_request(func, *args, **kwargs):\n","    max_retries = 5\n","    for attempt in range(max_retries):\n","        try:\n","            time.sleep(1)  # Prevent overloading NCBI servers\n","            handle = func(*args, **kwargs)\n","            return handle\n","        except Exception as e:\n","            if \"429\" in str(e):\n","                wait_time = 2 ** attempt\n","                logging.warning(f\"429 Error: Too Many Requests. Retrying in {wait_time} seconds...\")\n","                time.sleep(wait_time)\n","            else:\n","                logging.error(f\"Error fetching data: {e}\")\n","                break\n","    return None\n","\n","# Function to search PubMed\n","def search_pubmed(identifier, keywords, expanded_keywords, retmax):\n","    query = f'({identifier}) AND (\"{\" OR \".join(keywords)}\")'\n","    expanded_query = f'({identifier}) AND (\"{\" OR \".join(expanded_keywords)}\")'\n","\n","    print(f\"\\n🔍 Searching PubMed for: {identifier}\")\n","    #print(f\"  📌 Original Query: {query}\")\n","    #print(f\"  📌 Expanded Query: {expanded_query}\")\n","\n","    def execute_search(query_string):\n","        handle = safe_entrez_request(Entrez.esearch, db=\"pubmed\", term=query_string, retmax=retmax)\n","        if handle:\n","            record = Entrez.read(handle)\n","            handle.close()\n","            return set(record.get(\"IdList\", []))\n","        return set()\n","\n","    original_results = execute_search(query)\n","    expanded_results = execute_search(expanded_query)\n","\n","    print(f\"  ✅ Results for {identifier}: {len(original_results)} (original) | {len(expanded_results)} (expanded)\")\n","    return original_results, expanded_results\n","\n","# Function to fetch PubMed details\n","def fetch_pubmed_details(pubmed_id):\n","    try:\n","        handle = safe_entrez_request(Entrez.efetch, db=\"pubmed\", id=pubmed_id, rettype=\"medline\", retmode=\"xml\")\n","        records = Entrez.read(handle)\n","        handle.close()\n","\n","        if \"PubmedArticle\" in records and records[\"PubmedArticle\"]:\n","            article = records[\"PubmedArticle\"][0]\n","            citation = article[\"MedlineCitation\"]\n","            article_data = citation[\"Article\"]\n","            title = article_data[\"ArticleTitle\"]\n","            abstract = article_data.get(\"Abstract\", {}).get(\"AbstractText\", [\"\"])[0]\n","            pub_year = citation.get(\"Article\", {}).get(\"Journal\", {}).get(\"JournalIssue\", {}).get(\"PubDate\", {}).get(\"Year\", \"N/A\")\n","            doi = next((id_ for id_ in article_data.get(\"ELocationID\", []) if id_.attributes[\"EIdType\"] == \"doi\"), \"N/A\")\n","            return title, abstract, doi, pub_year\n","    except Exception as e:\n","        logging.error(f\"Error fetching PubMed details for {pubmed_id}: {e}\")\n","    return None, None, \"N/A\", \"N/A\"\n","\n","# Function to find matched keywords in the abstract\n","def find_matched_keywords(abstract, keywords):\n","    matched_keywords = [keyword for keyword in keywords if keyword.lower() in abstract.lower()]\n","    return \", \".join(matched_keywords) if matched_keywords else \"N/A\"\n","\n","# ✅ Step 1: User Input\n","tair_input = input(\"Enter comma-separated TAIR IDs or gene symbols: \").strip()\n","retmax = int(input(\"Enter the number of articles to retrieve per search (e.g., 5, 10, 15): \"))\n","print(f\"Using keywords: {', '.join(keywords)}\\n\")\n","\n","# ✅ Step 2: Separate TAIR IDs and Gene Symbols\n","tair_ids = [id_.strip() for id_ in tair_input.split(\",\") if re.match(r\"AT\\dG\\d{5}\", id_.strip())]\n","gene_symbols = [id_.strip() for id_ in tair_input.split(\",\") if id_.strip() not in tair_ids]\n","\n","print(\"\\n✅ Extracted Identifiers:\")\n","print(f\"  🔵 TAIR IDs: {tair_ids}\")\n","print(f\"  🟢 Gene Symbols: {gene_symbols}\")\n","\n","# ✅ Step 3: Convert TAIR IDs to Gene Symbols (if available)\n","converted_symbols = {}\n","for tair in tair_ids:\n","    gene_symbol = get_gene_symbol(tair)\n","    if gene_symbol:\n","        converted_symbols[tair] = gene_symbol\n","        gene_symbols.append(gene_symbol)  # Add the gene symbol to the search list\n","    else:\n","        print(f\"  ⚠️ No gene symbol found for {tair}. Using TAIR ID for search.\")\n","\n","print(\"\\n✅ Converted TAIR IDs to Gene Symbols:\")\n","for tair, symbol in converted_symbols.items():\n","    print(f\"  🔄 {tair} → {symbol}\")\n","\n","all_results = []\n","\n","# ✅ Step 4: Process both TAIR IDs and Gene Symbols\n","identifiers = tair_ids + gene_symbols  # Process them together\n","\n","for identifier in identifiers:\n","    print(f\"\\n🚀 Processing: {identifier}\")\n","\n","    original_results, expanded_results = search_pubmed(identifier, keywords, expanded_keywords, retmax)\n","\n","    # ✅ Step 5: Process Original Keyword Results\n","    for pubmed_id in original_results:\n","        title, abstract, doi, pub_year = fetch_pubmed_details(pubmed_id)\n","        if title and abstract:\n","            matched_keywords = find_matched_keywords(abstract, keywords)\n","            all_results.append({\n","                \"Identifier\": identifier,\n","                \"Query Type\": \"Original Keywords\",\n","                \"Matched Keywords\": matched_keywords,\n","                \"PubMed ID\": f'<a href=\"https://pubmed.ncbi.nlm.nih.gov/{pubmed_id}\">{pubmed_id}</a>',\n","                \"Title\": title,\n","                \"Year\": pub_year,\n","                \"DOI\": f'<a href=\"https://doi.org/{doi}\">{doi}</a>' if doi != \"N/A\" else \"N/A\"\n","            })\n","\n","    # ✅ Step 6: Process Expanded Keyword Results\n","    for pubmed_id in expanded_results:\n","        title, abstract, doi, pub_year = fetch_pubmed_details(pubmed_id)\n","        if title and abstract:\n","            matched_keywords = find_matched_keywords(abstract, expanded_keywords)\n","            all_results.append({\n","                \"Identifier\": identifier,\n","                \"Query Type\": \"Expanded Keywords\",\n","                \"Matched Keywords\": matched_keywords,\n","                \"PubMed ID\": f'<a href=\"https://pubmed.ncbi.nlm.nih.gov/{pubmed_id}\">{pubmed_id}</a>',\n","                \"Title\": title,\n","                \"Year\": pub_year,\n","                \"DOI\": f'<a href=\"https://doi.org/{doi}\">{doi}</a>' if doi != \"N/A\" else \"N/A\"\n","            })\n","\n","# ✅ Step 7: Convert results to DataFrame and remove duplicates\n","df = pd.DataFrame(all_results)\n","\n","if df.empty:\n","    print(\"\\n❌ No abstracts found.\")\n","else:\n","    # Remove duplicate PubMed IDs\n","    before_deduplication = len(df)\n","    df.drop_duplicates(subset=[\"PubMed ID\"], keep=\"first\", inplace=True)\n","    after_deduplication = len(df)\n","\n","    # Check if duplicates were removed and print message\n","    duplicates_removed = before_deduplication - after_deduplication\n","    if duplicates_removed > 0:\n","        print(f\"\\n⚠️ {duplicates_removed} duplicate PubMed ID(s) removed.\")\n","\n","    # Display final results\n","    display(HTML(df.to_html(escape=False)))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"mjYpdRmE288u","executionInfo":{"status":"ok","timestamp":1741623966605,"user_tz":-60,"elapsed":44919,"user":{"displayName":"Data Analysis","userId":"04405841940174047560"}},"outputId":"c0fdb818-0654-4e66-ed8c-46662b27ec38"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n"]},{"output_type":"stream","name":"stdout","text":["✅ MiniLM (BERT) model loaded successfully!\n","✅ Final expanded keyword list: {'Seed dormancy', 'drought stress', 'stomate', 'pore', 'stomatal closure', 'MAPK cascade', 'stomatal regulation', 'Guard cell signaling', 'ABA signaling pathway', 'Abscisic acid signaling', 'salt stress', 'aba', 'ABA', 'leaf senescence', 'stomata', 'GA signaling', 'stoma', 'abscisic acid', 'ABI1', 'MAPK', 'cold stress'}\n","Enter comma-separated TAIR IDs or gene symbols: AT4G26080, mapkkk18\n","Enter the number of articles to retrieve per search (e.g., 5, 10, 15): 5\n","Using keywords: ABA, abscisic acid, ABA signaling pathway, Abscisic acid signaling, ABI1, MAPK cascade, drought stress, salt stress, cold stress, Seed dormancy, leaf senescence, stomata, Guard cell signaling, stomatal closure, stomatal regulation, MAPK\n","\n","\n","✅ Extracted Identifiers:\n","  🔵 TAIR IDs: ['AT4G26080']\n","  🟢 Gene Symbols: ['mapkkk18']\n","\n","✅ Converted TAIR IDs to Gene Symbols:\n","  🔄 AT4G26080 → ABI1\n","\n","🚀 Processing: AT4G26080\n","\n","🔍 Searching PubMed for: AT4G26080\n","  ✅ Results for AT4G26080: 0 (original) | 0 (expanded)\n","\n","🚀 Processing: mapkkk18\n","\n","🔍 Searching PubMed for: mapkkk18\n","  ✅ Results for mapkkk18: 5 (original) | 5 (expanded)\n","\n","🚀 Processing: ABI1\n","\n","🔍 Searching PubMed for: ABI1\n","  ✅ Results for ABI1: 5 (original) | 5 (expanded)\n","\n","⚠️ 10 duplicate PubMed ID(s) removed.\n"]},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Identifier</th>\n","      <th>Query Type</th>\n","      <th>Matched Keywords</th>\n","      <th>PubMed ID</th>\n","      <th>Title</th>\n","      <th>Year</th>\n","      <th>DOI</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>mapkkk18</td>\n","      <td>Original Keywords</td>\n","      <td>ABA, abscisic acid, leaf senescence, MAPK</td>\n","      <td><a href=\"https://pubmed.ncbi.nlm.nih.gov/36841482\">36841482</a></td>\n","      <td>The abscisic acid-responsive element binding factors MAPKKK18 module regulates abscisic acid-induced leaf senescence in Arabidopsis.</td>\n","      <td>2023</td>\n","      <td><a href=\"https://doi.org/10.1016/j.jbc.2023.103060\">10.1016/j.jbc.2023.103060</a></td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>mapkkk18</td>\n","      <td>Original Keywords</td>\n","      <td>ABA, ABA signaling pathway, ABI1, stomata</td>\n","      <td><a href=\"https://pubmed.ncbi.nlm.nih.gov/34281207\">34281207</a></td>\n","      <td>Identification of Novel miRNAs and Their Target Genes in the Response to Abscisic Acid in <i>Arabidopsis</i>.</td>\n","      <td>2021</td>\n","      <td><a href=\"https://doi.org/10.3390/ijms22137153\">10.3390/ijms22137153</a></td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>mapkkk18</td>\n","      <td>Original Keywords</td>\n","      <td>ABA, abscisic acid, MAPK cascade, MAPK</td>\n","      <td><a href=\"https://pubmed.ncbi.nlm.nih.gov/38153765\">38153765</a></td>\n","      <td>Arabidopsis HECT and RING-type E3 Ligases Promote MAPKKK18 Degradation to Regulate Abscisic Acid Signaling.</td>\n","      <td>2024</td>\n","      <td><a href=\"https://doi.org/10.1093/pcp/pcad165\">10.1093/pcp/pcad165</a></td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>mapkkk18</td>\n","      <td>Original Keywords</td>\n","      <td>MAPK</td>\n","      <td><a href=\"https://pubmed.ncbi.nlm.nih.gov/39561685\">39561685</a></td>\n","      <td>Ethyl acetate extract of Artemisia argyi improves the resistance of cotton to Verticillium dahliae by activating the immune response.</td>\n","      <td>2024</td>\n","      <td><a href=\"https://doi.org/10.1016/j.plaphy.2024.109296\">10.1016/j.plaphy.2024.109296</a></td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>mapkkk18</td>\n","      <td>Original Keywords</td>\n","      <td>ABA, abscisic acid, MAPK</td>\n","      <td><a href=\"https://pubmed.ncbi.nlm.nih.gov/38096479\">38096479</a></td>\n","      <td>Stem-cell-expressed DEVIL-like small peptides maintain root growth under abiotic stress via abscisic acid signaling.</td>\n","      <td>2024</td>\n","      <td><a href=\"https://doi.org/10.1093/plphys/kiad659\">10.1093/plphys/kiad659</a></td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>ABI1</td>\n","      <td>Original Keywords</td>\n","      <td>ABA, abscisic acid</td>\n","      <td><a href=\"https://pubmed.ncbi.nlm.nih.gov/39879308\">39879308</a></td>\n","      <td>SnRK2 kinases sense molecular crowding and form condensates to disrupt ABI1 inhibition.</td>\n","      <td>2025</td>\n","      <td><a href=\"https://doi.org/10.1126/sciadv.adr8250\">10.1126/sciadv.adr8250</a></td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>ABI1</td>\n","      <td>Original Keywords</td>\n","      <td>ABA, abscisic acid, ABA signaling pathway, ABI1</td>\n","      <td><a href=\"https://pubmed.ncbi.nlm.nih.gov/40057048\">40057048</a></td>\n","      <td>Allelopathic inhibitory of thymol on Arabidopsis thaliana primary root growth is mediated by ABA signaling pathway.</td>\n","      <td>2025</td>\n","      <td><a href=\"https://doi.org/10.1016/j.plantsci.2025.112453\">10.1016/j.plantsci.2025.112453</a></td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>ABI1</td>\n","      <td>Original Keywords</td>\n","      <td>ABI1, drought stress</td>\n","      <td><a href=\"https://pubmed.ncbi.nlm.nih.gov/39898271\">39898271</a></td>\n","      <td>Effects of exogenous spraying of melatonin on the growth of <i>Platycrater arguta</i> under drought stress.</td>\n","      <td>2024</td>\n","      <td><a href=\"https://doi.org/10.3389/fpls.2024.1516302\">10.3389/fpls.2024.1516302</a></td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>ABI1</td>\n","      <td>Original Keywords</td>\n","      <td>ABI1</td>\n","      <td><a href=\"https://pubmed.ncbi.nlm.nih.gov/39568082\">39568082</a></td>\n","      <td>Programmed cell death-related ABI1 is a critical mediator of abdominal aortic aneurysm.</td>\n","      <td>2024</td>\n","      <td><a href=\"https://doi.org/10.1186/s40001-024-02128-4\">10.1186/s40001-024-02128-4</a></td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>ABI1</td>\n","      <td>Original Keywords</td>\n","      <td>ABI1, salt stress</td>\n","      <td><a href=\"https://pubmed.ncbi.nlm.nih.gov/39861571\">39861571</a></td>\n","      <td>Functional Characterization of the <i>PoWHY1</i> Gene from <i>Platycladus orientalis</i> and Its Role in Abiotic Stress Tolerance in Transgenic <i>Arabidopsis thaliana</i>.</td>\n","      <td>2025</td>\n","      <td><a href=\"https://doi.org/10.3390/plants14020218\">10.3390/plants14020218</a></td>\n","    </tr>\n","  </tbody>\n","</table>"]},"metadata":{}}]}],"metadata":{"colab":{"provenance":[{"file_id":"11B4eMIqleGlnYTc7qnmDWSacOixsnlhb","timestamp":1741624019623}]},"language_info":{"name":"python"},"kernelspec":{"name":"python3","display_name":"Python 3"}},"nbformat":4,"nbformat_minor":0}