Add files via upload

DeepTextSearch/DeepTextSearch.py

@@ -0,0 +1,77 @@
+import pandas as pd
+from sentence_transformers import SentenceTransformer, util
+import numpy as np
+import pickle
+import os
+
+
+corpus_list_data = os.path.join('embedding-data/','corpus_list_data.pickle')
+corpus_embeddings_data = os.path.join('embedding-data/','corpus_embeddings_data.pickle')
+
+class LoadData:
+    def __init__(self):
+        self.corpus_list = None
+    def from_csv(self,file_path:str):
+        self.file_path = file_path
+        csv_data = pd.read_csv(file_path)
+        column_name = str(input('Input the text Column Name Please ? : '))
+        self.corpus_list =  csv_data[column_name].dropna().to_list()
+        return self.corpus_list
+
+class TextEmbedder:
+    def __init__(self):
+        self.corpus_embeddings_data = corpus_embeddings_data
+        self.corpus_list_data = corpus_list_data
+        self.corpus_list = None
+        self.embedder = SentenceTransformer('paraphrase-xlm-r-multilingual-v1')
+        self.corpus_embeddings = None
+        if 'embedding-data' not in os.listdir():
+            os.makedirs("embedding-data")
+    def embed(self,corpus_list:list):
+        self.corpus_list = corpus_list
+        if len(os.listdir("embedding-data/"))==0:
+            self.corpus_embeddings = self.embedder.encode(self.corpus_list, convert_to_tensor=True,show_progress_bar=True)
+            pickle.dump(self.corpus_embeddings, open(self.corpus_embeddings_data, "wb"))
+            pickle.dump(self.corpus_list, open(self.corpus_list_data, "wb"))
+            print("Embedding data Saved Successfully!")
+            print(os.listdir("embedding-data/"))
+        else:
+            print("Embedding data allready present, Do you want Embed & Save Again? Enter yes or no")
+            flag  = str(input())
+            if flag.lower() == 'yes':
+                self.corpus_embeddings = self.embedder.encode(self.corpus_list, convert_to_tensor=True,show_progress_bar=True)
+                #np.savez(self.corpus_embeddings_data,self.corpus_embeddings.cpu().data.numpy())
+                #np.savez(self.corpus_list_data,self.corpus_list)
+                pickle.dump(self.corpus_embeddings, open(self.corpus_embeddings_data, "wb"))
+                pickle.dump(self.corpus_list, open(self.corpus_list_data, "wb"))
+                print("Embedding data Saved Successfully Again!")
+                print(os.listdir("embedding-data/"))
+            else:
+                print("Embedding data allready Present, Please Apply Search!")
+                print(os.listdir("embedding-data/"))
+    def load_embedding(self):
+        if len(os.listdir("embedding-data/"))==0:
+            print("Embedding data Not present, Please Run Embedding First")
+        else:
+            print("Embedding data Loaded Successfully!")
+            print(os.listdir("embedding-data/"))
+            return pickle.load(open(self.corpus_embeddings_data, "rb"))
+
+class TextSearch:
+    def __init__(self):
+        self.corpus_embeddings = pickle.load(open(corpus_embeddings_data, "rb"))
+        self.data = pickle.load(open(corpus_list_data, "rb"))
+    def find_similar(self,query_text:str,top_n=10):
+        self.top_n = top_n
+        self.query_text = query_text
+        self.query_embedding = TextEmbedder().embedder.encode(self.query_text, convert_to_tensor=True)
+        self.cos_scores = util.pytorch_cos_sim(self.query_embedding, self.corpus_embeddings)[0].cpu().data.numpy()
+        self.sort_list  = np.argsort(-self.cos_scores)
+        self.all_data  = []
+        for idx in self.sort_list[1:self.top_n+1]:
+            data_out = {}
+            data_out['index'] = int(idx)
+            data_out['text'] = self.data[idx]
+            data_out['score'] = self.cos_scores[idx]
+            self.all_data.append(data_out)
+        return self.all_data

DeepTextSearch/__init__.py

@@ -0,0 +1 @@
+from DeepTextSearch.DeepTextSearch import LoadData,TextEmbedder,TextSearch

DeepTextSearch/requirements.txt

@@ -0,0 +1,3 @@
+pandas==1.2.4
+sentence_transformers==1.2.0
+numpy==1.18.5

Demo/Deep Text Search Demo.ipynb

@@ -0,0 +1,134 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "49594b04",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Importing the proper classes\n",
+    "from DeepTextSearch import LoadData,TextEmbedder,TextSearch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "a5424e23",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Input the text Column Name Please ? : Question\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Load data from CSV file\n",
+    "data = LoadData().from_csv(\"../your_file_name.csv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "5ce9f30d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "26865bd100c948a6945f2e47ad3a9183",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Batches:   0%|          | 0/19 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Embedding data Saved Successfully!\n",
+      "['corpus_embeddings_data.pickle', 'corpus_list_data.pickle']\n"
+     ]
+    }
+   ],
+   "source": [
+    "# For Serching we need to Embed Data first, After Embedding all the data stored on the local path\n",
+    "TextEmbedder().embed(corpus_list=data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "5f349322",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'index': 575, 'text': 'What is Node.js?', 'score': 0.88481015},\n",
+       " {'index': 578, 'text': 'When should we use Node.js?', 'score': 0.8388137},\n",
+       " {'index': 581, 'text': 'Explain how does Node.js work?', 'score': 0.8064759},\n",
+       " {'index': 591, 'text': 'What are Globals in Node.js?', 'score': 0.7844132},\n",
+       " {'index': 602,\n",
+       "  'text': 'What is chaining process in Node.js?',\n",
+       "  'score': 0.7806176},\n",
+       " {'index': 596, 'text': 'What is NPM in Node.js?', 'score': 0.76716936},\n",
+       " {'index': 586, 'text': 'What is Callback in Node.js?', 'score': 0.7659653},\n",
+       " {'index': 579, 'text': 'When to not use Node.js?', 'score': 0.7643588},\n",
+       " {'index': 593,\n",
+       "  'text': 'What is EventEmitter in Node.js?',\n",
+       "  'score': 0.7514152},\n",
+       " {'index': 580,\n",
+       "  'text': 'What IDEs can you use for Node.js development?',\n",
+       "  'score': 0.74787086}]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# for searching, you need to give the query_text  and the number of the similar text you want\n",
+    "TextSearch().find_similar(query_text=\"What are the key features of Node.js?\",top_n=10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e8b4c035",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

Demo/DeepTextSearchDemo.py

@@ -0,0 +1,12 @@
+# Importing the proper classes
+from DeepTextSearch import LoadData,TextEmbedder,TextSearch
+
+# Load data from CSV file
+data = LoadData().from_csv("../your_file_name.csv")
+
+# For Serching we need to Embed Data first, After Embedding all the data stored on the local path
+TextEmbedder().embed(corpus_list=data)
+
+# for searching, you need to give the query_text  and the number of the similar text you want
+TextSearch().find_similar(query_text="What are the key features of Node.js?",top_n=10)
+

LICENSE.txt

@@ -0,0 +1,17 @@
+MIT License
+Copyright (c) 2021 Nilesh Verma
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.