Merge pull request #27 from agrepravin/dev

optimization for getting status of url(using threads)
DedSecInside · Jul 8, 2017 · ee4edc4 · ee4edc4
2 parents 638371f + ed3224c
commit ee4edc4
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 15 deletions.
diff --git a/modules/__init__.pyc b/modules/__init__.pyc
diff --git a/modules/bcolors.pyc b/modules/bcolors.pyc
diff --git a/modules/getemails.pyc b/modules/getemails.pyc
diff --git a/modules/getweblinks.py b/modules/getweblinks.py
@@ -4,16 +4,35 @@
 import urllib.request 
 from modules.bcolors import bcolors
 import bs4
+import time
+import threading
+import http
+
+def link_status(web):
+    link_live = False
+    try:
+        urllib.request.urlopen(web)    
+        link_live = True
+        print(web)
+    except urllib.error.HTTPError as e:
+        print(bcolors.On_Red+web+bcolors.ENDC)
+    except urllib.error.URLError as e:
+        print(bcolors.On_Red+web+bcolors.ENDC)
+    except http.client.RemoteDisconnected as e:
+        print(bcolors.On_Red+web+bcolors.ENDC)
+    return 
+
 
 """Get all onion links from the website"""
 def getLinks(soup,ext,live=0):
     _soup_instance = bs4.BeautifulSoup
     extensions = []
     if ext:
         for e in ext:
-            extensions.append(e)  
+            extensions.append(e) 
     if isinstance(type(soup), type(_soup_instance)):
         websites = []
+        start_time = time.time()
         for link in soup.find_all('a'):
             web_link = link.get('href')
             if web_link != None:
@@ -31,23 +50,14 @@ def getLinks(soup,ext,live=0):
         print (bcolors.OKGREEN+'Websites Found - '+bcolors.ENDC+str(len(websites)))
         print ('-------------------------------')
         if live:
+            threads = []
             for web in websites:
-                flag=1
-                try:
-                    urllib.request.urlopen(web)    
-                except urllib.error.HTTPError as e:
-                    if e.code:
-                        print(bcolors.On_Red+web+bcolors.ENDC)
-                        flag=0  
-                except urllib.error.URLError as e:
-                    print(bcolors.On_Red+web+bcolors.ENDC)
-                    flag=0  
-
-                if flag:
-                    print(web)          
+                t = threading.Thread(target=link_status, args=(web,))
+                threads.append(t)
+                t.start()
         else:
             for web in websites:
-                print(web) 
+                print(web)
         return websites           
     else:
         raise('Method parameter is not of instance bs4.BeautifulSoup')