File 2 - Data_Extraction_Phishing_Webpages.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This file contains codes which downloads a file of known phishing URLs, extract phishing webpage prediction features \n",
    "# from each webpage of the collected phishing websites that collects sensitive data. The features are then stored in \n",
    "# the database. All functions here are imported from the files Functions and SQL_functions.\n",
    "\n",
    "import csv\n",
    "import mysql.connector\n",
    "import mysql\n",
    "from bs4 import BeautifulSoup\n",
    "from Functions import *\n",
    "from SQL_functions import *\n",
    "import requests\n",
    "from requests.packages.urllib3.exceptions import InsecureRequestWarning\n",
    "import sys\n",
    "from datetime import datetime\n",
    "import urllib3\n",
    "from googleapiclient import discovery\n",
    "import googleapiclient\n",
    "import googleapiclient.discovery\n",
    "import time\n",
    "import json\n",
    "\n",
    "\n",
    "def download_PhishTank_data():\n",
    "    print('Starts downloading........')\n",
    "    PhishTankURL = 'https://data.phishtank.com/data/online-valid.csv'\n",
    "    file = 'C:/Users/tomna/OneDrive/Datasets/PhishTank/PhishTank.csv'\n",
    "    urllib.request.urlretrieve(PhishTankURL,file)\n",
    "    print('Download completed!')\n",
    "    \n",
    "    \n",
    "def get_phishing_data(conn,cur):\n",
    "    freeDomains = getFreeSubdomainList(cur)  # get a list of the most abused free domain names by attackers pre-loaded in the database\n",
    "    TLDs = getTLDs(cur)  # get a list of known gTLDs and ccTLDs pre-loaded in the database\n",
    "    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36'}  # make a server to see a request coming from a browser and not a python engine. Some servers block request from non-browsers\n",
    "    time_start = time.time()\n",
    "    time1 = datetime.now().strftime('%Y-%m-%d %H:%M')\n",
    "    print('--------Recording URLs starts at ' , time1,' ---------') \n",
    "    SN = 1\n",
    "    # retrieve and open a file contain known phishing URLs. To each URL, download its webpage and extract the features\n",
    "    file = 'C:/Users/tomna/OneDrive/Datasets/PhishTank/PhishTank.csv'\n",
    "    with open(file, encoding='utf-8') as memoryFile:\n",
    "        rowData = csv.reader(memoryFile, delimiter=',')       \n",
    "        for row in rowData:\n",
    "            recorded_URLs = []\n",
    "            URLs = getURLsInDataset(cur)\n",
    "            for url in URLs :\n",
    "                recorded_URLs.append(url[1])\n",
    "            URL = row[1]\n",
    "            if URL in recorded_URLs :\n",
    "                continue\n",
    "            else :\n",
    "                if len(URL) > 998 : # discard any URL above this long\n",
    "                    continue\n",
    "                else:\n",
    "                    # web scraping each webpage to read its contents\n",
    "                    try:\n",
    "                        webpage = requests.get(URL,verify=False,headers=headers,timeout=120)   # verify=False means disabling SSL verification. Also set the timeout in sec to avoid long waits (see http://docs.python-requests.org/en/master/user/advanced/#timeouts)\n",
    "                        webpage2 = webpage\n",
    "                        try:\n",
    "                            soup = BeautifulSoup(webpage.content,'html.parser')    \n",
    "                        except html.parser.HTMLParseError as e:\n",
    "                            continue\n",
    "                        except UnicodeError as e:\n",
    "                            continue              \n",
    "                        # check all types of redirections associated with a webpage\n",
    "                        url,URL,noRedirections,redirectType,shortURLFound  = detectAllURLRedirections(URL,webpage,cur,soup,headers)  # if there is redirection, url is the original url while URL is the final redirected url\n",
    "                    except TypeError as e:\n",
    "                        continue\n",
    "                    except IndexError as e:   # issues with structuring of meta tag values\n",
    "                        print('Error 0: ',e,'------',URL)\n",
    "                        continue   \n",
    "                    except requests.exceptions.RequestException as e:    # catch all types of exceptions related to connection and mistypying and skip to the next url (server is not reachable)\n",
    "                        print('Error 1: ',e,'------',URL)\n",
    "                        continue   \n",
    "                    except urllib3.exceptions.LocationValueError as e:  # no host specified error\n",
    "                        print('Error 2: ',e,'------',URL)  \n",
    "                        continue                \n",
    "                    if webpage2.status_code != 200:    # skip if the webpage is not found in the server (server is reachable)\n",
    "                        continue      \n",
    "                    # check if a search form exists in the webpage and drops down the webpage with one\n",
    "                    seachFormExist = checkSearchForm(soup)\n",
    "                    if seachFormExist == 'Yes' :   # filter out webpages with search forms  \n",
    "                        continue\n",
    "                    else:  \n",
    "                        # check if a webpage is of a type html, htm, javascript, json or xml. Drops it if is not\n",
    "                        try:\n",
    "                            validPage,page_type = checkWebpageType(webpage2)     # Does the page's content in html, htm, javascript, json or xml format?\n",
    "                        except KeyError as e:    # raise an error when ..........\n",
    "                            print('Error 3: ',e,'------',URL)\n",
    "                            continue\n",
    "                        if validPage == 'False':\n",
    "                            print('Error 4: ','------',URL)\n",
    "                            continue\n",
    "                        elif validPage == 'True':\n",
    "                            # check if the page has a form & input tag\n",
    "                            formChecker,inputChecker,countText,countEmail,countTel,countDate,countMonth,countPass,noPrompts,noWindowPrompts,noPopupWindow,noJSPrompts = checkFormInputTags(soup)                     \n",
    "                            # decoding contents from their respective encoded formats to a readable format\n",
    "                            try:                          \n",
    "                                content = decodeWebpageContents(webpage2,soup)\n",
    "                                content2 = content\n",
    "                            except UnicodeDecodeError as e:\n",
    "                                print('Error 5: ',e,'------',URL)\n",
    "                                continue\n",
    "                            # clean the contents by removing tags and white spaces\n",
    "                            newContent = removeTags(content)\n",
    "                            newContent = removeWhitespaces(newContent)    \n",
    "                            htmlEncodedJS = 'No'\n",
    "                            # check if the entire webpage is written in JavaScripts. If it is, return it in readable format\n",
    "                            if formChecker == 'No' and inputChecker == 'No' and noJSPrompts == 0:  \n",
    "                                 if 'document.write(unescape(' in content:    # if the whole webpage is encoded with JS\n",
    "                                     decodedContent = decodeJavaScriptContents(soup)\n",
    "                                     content2 = decodedContent\n",
    "                                     soup = BeautifulSoup(decodedContent, 'html.parser')\n",
    "                                     formChecker,inputChecker,countText,countEmail,countTel,countDate,countMonth,countPass,noPrompts,noWindowPrompts,noPopupWindow,noJSPrompts = checkFormInputTags(soup)\n",
    "                                     htmlEncodedJS = 'Yes'\n",
    "                                     decodedContent = removeTags(decodedContent)\n",
    "                                     newContent = removeWhitespaces(decodedContent)\n",
    "                            # tanslate the contents to English if they are not in English\n",
    "                            translator = Translator()\n",
    "                            try:\n",
    "                                language = checkLanguage(soup,translator,newContent)\n",
    "                            except AttributeError as e:    # if the webpage from the above function does not have a html or title tags\n",
    "                                #print('Error 6: ',e,'------',URL)\n",
    "                                language = 'en'\n",
    "                                #continue\n",
    "                                pass\n",
    "                            except ValueError as e:    # error on translator.translate \n",
    "                                print('Error 7: ',e,'------',URL)\n",
    "                                continue \n",
    "                            except requests.exceptions.ConnectionError as e:    # error on translator.translate \n",
    "                                print('Error 8: ',e,'------',URL)\n",
    "                                continue \n",
    "                            if 'en' not in language and 'EN' not in language and language is not None and language != '':  \n",
    "                                try:\n",
    "                                    translatedText = translate(lugha,newContent,translator)\n",
    "                                    newContent = translatedText\n",
    "                                except ValueError as e:   # handle missed translation\n",
    "                                    print('Error 8a: ',e,'------',URL)\n",
    "                                    continue\n",
    "                                except requests.exceptions.ConnectionError as e:   # handle missed translation\n",
    "                                    print('Error 8b: ',e,'------',URL)\n",
    "                                    continue\n",
    "                                except AttributeError as e:   # handle missed translation\n",
    "                                    #print('Error 8c: ',e,'------',URL)\n",
    "                                    #continue\n",
    "                                    pass\n",
    "                            # check for the existence of login related phrases\n",
    "                            allPhrases = getLoginPhrases(cur)\n",
    "                            wordsCount,phraseMatchList = searchLoginWord(allPhrases,newContent)   # words are searched from a content without script texts\n",
    "                            wordsCountScript = 0\n",
    "                            phraseMatchListScript = []\n",
    "                            if soup.findAll('script') and 'document.write(unescape(' not in str(webpage2.content):    # for webpages with script tags but not encoded\n",
    "                                scriptTexts = JavaScriptText(soup)\n",
    "                                wordsCountScript,phraseMatchListScript = searchLoginWord(allPhrases, scriptTexts)   # words are searched from javascript texts   \n",
    "                            # start extracting features from the webpages meeting the conditions\n",
    "                            if (formChecker == 'Yes' and inputChecker == 'Yes'  and (wordsCount >= 0 or wordsCountScript >= 0) and ((countText > 1 or countEmail != 0  or countTel != 0  or countDate != 0  or countMonth != 0 or countPass != 0) or (noJSPrompts != 0 and (wordsCount != 0 or wordsCountScript != 0)))) :\n",
    "                            FQDN,domainIdentity,mainDomain,URL_path,freeDomainUsed,FREEDomain,IPAddressUsed,IPAddress,PortNoUsed,PortNo,ccTLDUsed,ccTLD,gTLDUsed = getDomainIdentity(freeDomains,URL,TLDs)\n",
    "                            IPAddressUsed = check_IP4_in_hostname(hostname) \n",
    "                            # extract the feature related to IP address\n",
    "                            if IPAddressUsed == 'Yes':  \n",
    "                                try:\n",
    "                                    hostname = reverseIPLookup(IPAddress) # This will be the host/sub domain of the web page\n",
    "                                except socket.gaierror as e:   # socket connection error\n",
    "                                    print('Error 10: ',e,'------',URL)\n",
    "                                    continue\n",
    "                                except UnicodeError as e:   \n",
    "                                    print('Error 11: ',e,'------',URL)\n",
    "                                    continue\n",
    "                                except OSError as e:   \n",
    "                                    print('Error 11b: ',e,'------',URL)\n",
    "                                    continue\n",
    "                                if hostname != '':\n",
    "                                    URLPrefix = getURLPrefix(URL)\n",
    "                                    URL = URLPrefix + ':'  + '//' + hostname + '/' + URL_path\n",
    "                                    FQDN,domainIdentity,mainDomain,URL_path,freeDomainUsed,FREEDomain,IPAddressUsed,IPAddress,PortNoUsed,PortNo,ccTLDUsed,ccTLD,gTLDUsed = getDomainIdentity(freeDomains,URL,TLDs)\n",
    "                                else:  \n",
    "                                    print('Hostname not returned! --- ', URL)\n",
    "                                    continue\n",
    "                            # extract the feature related to void and same webpage hyperlink ratio\n",
    "                            try:\n",
    "                                dominantDomain,totalLinks,voidLinks,samePageLinks,hyperlinkList = getHyperlinkDominantDomain(soup,freeDomains,TLDs,mainDomain)\n",
    "                            except IndexError as e: # in strange few webpages, all links may not to be extracted (e.g https://www.bible.com/en-GB/sign-in). We need to skip such webpages\n",
    "                                print('No hyperlinks found --- ', URL)\n",
    "                                continue\n",
    "                            if mainDomain == dominantDomain: \n",
    "                                hyperlinkDomainMatched = 'Yes'\n",
    "                            else:\n",
    "                                hyperlinkDomainMatched = 'No'  \n",
    "                            if totalLinks != 0 :  \n",
    "                                linkRatio = (voidLinks + samePageLinks)/totalLinks\n",
    "                                linkRatio = float(str(round(linkRatio,2)))\n",
    "                                otherLinks = totalLinks - (voidLinks + samePageLinks)\n",
    "                            else:\n",
    "                                linkRatio = -1\n",
    "                                otherLinks = -1                \n",
    "                            hostIPList = getIPAdresses(FQDN) # we avoid collecting URLs which can not be resolved as IPs are part of the features. Subdomains may be hosted in separate servers therefore we need to query using FQDN and not mainDomain    \n",
    "                            if not hostIPList: # IPs list is empty\n",
    "                                print('No returned IP addresses --- ', URL)\n",
    "                                continue\n",
    "                            # extract features related to URL encoded characters\n",
    "                            else:  # IPs of a domain were returned\n",
    "                                redirectionCharacter = checkRedirectionCharacter(URL)  \n",
    "                                domainEncoded,pathEncodedCharacters = checkURLEncoding(FQDN,URL_path) \n",
    "                                #extract feature related to URL's port number\n",
    "                                URLPrefix = getURLPrefix(URL)                                                                \n",
    "                                if PortNoUsed == 'Yes': \n",
    "                                    standardPort = determineStandardPort(URLPrefix,PortNo)\n",
    "                                else:\n",
    "                                    standardPort = 'No'  \n",
    "                                # extract features related to domain identity, and canonical and alternative hyperlinks\n",
    "                                domainIdentityCounts = domainIdentityInWebapge(content2,domainIdentity)   # count number of appearances of a domain identity in the content of a webpage with tags  \n",
    "                                domainInCopyright = domainIdentityInCopyright(soup,domainIdentity)  # check if the domain identity exists in a copyright info \n",
    "                                try:\n",
    "                                    domainInCanonical = canonicalURL(URL,soup,mainDomain,freeDomains,TLDs)  # check if the webpage's URL and canonical URL share the same domain name \n",
    "                                    domainInAlternate = alternateURL(URL,soup,mainDomain,freeDomains,TLDs)  # check if the webpage's URL and alternate URL share the same domain name                            \n",
    "                                except KeyError as e:\n",
    "                                    print('Error in canonical or alternate URL! --- ', URL)\n",
    "                                    continue\n",
    "                                # extract features related to domain positioning, number of dots in the URL, use of obfuscation characters in the URL and URL length\n",
    "                                outPosition = domainOutPosition(URL,FQDN,URL_path,TLDs)  \n",
    "                                noDotsFQDN,noDotsURLPath = numberOfDots(FQDN,URL_path) \n",
    "                                obfuscFQDN,obfuscPath = URLObfuscationCharacters(FQDN,URL_path) \n",
    "                                lengthFQDN,lengthPath,noSlashes = checkURLLength(URL,FQDN,URL_path) \n",
    "                                # extract features related to domain validity and domain age\n",
    "                                try:\n",
    "                                    domainValidity,domainAge = domainWHOISrecords(mainDomain) \n",
    "                                except ConnectionRefusedError as e:\n",
    "                                    print('Connection Refused Error --- ', URL)\n",
    "                                    continue\n",
    "                                except ConnectionResetError as e:\n",
    "                                    print('Connection Reset Error --- ', URL)\n",
    "                                    continue\n",
    "                                except OSError as e:\n",
    "                                    print('OS Error --- ', URL)\n",
    "                                    continue\n",
    "                                formHandlerStatus,FHDomainList = foreignFormHandler(soup,mainDomain,dominantDomain,freeDomains,TLDs)  # One login webpage may have more than one form submissions\n",
    "                                if FHDomainList:   \n",
    "                                    noOfFHdomains = len(FHDomainList)\n",
    "                                    if noOfFHdomains == 1:\n",
    "                                        if FHDomainList[0] != mainDomain:\n",
    "                                            FHDomainValidity,FHDomainAge = domainWHOISrecords(FHDomainList[0]) \n",
    "                                        else:\n",
    "                                            FHDomainValidity = domainValidity\n",
    "                                            FHDomainAge = domainAge\n",
    "                                    elif noOfFHdomains > 1:\n",
    "                                        FHDomainValidityList = []\n",
    "                                        ages = []\n",
    "                                        for FHDomain in FHDomainList:\n",
    "                                            try:\n",
    "                                                FHDomainValidity,FHDomainAge = domainWHOISrecords(FHDomain)\n",
    "                                            except ConnectionResetError as e: \n",
    "                                                continue\n",
    "                                            except ConnectionAbortedError as e: \n",
    "                                                print('ConnectionAbortedError ------ ',URL)\n",
    "                                                continue\n",
    "                                            if FHDomainAge != -1 :\n",
    "                                                FHDomainValidityList.append(FHDomainValidity)\n",
    "                                                ages.append(FHDomainAge)\n",
    "                                            else:\n",
    "                                                continue\n",
    "                                        try:\n",
    "                                            FHDomainValidity = mostCommonItem(FHDomainValidityList)\n",
    "                                            FHDomainAge = min(ages)  # find the least age\n",
    "                                        except IndexError as e:   # if one of the above lists are empty, meaning no FHs were found with whois records (though they might be several FHs)\n",
    "                                            print('Error 12: ',e,'----',URL,'-----',FHDomain,'-----',FHDomainAge)  \n",
    "                                            FHDomainValidity = 'Unknown'\n",
    "                                            FHDomainAge = -1\n",
    "                                            pass\n",
    "                                else:  # if no FHs were found. This is possible if the webpage uses javascript to send data and not a html form\n",
    "                                    FHDomainValidity = 'Unknown'\n",
    "                                    FHDomainAge = -1         \n",
    "                                # extract data related to SSL certificates\n",
    "                                try:                        \n",
    "                                    certificateExist,OID,organizationName,countryName,jurisdictionCountryName,certificateVerificationErrorType = verifySSLCertificate(URL,FQDN,headers)   # FEATURES 23  \n",
    "                                except TimeoutError as e: # error in connecting to the server to load the certificate\n",
    "                                    print('Error 13:a ',e,'------',URL)\n",
    "                                    continue\n",
    "                                try:\n",
    "                                    certificateType = checkSSLCertificateType(certificateExist,OID,organizationName,jurisdictionCountryName,certificateVerificationErrorType)\n",
    "                                except UnboundLocalError as e: \n",
    "                                    print('Error 13b: ',e,'------',URL)\n",
    "                                    continue\n",
    "                                try:\n",
    "                                    countryMatch = verifyRegisteredCountry(hostIPList,ccTLDUsed,ccTLD,gTLDUsed,certificateType,jurisdictionCountryName,countryName) # FEATURES 24 \n",
    "                                except AttributeError as e:  # geoCountry may not be found\n",
    "                                    countryMatch = 'Unknown' \n",
    "                                # extract features related to webpage reputation in search engines\n",
    "                                try:\n",
    "                                    googleResultsReturned,googleResults = googleCustomSearchEngine(URL)\n",
    "                                except googleapiclient.errors.HttpError as e: # error in google search engine\n",
    "                                    print('Error 14a: ',e,'------',URL)\n",
    "                                    continue\n",
    "                                try :\n",
    "                                    bingResultsReturned,bingResults = bingSearch(URL)\n",
    "                                    URLMatch,engine = checkURLSearchEngineReputation(URL,googleResultsReturned,bingResultsReturned,googleResults,bingResults) # FEATURES 25\n",
    "                                except json.decoder.JSONDecodeError as e: # error in bing search engine\n",
    "                                    print('Error 14b: ',e,'------',URL)\n",
    "                                    continue\n",
    "                                domainMatch,FQDNMatch = checkDomainFQDNSearchEngineReputation(mainDomain,FQDN,googleResultsReturned,bingResultsReturned,googleResults,bingResults)\n",
    "                                if FQDN != mainDomain :\n",
    "                                    domainIPList = getIPAdresses(mainDomain) \n",
    "                                else:\n",
    "                                    domainIPList = hostIPList\n",
    "                                FQDNBlacklistedIP,domainBlacklistedIP,FQDNBlacklistIPCounts,domainBlacklistIPCounts = checkBlacklistedIPs(hostIPList,domainIPList,cur) \n",
    "                                 \n",
    "                                # storing data of the extracted features into the database\n",
    "                                URLID = SN\n",
    "                                classLabel = 'Phishing'\n",
    "                                try :\n",
    "                                    recordDataset(cur,conn,URLID,URL,domainIdentityCounts,domainInCopyright,domainInCanonical,domainInAlternate,hyperlinkDomainMatched,linkRatio,formHandlerStatus,IPAddressUsed,domainEncoded,pathEncodedCharacters,redirectionCharacter,outPosition,noDotsFQDN,noDotsURLPath,standardPort,obfuscFQDN,obfuscPath,noSlashes,lengthFQDN,lengthPath,shortURLFound,freeDomainUsed,domainValidity,domainAge,FHDomainValidity,FHDomainAge,certificateType,countryMatch,URLMatch,domainMatch,FQDNMatch,FQDNBlacklistIPCounts,domainBlacklistIPCounts,classLabel)\n",
    "                                except mysql.connector.errors.IntegrityError as e:  # skip duplicated records\n",
    "                                    print(' Duplicate error 1! ---- ', URL)\n",
    "                                    continue \n",
    "                                except mysql.connector.errors.DataError as e:  # skip duplicated records\n",
    "                                    print(' Data too long for a column! ---- ', URL)\n",
    "                                    continue\n",
    "                                # record URL details in a separate database table\n",
    "                                try:\n",
    "                                    recordDetailedURL(cur,conn,URLID,URL,page_type,htmlEncodedJS,countText,countEmail,countTel,countDate,countMonth,countPass,noJSPrompts,lugha,voidLinks,samePageLinks,otherLinks)\n",
    "                                except mysql.connector.errors.IntegrityError as e:  # skip duplicated records\n",
    "                                    print(' Duplicate error 2! ---- ', URL)\n",
    "                                    pass \n",
    "                                except mysql.connector.errors.DatabaseError as e:  # skip duplicated records\n",
    "                                    print(' Database error! ---- ', URL)\n",
    "                                    continue\n",
    "                                \n",
    "                                print(SN,'. ',URL)\n",
    "                                SN = SN + 1\n",
    "                        else:   # if a webpage does not have credentials prompting features or phrases, skip it go to the next one\n",
    "                            print('No form or login phrases! --- ', URL)\n",
    "                            continue   \n",
    "    time_end = time.time()\n",
    "    time_taken = (time_end - time_start) / 3600  \n",
    "    time_taken = round(time_taken, 2)\n",
    "    print ('----------- Recording of ', SN-1,' URLs completed after ', time_taken, ' hrs------------')        \n",
    "    \n",
    "                                                                      \n",
    "urllib3.disable_warnings(InsecureRequestWarning)    # disable all warnings for disabling SSL verification\n",
    "download_PhishTank_data()\n",
    "conn,cur = openDBConnection2()\n",
    "get_phishing_data(conn,cur)\n",
    "closeDBConnection(conn,cur)   \n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}