CI/CD integration support

API/api.py

@@ -7,23 +7,34 @@
 import json
 import threading
 import logging
+import requests
+import socket
+import urlparse
+import re
+
+from dbconnection import db_connect
+from scanstatus import check_scan_status, scan_status
 
 sys.path.append('../')
 
 from flask import Flask, render_template
 from flask import Response, make_response
 from flask import request
 from flask import Flask
-#from astra import scan_single_api
 from flask import jsonify
 from pymongo import MongoClient
 from pymongo.errors import ServerSelectionTimeoutError
+
+
+
 from utils.vulnerabilities import alerts
+#from utils.sendemail import send_email
 from jinja2 import utils
+from utils.email_cron import send_email_notification
 
 
 if os.getcwd().split('/')[-1] == 'API':
-    from astra import scan_single_api
+    from astra import scan_single_api, scan_postman_collection
 
 
 app = Flask(__name__, template_folder='../Dashboard/templates', static_folder='../Dashboard/static')
@@ -40,26 +51,9 @@ def run(self):
     app.run(host='0.0.0.0', port= 8094)
 
 
-# Mongo DB connection
-maxSevSelDelay = 1
-try:
-    mongo_host = 'localhost'
-    mongo_port = 27017
-
-    if 'MONGO_PORT_27017_TCP_ADDR' in os.environ :
-        mongo_host = os.environ['MONGO_PORT_27017_TCP_ADDR']
-
-    if 'MONGO_PORT_27017_TCP_PORT' in os.environ:
-        mongo_port = int(os.environ['MONGO_PORT_27017_TCP_PORT'])
-
-    client = MongoClient(mongo_host, mongo_port, serverSelectionTimeoutMS=maxSevSelDelay)
-    client.server_info()
-
-except ServerSelectionTimeoutError as err:
-    exit("Failed to connect to MongoDB.")
-
+db_object = db_connect()
 global db
-db = client.apiscan
+db = db_object.apiscan
 
 
 ############################# Start scan API ######################################
@@ -108,20 +102,6 @@ def start_scan():
     return jsonify(msg)
 
 
-#############################  Fetch ScanID API #########################################
-def check_scan_status(data):
-    # Return the Scan status
-    total_scan = data['total_scan']
-    count = 0
-    for key,value in data.items():
-        if value == 'Y' or value == 'y':
-            count += 1
-
-    if total_scan == count:
-        return "Completed"
-    else:
-        return "In progress"
-
 @app.route('/scan/scanids/', methods=['GET'])
 def fetch_scanids():
     scanids = []
@@ -204,7 +184,90 @@ def view_dashboard(page):
 def start_server():
     app.run(host='0.0.0.0', port= 8094)
 
-#if __name__ == "__main__":
+
+############################Postman collection################################
+
+def postman_collection_download(url):
+    # Download postman collection from URL
+    postman_req = requests.get(url,allow_redirects=True, verify=False)
+    try:
+        filename = url[url.rfind("/")+1:]+"_"+generate_hash()
+        open("../Files/"+filename, 'wb').write(postman_req.content)
+        return "../Files/"+filename
+    except:
+        return False
+
+
+def verify_email(email):
+    # credit : www.scottbrady91.com
+    match = re.match('^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,4})$', email)
+    return match
+
+
+@app.route('/scan/postman/', methods = ['POST'])
+def scan_postman():
+    content = request.get_json()
+    try:
+        # mandatory inputs
+        appname = content['appname']
+        postman_url = content['postman_url']
+        env_type = content['env_type']
+        if "email" in content.keys():
+            email_verify_result = verify_email(content['email'])
+            if email_verify_result == None:
+                # Not a valid email id
+                email = "NA"
+            else:
+                email = content['email']
+        else:
+            email = "NA"
+
+        try:
+            # IP address param is optional.
+            url = "NA"
+            if "ip" in content.keys():
+                url = content['ip']
+                if urlparse.urlparse(url).scheme == "http" or urlparse.urlparse(url).scheme == "https":
+                    ip = urlparse.urlparse(url).netloc
+                    socket.inet_aton(ip)
+                    ip_result = 1
+
+            else:
+                ip_result = 0
+        except:
+            print "Missing Arugument or invalid IP address!"
+            ip_result = 0
+
+
+        result = postman_collection_download(postman_url)
+
+        if result is False:
+            msg = {"status" : "Failed to Download Postman collection"}
+            return msg
+        else:
+            try:
+                scan_id = generate_hash()
+                db.scanids.insert({"scanid" : scan_id, "name" : appname, "url" : postman_url,"env_type": env_type, "url" : url,"email" : email})
+                if ip_result == 1:
+                    scan_result = scan_postman_collection(result,scan_id,url)
+                else:
+                    scan_result = scan_postman_collection(result,scan_id)
+            except:
+                #Failed to update the DB
+                pass
+
+            if scan_result == True:
+                 # Update the email notification collection 
+                db.email.insert({"email" : email, "scanid" : scan_id, "to_email" : email, "email_notification" : 'N'})
+                msg = {"status" : "Success", "scanid" : scan_id}
+            else:
+                msg = {"status" : "Failed!"}
+
+
+    except:
+        msg = {"status" "Failed. Application name and postman URL is required!"}
+
+    return jsonify(msg)
 
 def main():
     if os.getcwd().split('/')[-1] == 'API':
@@ -214,4 +277,5 @@ def main():
         thread.daemon = True
         thread.start()
 
-main()
+main()
+

API/dbconnection.py

@@ -0,0 +1,25 @@
+import os
+
+from pymongo import MongoClient
+from pymongo.errors import ServerSelectionTimeoutError
+
+
+# Mongo DB connection
+def db_connect():
+    maxSevSelDelay = 1
+    try:
+        mongo_host = 'localhost'
+        mongo_port = 27017
+
+        if 'MONGO_PORT_27017_TCP_ADDR' in os.environ :
+            mongo_host = os.environ['MONGO_PORT_27017_TCP_ADDR']
+
+        if 'MONGO_PORT_27017_TCP_PORT' in os.environ:
+            mongo_port = int(os.environ['MONGO_PORT_27017_TCP_PORT'])
+
+        client = MongoClient(mongo_host, mongo_port, serverSelectionTimeoutMS=maxSevSelDelay)
+        client.server_info()
+        return client
+
+    except ServerSelectionTimeoutError as err:
+        exit("Failed to connect to MongoDB.")

API/scanstatus.py

@@ -0,0 +1,29 @@
+import ast
+import json
+
+from dbconnection import db_connect
+
+db_object = db_connect()
+global db
+db = db_object.apiscan
+
+def check_scan_status(data):
+    # Return the Scan status
+    total_scan = data['total_scan']
+    count = 0
+    for key,value in data.items():
+        if value == 'Y' or value == 'y':
+            count += 1
+
+    if total_scan == count:
+        return "Completed"
+    else:
+        return "In progress"
+
+def scan_status(scan_id):
+    # Return scan status based on scan id
+    data = db.scanids.find({"scanid": scan_id})
+    data = data[0]
+    data.pop('_id')
+    scan_result = check_scan_status(ast.literal_eval(json.dumps(data)))
+    return scan_result

astra.py

@@ -29,6 +29,7 @@
 from core.zap_config import zap_start
 from multiprocessing import Process
 from utils.db import Database_update
+from utils.email_cron import email_start_cron
 
 
 if os.getcwd().split('/')[-1] != 'API':
@@ -43,6 +44,43 @@ def parse_collection(collection_name,collection_type):
         print "[-]Failed to Parse collection"
         sys.exit(1)
 
+def scan_postman_collection(file_name,scanid,new_url=None):
+    # Read and parse postman collection file
+    try:
+        parse_data = PostmanParser()
+        parse_data.postman_parser(file_name)
+        for data in parse_data.api_lst:
+            try:
+                url = data['url']['raw']
+            except:
+                url = data['url']
+            headers,method,body = data['headers'],data['method'],''
+            if headers:
+                try:
+                    headers = add_headers(headers)
+                except:
+                    pass
+
+            if data['body'] != '':
+                body = json.loads(base64.b64decode(data['body']))
+
+
+            if new_url is not None and new_url is not "NA":
+                uri = url[[m.start() for m in re.finditer('/',url)][2]:]
+                new_url = new_url+uri
+            else:
+                new_url = url
+
+            p = Process(target=modules_scan,args=(new_url,method,headers,body,scanid),name='module-scan')
+            p.start()
+
+        email_start_cron()
+        return True
+
+    except:
+        return False
+
+
 def scan_complete():
     print "[+]Scan has been completed"
     webbrowser.open("http://127.0.0.1:8094/reports.html#"+scanid)
@@ -88,7 +126,6 @@ def read_scan_policy():
 
     except Exception as e:
         print e
-        print "Failed to parse scan property file."
 
     return attack
 
@@ -108,12 +145,15 @@ def modules_scan(url,method,headers,body,scanid=None):
         print "Failed to start scan."
         sys.exit(1)
 
-    if scanid is not None:
-        count = 0
-        for key,value in attack.items():
-            if value == 'Y' or value =='y':
-                count += 1
-        update_scan_status(scanid,"",count)
+    if scanid is None:
+        scanid = generate_scanid()
+
+    count = 0
+    for key,value in attack.items():
+        if value == 'Y' or value =='y':
+            count += 1
+
+    update_scan_status(scanid,"",count)
 
 
     if attack['zap'] == "Y" or attack['zap'] == "y":
@@ -124,37 +164,45 @@ def modules_scan(url,method,headers,body,scanid=None):
 
     # Custom modules scan
     if attack['cors'] == 'Y' or attack['cors'] == 'y':
-        cors_main(url,method,headers,body,scanid)
+        handleException(lambda: cors_main(url,method,headers,body,scanid), "CORS")
         update_scan_status(scanid, "cors")
     if attack['Broken auth'] == 'Y' or attack['Broken auth'] == 'y':
-        auth_check(url,method,headers,body,scanid)
+        handleException(lambda: auth_check(url,method,headers,body,scanid), "Authentication")
         update_scan_status(scanid, "auth")
     if attack['Rate limit'] == 'Y' or attack['Rate limit'] == 'y':
-        rate_limit(url,method,headers,body,scanid)
+        handleException(lambda: rate_limit(url,method,headers,body,scanid), "Rate limit")
         update_scan_status(scanid, "Rate limit")
     if attack['csrf'] == 'Y' or attack['csrf'] == 'y':
-        csrf_check(url,method,headers,body,scanid)
+        handleException(lambda: csrf_check(url,method,headers,body,scanid), "CSRf")
         update_scan_status(scanid, "csrf")
     if attack['jwt'] == 'Y' or attack['jwt'] == 'y':
-        jwt_check(url,method,headers,body,scanid)
+        handleException(lambda: jwt_check(url,method,headers,body,scanid), "JWT")
         update_scan_status(scanid, "jwt")
     if attack['sqli'] == 'Y' or attack['sqli'] == 'y':
-        sqli_check(url,method,headers,body,scanid)
+        handleException(lambda: sqli_check(url,method,headers,body,scanid), "SQL injection")
         update_scan_status(scanid, "sqli")
     if attack['xss'] == 'Y' or attack['xss'] == 'y':
-        xss_check(url,method,headers,body,scanid)
+        handleException(lambda: xss_check(url,method,headers,body,scanid), "XSS")
         update_scan_status(scanid, "xss")
     if attack['open-redirection'] == 'Y' or attack['open-redirection'] == 'y':
-        open_redirect_check(url,method,headers,body,scanid)
+        handleException(lambda: open_redirect_check(url,method,headers,body,scanid), "Open redirect")
         update_scan_status(scanid, "open-redirection")
     if attack['xxe'] == 'Y' or attack['xxe'] == 'y':
         xxe = xxe_scan()
-        xxe.xxe_test(url,method,headers,body,scanid)
+        handleException(lambda: xxe.xxe_test(url,method,headers,body,scanid), "XXE")
         update_scan_status(scanid, "xxe")
     if attack['crlf'] == 'Y' or attack['crlf'] == 'y':
-        crlf_check(url,method,headers,body,scanid)
+        handleException(lambda: crlf_check(url,method,headers,body,scanid), "CRLF")
         update_scan_status(scanid, "crlf") 
 
+
+def handleException(method, module_name):
+    try:
+        #raise Exception("handle exception")
+        method()
+    except Exception:
+        print "exception in", module_name
+
 def validate_data(url,method):
     ''' Validate HTTP request data and return boolean value'''
     validate_url = urlparse.urlparse(url)
@@ -222,7 +270,7 @@ def scan_core(collection_type,collection_name,url,headers,method,body,loginurl,l
             if data['body'] != '':
                 body = json.loads(base64.b64decode(data['body']))
 
-            modules_scan(url,method,headers,body,scanid)        
+            modules_scan(url,method,headers,body,scanid)      
 
     else:
         print "%s [-]Invalid Collection. Please recheck collection Type/Name %s" %(api_logger.G, api_logger.W)