Add masscan support

grinder/dbhandling.py

@@ -46,6 +46,7 @@ def create_db(self) -> None:
         - scan_information - basic table with all main information about scan
         - scan_data - information about product, vendor, script running and confidence
         - shodan_results/censys_results - for results from backend search engines
+        - masscan_results - for results from masscan scan
         :return: None
         """
         with self.connection as db_connection:
@@ -113,6 +114,23 @@ def create_db(self) -> None:
                 )
                 """
             )
+            db_connection.execute(
+                """
+                CREATE TABLE IF NOT EXISTS
+                masscan_results(
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    scan_data_id INTEGER,
+                    scan_information_id INTEGER,
+                    query TEXT,
+                    query_confidence TEXT,
+                    results_count INTEGER,
+                    results TEXT,
+
+                    FOREIGN KEY (scan_data_id) REFERENCES scan_data(id),
+                    FOREIGN KEY (scan_information_id) REFERENCES scan_information(id)
+                )
+                """
+            )
 
     @exception_handler(expected_exception=GrinderDatabaseInitialScanError)
     def initiate_scan(self, queries_filename: str) -> None:
@@ -321,14 +339,53 @@ def add_censys_scan_data(
                 ),
             )
 
+    @exception_handler(expected_exception=GrinderDatabaseAddScanDataError)
+    def add_masscan_scan_data(
+        self, query: dict, results_count: int, results: dict or list
+    ) -> None:
+        """
+        Add results from masscan for current query
+        :param query: result for current query
+        :param results_count: quantity of results
+        :param results: results itself
+        :return: None
+        """
+        with self.connection as db_connection:
+            db_connection.execute(
+                """
+                INSERT OR REPLACE INTO
+                masscan_results(
+                    scan_data_id,
+                    scan_information_id,
+                    query,
+                    query_confidence,
+                    results_count,
+                    results
+                ) VALUES (
+                    (SELECT max(id) FROM scan_data),
+                    (SELECT max(id) FROM scan_information),
+                    ?,
+                    ?,
+                    ?,
+                    json(?)
+                )
+                """,
+                (
+                    query.get("hosts"),
+                    query.get("query_confidence"),
+                    results_count,
+                    json_dumps(results),
+                ),
+            )
+
     @exception_handler(expected_exception=GrinderDatabaseLoadResultsError)
     def load_last_results(self) -> dict:
         """
         Load latest scan results from database, without scan linking.
-        This function collects last result from censys scan and
-        last result from shodan scan, and combine it together
-        with union select. Needed if you only need to load
-        any last results combination.
+        This function collects last result from censys scan, last
+        result from shodan scan and last result from masscan scan,
+        and combine it together with union select. Needed if you
+        only need to load any last results combination.
         :return: dict with results
         """
         with self.connection as db_connection:
@@ -346,6 +403,12 @@ def load_last_results(self) -> dict:
                     SELECT max(id) FROM scan_information
                     WHERE scan_total_results != 0
                 )
+                UNION SELECT json_extract(results, '$')
+                FROM masscan_results
+                WHERE scan_information_id = (
+                    SELECT max(id) FROM scan_information
+                    WHERE scan_total_results != 0
+                )
                 """
             ).fetchall()
             if not sql_results:
@@ -361,11 +424,11 @@ def load_last_results_by_name(self, engine_table: str, scan_name: str = "") -> d
         """
         Load last results with some particular scan that can be
         passed via 'scan_name' variable. This function returns results
-        only from one backend system (censys, shodan) at time,
+        only from one backend system (censys, shodan, masscan) at time,
         and only the latest _one_.
         If 'scan_name' is not setted, any last result from
         censys or shodan scan will be loaded.
-        :param engine_table: shodan_results, censys_results, etc.
+        :param engine_table: shodan_results, censys_results, masscan_results, etc.
         :param scan_name: name of scanning - "servers", "sd-wans", etc.
         :return: dict with results
         """
@@ -397,10 +460,10 @@ def load_last_results_by_name(self, engine_table: str, scan_name: str = "") -> d
     @exception_handler(expected_exception=GrinderDatabaseLoadResultsError)
     def load_all_results_by_name(self, engine_table: str, scan_name: str = "") -> dict:
         """
-        Load collection of all results from one backend system (censys, shodan).
-        For exampe, you can load all records from Shodan with 'servers' scan,
-        and this function will sort only unique hosts from all of the history
-        of 'servers' scanning
+        Load collection of all results from one backend system (censys, shodan,
+        masscan). For example, you can load all records from Shodan with
+        'servers' scan, and this function will sort only unique hosts from
+        all of the history of 'servers' scanning
         :param engine_table: shodan_results, censys_results, etc.
         :param scan_name: name of scanning - "servers", "sd-wans", etc.
         :return: dict with results
@@ -435,14 +498,15 @@ def load_all_results_by_name(self, engine_table: str, scan_name: str = "") -> di
     def load_multiple_last_results_by_name(self) -> dict:
         """
         Load last results with some 'scan_name' from multiple
-        backend systems (shodan + censys at once). This function
+        backend systems (shodan + censys + masscan at once). This function
         sort all of the host into one dictionary and returns
         unique results from last scan of some 'scan_name'
         :return: dictionary with all results, like "combined" results
         """
         shodan_results = self.load_last_results_by_name(engine_table="shodan_results", scan_name=self.scan_name)
         censys_results = self.load_last_results_by_name(engine_table="censys_results", scan_name=self.scan_name)
-        return {**shodan_results, **censys_results}
+        masscan_results = self.load_last_results_by_name(engine_table="masscan_results", scan_name=self.scan_name)
+        return {**shodan_results, **censys_results, **masscan_results}
 
     @exception_handler(expected_exception=GrinderDatabaseLoadResultsError)
     def load_last_shodan_results(self) -> dict:
@@ -460,6 +524,14 @@ def load_last_censys_results(self) -> dict:
         """
         return self.load_last_results_by_name(engine_table="censys_results")
 
+    @exception_handler(expected_exception=GrinderDatabaseLoadResultsError)
+    def load_last_masscan_results(self) -> dict:
+        """
+        Return latest results from masscan only
+        :return: dict with masscan results
+        """
+        return self.load_last_results_by_name(engine_table="masscan_results")
+
     @exception_handler(expected_exception=GrinderDatabaseLoadResultsError)
     def load_last_shodan_results_by_scan_name(self) -> dict:
         """
@@ -476,6 +548,14 @@ def load_last_censys_results_by_scan_name(self) -> dict:
         """
         return self.load_last_results_by_name(engine_table="censys_results", scan_name=self.scan_name)
 
+    @exception_handler(expected_exception=GrinderDatabaseLoadResultsError)
+    def load_last_masscan_results_by_scan_name(self) -> dict:
+        """
+        Return latest masscan results by some scan name (filename.json)
+        :return: dict with results by name
+        """
+        return self.load_last_results_by_name(engine_table="masscan_results", scan_name=self.scan_name)
+
     @exception_handler(expected_exception=GrinderDatabaseLoadResultsError)
     def load_all_shodan_results_by_scan_name(self) -> dict:
         """
@@ -492,6 +572,14 @@ def load_all_censys_results_by_scan_name(self) -> dict:
         """
         return self.load_all_results_by_name(engine_table="censys_results", scan_name=self.scan_name)
 
+    @exception_handler(expected_exception=GrinderDatabaseLoadResultsError)
+    def load_all_masscan_results_by_scan_name(self) -> dict:
+        """
+        Return all combined masscan results by some scan name (filename.json)
+        :return: dict with results by name
+        """
+        return self.load_all_results_by_name(engine_table="masscan_results", scan_name=self.scan_name)
+
     @exception_handler(expected_exception=GrinderDatabaseCloseError)
     def close(self) -> None:
         """

grinder/defaultvalues.py

@@ -6,6 +6,7 @@ class DefaultValues:
     Values that used almost everywhere,
     most basic default values class
     """
+
     SHODAN_API_KEY: str = "YOUR_DEFAULT_API_KEY"
     CENSYS_API_ID: str = "YOUR_CENSYS_API_ID"
     CENSYS_API_SECRET: str = "YOUR_CENSYS_API_SECRET"
@@ -55,13 +56,15 @@ class DefaultScriptCheckerValues:
     """
     Default values for script scanners
     """
+
     WORKERS = 50
 
 
 class DefaultTlsParserValues:
     """
     Default values for TLS-Parser
     """
+
     PARSED_RESULTS_DIR = "tls_processed_data"
 
     FULL_RESULTS_JSON = "tls_scanner_results.json"
@@ -81,6 +84,7 @@ class DefaultTlsScannerValues:
     """
     Default values for TLS-Scanner
     """
+
     PRODUCT_LIMIT = 50
     LENGTH_OF_HOSTS_SUBGROUPS = 100
     NMAP_PING_SCAN_ARGS = "-n -sP"
@@ -98,6 +102,7 @@ class DefaultVulnersScanValues:
     """
     Default values for Nmap Vulners scan
     """
+
     SUDO = False
     PORTS = None
     TOP_PORTS = None
@@ -106,10 +111,23 @@ class DefaultVulnersScanValues:
     VULNERS_SCRIPT_PATH = "/plugins/vulners.nse"
 
 
+class DefaultMasscanScanValues:
+    """
+    Default values for Masscan scan itself
+    """
+
+    PORTS = "1-1024"
+    TOP_PORTS = None
+    RATE = 1000
+    ARGUMENTS = ""
+    SUDO = True
+
+
 class DefaultNmapScanValues:
     """
     Default values for Nmap scan itself
     """
+
     PORTS = None
     TOP_PORTS = None
     SUDO = False
@@ -122,6 +140,7 @@ class DefaultProcessManagerValues:
     """
     Default values for process manager
     """
+
     PORTS = None
     SUDO = False
     ARGUMENTS = "-Pn -A --open"
@@ -132,6 +151,7 @@ class DefaultPlotValues:
     """
     Default plot values
     """
+
     PLOT_DEFAULT_AUTOPCT = "%1.1f%%"
     PLOT_LABEL_FONT_SIZE = 6
     PLOT_SUPTITLE_FONT_SIZE = 10
@@ -143,4 +163,5 @@ class DefaultDatabaseValues:
     """
     Default database values
     """
+
     DB_NAME = "database.db"

grinder/errors.py

@@ -27,6 +27,19 @@ def __str__(self):
         return f"Error occured in Censys Connector module: {self._error_args}"
 
 
+class MasscanConnectorException(Exception):
+    def __init__(self, error_args: Exception or str):
+        super().__init__(self)
+        self._error_args = error_args
+
+    @property
+    def error_args(self):
+        return self._error_args
+
+    def __str__(self):
+        return f"Error occured in Masscan Connector module: {self._error_args}"
+
+
 class NmapConnectorException(Exception):
     def __init__(self, error_args: Exception or str):
         super().__init__(self)
@@ -175,7 +188,9 @@ def __init__(self, error_args: Exception or str):
         super().__init__(error_args)
 
 
-class PyScriptExecutorOrganizeProcessesError(PyScriptExecutoryProcessingManagerException):
+class PyScriptExecutorOrganizeProcessesError(
+    PyScriptExecutoryProcessingManagerException
+):
     def __init__(self, error_args: Exception or str):
         super().__init__(error_args)
 
@@ -267,6 +282,26 @@ def __init__(self, error_args: Exception or str):
         super().__init__(error_args)
 
 
+class MasscanConnectorInitError(MasscanConnectorException):
+    def __init__(self, error_args: Exception or str):
+        super().__init__(error_args)
+
+
+class MasscanConnectorScanError(MasscanConnectorException):
+    def __init__(self, error_args: Exception or str):
+        super().__init__(error_args)
+
+
+class MasscanConnectorGetResultsCountError(MasscanConnectorException):
+    def __init__(self, error_args: Exception or str):
+        super().__init__(error_args)
+
+
+class MasscanConnectorGetResultsError(MasscanConnectorException):
+    def __init__(self, error_args: Exception or str):
+        super().__init__(error_args)
+
+
 class NmapConnectorInitError(NmapConnectorException):
     def __init__(self, error_args: Exception or str):
         super().__init__(error_args)
@@ -337,6 +372,11 @@ def __init__(self, error_args: Exception or str):
         super().__init__(error_args)
 
 
+class GrinderCoreHostMasscanResultsError(GrinderCoreException):
+    def __init__(self, error_args: Exception or str):
+        super().__init__(error_args)
+
+
 class GrinderCoreUpdateMapMarkersError(GrinderCoreException):
     def __init__(self, error_args: Exception or str):
         super().__init__(error_args)
@@ -447,6 +487,11 @@ def __init__(self, error_args: Exception or str):
         super().__init__(error_args)
 
 
+class GrinderCoreMasscanSaveToDatabaseError(GrinderCoreException):
+    def __init__(self, error_args: Exception or str):
+        super().__init__(error_args)
+
+
 class GrinderCoreSaveResultsToDatabaseError(GrinderCoreException):
     def __init__(self, error_args: Exception or str):
         super().__init__(error_args)