Merge branch 'develop' into arcsight_mapping_update

deployment/ibm_cloud_pak_for_security/Dockerfile

@@ -12,6 +12,7 @@ RUN microdnf update -y && rm -fr /var/cache/yum && \
 
 COPY ./bundle/ /opt/app/
 RUN chown -R 1001 /opt/app/
+RUN python3 -m pip install --upgrade pip
 RUN pip3 install 'cryptography==3.4.7'
 RUN pip3 install 'pyopenssl==21.0.0'
 

stix_shifter/stix_transmission/stix_transmission.py

@@ -39,33 +39,42 @@ def query(self, query):
             ErrorResponder.fill_error(return_obj, error=ex, connector=self.connector)
             return return_obj
 
-    def status(self, search_id):
+    def status(self, search_id, metadata=None):
         # Creates and sends a status query to the correct datasource asking for the status of the specific query
         try:
             if self.init_error:
                 raise self.init_error
-            return self.entry_point.create_status_connection(search_id)
+            if metadata:
+                return self.entry_point.create_status_connection(search_id, metadata)
+            else:
+                return self.entry_point.create_status_connection(search_id)
         except Exception as ex:
             return_obj = dict()
             ErrorResponder.fill_error(return_obj, error=ex, connector=self.connector)
             return return_obj
 
-    def results(self, search_id, offset, length):
+    def results(self, search_id, offset, length, metadata=None):
         # Creates and sends a query to the correct datasource asking for results of the specific query
         try:
             if self.init_error:
                 raise self.init_error
-            return self.entry_point.create_results_connection(search_id, offset, length)
+            if metadata:
+                return self.entry_point.create_results_connection(search_id, offset, length, metadata)
+            else:
+                return self.entry_point.create_results_connection(search_id, offset, length)
         except Exception as ex:
             return_obj = dict()
             ErrorResponder.fill_error(return_obj, error=ex, connector=self.connector)
             return return_obj
 
-    def results_stix(self, search_id, offset, length, data_source):
+    def results_stix(self, search_id, offset, length, data_source, metadata=None):
         try:
             if self.init_error:
                 raise self.init_error
-            return self.entry_point.create_results_stix_connection(search_id, offset, length, data_source)
+            if metadata:
+                return self.entry_point.create_results_stix_connection(search_id, offset, length, data_source, metadata)
+            else:
+                return self.entry_point.create_results_stix_connection(search_id, offset, length, data_source)
         except Exception as ex:
             return_obj = dict()
             ErrorResponder.fill_error(return_obj, error=ex, connector=self.connector)

stix_shifter_modules/mysql/README.md

@@ -1,6 +1,6 @@
 # MySQL Connector
 
-This is a connector for querying a MySQL database. For demo purposes, the [mappings](./stix_translation/json) for this connector are based on the following sample schema for a database named `security_system` with a table named `demo_siem`. The mappings should be changed to fit with the target database schema.
+This is a connector for querying a MySQL database. For demo purposes, the [mappings](./stix_translation/json) for this connector are based on the following sample schema. The mappings should be changed to fit with the target database schema.
 
 |  FIELD  |  DATA TYPE 
 | ---------- | --------- 
@@ -19,4 +19,14 @@ This is a connector for querying a MySQL database. For demo purposes, the [mappi
 | system_name | varchar(100)
 | severity  |  int
 
-STIX-shifter provides a [script](https://github.com/opencybersecurityalliance/stix-shifter/tree/develop/stix_shifter/scripts/mysql_populate_script) to easily create and populate a table on a MySQL database.
+## Running the connector from the CLI
+
+This connector can be called from the STIX-shifter CLI using the `execute` command. The following is an example for a database (`demo_db`) that is running locally with a target table (`demo_table`).
+
+```
+python main.py execute mysql mysql '{"type": "identity","id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff","name": "mysql","identity_class": "system"}' '{"host": "localhost", "database":"demo_db", "options": {"table":"demo_table"}}' '{"auth": {"username":"<DATABASE_USERNAME>", "password":"<DATABASE_PASSWORD>"}}' "[ipv4-addr:value = '213.213.142.5']"
+```
+
+## Sample Table Creation Script
+
+STIX-shifter provides a [script](https://github.com/opencybersecurityalliance/stix-shifter/tree/develop/stix_shifter/scripts/mysql_populate_script) to easily create and populate a table with sample data on an existing database.

stix_shifter_modules/mysql/configuration/config.json

@@ -16,9 +16,6 @@
         "database": {
             "type": "text"
         },
-        "table": {
-            "type": "text"
-        },
         "help": {
             "type": "link",
             "default": "https://www.ibm.com/docs/en/cloud-paks/cp-security/1.6.0?topic=administering-universal-data-insights-connectors"
@@ -38,6 +35,10 @@
         "options": {
             "unmapped_fallback": {
                 "default": true
+            },
+            "table": {
+                "type": "text",
+                "optional": false
             }
         }
     },

stix_shifter_modules/mysql/configuration/lang_en.json

@@ -14,10 +14,6 @@
             "label": "Database",
             "description": "The name of the database"
         },
-        "table": {
-            "label": "Table",
-            "description": "The table this connector will query"
-        },
         "help": {
             "label": "Need additional help?",
             "description": "More details on the data source setting can be found in the specified link"
@@ -29,6 +25,12 @@
         "sni": {
             "label": "Server Name Indicator",
             "description": "The Server Name Indicator (SNI) enables a separate hostname to be provided for SSL authentication"
+        },
+        "options": {
+            "table": {
+                "label": "Table Name",
+                "description": "The name of the MySQL table to be queried by this connector."
+            }
         }
     },
     "configuration": {

stix_shifter_modules/mysql/stix_translation/query_constructor.py

@@ -236,8 +236,6 @@ def translate_pattern(pattern: Pattern, data_model_mapping, options):
     query = re.sub("START", "START ", query)
     query = re.sub("STOP", " STOP ", query)
     table = options.get('table')
-    if not table:
-        table = 'demo_siem'
 
 
     # This sample return statement is in an SQL format. This should be changed to the native data source query language.

stix_shifter_modules/mysql/stix_transmission/api_client.py

@@ -12,7 +12,7 @@ def __init__(self, connection, configuration):
         self.result_limit = connection['options'].get('result_limit')
         self.host = connection.get("host")
         self.database = connection.get("database")
-        self.table = connection.get("table")
+        self.table = connection['options'].get("table")
         self.port = connection.get("port")
         self.auth_plugin = 'mysql_native_password'
 

stix_shifter_modules/mysql/tests/stix_translation/test_mysql_stix_to_query.py

@@ -72,9 +72,9 @@ def _test_mappings(mappings, stix_spec='2.0'):
                 field_count -= 1
             stix_pattern += "]"
             if stix_spec == "2.1":
-                pattern_translation = _translate_query(stix_pattern, {"stix_2.1": True})
+                pattern_translation = _translate_query(stix_pattern, {"table": "demo_table", "stix_2.1": True})
             else:
-                pattern_translation = _translate_query(stix_pattern)
+                pattern_translation = _translate_query(stix_pattern, {"table": "demo_table"})
             assert pattern_translation.get("queries"), "failed to translate {}".format(stix_pattern)
             for field in field_list:
                 _test_query_assertions(field, pattern_translation["queries"])

stix_shifter_modules/proxy/stix_transmission/connector.py

@@ -21,13 +21,13 @@ def create_query_connection(self, query):
         response = self.client.call_api('/create_query_connection', 'POST', data=data, timeout=self.timeout)
         return json.loads(response.bytes)
 
-    def create_results_connection(self, search_id, offset, length):
-        data = json.dumps({"connection": self.connection, "configuration": self.configuration, "search_id": search_id, "offset": offset, "length": length})
+    def create_results_stix_connection(self, entry_point, search_id, offset, length, data_source, metadata=None):
+        data = json.dumps({"connection": self.connection, "configuration": self.configuration, "search_id": search_id, "offset": offset, "length": length, "data_source": data_source, "metadata": metadata})
         response = self.client.call_api('/create_results_connection', 'POST', data=data, timeout=self.timeout)
         return json.loads(response.bytes)
 
-    def create_results_stix_connection(self, entry_point, search_id, offset, length, data_source):
-        data = json.dumps({"connection": self.connection, "configuration": self.configuration, "search_id": search_id, "offset": offset, "length": length, "data_source": data_source})
+    def create_results_stix_connection(self, entry_point, search_id, offset, length, data_source, metadata=None):
+        data = json.dumps({"connection": self.connection, "configuration": self.configuration, "search_id": search_id, "offset": offset, "length": length, "data_source": data_source, "metadata": metadata})
         response = self.client.call_api('/create_results_stix_connection', 'POST', data=data, timeout=self.timeout)
         return json.loads(response.bytes)
 

stix_shifter_modules/synchronous_template/stix_transmission/results_connector.py

@@ -8,8 +8,15 @@ def __init__(self, api_client):
         self.logger = logger.set_logger(__name__)
         self.connector = __name__.split('.')[1]
 
-    def create_results_connection(self, search_id, offset, length):
+    def create_results_connection(self, search_id, offset, length, metadata=None):
         try:
+            #METADATA_SAMPLE metadata_result_id = 0
+            #METADATA_SAMPLE if metadata:
+            #METADATA_SAMPLE     metadata_result_id = int(metadata.get('metadata_result_id', 0))
+            #METADATA_SAMPLE else:
+            #METADATA_SAMPLE     metadata = {}
+            #METADATA_SAMPLE metadata_result_id += 1
+            #METADATA_SAMPLE metadata['metadata_result_id'] = metadata_result_id
             min_range = offset
             max_range = offset + length
             # Grab the response, extract the response code, and convert it to readable json
@@ -23,6 +30,8 @@ def create_results_connection(self, search_id, offset, length):
                 return_obj['data'] = response_dict['data']
             else:
                 ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector)
+            if metadata:
+                return_obj['metadata'] = metadata
             return return_obj
         except Exception as err:
             self.logger.error('error when getting search results: {}'.format(err))

stix_shifter_utils/modules/base/stix_transmission/base_connector.py

@@ -24,7 +24,7 @@ def create_query_connection(self, query):
         """
         raise NotImplementedError()
 
-    def create_status_connection(self, search_id):
+    def create_status_connection(self, search_id, metadata=None):
         """
         Creates a connection to the specified datasource to determine the status of a given query
 
@@ -41,7 +41,7 @@ def create_status_connection(self, search_id):
         """
         raise NotImplementedError()
 
-    def create_results_connection(self, search_id, offset, length):
+    def create_results_connection(self, search_id, offset, length, metadata=None):
         """
         Creates a connection to the specified datasource to retrieve query results
 
@@ -89,14 +89,23 @@ def ping_connection(self):
         """
         raise NotImplementedError()
 
-    def create_results_stix_connection(self, entry_point, search_id, offset, length, data_source):
+    def create_results_stix_connection(self, entry_point, search_id, offset, length, data_source, metadata=None):
         stats = []
-        result = entry_point.create_results_connection(search_id, offset, length)
+        if metadata:
+            result = entry_point.create_results_connection(search_id, offset, length, metadata)
+        else:
+            result = entry_point.create_results_connection(search_id, offset, length)
         stats.append({'action': 'transmission', 'time': int(time.time()*1000)})
+        metadata = None
+        if 'metadata' in result:            
+            metadata = result['metadata']
+            del result['metadata']
         if result.get('success'):
             data = result['data']
             data = data[:length]
             result = entry_point.translate_results(data_source, json.dumps(data))
             stats.append({'action': 'translation', 'time': int(time.time()*1000)})
         result['stats'] = stats
+        if metadata:
+            result['metadata'] = metadata
         return result

stix_shifter_utils/modules/base/stix_transmission/base_results_connector.py

@@ -5,7 +5,7 @@
 
 class BaseResultsConnector(object, metaclass=ABCMeta):
     @abstractmethod
-    def create_results_connection(self, search_id, offset, length):
+    def create_results_connection(self, search_id, offset, length, metadata=None):
         """
         Creates a connection to the specified datasource to retrieve query results
 
@@ -23,14 +23,23 @@ def create_results_connection(self, search_id, offset, length):
         """
         raise NotImplementedError()
 
-    def create_results_stix_connection(self, entry_point, search_id, offset, length, data_source):
+    def create_results_stix_connection(self, entry_point, search_id, offset, length, data_source, metadata=None):
         stats = []
-        result = entry_point.create_results_connection(search_id, offset, length)
+        if metadata:
+            result = entry_point.create_results_connection(search_id, offset, length, metadata)
+        else:
+            result = entry_point.create_results_connection(search_id, offset, length)
+        metadata = None
+        if 'metadata' in result:            
+            metadata = result['metadata']
+            del result['metadata']
         stats.append({'action': 'transmission', 'time': int(time.time()*1000)})
         if result.get('success'):
             data = result['data']
             data = data[:int(length)]
             result = entry_point.translate_results(data_source, json.dumps(data))
             stats.append({'action': 'translation', 'time': int(time.time()*1000)})
         result['stats'] = stats
+        if metadata:
+            result['metadata'] = metadata
         return result

stix_shifter_utils/modules/base/stix_transmission/base_status_connector.py

@@ -12,7 +12,7 @@ class Status(Enum):
 
 class BaseStatusConnector(object, metaclass=ABCMeta):
     @abstractmethod
-    def create_status_connection(self, search_id):
+    def create_status_connection(self, search_id, metadata=None):
         """
         Creates a connection to the specified datasource to determine the status of a given query
 

stix_shifter_utils/utils/base_entry_point.py

@@ -251,7 +251,9 @@ def set_status_connector(self, connector):
         self.__status_connector = connector
 
     @transmission
-    def create_status_connection(self, search_id):
+    def create_status_connection(self, search_id, metadata=None):
+        if metadata:
+            return self.__status_connector.create_status_connection(search_id, metadata)
         return self.__status_connector.create_status_connection(search_id)
 
     def set_results_connector(self, connector):
@@ -260,11 +262,15 @@ def set_results_connector(self, connector):
         self.__results_connector = connector
 
     @transmission
-    def create_results_connection(self, search_id, offset, length):
+    def create_results_connection(self, search_id, offset, length, metadata=None):
+        if metadata:
+            return self.__results_connector.create_results_connection(search_id, offset, length, metadata)
         return self.__results_connector.create_results_connection(search_id, offset, length)
 
     @transmission
-    def create_results_stix_connection(self, search_id, offset, length, data_source):
+    def create_results_stix_connection(self, search_id, offset, length, data_source, metadata=None):
+        if metadata:
+            return self.__results_connector.create_results_stix_connection(self, search_id, offset, length, data_source, metadata) 
         return self.__results_connector.create_results_stix_connection(self, search_id, offset, length, data_source)
 
     def set_delete_connector(self, connector):