Dev (#2)

* change to SQL to no longer get all partitions from partition table, only the online and online_active tables. Small change to main thread for better cpu management

* added modified library for azure caller

* Added extra logging to cover errors to attempt calls to azure, added a health check to see if azure can be reached, changed the database calls to only get the last 3 months opf data. including the PasswordHandler.py as this was missed in the include in previous versions, but was in released versions

* changed health check timeout to 10 seconds, removed second call to health check

CloudHandlers/AzureDataCollector.py

@@ -6,20 +6,22 @@
 
 from requests import ReadTimeout
 
-from Config import Configurations
+from Config import Configurations, Persistence
 
 
-def azure_api_call(customer_id, shared_key, log_type, send_list):
+def azure_api_call(customer_id, shared_key, log_type, send_list, offset_time):
     api = client.DataCollectorAPIClient(customer_id, shared_key)
-    try:
-        result = api.post_data(log_type, send_list, timeout=30.0)
-        if result:
-            logging.info(f'azure api response: {result}')
-    except (requests.exceptions.ConnectionError, ReadTimeout) as e:
-        logging.error(f'{e}: error occurred')
+    result = api.post_data(log_type, send_list, timeout=30.0)
+    if result.status_code != 200:
+        logging.info(f'Error azure api response: {result.status_code}')
+    else:
+        logging.info(f'Successful azure api response: {result.status_code}')
+        if offset_time is not None:
+            Persistence().set_date(str(offset_time), 'AzureUpdateDate')
 
 
-def azure_data_collector(json_records):
+
+def azure_data_collector(json_records, offset_time):
     customer_id = Configurations.get_configurations()['AzureCustomerId']
     shared_key = Configurations.get_configurations()['AzureSharedKey']
     log_type = Configurations.get_configurations()['LogName']
@@ -30,8 +32,8 @@ def azure_data_collector(json_records):
 
         if len(numbers_deque) > 50:
             send_list = [numbers_deque.popleft() for _i in range(49)]
-            azure_api_call(customer_id, shared_key, log_type, send_list)
+            azure_api_call(customer_id, shared_key, log_type, send_list, offset_time)
 
         else:
             send_list = ([numbers_deque.popleft() for _i in range(len(numbers_deque))])
-            azure_api_call(customer_id, shared_key, log_type, send_list)
+            azure_api_call(customer_id, shared_key, log_type, send_list, offset_time)

DLPExporter.py

@@ -2,8 +2,12 @@
 import os
 import time
 import pyodbc
+import requests
+from requests import ReadTimeout
+
 import ASFFMapper as Mapper
 import DatabaseConnector
+from datacollectorapiMOD import client
 from logger import LogConfig
 from CloudHandlers.AzureDataCollector import azure_data_collector
 from CloudHandlers.SecurityHubTool import amazon_security_hub, insight_creator, amazon_security_hub_xml, \
@@ -55,12 +59,27 @@ def __init__(self):
 
     def run(self):
         while True:
+            customer_id = Configurations.get_configurations()['AzureCustomerId']
+            shared_key = Configurations.get_configurations()['AzureSharedKey']
+            log_type = Configurations.get_configurations()['LogName']
+            api = client.DataCollectorAPIClient(customer_id, shared_key)
 
-            json_file = Mapper.map_sql_to_azure()
-            if not json_file:
+            try:
+                health = api.health_check(log_type)
+                if health.status_code != 500:
+                    #Do logging for test version
+                    json_file, offset_time = Mapper.map_sql_to_azure()
+                    if not json_file:
+                        #logging.info("No Data received, azure thread is sleeping for 5 minutes before retrying")
+                        time.sleep(300)
+                    elif (len(json_file)) >= 1:
+                        azure_data_collector(json_file, offset_time)
+                else:
+                    logging.error("Azure cannot be reached, azure thread is sleeping for 5 minutes before retrying")
+                    time.sleep(300)
+            except (requests.exceptions.ConnectionError, ReadTimeout) as e:
+                logging.error(f'{e}: error occurred, Azure threading is sleeping for 5 minutes before retrying')
                 time.sleep(300)
-            elif (len(json_file)) >= 1:
-                azure_data_collector(json_file)
 
 
 class AWSAutoCheck(Thread):
@@ -140,6 +159,9 @@ def run(self):
                     logging.info('Database Connection established - Azure thread starting')
 
                     AzureAutoCheck()
+                else:
+                    logging.info('Database Connection cannot be established, please check your installation')
+
             except pyodbc.Error as ex:
                 logging.error(ex.args[1])
         else:
@@ -148,5 +170,4 @@ def run(self):
         logging.info("Ignore if not using Sentinel. Some fields are missing from the config (AzureCustomerId, "
                      "AzureSharedKey)")
 
-    while True:
-        pass
+    os.system("pause")

DatabaseConnector.py

@@ -1,4 +1,6 @@
+from datetime import datetime
 import pyodbc
+from future.backports.datetime import timedelta
 from Config import Persistence, DatabaseConnection
 
 
@@ -17,13 +19,24 @@ def table_fields_by_name(sql):
 
 
 def get_partitions():
-    get_partitions = 'SELECT [PARTITION_INDEX] FROM [wbsn-data-security].[dbo].[PA_EVENT_PARTITION_CATALOG]'
+
+    get_partitions = 'SELECT [PARTITION_INDEX] FROM [wbsn-data-security].[dbo].[PA_EVENT_PARTITION_CATALOG]  where ' \
+                     'STATUS = \'ONLINE_ACTIVE\' or  STATUS = \'ONLINE\''
 
     partitions = execute_sql(get_partitions).fetchall()
     normalised_partitions = []
     for partition in partitions:
         normalised_partitions.append(partition[0])
 
+    iterator = 0
+    for partitions in normalised_partitions:
+        date_string = f'{str(partitions)[0:4]}-{str(partitions)[4:6]}-{str(partitions)[6:]}'
+        date_time_obj = datetime.strptime(date_string, '%Y-%m-%d')
+        now_minus_90 = datetime.now() - timedelta(days=120)
+        if date_time_obj < now_minus_90:
+            normalised_partitions.pop(iterator)
+        iterator += 1
+
     return normalised_partitions
 
 

PasswordEncryption/PasswordHandler.py

@@ -0,0 +1,36 @@
+import base64
+import os
+from cryptography.fernet import Fernet
+
+
+def encrypt_password(pass_phrase):
+    key = base64.urlsafe_b64encode(os.urandom(32))
+    string_key = key.decode("utf-8")
+    cipher_suite = Fernet(string_key)
+    ciphered_text = cipher_suite.encrypt(str.encode(pass_phrase))
+    '''f = open("password.txt", "w")
+    f.write(ciphered_text.decode("utf-8"))
+    f.close()'''
+    return string_key, ciphered_text.decode("utf-8")
+
+
+def get_encrypted_password():
+    f = open("password.txt", "r")
+    return f.read()
+
+
+def decrypt_password(key, encypted_pass_phrase):
+    cipher_suite = Fernet(key)
+    return cipher_suite.decrypt(str.encode(encypted_pass_phrase))
+
+
+'''if get_encrypted_password():
+    pass_phrase = get_encrypted_password()
+    key = input("Please enter your key:\n")
+    print((decrypt_password(key, pass_phrase)).decode("utf-8"))
+else:
+    pass_phrase = input("Please enter your pass phrase:\n")
+    key = save_encrypted_password(pass_phrase)
+    print(f'Please save this key somewhere safe if you need to log in again : {key}')'''
+
+

Resources/install.bat

@@ -14,6 +14,8 @@ echo.
 echo [Log into Database]
 echo [-----------------]
 echo.
+set /P option="Are you using a trusted Database connection in your config? (y/n): "
+if %option%==y goto continue
 set /P option="Is this your first login? (y/n): "
 if %option%==y goto passwordLogin
 if %option%==n goto keyLogin

datacollectorapiMOD/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Yoichi Kawasaki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

datacollectorapiMOD/__init__.py

@@ -0,0 +1,2 @@
+version = '0.1.0'
+#from handlersocket.client import Client, TransportError, RemoteError

datacollectorapiMOD/client.py

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+import requests
+import datetime
+import hashlib
+import hmac
+import base64
+import simplejson as json
+
+"""
+This is Azure Log Analytics Data Collector API Client libraries
+
+Data Collector API can be refered in the following document:
+https://docs.microsoft.com/en-us/azure/log-analytics/log-analytics-data-collector-api
+"""
+
+__author__ = 'Yoichi Kawasaki'
+
+### Global Defines
+_LOG_ANALYTICS_DATA_COLLECTOR_API_VERSION = '2016-04-01'
+
+
+class DataCollectorAPIClient:
+    """
+    Azure Log Analytics Data Collector API Client Class
+    """
+
+    def __init__(self, customer_id, shared_key):
+        self.customer_id = customer_id
+        self.shared_key = shared_key
+
+    # Build the API signature
+    def __signature(self, date, content_length):
+        try:
+            sigs = "POST\n{}\napplication/json\nx-ms-date:{}\n/api/logs".format(
+                str(content_length), date)
+            utf8_sigs = sigs.encode('utf-8')
+            decoded_shared_key = base64.b64decode(self.shared_key)
+            hmac_sha256_sigs = hmac.new(
+                decoded_shared_key, utf8_sigs, digestmod=hashlib.sha256).digest()
+            encoded_hash = base64.b64encode(hmac_sha256_sigs).decode('utf-8')
+            authorization = "SharedKey {}:{}".format(self.customer_id, encoded_hash)
+            return authorization
+        except Exception as e:
+            raise e
+
+    def __rfc1123date(self):
+        return datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
+
+    # Build and send a request to the POST API
+    def post_data(self, log_type, json_records, record_timestamp='', timeout=None):
+        # Check if string contains other than alpha characters
+        if not log_type.isalpha():
+            raise Exception(
+                "ERROR: log_type supports only alpha characters: {}".format(log_type))
+        try:
+            body = json.dumps(json_records)
+            rfc1123date = self.__rfc1123date()
+            content_length = len(body)
+            signature = self.__signature(rfc1123date, content_length)
+            uri = "https://{}.ods.opinsights.azure.com/api/logs?api-version={}".format(
+                self.customer_id, _LOG_ANALYTICS_DATA_COLLECTOR_API_VERSION)
+
+            """
+            time-generated-field
+            The name of a field in the data that contains the timestamp of the data item.
+            If this isn’t specified, the default is the time that the message is ingested.
+            The field format is ISO 8601 format YYYY-MM-DDThh:mm:ssZ
+            """
+            headers = {
+                'content-type': 'application/json',
+                'Authorization': signature,
+                'Log-Type': log_type,
+                'x-ms-date': rfc1123date,
+                'time-generated-field': record_timestamp,
+                'User-Agent': 'SentinelPartner-Forcepoint-DLP/1.8',
+            }
+            return requests.post(uri, data=body, headers=headers, timeout=timeout)
+        except Exception as e:
+            raise
+
+    def health_check(self, log_type, record_timestamp='', timeout=10):
+        try:
+
+            rfc1123date = self.__rfc1123date()
+            content_length = 0
+            signature = self.__signature(rfc1123date, content_length)
+            uri = "https://{}.ods.opinsights.azure.com/api/logs?api-version={}".format(
+                self.customer_id, _LOG_ANALYTICS_DATA_COLLECTOR_API_VERSION)
+
+            """
+            time-generated-field
+            The name of a field in the data that contains the timestamp of the data item.
+            If this isn’t specified, the default is the time that the message is ingested.
+            The field format is ISO 8601 format YYYY-MM-DDThh:mm:ssZ
+            """
+            headers = {
+                'content-type': 'application/json',
+                'Authorization': signature,
+                'Log-Type': log_type,
+                'x-ms-date': rfc1123date,
+                'time-generated-field': record_timestamp,
+                'User-Agent': 'SentinelPartner-Forcepoint-DLP/1.8',
+            }
+
+            return requests.get(uri, headers=headers, timeout=timeout)
+        except Exception as e:
+            raise

datacollectorapiMOD/helper.py

@@ -0,0 +1,2 @@
+def is_success(status_code):
+    return True if (status_code == 200) else False

datacollectorapiMOD/version.py

@@ -0,0 +1 @@
+VERSION = "0.2.0"

makeFile.ps1

@@ -91,12 +91,12 @@ if ($azureInstall -eq $false) {
 }
 
 
-Write-Output [96m Creating fp-dlp-exporter-aws-azure-v1-8.zip[0m
+Write-Output [96m Creating fp-dlp-exporter-aws-azure-v1-8-2.zip[0m
 Write-Output -----------------------------
 Write-Output -
 Write-Output -
 
-Compress-Archive .\fp-dlp-exporter-aws-azure-v1 .\fp-dlp-exporter-aws-azure-v1-8.zip
+Compress-Archive .\fp-dlp-exporter-aws-azure-v1 .\fp-dlp-exporter-aws-azure-v1-8-2.zip
 Write-Output  Clean up[0m
 Write-Output -----------------------------
 Write-Output -

requirements.txt

@@ -1,6 +1,6 @@
-botocore==1.15.32
+botocore==1.20.51
 pyodbc==4.0.28
-boto3==1.12.32
+boto3==1.17.51
 xmltodict==0.12.0
 requests==2.22.0
 simplejson~=3.17.0