Merge pull request #21 from nbeguier/blacklist

Add BlacklistKeywords feature to ignore some keywords

CertStreamMonitor.py

@@ -24,8 +24,7 @@
 from utils.utils import TimestampNow, VerifyPath
 from utils.sqlite import SqliteCmd
 
-VERSION = "0.5.0"
-
+VERSION = "0.6.0"
 
 def usage():
     """
@@ -51,6 +50,7 @@ def ConfAnalysis(ConfFile):
     global TABLEname
     global LogFile
     global SearchKeywords
+    global BlacklistKeywords
     global DetectionThreshold
     global ACTServer
     global Proxy_Host
@@ -65,6 +65,7 @@ def ConfAnalysis(ConfFile):
         TABLEname = CONF.TABLEname
         LogFile = CONF.LogFile
         SearchKeywords = CONF.SearchKeywords
+        BlacklistKeywords = CONF.BlacklistKeywords
         DetectionThreshold = CONF.DetectionThreshold
         ACTServer = CONF.ACTServer
         Proxy_Host = CONF.Proxy_Host
@@ -119,9 +120,16 @@ def print_callback(message, context):
 
     # look for pattern on *each* hostname
     for host in all_domains:
+        is_blacklisted = False
+        if BlacklistKeywords != str():
+            is_blacklisted = re.findall(BlacklistKeywords, host)
         results = re.findall(SearchKeywords, host)
         FindNb = len(set(results))
 
+        # Matching host whith blacklisted keywords are ignored
+        if is_blacklisted and FindNb >= DetectionThreshold:
+            continue
+
         # If search keywords occurence in the hostname is greater or equal to DetectionThreshold
         # we store the hostname into DB
         if FindNb >= DetectionThreshold:

README.md

@@ -50,6 +50,7 @@ You can find a configuration file example placed into 'conf' directory.
 Configurable parameters are:
 - `SearchKeywords`: Keywords to look for (with '|' (or) as separator)
 - `DetectionThreshold`: set the minimum number of detected SearchKeywords in a hostname before writing it to DB. Under this value but above zero, detected hostnames are only written to logfile. Default value: 2.
+- `BlacklistKeywords`: Keywords to ignore matched hosts (with '|' (or) as separator)
 - `DBFile`: SQLite3 database file (the path and file will be created if don't exist)
 - `TABLEname`: The name of the database table
 - `LogFile`: The logging file (the path and file will be created if don't exist)

conf/example.conf

@@ -12,6 +12,10 @@ SearchKeywords = paypal|apple|account|secure|login
 # Under 2 matching SearchKeywords but above 0, the matching hostname will only be logged to the log file.
 DetectionThreshold = 2
 
+# Keywords to ignore matched hosts (with '|' (or) as separator).
+# Leave it empty or comment it to disable this feature.
+# BlacklistKeywords = mail\.|\.com|bitcoin
+
 [DATABASE]
 # SQLite3 database path (will be created if not exist)
 DBFile = ./db/CertStreamMonitor.sqlite3

utils/confparser.py

@@ -21,6 +21,10 @@ def __init__(self, Confile=None):
 
                 # search strings
                 self.SearchKeywords = self.config['SEARCH']['SearchKeywords']
+                try:
+                    self.BlacklistKeywords = self.config['SEARCH']['BlacklistKeywords']
+                except KeyError:
+                    self.BlacklistKeywords = str()
 
                 # Databases
                 self.DBFile = self.config['DATABASE']['DBFile']