HARM: remove type malware

align with RSIT replaced by either 'malware-distribution' or 'infected-system' see #1409
certtools · Mar 24, 2020 · c9e89e7 · c9e89e7
1 parent f24b15f
commit c9e89e7
Show file tree

Hide file tree

Showing 61 changed files with 143 additions and 135 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -38,6 +38,8 @@ Update allowed classification fields to 2020-01-28 version (#1409, #1476). Old n
 - The taxonomy `information gathering` has been rename to `information-gathering`.
 - The taxonomy `malicious code` has been renamed to `malicious-code`.
   - The type `c2server` has been renamed to `c2-server`.
+  - The type `malware` has been integrated into `infected-system` and `malware-distribution`, respectively.
+  - The type `ransomware` has been integrated into `infected-system`.
 - For the taxonomy 'availability', the type `misconfiguration` is new.
 - For the taxonomy 'other', the type `undetermined` is new.
 - For the taxonomy 'vulnerable':

diff --git a/NEWS.md b/NEWS.md
@@ -17,15 +17,17 @@ A few classification scheme has been updated to better match the [Reference Secu
 
 | old taxonomy name | old type name | new taxonomy name | new type name |
 |-|-|-|-|-|-|-|-|
-| abusive content              |                                       | abusive-content               |                                       |
-| information content security |                                       | information-content-security  |                                       |
-| information content security | leak                                  | information-content-security  | data-leak                             |
-| intrusion attempts           |                                       | intrusion-attempts            |                                       |
-| information gathering        |                                       | information-gathering         |                                       |
-| malicious code               |                                       | malicious-code                |                                       |
-| malicious code               | c2server                              | malicious-code                | c2-server                             |
-| vulnerable                   | vulnerable client                     | vulnerable                    | vulnerable-system                     |
-| vulnerable                   | vulnerable service                    | vulnerable                    | vulnerable-system                     |
+| abusive content              |                    | abusive-content              |                                        |
+| information content security |                    | information-content-security |                                        |
+| information content security | leak               | information-content-security | data-leak                              |
+| intrusion attempts           |                    | intrusion-attempts           |                                        |
+| information gathering        |                    | information-gathering        |                                        |
+| malicious code               |                    | malicious-code               |                                        |
+| malicious code               | c2server           | malicious-code               | c2-server                              |
+| malicious code               | malware            | malicious-code               | infected-system / malware-distribution |
+| malicious code               | ransomware         | malicious-code               | infected-system                        |
+| vulnerable                   | vulnerable client  | vulnerable                   | vulnerable-system                      |
+| vulnerable                   | vulnerable service | vulnerable                   | vulnerable-system                      |
 
 - For the taxonomy 'availability', the type `misconfiguration` is new.
 - For the taxonomy 'other', the type `undetermined` is new.
@@ -64,6 +66,18 @@ UPDATE events
    SET "classification.type" = 'vulnerable-system'
    WHERE "classification.taxonomy" = 'vulnerable' AND ("classification.type" = 'vulnerable service' OR "classification.type" = 'vulnerable client');
 ```
+Depending on the data (e.g. feed), the correct statement for the `malware` type deprecation may be either this:
+```sql
+UPDATE events
+   SET "classification.type" = 'infected-system'
+   WHERE "classification.taxonomy" = 'malicious-code' AND ("classification.type" = 'malware' OR "classification.type" = 'ransomware');
+```
+or this:
+```sql
+UPDATE events
+   SET "classification.type" = 'malware-distribution'
+   WHERE "classification.taxonomy" = 'malicious-code' AND ("classification.type" = 'malware' OR "classification.type" = 'ransomware');
+```
 
 2.1.3 Bugfix release (unreleased)
 ---------------------------------

diff --git a/docs/Bots.md b/docs/Bots.md
@@ -2199,7 +2199,7 @@ if source.ip << '192.0.0.0/24' {
     add! comment = 'bogon'
 }
 
-if classification.type == ['phishing', 'malware'] && source.fqdn =~ '.*\.(ch|li)$' {
+if classification.type == ['phishing', 'malware-distribution'] && source.fqdn =~ '.*\.(ch|li)$' {
   add! comment = 'domainabuse'
   keep
 } elif classification.type == 'scanner' {
@@ -2982,7 +2982,7 @@ Multihreading is disabled for this bot.
 
 Consider the following event:
 ```json
-{"raw": "MjAxNi8wNC8yNV8xMTozOSxzY2hpenppbm8ub21hcmF0aG9uLmNvbS9na0NDSnVUSE0vRFBlQ1pFay9XdFZOSERLbC1tWFllRk5Iai8sODUuMjUuMTYwLjExNCxzdGF0aWMtaXAtODUtMjUtMTYwLTExNC5pbmFkZHIuaXAtcG9vbC5jb20uLEFuZ2xlciBFSywtLDg5NzI=", "source": {"asn": 8972, "ip": "85.25.160.114", "url": "http://schizzino.omarathon.com/gkCCJuTHM/DPeCZEk/WtVNHDKl-mXYeFNHj/", "reverse_dns": "static-ip-85-25-160-114.inaddr.ip-pool.com"}, "classification": {"type": "malware"}, "event_description": {"text": "Angler EK"}, "feed": {"url": "http://www.malwaredomainlist.com/updatescsv.php", "name": "Malware Domain List", "accuracy": 100.0}, "time": {"observation": "2016-04-29T10:59:34+00:00", "source": "2016-04-25T11:39:00+00:00"}}
+{"raw": "MjAxNi8wNC8yNV8xMTozOSxzY2hpenppbm8ub21hcmF0aG9uLmNvbS9na0NDSnVUSE0vRFBlQ1pFay9XdFZOSERLbC1tWFllRk5Iai8sODUuMjUuMTYwLjExNCxzdGF0aWMtaXAtODUtMjUtMTYwLTExNC5pbmFkZHIuaXAtcG9vbC5jb20uLEFuZ2xlciBFSywtLDg5NzI=", "source": {"asn": 8972, "ip": "85.25.160.114", "url": "http://schizzino.omarathon.com/gkCCJuTHM/DPeCZEk/WtVNHDKl-mXYeFNHj/", "reverse_dns": "static-ip-85-25-160-114.inaddr.ip-pool.com"}, "classification": {"type": "malware-distribution"}, "event_description": {"text": "Angler EK"}, "feed": {"url": "http://www.malwaredomainlist.com/updatescsv.php", "name": "Malware Domain List", "accuracy": 100.0}, "time": {"observation": "2016-04-29T10:59:34+00:00", "source": "2016-04-25T11:39:00+00:00"}}
 ```
 With the following Parameters:
 
@@ -2996,7 +2996,7 @@ With the following Parameters:
 Resulting line in syslog:
 
 ```
-Apr 29 11:01:29 header example {"raw": "MjAxNi8wNC8yNV8xMTozOSxzY2hpenppbm8ub21hcmF0aG9uLmNvbS9na0NDSnVUSE0vRFBlQ1pFay9XdFZOSERLbC1tWFllRk5Iai8sODUuMjUuMTYwLjExNCxzdGF0aWMtaXAtODUtMjUtMTYwLTExNC5pbmFkZHIuaXAtcG9vbC5jb20uLEFuZ2xlciBFSywtLDg5NzI=", "source": {"asn": 8972, "ip": "85.25.160.114", "url": "http://schizzino.omarathon.com/gkCCJuTHM/DPeCZEk/WtVNHDKl-mXYeFNHj/", "reverse_dns": "static-ip-85-25-160-114.inaddr.ip-pool.com"}, "classification": {"type": "malware"}, "event_description": {"text": "Angler EK"}, "feed": {"url": "http://www.malwaredomainlist.com/updatescsv.php", "name": "Malware Domain List", "accuracy": 100.0}, "time": {"observation": "2016-04-29T10:59:34+00:00", "source": "2016-04-25T11:39:00+00:00"}}
+Apr 29 11:01:29 header example {"raw": "MjAxNi8wNC8yNV8xMTozOSxzY2hpenppbm8ub21hcmF0aG9uLmNvbS9na0NDSnVUSE0vRFBlQ1pFay9XdFZOSERLbC1tWFllRk5Iai8sODUuMjUuMTYwLjExNCxzdGF0aWMtaXAtODUtMjUtMTYwLTExNC5pbmFkZHIuaXAtcG9vbC5jb20uLEFuZ2xlciBFSywtLDg5NzI=", "source": {"asn": 8972, "ip": "85.25.160.114", "url": "http://schizzino.omarathon.com/gkCCJuTHM/DPeCZEk/WtVNHDKl-mXYeFNHj/", "reverse_dns": "static-ip-85-25-160-114.inaddr.ip-pool.com"}, "classification": {"type": "malware-distribution"}, "event_description": {"text": "Angler EK"}, "feed": {"url": "http://www.malwaredomainlist.com/updatescsv.php", "name": "Malware Domain List", "accuracy": 100.0}, "time": {"observation": "2016-04-29T10:59:34+00:00", "source": "2016-04-25T11:39:00+00:00"}}
 ```
 With the following Parameters:
 
@@ -3010,7 +3010,7 @@ With the following Parameters:
 Resulting line in syslog:
 
 ```
-Apr 29 11:17:47 localhost IntelMQ-event|source.ip: 85.25.160.114|time.source:2016-04-25T11:39:00+00:00|feed.url:http://www.malwaredomainlist.com/updatescsv.php|time.observation:2016-04-29T11:17:44+00:00|source.reverse_dns:static-ip-85-25-160-114.inaddr.ip-pool.com|feed.name:Malware Domain List|event_description.text:Angler EK|source.url:http://schizzino.omarathon.com/gkCCJuTHM/DPeCZEk/WtVNHDKl-mXYeFNHj/|source.asn:8972|classification.type:malware|feed.accuracy:100.0
+Apr 29 11:17:47 localhost IntelMQ-event|source.ip: 85.25.160.114|time.source:2016-04-25T11:39:00+00:00|feed.url:http://www.malwaredomainlist.com/updatescsv.php|time.observation:2016-04-29T11:17:44+00:00|source.reverse_dns:static-ip-85-25-160-114.inaddr.ip-pool.com|feed.name:Malware Domain List|event_description.text:Angler EK|source.url:http://schizzino.omarathon.com/gkCCJuTHM/DPeCZEk/WtVNHDKl-mXYeFNHj/|source.asn:8972|classification.type:malware-distribution|feed.accuracy:100.0
 ```
 
 * * *

diff --git a/docs/Data-Harmonization.md b/docs/Data-Harmonization.md
@@ -124,10 +124,8 @@ The taxonomy can be automatically added by the taxonomy expert bot based on the
 |malicious-code|c2-server|This is a command and control server in charge of a given number of botnet drones.|
 |malicious-code|dga domain|DGA Domains are seen various families of malware that are used to periodically generate a large number of domain names that can be used as rendezvous points with their command and control servers. Not in ENISA eCSIRT-II taxonomy.|
 |malicious-code|infected-system|This is a compromised machine, which has been observed to make a connection to a command and control server.|
-|malicious-code|malware|A URL is the most common resource with reference to malware binary distribution. Not in ENISA eCSIRT-II taxonomy.|
 |malicious-code|malware-configuration|This is a resource which updates botnet drones with a new configuration.|
 |malicious-code|malware-distribution|URI used for malware distribution, e.g. a download URL included in fake invoice malware spam.|
-|malicious-code|ransomware|This IOC refers to a specific type of compromised machine, where the computer has been hijacked for ransom by the criminals. Not in ENISA eCSIRT-II taxonomy and deprecated, use 'infected system instead'.|
 |other|blacklist|Some sources provide blacklists, which clearly refer to abusive behavior, such as spamming, but fail to denote the exact reason why a given identity has been blacklisted. The reason may be that the justification is anecdotal or missing entirely. This type should only be used if the typing fits the definition of a blacklist, but an event specific denomination is not possible for one reason or another.|
 |other|other|All incidents which don't fit in one of the given categories should be put into this class.|
 |other|proxy|This refers to the use of proxies from inside your network. Not in ENISA eCSIRT-II taxonomy.|
@@ -156,13 +154,11 @@ Meaning of source, destination and local values for each classification type and
 |dropzone|*server hosting stolen data*||||
 |exploit|*hosting server*||||
 |ids-alert|*triggering device*||||
-|infected system|*infected device*|*contacted c2c server*|||
-|malware|*infected device*||internal at source|zeus, palevo, feodo|
-|malware configuration|*infected device*||||
+|infected-system|*infected device*|*contacted c2c server*|||
+|malware-configuration|*infected device*||||
 |other||||||
 |phishing|*phishing website*||||
 |proxy|*server allowing policy and security bypass*||||
-|ransomware|*infected device*||||
 |scanner|*scanning device*|scanned device||http,modbus,wordpress|
 |spam|*infected device*|targeted server|internal at source||
 |test||||||

diff --git a/docs/Feeds.md b/docs/Feeds.md
@@ -1617,7 +1617,7 @@ server {
 *  * `columns`: `['source.ip|source.url', 'time.source']`
 *  * `default_url_protocol`: `http://`
 *  * `skip_table_head`: `True`
-*  * `type`: `malware`
+*  * `type`: `malware-distribution`
 
 
 # ShadowServer
@@ -2082,7 +2082,7 @@ server {
 *  * `columns`: `['malware.name', 'source.url', 'source.ip', 'time.source']`
 *  * `html_parser`: `lxml`
 *  * `time_format`: `from_format_midnight|%d-%m-%Y`
-*  * `type`: `malware`
+*  * `type`: `malware-distribution`
 
 
 # WebInspektor

diff --git a/docs/Harmonization-fields.md b/docs/Harmonization-fields.md
@@ -168,6 +168,10 @@ These old values are automatically mapped to the new ones:
     'leak' -> 'data-leak'
     'vulnerable client' -> 'vulnerable-system'
     'vulnerable service' -> 'vulnerable-system'
+    'ransomware' -> 'infected-system'
+
+This old values can not be automatically mapped as they are ambiguous:
+    'malware': Either 'infected-system' or 'malware-distribution'
 
 Allowed values are:
  * application-compromise
@@ -191,7 +195,6 @@ Allowed values are:
  * infected-system
  * information-disclosure
  * data-leak
- * malware
  * malware-configuration
  * malware-distribution
  * masquerade
@@ -202,7 +205,6 @@ Allowed values are:
  * potentially-unwanted-accessible
  * privileged-account-compromise
  * proxy
- * ransomware
  * sabotage
  * scanner
  * sniffing

diff --git a/intelmq/bots/experts/idea/expert.py b/intelmq/bots/experts/idea/expert.py
@@ -32,8 +32,6 @@ class IdeaExpertBot(Bot):
         "dropzone": "Information.UnauthorizedAccess",
         "infected-system": "Malware",
         "malware-configuration": "Malware",
-        "ransomware": "Malware",
-        "malware": "Malware",
         "c2-server": "Intrusion.Botnet",
         "exploit": "Attempt.Exploit",
         "brute-force": "Attempt.Login",

diff --git a/intelmq/bots/experts/taxonomy/expert.py b/intelmq/bots/experts/taxonomy/expert.py
@@ -48,10 +48,8 @@
     "c2-server": "malicious-code",
     "dga domain": "malicious-code",  # not in ENISA eCSIRT-II taxonomy
     "infected-system": "malicious-code",
-    "malware": "malicious-code",  # not in ENISA eCSIRT-II taxonomy
     "malware-configuration": "malicious-code",
     "malware-distribution": "malicious-code",
-    "ransomware": "malicious-code",  # not in ENISA eCSIRT-II taxonomy
     "blacklist": "other",  # not in ENISA eCSIRT-II taxonomy
     "other": "other",
     "undetermined": "other",

diff --git a/intelmq/bots/parsers/alienvault/parser.py b/intelmq/bots/parsers/alienvault/parser.py
@@ -5,11 +5,11 @@
 CLASSIFICATION = {
     "c2server": "c2-server",
     "scanning host": "scanner",
-    "malicious host": "malware",
+    "malicious host": "infected-system",
     "spamming": "spam",
-    "malware domain": "malware",
-    "malware ip": "malware",
-    "malware distribution": "malware",
+    "malware domain": "malware-distribution",
+    "malware ip": "malware-distribution",
+    "malware distribution": "malware-distribution",
 }
 
 

diff --git a/intelmq/bots/parsers/anubisnetworks/parser.py b/intelmq/bots/parsers/anubisnetworks/parser.py
@@ -66,7 +66,7 @@ def process(self):
         event = self.new_event(report)
         event.change("feed.url", event["feed.url"].split("?key=")[0])
         event.add("raw", report.get('raw'), sanitize=False)
-        event.add('classification.type', 'malware')
+        event.add('classification.type', 'infected-system')
         event.add('event_description.text', 'Sinkhole attempted connection')
 
         for key, value in raw_report.items():

diff --git a/intelmq/bots/parsers/blueliv/parser_crimeserver.py b/intelmq/bots/parsers/blueliv/parser_crimeserver.py
@@ -9,7 +9,7 @@
 
 TYPES = {
     'PHISHING': 'phishing',
-    'MALWARE': 'malware',
+    'MALWARE': 'malware-distribution',
     'EXPLOIT_KIT': 'exploit',
     'BACKDOOR': 'backdoor',
     'TOR_IP': 'proxy',

diff --git a/intelmq/bots/parsers/cert_eu/parser_csv.py b/intelmq/bots/parsers/cert_eu/parser_csv.py
@@ -30,9 +30,9 @@ class CertEUCSVParserBot(ParserBot):
         "exploit url": "exploit",
         "ids alert": "ids-alert",
         "malware-configuration": "malware-configuration",
-        "malware url": "malware",
+        "malware url": "malware-distribution",
         "phishing": "phishing",
-        "ransomware": "ransomware",
+        "ransomware": "infected-system",
         "scanner": "scanner",
         "spam infrastructure": "spam",
         "test": "test",

diff --git a/intelmq/bots/parsers/cleanmx/parser.py b/intelmq/bots/parsers/cleanmx/parser.py
@@ -76,7 +76,7 @@ def get_mapping_and_type(self, url):
             return PHISHING, 'phishing'
 
         elif 'xmlviruses' in url:
-            return VIRUS, 'malware'
+            return VIRUS, 'malware-distribution'
 
         else:
             raise ValueError('Unknown report.')

diff --git a/intelmq/bots/parsers/cymru/parser_cap_program.py b/intelmq/bots/parsers/cymru/parser_cap_program.py
@@ -96,7 +96,7 @@ def parse_line_old(self, line, report):
         event.add('raw', self.recover_line(line))
         if report_type == 'beagle':  # TODO: verify
             # beagle|192.0.2.1|ASN|YYYY-MM-DD HH:MM:SS|[<GET REQUEST>] [srcport <PORT>]|ASNAME
-            event.add('classification.type', 'malware')
+            event.add('classification.type', 'infected-system')
             event.add('classification.identifier', 'beagle')
             event.add('malware.name', 'beagle')
             if len(comments):
@@ -113,7 +113,7 @@ def parse_line_old(self, line, report):
             # phatbot|192.0.2.1|ASN|YYYY-MM-DD HH:MM:SS||ASNAME
             # sinit|192.0.2.1|ASN|YYYY-MM-DD HH:MM:SS||ASNAME
             # slammer|192.0.2.1|ASN|YYYY-MM-DD HH:MM:SS||ASNAME
-            event.add('classification.type', 'malware')
+            event.add('classification.type', 'infected-system')
             event.add('classification.identifier', report_type)
             event.add('malware.name', report_type)
         elif report_type == 'bots':
@@ -150,7 +150,7 @@ def parse_line_old(self, line, report):
         elif report_type == 'malwareurl':  # TODO: verify
             # malwareurl|192.0.2.1|ASN|YYYY-MM-DD HH:MM:SS|<URL> <SCAN-ID>|ASNAME
             event['source.url'] = comment_split[0]
-            event.add('classification.type', 'malware')
+            event.add('classification.type', 'malware-distribution')
             event.add('classification.identifier', 'malwareurl')
         elif report_type == 'openresolvers':
             # openresolvers|192.0.2.1|ASN|YYYY-MM-DD HH:MM:SS||ASNAME
@@ -199,7 +199,7 @@ def parse_line_old(self, line, report):
             event.add('source.url', comment_split[0])
             if len(comment_split == 2):
                 event.add('malware.hash.md5', comment_split[1])
-            event.add('classification.type', 'malware')
+            event.add('classification.type', 'malware-distribution')
             event.add('classification.identifier', 'spreader')
         elif report_type == 'stormworm':  # TODO: verify
             # stormworm|192.0.2.1|ASN|YYYY-MM-DD HH:MM:SS|confidence:<NUMBER> [legacy|crypto] [srcport <SOURCE PORT>]|ASNAME

diff --git a/intelmq/bots/parsers/dshield/parser_domain.py b/intelmq/bots/parsers/dshield/parser_domain.py
@@ -34,7 +34,7 @@ def process(self):
 
             event = self.new_event(report)
 
-            event.add('classification.type', 'malware')
+            event.add('classification.type', 'malware-distribution')
             event.add('source.fqdn', row.strip())
             event.add('time.source', time)
             event.add("raw", row)

diff --git a/intelmq/bots/parsers/dyn/parser.py b/intelmq/bots/parsers/dyn/parser.py
@@ -35,7 +35,7 @@ def process(self):
 
             event_infected = self.new_event(report)
             event_infected.add('time.source', source_time)
-            event_infected.add('classification.type', 'malware')
+            event_infected.add('classification.type', 'malware-distribution')
             if row_split[0] != '/':
                 event_infected.add('source.ip', row_split[0])
             event_infected.add('source.fqdn', row_split[1])

diff --git a/intelmq/bots/parsers/malc0de/parser.py b/intelmq/bots/parsers/malc0de/parser.py
@@ -30,7 +30,7 @@ def parse_line(self, line, report):
             event = self.new_event(report)
             if self.lastgenerated:
                 event.add('time.source', self.lastgenerated)
-            event.add('classification.type', 'malware')
+            event.add('classification.type', 'malware-distribution')
             event.add('raw', line)
 
             if report['feed.url'] in Malc0deParserBot.WINDOWS_FORMAT:

diff --git a/intelmq/bots/parsers/malwaredomainlist/parser.py b/intelmq/bots/parsers/malwaredomainlist/parser.py
@@ -37,7 +37,7 @@ def parse_line(self, row, report):
 
         event.add("source.asn", int(row[6]))
 
-        event.add('classification.type', 'malware')
+        event.add('classification.type', 'malware-distribution')
         event.add("raw", self.recover_line(row))
         yield event
 

diff --git a/intelmq/bots/parsers/malwaredomains/parser.py b/intelmq/bots/parsers/malwaredomains/parser.py
@@ -38,7 +38,7 @@ def process(self):
                               values[i] + "T00:00:00+00:00", overwrite=True)
                     break
 
-            event.add('classification.type', 'malware')
+            event.add('classification.type', 'malware-distribution')
             event.add('raw', row)
 
             self.send_message(event)

diff --git a/intelmq/bots/parsers/malwarepatrol/parser_dansguardian.py b/intelmq/bots/parsers/malwarepatrol/parser_dansguardian.py
@@ -25,7 +25,7 @@ def parse_line(self, row, report):
                 event.add('source.url', 'http://' + row)
             else:
                 event.add('source.url', row)
-            event.add('classification.type', 'malware')
+            event.add('classification.type', 'malware-distribution')
             event.add('time.source', self.sourcetime)
             event.add("raw", self.recover_line(row))