Collect vulnerabilities from arch linux aboutcode-org#20

Signed-off-by: Ayush Lohani <lohani.ayush01@gmail.com>
lohani2280 · Mar 12, 2019 · 6b4ddf1 · 6b4ddf1
1 parent 55a633d
commit 6b4ddf1
Show file tree

Hide file tree

Showing 5 changed files with 350 additions and 0 deletions.
diff --git a/vulnerabilities/data_dump.py b/vulnerabilities/data_dump.py
@@ -68,3 +68,29 @@ def ubuntu_dump(html):
             vulnerability=vulnerability,
             package=package
         )
+
+
+def archlinux_dump(extract_data):
+    """
+    Save data scraped from archlinux' security tracker.
+    Args:
+        extract_data(generator): data collected from archlinux' security tracker
+    """
+    for raw_data in extract_data:
+        for data in raw_data:
+            vulnerability = Vulnerability.objects.create(
+                summary=data['description'],
+            )
+            VulnerabilityReference.objects.create(
+                vulnerability=vulnerability,
+                reference_id=data['vulnerability_id'],
+                source='archlinux',
+            )
+            package = Package.objects.create(
+                name=data['package_name'],
+                version=data['version'],
+            )
+            ImpactedPackage.objects.create(
+                vulnerability=vulnerability,
+                package=package
+            )
diff --git a/vulnerabilities/scraper/archlinux.py b/vulnerabilities/scraper/archlinux.py
@@ -0,0 +1,77 @@
+#
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+import json
+from urllib.request import urlopen
+
+
+ARCHLINUX_TRACKER_URL = 'https://security.archlinux.org/json'
+
+
+def extract_vulnerabilities(arch_data):
+    """
+    Return a sequence of mappings for each existing combination of
+    package and vulnerability from a mapping of arch linux vulnerabilities
+    data.
+    """
+    for item in arch_data:
+        package_vulnerabilities = []
+        vulnerabilities = item['issues']
+        packages_name = item['packages']
+
+        if not vulnerabilities or not packages_name:
+            continue
+
+        for package in packages_name:
+            affected_version = item['affected']
+            fixed_version = item['fixed']
+            for vulnerability in vulnerabilities:
+                package_vulnerabilities.append({
+                    'package_name': package,
+                    'vulnerability_id': vulnerability,
+                    'description': item['type'],
+                    'status': item['status'],
+                    'severity': item['severity'],
+                    'version': affected_version
+                })
+
+            for vulnerability in vulnerabilities:
+                if not fixed_version:
+                    fixed_version = ''
+                package_vulnerabilities.append({
+                    'package_name': package,
+                    'vulnerability_id': vulnerability,
+                    'description': item['type'],
+                    'status': item['status'],
+                    'severity': item['severity'],
+                    'version': fixed_version
+                })
+        yield package_vulnerabilities
+
+
+def scrape_vulnerabilities():
+    """
+    Scrape arch linux' security tracker.
+    """
+    json_content = urlopen(ARCHLINUX_TRACKER_URL).read()
+    return extract_vulnerabilities(json.loads(json_content))
diff --git a/vulnerabilities/tests/test_data/archlinux.json b/vulnerabilities/tests/test_data/archlinux.json
@@ -0,0 +1,44 @@
+[
+  {
+    "name": "AVG-837",
+    "packages": [
+      "libarchive"
+    ],
+    "status": "Vulnerable",
+    "severity": "High",
+    "type": "multiple issues",
+    "affected": "3.3.3-1",
+    "fixed": "3.3.3-2",
+    "ticket": null,
+    "issues": [
+      "CVE-2019-1000020",
+      "CVE-2019-1000019",
+      "CVE-2018-1000880",
+      "CVE-2018-1000879",
+      "CVE-2018-1000878",
+      "CVE-2018-1000877"
+    ],
+    "advisories": []
+  },
+  {
+    "name": "AVG-57",
+    "packages": [
+      "python2-django",
+      "python-django"
+    ],
+    "status": "Fixed",
+    "severity": "High",
+    "type": "multiple issues",
+    "affected": "1.10.2-1",
+    "fixed": "1.10.3-1",
+    "ticket": null,
+    "issues": [
+      "CVE-2016-9014",
+      "CVE-2016-9013"
+    ],
+    "advisories": [
+      "ASA-201611-15",
+      "ASA-201611-14"
+    ]
+  }
+]
diff --git a/vulnerabilities/tests/test_data_dump.py b/vulnerabilities/tests/test_data_dump.py
@@ -31,8 +31,10 @@
 from vulnerabilities.models import Package
 from vulnerabilities.data_dump import debian_dump
 from vulnerabilities.data_dump import ubuntu_dump
+from vulnerabilities.data_dump import archlinux_dump
 from vulnerabilities.scraper import debian
 from vulnerabilities.scraper import ubuntu
+from vulnerabilities.scraper import archlinux
 
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -85,3 +87,24 @@ def test_ubuntu_data_dump(self):
         reference = VulnerabilityReference.objects.filter(reference_id='CVE-2002-2439')[0]
         self.assertEqual(reference.reference_id, 'CVE-2002-2439')
         self.assertTrue(Package.objects.filter(name='gcc-4.6')[0].name, 'gcc-4.6')
+
+    def test_archlinux_data_dump(self):
+        """
+        Scrape data from Archlinux' main tracker, save it
+        in the database and verify entries.
+        """
+        with open(os.path.join(TEST_DATA, 'archlinux.json')) as f:
+            test_data = json.loads(f.read())
+
+        extract_data = archlinux.extract_vulnerabilities(test_data)
+        archlinux_dump(extract_data)
+
+        self.assertEqual(20, Vulnerability.objects.count())
+        self.assertEqual(20, VulnerabilityReference.objects.count())
+        self.assertEqual(20, Package.objects.count())
+
+        self.assertTrue(Vulnerability.objects.filter(
+                        summary='multiple issues'))
+
+        self.assertEqual(Package.objects.filter(name='libarchive')[0].name, 'libarchive')
+        self.assertEqual(Package.objects.filter(name='python-django')[0].name, 'python-django')
diff --git a/vulnerabilities/tests/test_scrapers.py b/vulnerabilities/tests/test_scrapers.py
@@ -27,6 +27,7 @@
 
 from vulnerabilities.scraper import debian
 from vulnerabilities.scraper import ubuntu
+from vulnerabilities.scraper import archlinux
 
 
 def test_ubuntu_extract_cves():
@@ -93,3 +94,182 @@ def test_debian_extract_vulnerabilities():
     ]
 
     assert expected == debian.extract_vulnerabilities(test_data)
+
+
+def test_archlinux_extract_vulnerabilities():
+    archlinux_test_file = join(dirname(__file__), 'test_data', 'archlinux.json')
+
+    with open(archlinux_test_file) as f:
+        test_data = json.loads(f.read())
+
+    G = archlinux.extract_vulnerabilities(test_data)
+
+    expected = [
+        {
+            'package_name': 'libarchive',
+            'vulnerability_id': 'CVE-2019-1000020',
+            'description': 'multiple issues',
+            'status': 'Vulnerable',
+            'severity': 'High',
+            'version': '3.3.3-1'
+        },
+        {
+            'package_name': 'libarchive',
+            'vulnerability_id': 'CVE-2019-1000019',
+            'description': 'multiple issues',
+            'status': 'Vulnerable',
+            'severity': 'High',
+            'version': '3.3.3-1'
+        },
+        {
+            'package_name': 'libarchive',
+            'vulnerability_id': 'CVE-2018-1000880',
+            'description': 'multiple issues',
+            'status': 'Vulnerable',
+            'severity': 'High',
+            'version': '3.3.3-1'
+        },
+        {
+            'package_name': 'libarchive',
+            'vulnerability_id': 'CVE-2018-1000879',
+            'description': 'multiple issues',
+            'status': 'Vulnerable',
+            'severity': 'High',
+            'version': '3.3.3-1'
+        },
+        {
+            'package_name': 'libarchive',
+            'vulnerability_id': 'CVE-2018-1000878',
+            'description': 'multiple issues',
+            'status': 'Vulnerable',
+            'severity': 'High',
+            'version': '3.3.3-1'
+        },
+        {
+            'package_name': 'libarchive',
+            'vulnerability_id': 'CVE-2018-1000877',
+            'description': 'multiple issues',
+            'status': 'Vulnerable',
+            'severity': 'High',
+            'version': '3.3.3-1'
+        },
+        {
+            'package_name': 'libarchive',
+            'vulnerability_id': 'CVE-2019-1000020',
+            'description': 'multiple issues',
+            'status': 'Vulnerable',
+            'severity': 'High',
+            'version': '3.3.3-2'
+        },
+        {
+            'package_name': 'libarchive',
+            'vulnerability_id': 'CVE-2019-1000019',
+            'description': 'multiple issues',
+            'status': 'Vulnerable',
+            'severity': 'High',
+            'version': '3.3.3-2'
+        },
+        {
+            'package_name': 'libarchive',
+            'vulnerability_id': 'CVE-2018-1000880',
+            'description': 'multiple issues',
+            'status': 'Vulnerable',
+            'severity': 'High',
+            'version': '3.3.3-2'
+        },
+        {
+            'package_name': 'libarchive',
+            'vulnerability_id': 'CVE-2018-1000879',
+            'description': 'multiple issues',
+            'status': 'Vulnerable',
+            'severity': 'High',
+            'version': '3.3.3-2'
+        },
+        {
+            'package_name': 'libarchive',
+            'vulnerability_id': 'CVE-2018-1000878',
+            'description': 'multiple issues',
+            'status': 'Vulnerable',
+            'severity': 'High',
+            'version': '3.3.3-2'
+        },
+        {
+            'package_name': 'libarchive',
+            'vulnerability_id': 'CVE-2018-1000877',
+            'description': 'multiple issues',
+            'status': 'Vulnerable',
+            'severity': 'High',
+            'version': '3.3.3-2'
+        }
+    ]
+
+    assert expected == next(G)
+
+    expected = [
+        {
+            'package_name': 'python2-django',
+            'vulnerability_id': 'CVE-2016-9014',
+            'description': 'multiple issues',
+            'status': 'Fixed',
+            'severity': 'High',
+            'version': '1.10.2-1'
+        },
+        {
+            'package_name': 'python2-django',
+            'vulnerability_id': 'CVE-2016-9013',
+            'description': 'multiple issues',
+            'status': 'Fixed',
+            'severity': 'High',
+            'version': '1.10.2-1'
+        },
+        {
+            'package_name': 'python2-django',
+            'vulnerability_id': 'CVE-2016-9014',
+            'description': 'multiple issues',
+            'status': 'Fixed',
+            'severity': 'High',
+            'version': '1.10.3-1'
+        },
+        {
+            'package_name': 'python2-django',
+            'vulnerability_id': 'CVE-2016-9013',
+            'description': 'multiple issues',
+            'status': 'Fixed',
+            'severity': 'High',
+            'version': '1.10.3-1'
+        },
+        {
+            'package_name': 'python-django',
+            'vulnerability_id': 'CVE-2016-9014',
+            'description': 'multiple issues',
+            'status': 'Fixed',
+            'severity': 'High',
+            'version': '1.10.2-1'
+        },
+        {
+            'package_name': 'python-django',
+            'vulnerability_id': 'CVE-2016-9013',
+            'description': 'multiple issues',
+            'status': 'Fixed',
+            'severity': 'High',
+            'version': '1.10.2-1'
+        },
+        {
+            'package_name': 'python-django',
+            'vulnerability_id': 'CVE-2016-9014',
+            'description': 'multiple issues',
+            'status': 'Fixed',
+            'severity': 'High',
+            'version': '1.10.3-1'
+        },
+        {
+            'package_name': 'python-django',
+            'vulnerability_id': 'CVE-2016-9013',
+            'description': 'multiple issues',
+            'status': 'Fixed',
+            'severity': 'High',
+            'version': '1.10.3-1'
+        }
+    ]
+
+    assert expected == next(G)