Collect vulnerabilities from arch linux aboutcode-org#20

Signed-off-by: Ayush Lohani <lohani.ayush01@gmail.com>

vulnerabilities/data_dump.py

@@ -68,3 +68,25 @@ def ubuntu_dump(html):
             vulnerability=vulnerability,
             package=package
         )
+
+
+def archlinux_dump(extract_data):
+    """
+    Save data scraped from archlinux' security tracker.
+    """
+    for data in extract_data:
+        vulnerability = Vulnerability.objects.create(
+            summary=data.get('description', ''),
+        )
+        VulnerabilityReference.objects.create(
+            vulnerability=vulnerability,
+            reference_id=data.get('vulnerability_id', ''),
+        )
+        package = Package.objects.create(
+            name=data.get('package_name', ''),
+            version=data.get('fixed_version', ''),
+        )
+        ImpactedPackage.objects.create(
+            vulnerability=vulnerability,
+            package=package
+        )

vulnerabilities/scraper/archlinux.py

@@ -0,0 +1,60 @@
+#
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+import json
+from urllib.request import urlopen
+
+
+ARCHLINUX_TRACKER_URL = 'https://security.archlinux.org/json'
+
+
+def extract_vulnerabilities(arch_data):
+    """
+    Return a sequence of mappings for each existing combination of
+    package and vulnerability from a mapping of arch linux vulnerabilities
+    data.
+    """
+    package_vulnerabilities = []
+
+    for item in arch_data:
+        if not item["name"] or not item["packages"][0] or not item["fixed"]:
+            continue
+
+        package_vulnerabilities.append({
+                'package_name': item["packages"][0],
+                'vulnerability_id': item["name"],
+                'description': item["type"],
+                'status': item["status"],
+                'severity': item["severity"],
+                'affected_version': item["affected"],
+                'fixed_version': item["fixed"]
+            })
+    return package_vulnerabilities
+
+
+def scrape_vulnerabilities():
+    """
+    Scrape arch linux' security tracker.
+    """
+    json_content = urlopen(ARCHLINUX_TRACKER_URL).read()
+    return extract_vulnerabilities(json.loads(json_content))

vulnerabilities/tests/test_data/archlinux.json

@@ -0,0 +1,56 @@
+[
+  {
+    "name": "AVG-837",
+    "packages": [
+      "libarchive"
+    ],
+    "status": "Vulnerable",
+    "severity": "High",
+    "type": "multiple issues",
+    "affected": "3.3.3-1",
+    "fixed": "3.3.3-2",
+    "ticket": null,
+    "issues": [
+      "CVE-2019-1000020",
+      "CVE-2019-1000019",
+      "CVE-2018-1000880",
+      "CVE-2018-1000879",
+      "CVE-2018-1000878",
+      "CVE-2018-1000877"
+    ],
+    "advisories": []
+  },
+  {
+    "name": "AVG-886",
+    "packages": [
+      "libtiff"
+    ],
+    "status": "Vulnerable",
+    "severity": "Medium",
+    "type": "multiple issues",
+    "affected": "4.0.10-1",
+    "fixed": "4.0.10-2",
+    "ticket": null,
+    "issues": [
+      "CVE-2019-7663",
+      "CVE-2019-6128"
+    ],
+    "advisories": []
+  },
+  {
+   "name": "AVG-340",
+   "packages": [
+     "ipsec-tools"
+   ],
+   "status": "Vulnerable",
+   "severity": "Medium",
+   "type": "denial of service",
+   "affected": "0.8.2-8",
+   "fixed": "0.8.3-1",
+   "ticket": null,
+   "issues": [
+     "CVE-2016-10396"
+   ],
+   "advisories": []
+  }
+]

vulnerabilities/tests/test_data_dump.py

@@ -33,6 +33,7 @@
 from vulnerabilities.data_dump import ubuntu_dump
 from vulnerabilities.scraper import debian
 from vulnerabilities.scraper import ubuntu
+from vulnerabilities.scraper import archlinux
 
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -85,3 +86,31 @@ def test_ubuntu_data_dump(self):
         reference = VulnerabilityReference.objects.filter(reference_id='CVE-2002-2439')[0]
         self.assertEqual(reference.reference_id, 'CVE-2002-2439')
         self.assertTrue(Package.objects.filter(name='gcc-4.6')[0].name, 'gcc-4.6')
+
+    def test_archlinux_data_dump(self):
+        """
+        Scrape data from Archlinux' main tracker, save it
+        in the database and verify entries.
+        """
+        with open(os.path.join(TEST_DATA, 'archlinux.json')) as f:
+            test_data = json.loads(f.read())
+
+        extract_data = archlinux.extract_vulnerabilities(test_data)
+        debian_dump(extract_data)
+
+        self.assertEqual(3, Vulnerability.objects.count())
+        self.assertEqual(3, VulnerabilityReference.objects.count())
+        self.assertEqual(3, Package.objects.count())
+
+        self.assertTrue(Vulnerability.objects.get(
+                        summary='denial of service'))
+
+        self.assertTrue(VulnerabilityReference.objects.get(reference_id='AVG-837'))
+
+        self.assertTrue(VulnerabilityReference.objects.get(reference_id='AVG-886'))
+
+        self.assertTrue(VulnerabilityReference.objects.get(reference_id='AVG-340'))
+
+        self.assertEqual(Package.objects.filter(name='libarchive')[0].name, 'libarchive')
+        self.assertTrue(Package.objects.get(name='libtiff'))
+        self.assertEqual(Package.objects.filter(version='0.8.3-1')[0].version, '0.8.3-1')

vulnerabilities/tests/test_scrapers.py

@@ -27,6 +27,7 @@
 
 from vulnerabilities.scraper import debian
 from vulnerabilities.scraper import ubuntu
+from vulnerabilities.scraper import archlinux
 
 
 def test_ubuntu_extract_cves():
@@ -93,3 +94,42 @@ def test_debian_extract_vulnerabilities():
     ]
 
     assert expected == debian.extract_vulnerabilities(test_data)
+
+
+def test_archlinux_extract_vulnerabilities():
+    archlinux_test_file = join(dirname(__file__), 'test_data', 'archlinux.json')
+
+    with open(archlinux_test_file) as f:
+        test_data = json.loads(f.read())
+
+    expected = [
+        {
+            'package_name': 'libarchive',
+            'vulnerability_id': 'AVG-837',
+            'description': 'multiple issues',
+            'status': 'Vulnerable',
+            'severity': 'High',
+            'affected_version': '3.3.3-1',
+            'fixed_version': '3.3.3-2'
+        },
+        {
+            'package_name': 'libtiff',
+            'vulnerability_id': 'AVG-886',
+            'description': 'multiple issues',
+            'status': 'Vulnerable',
+            'severity': 'Medium',
+            'affected_version': '4.0.10-1',
+            'fixed_version': '4.0.10-2'
+        },
+        {
+            'package_name': 'ipsec-tools',
+            'vulnerability_id': 'AVG-340',
+            'description': 'denial of service',
+            'status': 'Vulnerable',
+            'severity': 'Medium',
+            'affected_version': '0.8.2-8',
+            'fixed_version': '0.8.3-1'
+        },
+    ]
+
+    assert expected == archlinux.extract_vulnerabilities(test_data)