Collect vulnerabilities from arch linux aboutcode-org#20

Signed-off-by: Ayush Lohani <lohani.ayush01@gmail.com>

vulnerabilities/data_dump.py

@@ -22,6 +22,7 @@
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
 from vulnerabilities.models import ImpactedPackage
+from vulnerabilities.models import ResolvedPackage
 from vulnerabilities.models import Package
 from vulnerabilities.models import Vulnerability
 from vulnerabilities.models import VulnerabilityReference
@@ -68,3 +69,46 @@ def ubuntu_dump(html):
             vulnerability=vulnerability,
             package=package
         )
+
+
+def archlinux_dump(extract_data):
+    """
+    Save data scraped from archlinux' security tracker.
+    Args:
+        extract_data(generator): data collected from archlinux' security tracker
+    """
+    for data in extract_data:
+        print(data)
+        packages_name = data['packages_name']
+        vulnerabilities = data['vulnerability_id']
+        affected_version = data['version'][0]
+        fixed_version = data['version'][1]
+
+        vulnerability = Vulnerability.objects.create(
+            summary=data['description'],
+        )
+
+        for vulnerability_id in vulnerabilities:
+            VulnerabilityReference.objects.create(
+                vulnerability=vulnerability,
+                reference_id=vulnerability_id,
+                source='archlinux',
+            )
+
+        for package_name in packages_name:
+            package_affected = Package.objects.create(
+                name=package_name,
+                version=fixed_version
+            )
+            ImpactedPackage.objects.create(
+                vulnerability=vulnerability,
+                package=package_affected
+            )
+            package_fixed = Package.objects.create(
+                name=package_name,
+                version=affected_version
+            )
+            ResolvedPackage.objects.create(
+                vulnerability=vulnerability,
+                package=package_fixed
+            )

vulnerabilities/scraper/archlinux.py

@@ -0,0 +1,68 @@
+#
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+import json
+from urllib.request import urlopen
+
+
+ARCHLINUX_TRACKER_URL = 'https://security.archlinux.org/json'
+
+
+def extract_vulnerabilities(arch_data):
+    """
+    Return a series of mappings for each record of arch linux security tracker
+    data.
+    """
+    for item in arch_data:
+        package_vulnerabilities = []
+        cves = item['issues']
+        advisories = set(item['advisories'])
+        vulnerabilities = cves + list(advisories)
+        vulnerabilities.append(item['name'])
+        packages_name = item['packages']
+
+        if not vulnerabilities or not packages_name:
+            continue
+
+        affected_version = item['affected']
+        fixed_version = item['fixed']
+        if not fixed_version:
+            fixed_version = 'None'
+
+        package_vulnerabilities = {
+            'packages_name': packages_name,
+            'vulnerability_id': vulnerabilities,
+            'description': item['type'],
+            'status': item['status'],
+            'severity': item['severity'],
+            'version': [affected_version,fixed_version]
+        }
+        yield package_vulnerabilities
+
+
+def scrape_vulnerabilities():
+    """
+    Scrape arch linux' security tracker.
+    """
+    json_content = urlopen(ARCHLINUX_TRACKER_URL).read()
+    return extract_vulnerabilities(json.loads(json_content))