Merge pull request #90 from kreatolinux/audit

kpkg: feat: kpkg audit

.github/workflows/build-db.yml

@@ -0,0 +1,43 @@
+name: Build Vulnerability Database
+
+# Controls when the workflow will run
+on:
+  schedule:
+    - cron: '0 0 * * 0'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  ci:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    container: ghcr.io/kreatolinux/builder:latest
+
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Update cache on every commit
+        uses: actions/cache@v3
+        with:
+          path: /var/cache/kpkg/archives
+          key: binary-cache-${{ github.run_id }}
+          restore-keys: |
+            binary-cache
+    
+
+      - name: build audit database (amd64)
+        run: |
+          IS_ACTIONS=y sh $GITHUB_WORKSPACE/scripts/build-ci.sh init
+          export PATH=$PATH:$HOME/.nimble/bin
+          make deps kpkg
+          ./out/kpkg audit --fetch --fetchBinary=false
+        
+      
+      - uses: actions/upload-artifact@v3
+        with:
+          name: vulndb
+          path: /var/cache/kpkg/vulns.db

build.sh

@@ -15,7 +15,7 @@ buildNimMain() {
 	if [ "$1" = "chkupd" ]; then
 		nim c -d:$buildType $args -d:branch=$branch --threads:on -d:ssl -o="$prefix/$1" "$srcdir/$1/$1.nim" || err "building $1 failed"
 	else
- 		nim c -d:$buildType $args --passL:-larchive -d:branch=$branch --threads:on -d:ssl -o="$prefix/$1" "$srcdir/$1/$1.nim" || err "building $1 failed"
+ 		nim c -d:$buildType $args --deepcopy:on --passL:-larchive -d:branch=$branch --threads:on -d:ssl -o="$prefix/$1" "$srcdir/$1/$1.nim" || err "building $1 failed"
    	fi
 }
 
@@ -74,7 +74,7 @@ while [ "$#" -gt 0 ]; do
 done
 
 if [ "$installDeps" = "1" ]; then
-    nimble install fuzzy futhark cligen libsha fusion -y || err "Installing depndencies failed"
+    nimble install fuzzy futhark cligen libsha norm fusion -y || err "Installing depndencies failed"
 fi
 
 IFS=","

kpkg/commands/auditcmd.nim

@@ -0,0 +1,88 @@
+import os
+import strutils
+import times
+import ../modules/cveparser
+import ../modules/runparser
+import ../modules/logger
+import ../modules/downloader
+import ../modules/libarchive
+import norm/[model, sqlite]
+
+proc verChecker*(ver1, ver2: string): bool =
+    # Check version, and return true if ver1 >= ver2.
+    let ver1Split = ver1.split(".")
+    let ver2Split = ver2.split(".")
+    if ver1Split.len == ver2Split.len and ver1Split.len >= 3:
+        for count in 0..ver1Split.len:
+            # MAJOR.MINOR.PATCH
+            if ver1Split[count] >= ver2Split[count]:
+                return true
+
+    return false
+
+proc vulnChecker(package, runfPath: string, dbConn: DbConn, description: bool) =
+    let runf = parseRunfile(runfPath, removeLockfileWhenErr = false)
+    var packageVulns = @[newVulnerability()]
+    dbConn.select(packageVulns, "vulnerability.package = ? LIMIT 10", lastPathPart(runfPath)) # TODO: get name from runFile AUDIT_NAME variable, etc.
+    for vuln in packageVulns:
+        for version in vuln.versionEndExcluding.split("::"):
+            if verChecker(runf.version, version) or runf.version >= version:
+                continue
+            elif version != "false":
+                info "vulnerability found in package '"&lastPathPart(runfPath)&"', "&vuln.cve 
+
+                if description:
+                    echo "\nDescription of '"&vuln.cve&"': \n\n"&vuln.description.strip()&"\n"
+
+                break
+
+
+proc audit*(package: seq[string], description = false, fetch = false, fetchBinary = true) = # TODO: binary = true on release
+    ## Check vulnerabilities in installed packages.
+
+    const dbPath = "/var/cache/kpkg/vulns.db"
+
+    if not fileExists("/var/cache/kpkg/vulns.db") and not fetch:
+        err("Vulnerability database doesn't exist. Please create one using `kpkg audit --fetch`.", false)
+
+    if fetch and not isAdmin():
+        err("you have to be root for this action.", false)
+
+    if fetch:
+        removeFile(dbPath)
+
+    let dbConn = open(dbPath, "", "", "")
+
+    if fetch:
+        removeFile("/var/cache/kpkg/vulns.json")
+        setCurrentDir("/var/cache/kpkg")
+
+        if not fetchBinary:
+            # I use fkie-cad/nvd-json-data-feeds since it makes collecting the data really simple.
+            download("https://github.com/fkie-cad/nvd-json-data-feeds/releases/latest/download/CVE-"&($year(now()))&".json.xz", "/var/cache/kpkg/vulns.json.xz")
+
+            if execShellCmd("xz -d vulns.json.xz") != 0:
+                err("Couldn't extract compressed file, is `xz` installed?", false)
+
+            success("'"&($updateVulns(dbConn, "/var/cache/kpkg/vulns.json", true))&"' vulnerabilities parsed.")
+            removeFile("/var/cache/kpkg/vulns.json.xz")
+            removeFile("/var/cache/kpkg/vulns.json")
+        else:
+            download("https://nightly.link/kreatolinux/src/workflows/build-db/master/vulndb.zip", "/var/cache/kpkg/vulndb.zip")
+            discard extract("/var/cache/kpkg/vulndb.zip")
+            success("Vulnerability database installed.")
+            removeFile("/var/cache/kpkg/vulndb.zip")
+
+        return
+
+
+    if not fileExists("/var/cache/kpkg/vulns.db"):
+        err("Vulnerability database doesn't exist. Please create one using `kpkg audit --fetch`.", false)
+
+    if isEmptyOrWhitespace(package.join("")):
+        for i in walkDirs("/var/cache/kpkg/installed/*"):
+            vulnChecker(lastPathPart(i), i, dbConn, description)
+    else:
+        for i in package:
+            vulnChecker(i, "/var/cache/kpkg/installed/"&i, dbConn, description)
+

kpkg/kpkg.nim

@@ -25,6 +25,7 @@ import commands/upgradecmd
 import commands/installcmd
 import commands/searchcmd
 import commands/cleancmd
+import commands/auditcmd
 import ../common/version
 
 if commitVer != "unavailable":
@@ -94,6 +95,10 @@ dispatchMulti(
     "binaries": "Remove binary tarballs from cache"
   }
   ],
+
+  [
+  audit
+  ],
 
   [
   info, help = {

kpkg/modules/cveparser.nim

@@ -0,0 +1,73 @@
+import os
+import json
+import strutils
+import norm/[model, sqlite]
+import std/options
+
+type vulnerability* = ref object of Model
+    cve*: string
+    description*: string
+    package*: string
+    versionEndExcluding*: string
+
+func newVulnerability*(cve = "", description = "", package = "", versionEndExcluding = ""): vulnerability =
+  vulnerability(cve: cve, description: description, package: package, versionEndExcluding: versionEndExcluding)
+
+proc updateVulns*(dbConn: DbConn, file: string, removeFileAfterwards = false): int =
+    # Parses vulnerabilities. Creates a vulnerability database and returns the count.
+    let json = parseJson(readFile(file))
+
+    var counter: int
+    for i in json["cve_items"]:
+        counter = counter + 1
+
+        var 
+            cve: string
+            description: string
+            package: string
+            versionEndExcluding: seq[string]
+            cpeMatch: JsonNode
+
+        try:
+            cpeMatch = json["cve_items"][counter]["configurations"][0]["nodes"][0]["cpeMatch"]
+        except Exception:
+            continue
+
+        try:
+            package = cpeMatch[0]["criteria"].getStr().split(":")[4]
+        except Exception:
+            continue
+
+        cve = json["cve_items"][counter]["id"].getStr()
+
+        for i in json["cve_items"][counter]["descriptions"]:
+            if i["lang"].getStr() == "en":
+                description = i["value"].getStr()
+
+        for i in 0..cpeMatch.len:
+            try:
+                versionEndExcluding = versionEndExcluding&cpeMatch[i]["versionEndExcluding"].getStr()
+            except Exception:
+                versionEndExcluding = versionEndExcluding&"false"
+
+        var vulnFinal = newVulnerability(cve = cve, description = description, package = package, versionEndExcluding = versionEndExcluding.join("::"))
+        dbConn.createTables(newVulnerability())
+        dbConn.insert(vulnFinal)
+
+    if removeFileAfterwards:
+        removeFile(file)
+
+    return counter
+
+#[let vulns = updateVulns()
+#echo "cveparse: '"&($vulns)&"' vulnerabilities parsed."
+#var customersFoo = @[newVulnerability()]
+dbConn.select(customersFoo, "vulnerability.package = ? LIMIT 10", "linux_kernel")
+
+for i in customersFoo:
+    echo i.cve
+    echo i.package
+    echo i.description
+    echo i.versionEndExcluding
+]#
+

scripts/build-ci.sh

@@ -19,6 +19,9 @@ case $1 in
       		kpkg build ninja -y
     		kpkg build llvm -y # Required by futhark
 
+		# TEMPORARY
+    		kpkg build sqlite -y # Required by kpkg audit
+
 		# temp
       		kpkg build meson -yu || exit 1
 		# end temp