Migrate from go-yara to yara-x; improve performance and readability

.github/workflows/go-tests.yaml

@@ -16,31 +16,67 @@ permissions:
 
 jobs:
   test:
+    if: ${{ github.repository }} == 'chainguard-dev/malcontent'
     runs-on: mal-ubuntu-latest-8-core
-
+    container:
+      image: cgr.dev/chainguard/wolfi-base@sha256:eeb70e74e2ac07d3c80a30150bf473970c8b51a57f06daef3e4d065ac52489bc
+      options: >-
+        --cap-add DAC_OVERRIDE
+        --cap-add SETGID
+        --cap-add SETUID
+        --cap-drop ALL
+        --cgroupns private
+        --cpu-shares=8192
+        --memory-swappiness=0
+        --security-opt no-new-privileges
+        --ulimit core=0
+        --ulimit nofile=1024:1024
+        --ulimit nproc=4096:4096
     steps:
-      - uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
-        with:
-          egress-policy: audit
+      - name: Install dependencies
+        run: |
+          apk update
+          apk add curl findutils git go nodejs upx xz yara-x-compat
 
       - name: Checkout code
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-      - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
-        with:
-          go-version-file: go.mod
-          check-latest: true
-
-      - uses: chainguard-dev/actions/goimports@9d943fc9889a0c0795e3c2bd4b949a9b610ac02e # main
-
-      - name: install dependencies
-        run: |
-          sudo apt update && sudo apt install libyara-dev xz-utils -y
+      - name: Trust repository
+        run: git config --global --add safe.directory $GITHUB_WORKSPACE
 
       - name: Unit tests
         run: |
           make test
 
+  integration:
+    if: ${{ github.repository }} == 'chainguard-dev/malcontent'
+    runs-on: mal-ubuntu-latest-8-core
+    container:
+      image: cgr.dev/chainguard/wolfi-base@sha256:eeb70e74e2ac07d3c80a30150bf473970c8b51a57f06daef3e4d065ac52489bc
+      options: >-
+        --cap-add DAC_OVERRIDE
+        --cap-add SETGID
+        --cap-add SETUID
+        --cap-drop ALL
+        --cgroupns private
+        --cpu-shares=8192
+        --memory-swappiness=0
+        --security-opt no-new-privileges
+        --ulimit core=0
+        --ulimit nofile=1024:1024
+        --ulimit nproc=4096:4096
+    steps:
+      - name: Install dependencies
+        run: |
+          apk update
+          apk add curl findutils git go nodejs upx xz yara-x-compat
+
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Trust repository
+        run: git config --global --add safe.directory $GITHUB_WORKSPACE
+
       - name: Integration tests
         run: |
           make integration

.github/workflows/style.yaml

@@ -114,29 +114,33 @@ jobs:
 
   golangci-lint:
     name: golangci-lint
-    runs-on: ubuntu-24.04
-
+    runs-on: ubuntu-latest
+    container:
+      image: cgr.dev/chainguard/wolfi-base@sha256:eeb70e74e2ac07d3c80a30150bf473970c8b51a57f06daef3e4d065ac52489bc
+      options: >-
+        --cap-add DAC_OVERRIDE
+        --cap-add SETGID
+        --cap-add SETUID
+        --cap-drop ALL
+        --cgroupns private
+        --cpu-shares=4096
+        --memory-swappiness=0
+        --security-opt no-new-privileges
+        --ulimit core=0
+        --ulimit nofile=1024:1024
+        --ulimit nproc=4096:4096
     steps:
-      - name: Harden Runner
-        uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
-        with:
-          egress-policy: audit
+      - name: Install dependencies
+        run: |
+          apk update
+          apk add curl findutils git go nodejs yara-x-compat
 
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-      - name: Set up Go
-        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
-        with:
-          go-version-file: go.mod
-          check-latest: true
+      - name: Trust repository
+        run: git config --global --add safe.directory $GITHUB_WORKSPACE
 
-      - name: install libyara-dev
+      - name: Run golangci-lint
         run: |
-          sudo apt update && sudo apt install libyara-dev -y
-
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
-        with:
-          version: v1.62.0
-          args: --timeout=5m
-
+          make golangci-lint-lint

.github/workflows/third-party.yaml

@@ -12,14 +12,29 @@ jobs:
   update:
     if: ${{ github.repository }} == 'chainguard-dev/malcontent'
     runs-on: mal-ubuntu-latest-8-core
+    container:
+      image: cgr.dev/chainguard/wolfi-base@sha256:eeb70e74e2ac07d3c80a30150bf473970c8b51a57f06daef3e4d065ac52489bc
+      options: >-
+        --cap-add DAC_OVERRIDE
+        --cap-add SETGID
+        --cap-add SETUID
+        --cap-drop ALL
+        --cgroupns private
+        --cpu-shares=8192
+        --memory-swappiness=0
+        --security-opt no-new-privileges
+        --ulimit core=0
+        --ulimit nofile=1024:1024
+        --ulimit nproc=4096:4096
     permissions:
       contents: write
       id-token: write
       pull-requests: write
     steps:
-      - uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f
-        with:
-          egress-policy: audit
+      - name: Install dependencies
+        run: |
+          apk update
+          apk add bash curl findutils gh git go nodejs upx xz yara-x-compat
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       - uses: chainguard-dev/actions/setup-gitsign@e82b4e5ae10182af72972addcb3fedf7454621c8
       - name: Set up Octo-STS
@@ -28,9 +43,7 @@ jobs:
         with:
           scope: chainguard-dev/malcontent
           identity: third-party
-      - name: Install yara and libyara-dev
-        run: |
-          sudo apt update && sudo apt install yara libyara-dev -y
+
       - name: Run make update-third-party
         run: |
           make update-third-party

Makefile

@@ -50,7 +50,7 @@ $(GOLANGCI_LINT_BIN):
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(LINT_ROOT)/out/linters $(GOLANGCI_LINT_VERSION)
 	mv $(LINT_ROOT)/out/linters/golangci-lint $@
 
-YARA_X_VERSION ?= v0.10.0
+YARA_X_VERSION ?= v0.12.0
 YARA_X_BIN := $(LINT_ROOT)/out/linters/yr-$(YARA_X_VERSION)-$(LINT_ARCH)
 $(YARA_X_BIN):
 	mkdir -p $(LINT_ROOT)/out/linters

README.md

@@ -118,16 +118,19 @@ The analyze mode emits a list of capabilities often seen in malware, categorized
 Requirements:
 
 * [go](https://go.dev/) - the programming language
-* [yara](https://virustotal.github.io/yara/) - the rule language
+* [yara-x](https://virustotal.github.io/yara-x/) - the rule language
 * [pkgconf](http://pkgconf.org/) - required by Go to find C dependencies, included in many UNIX distributions
 
-Linux or macOS users can run the following command to install the necessary dependencies, other than Go:
+`yara-x` requires an underlying C API to function. For Wolfi users, this can be installed by running `apk add yara-x-compat`.
 
-```shell
-brew install yara || sudo apt install libyara-dev \
- || sudo dnf install yara-devel || sudo pacman -S yara \
- || sudo zypper install yara
+For other devices, reference the documentation here for installation instructions: https://virustotal.github.io/yara-x/docs/api/c/c-/#building-the-c-library
+
+Running `cargo cinstall -p yara-x-capi --release` may encounter permission denied errors. If this is the case, run the following:
 ```
+cargo cinstall -p yara-x-capi --release --prefix=/tmp/yara-x
+```
+
+And then copy the resulting files from `/tmp/yara-x` to `/usr/local/lib` and `/usr/local/include` (or the preferred `PKG_CONFIG_PATH`/`LD_LIBRARY_PATH` location(s)).
 
 Install malcontent:
 

cmd/mal/mal.go

@@ -239,8 +239,26 @@ func main() {
 				rfs = append(rfs, thirdparty.FS)
 			}
 
+			yrs, err := action.CachedRules(ctx, rfs)
+			if err != nil {
+				returnCode = ExitInvalidRules
+			}
+
+			concurrency := concurrencyFlag
+			if concurrency < 1 {
+				concurrency = 1
+			}
+
+			var pool *malcontent.ScannerPool
+			if mc.ScannerPool == nil {
+				pool, err = malcontent.NewScannerPool(yrs, concurrency)
+				if err != nil {
+					returnCode = ExitInvalidRules
+				}
+			}
+
 			mc = malcontent.Config{
-				Concurrency:           concurrencyFlag,
+				Concurrency:           concurrency,
 				ExitFirstHit:          exitFirstHitFlag,
 				ExitFirstMiss:         exitFirstMissFlag,
 				IgnoreSelf:            ignoreSelfFlag,
@@ -251,8 +269,9 @@ func main() {
 				OCI:                   ociFlag,
 				QuantityIncreasesRisk: quantityIncreasesRiskFlag,
 				Renderer:              renderer,
-				RuleFS:                rfs,
+				Rules:                 yrs,
 				ScanPaths:             scanPaths,
+				ScannerPool:           pool,
 				Stats:                 statsFlag,
 			}
 

go.mod

@@ -1,8 +1,9 @@
 module github.com/chainguard-dev/malcontent
 
-go 1.23.3
+go 1.23.4
 
 require (
+	github.com/VirusTotal/yara-x/go v0.12.0
 	github.com/agext/levenshtein v1.2.3
 	github.com/cavaliergopher/cpio v1.0.1
 	github.com/cavaliergopher/rpm v1.2.0
@@ -15,7 +16,6 @@ require (
 	github.com/gabriel-vasile/mimetype v1.4.8
 	github.com/google/go-cmp v0.6.0
 	github.com/google/go-containerregistry v0.20.2
-	github.com/hillu/go-yara/v4 v4.3.3
 	github.com/klauspost/compress v1.17.11
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/shirou/gopsutil/v4 v4.24.12
@@ -31,21 +31,21 @@ require (
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
-	github.com/charmbracelet/x/ansi v0.4.5 // indirect
+	github.com/charmbracelet/x/ansi v0.6.0 // indirect
 	github.com/charmbracelet/x/term v0.2.1 // indirect
-	github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
-	github.com/docker/cli v27.3.1+incompatible // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
+	github.com/docker/cli v27.4.1+incompatible // indirect
 	github.com/docker/distribution v2.8.3+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.8.2 // indirect
-	github.com/ebitengine/purego v0.8.1 // indirect
+	github.com/ebitengine/purego v0.8.2 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/kr/pretty v0.2.1 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect
-	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mailru/easyjson v0.9.0 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
@@ -65,9 +65,10 @@ require (
 	github.com/vbatts/tar-split v0.11.6 // indirect
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
-	golang.org/x/crypto v0.31.0 // indirect
-	golang.org/x/net v0.33.0 // indirect
+	golang.org/x/crypto v0.32.0 // indirect
+	golang.org/x/net v0.34.0 // indirect
 	golang.org/x/sys v0.29.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
+	google.golang.org/protobuf v1.36.2 // indirect
 	pault.ag/go/topsort v0.1.1 // indirect
 )