Merge branch 'microsoft:main' into main

.github/workflows/images.yaml

@@ -69,6 +69,7 @@ jobs:
       matrix:
         platform: ["windows"]
         arch: ["amd64"]
+        year: ["2019", "2022"]
 
     steps:
       - name: Checkout code
@@ -101,11 +102,13 @@ jobs:
               IMAGE_NAMESPACE=${{ github.repository }} \
               PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \
               IMAGE_REGISTRY=${{ vars.ACR_NAME }}  \
+              YEAR=${{ matrix.year }} \
               BUILDX_ACTION=--push
           else
             make retina-image-win \
               IMAGE_NAMESPACE=${{ github.repository }} \
-              PLATFORM=${{ matrix.platform }}/${{ matrix.arch }}
+              PLATFORM=${{ matrix.platform }}/${{ matrix.arch }} \
+              WINDOWS_YEARS=${{ matrix.year }} 
           fi
         env:
           IS_MERGE_GROUP: ${{ github.event_name == 'merge_group' }}

.github/workflows/trivy.yaml

@@ -30,7 +30,7 @@ jobs:
           echo "TAG=$(make version)" >> $GITHUB_ENV
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@fd25fed6972e341ff0007ddb61f77e88103953c2
+        uses: aquasecurity/trivy-action@595be6a0f6560a0a8fc419ddf630567fc623531d
         with:
           image-ref: "ghcr.io/${{ github.repository }}/${{ matrix.image }}:${{ env.TAG }}"
           format: "template"

.pipelines/cg-pipeline.yaml

@@ -2,16 +2,14 @@ pr: none
 trigger: none
 
 schedules:
-- cron: '0 0 * * *'
-  displayName: Daily midnight Retina OSS CG pipeline
-  branches:
-    include:
-    - main
-    - vakr/newadocg
-
+  - cron: "0 0 * * *"
+    displayName: Daily midnight Retina OSS CG pipeline
+    branches:
+      include:
+        - main
 stages:
   - stage: setup
-    displayName: Retina Setup
+    displayName: Linux Buildx
     jobs:
       - job: env
         displayName: Setup
@@ -43,4 +41,159 @@ stages:
               fi
             name: "EnvironmentalVariables"
             displayName: "Set environmental variables"
-            condition: always()
+            condition: always()
+
+      - job: retinaagentimages
+        displayName: Build Retina Agent Images
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
+        strategy:
+          matrix:
+            agent-linux-arm:
+              platform: "linux"
+              arch: "arm64"
+              target: "agent"
+
+            agent-linux-amd64:
+              platform: "linux"
+              arch: "amd64"
+              target: "agent"
+
+            init-linux-arm:
+              platform: "linux"
+              arch: "arm64"
+              target: "init"
+
+            init-linux-amd64:
+              platform: "linux"
+              arch: "amd64"
+              target: "init"
+
+        steps:
+          - checkout: self
+            fetchTags: true
+          - script: |
+              set -euo pipefail
+              echo "VERSION=$(make version)"
+              export VERSION=$(make version)
+              mkdir -p ./output/images/$(platform)/$(arch)/$(year)
+              make retina-image \
+                AGENT_TARGETS=$(target) \
+                TAG=$(make version) \
+                IMAGE_NAMESPACE=retina \
+                PLATFORM=$(platform)/$(arch) \
+                IMAGE_REGISTRY=ghcr.io/microsoft  \
+                BUILDX_ACTION="-o type=docker,dest=./output/images/$(platform)/$(arch)/retina-$(target)-$VERSION-$(platform)-$(arch).tar"
+            # make manifest-skopeo-archive IMAGE_ARCHIVE_DIR=$(Build.ArtifactStagingDirectory)/images
+            displayName: "Build Retina Linux Images"
+
+          - bash: |
+              mkdir -p $(Build.ArtifactStagingDirectory)
+              cp -r output/* $(Build.ArtifactStagingDirectory)
+            displayName: "Copy binaries to the binaries artifact folder"
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              artifactName: output
+              pathtoPublish: "$(Build.ArtifactStagingDirectory)"
+            condition: succeeded()
+
+      - job: retinaoperatorimages
+        displayName: Build Retina Operator Images
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
+        strategy:
+          matrix:
+            operator-linux-amd64:
+              platform: "linux"
+              arch: "amd64"
+
+        steps:
+          - checkout: self
+            fetchTags: true
+          - script: |
+              set -euo pipefail
+              echo "VERSION=$(make version)"
+              export VERSION=$(make version)
+              mkdir -p ./output/images/$(platform)/$(arch)/$(year)
+              make retina-operator-image \
+                TAG=$(make version) \
+                IMAGE_NAMESPACE=retina \
+                PLATFORM=$(platform)/$(arch) \
+                IMAGE_REGISTRY=ghcr.io/microsoft  \
+                BUILDX_ACTION="-o type=docker,dest=./output/images/$(platform)/$(arch)/retina-operator-$VERSION-$(platform)-$(arch).tar"
+            displayName: "Build Retina Operator Linux Images"
+
+          - bash: |
+              mkdir -p $(Build.ArtifactStagingDirectory)
+              cp -r output/* $(Build.ArtifactStagingDirectory)
+            displayName: "Copy binaries to the binaries artifact folder"
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              artifactName: output
+              pathtoPublish: "$(Build.ArtifactStagingDirectory)"
+            condition: succeeded()
+
+      # windows 2019 only in buildx, windows server 2022 requires native windows container build because of cgo
+      - job: retinaagentimageswin2019
+        displayName: Build Retina Windows Images (buildx)
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
+        strategy:
+          matrix:
+            windows-ltsc2019:
+              platform: "windows"
+              arch: "amd64"
+              year: "2019"
+
+        steps:
+          - checkout: self
+            fetchTags: true
+          - script: |
+              set -euo pipefail
+              echo "VERSION=$(make version)"
+              export VERSION=$(make version)
+              mkdir -p ./output/images/$(platform)/$(arch)/$(year)
+              make retina-image-win \
+                TARGET=final \
+                WINDOWS_YEARS=$(year) \
+                TAG=$(make version) \
+                BUILDX_ACTION="-o type=docker,dest=./output/images/$(platform)/$(arch)/$(year)/retina-agent-$VERSION-windows-ltsc$(year)-$(arch).tar"
+            displayName: "Build Retina Windows Image"
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              artifactName: output
+              pathtoPublish: ./output
+            condition: succeeded()
+
+      - job: windowsnative
+        displayName: Build Retina Windows Images (native)
+        pool:
+          name: "$(BUILD_POOL_NAME_WINDOWS_DEFAULT)"
+        steps:
+          - checkout: self
+            fetchTags: true
+
+          - task: Docker@2
+            displayName: Docker Login
+            inputs:
+              containerRegistry: $(WINDOWS_BUILDER_REGISTRY)
+              command: "login"
+              addPipelineData: false
+
+          - task: PowerShell@2
+            displayName: "Build Retina Windows Image (LTSC2022)"
+            inputs:
+              targetType: "inline"
+              script: |
+                Import-Module -Name "$(Build.SourcesDirectory)\windows\docker\DockerBuildModule.psm1" -Force
+                Build-RetinaAgentImage -fullBuilderImageName $(WINDOWS_BUILDER_IMAGE) -registry $(BUILD_REGISTRY)
+                Save-Image -imageName retina-agent -registry $(BUILD_REGISTRY)
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              artifactName: output
+              pathtoPublish: ./output
+            condition: succeeded()

Makefile

@@ -45,6 +45,11 @@ ALL_ARCH.windows = amd64
 # while RETINA_PLATFORM_TAG is platform specific, which can be used for image built for specific platforms.
 RETINA_PLATFORM_TAG        ?= $(TAG)-$(subst /,-,$(PLATFORM))
 
+# used for looping through components in container build
+AGENT_TARGETS ?= init agent
+
+WINDOWS_YEARS ?= "2019 2022"
+
 # for windows os, add year to the platform tag
 ifeq ($(OS),windows)
 RETINA_PLATFORM_TAG        = $(TAG)-windows-ltsc$(YEAR)-amd64
@@ -53,8 +58,14 @@ endif
 qemu-user-static: ## Set up the host to run qemu multiplatform container builds.
 	sudo $(CONTAINER_RUNTIME) run --rm --privileged multiarch/qemu-user-static --reset -p yes
 
+.PHONY: version
 version: ## prints the root version
-	@echo $(TAG)
+	@if [ "$(shell git tag --points-at HEAD)" != "" ]; then \
+		export VERSION="$$(git tag --points-at HEAD)"; \
+	else \
+		export VERSION="$$(git rev-parse --short HEAD)"; \
+	fi; \
+	echo "$${VERSION}"
 
 ##@ Help 
 
@@ -192,6 +203,11 @@ retina-skopeo-export:
 		REF=$(IMAGE_REGISTRY)/$(RETINA_IMAGE):$(RETINA_PLATFORM_TAG) \
 		IMG=$(RETINA_IMAGE)
 		TAG=$(RETINA_PLATFORM_TAG)
+
+
+manifest-skopeo-archive: # util target to export tar archive of multiarch container manifest.
+	skopeo copy --all docker://$(IMAGE_REGISTRY)/$(IMAGE):$(TAG) oci-archive:$(IMAGE_ARCHIVE_DIR)/$(IMAGE)-$(TAG).tar --debug
+
 
 buildx:
 	if docker buildx inspect retina > /dev/null 2>&1; then \
@@ -203,6 +219,8 @@ buildx:
 		echo "Buildx instance retina created."; \
 	fi;
 
+
+
 container-docker: buildx # util target to build container images using docker buildx. do not invoke directly.
 	os=$$(echo $(PLATFORM) | cut -d'/' -f1); \
 	arch=$$(echo $(PLATFORM) | cut -d'/' -f2); \
@@ -211,7 +229,6 @@ container-docker: buildx # util target to build container images using docker bu
 	touch $$image_metadata_filename; \
 	echo "Building $$image_name for $$os/$$arch "; \
 	docker buildx build \
-		$(BUILDX_ACTION) \
 		--platform $(PLATFORM) \
 		--metadata-file=$$image_metadata_filename \
 		-f $(DOCKERFILE) \
@@ -222,12 +239,13 @@ container-docker: buildx # util target to build container images using docker bu
 		--build-arg VERSION=$(VERSION) $(EXTRA_BUILD_ARGS) \
 		--target=$(TARGET) \
 		-t $(IMAGE_REGISTRY)/$(IMAGE):$(TAG) \
-		$(CONTEXT_DIR)
+		$(BUILDX_ACTION) \
+		$(CONTEXT_DIR) 
+
 
 retina-image: ## build the retina linux container image.
 	echo "Building for $(PLATFORM)"
-	set -e ; \
-	for target in init agent; do \
+	for target in $(AGENT_TARGETS); do \
 		echo "Building for $$target"; \
 		if [ "$$target" = "init" ]; then \
 			image_name=$(RETINA_INIT_IMAGE); \
@@ -247,7 +265,7 @@ retina-image: ## build the retina linux container image.
 	done
 
 retina-image-win: ## build the retina Windows container image.
-	for year in 2019 2022; do \
+	for year in $(WINDOWS_YEARS); do \
 		tag=$(TAG)-windows-ltsc$$year-amd64; \
 		echo "Building $(RETINA_PLATFORM_TAG)"; \
 		set -e ; \

controller/Dockerfile.windows-2019

@@ -0,0 +1,25 @@
+FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.21 as builder
+# Build args
+ARG VERSION
+ARG APP_INSIGHTS_ID
+
+ENV GOOS=windows
+ENV GOARCH=amd64 
+
+WORKDIR /usr/src/retina
+# Copy the source
+COPY . .
+
+RUN --mount=type=cache,target="/root/.cache/go-build" go build -v -o /usr/bin/controller.exe -ldflags "-X main.version="$VERSION" -X "main.applicationInsightsID"="$APP_INSIGHTS_ID""  ./controller/
+RUN --mount=type=cache,target="/root/.cache/go-build" go build -v -o /usr/bin/captureworkload.exe ./captureworkload/
+
+# Copy into final image
+FROM  mcr.microsoft.com/windows/servercore:ltsc2019 as final
+COPY --from=builder /usr/src/retina/windows/kubeconfigtemplate.yaml kubeconfigtemplate.yaml
+COPY --from=builder /usr/src/retina/windows/setkubeconfigpath.ps1 setkubeconfigpath.ps1
+COPY --from=builder /usr/bin/controller.exe controller.exe
+COPY --from=builder /usr/bin/captureworkload.exe captureworkload.exe
+
+ADD https://github.com/microsoft/etl2pcapng/releases/download/v1.10.0/etl2pcapng.exe /etl2pcapng.exe
+
+CMD ["controller.exe", "start", "--kubeconfig=.\\kubeconfig"]

controller/Dockerfile.windows-2022

@@ -0,0 +1,26 @@
+FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.21 as builder
+
+# Build args
+ARG VERSION
+ARG APP_INSIGHTS_ID
+
+ENV GOOS=windows
+ENV GOARCH=amd64 
+
+WORKDIR /usr/src/retina
+# Copy the source
+COPY . .
+
+RUN --mount=type=cache,target="/root/.cache/go-build" go build -v -o /usr/bin/controller.exe -ldflags "-X main.version="$VERSION" -X "main.applicationInsightsID"="$APP_INSIGHTS_ID"" ./controller/
+RUN --mount=type=cache,target="/root/.cache/go-build" go build -v -o /usr/bin/captureworkload.exe ./captureworkload/
+
+# Copy into final image
+FROM  mcr.microsoft.com/windows/servercore:ltsc2022 as final
+COPY --from=builder /usr/src/retina/windows/kubeconfigtemplate.yaml kubeconfigtemplate.yaml
+COPY --from=builder /usr/src/retina/windows/setkubeconfigpath.ps1 setkubeconfigpath.ps1
+COPY --from=builder /usr/bin/controller.exe controller.exe
+COPY --from=builder /usr/bin/captureworkload.exe captureworkload.exe
+
+ADD https://github.com/microsoft/etl2pcapng/releases/download/v1.10.0/etl2pcapng.exe /etl2pcapng.exe
+
+CMD ["controller.exe", "start", "--kubeconfig=.\\kubeconfig"]

controller/Dockerfile.windows-cgo

@@ -0,0 +1,20 @@
+FROM  --platform=windows/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.22.2-windowsservercore-ltsc2022 as cgo
+
+SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+
+RUN [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; \
+    Invoke-WebRequest -UseBasicParsing -uri "https://github.com/msys2/msys2-installer/releases/download/2024-01-13/msys2-base-x86_64-20240113.sfx.exe" -OutFile msys2.exe; \
+    .\msys2.exe -y -oC:\; \
+    Remove-Item msys2.exe ; \
+    function msys() { C:\msys64\usr\bin\bash.exe @('-lc') + @Args; } \
+    msys ' '; \
+    msys 'pacman --noconfirm -Syuu'; \
+    msys 'pacman --noconfirm -S mingw-w64-x86_64-gcc'; \
+    msys 'pacman --noconfirm -Scc'; 
+
+# pure magic: https://github.com/MicrosoftDocs/Virtualization-Documentation/blob/3f7c7ed7ef8d582c74ec740414c54f25bf5850c0/windows-container-samples/golang/Dockerfile#L15C1-L15C179
+RUN setx path "C:\msys64\mingw64\bin"
+
+FROM cgo as builder
+WORKDIR C:\\pktmon
+COPY .\\pkg\\plugin\\windows\\pktmon\\packetmonitorsupport .