From eee18aa16d0352da522ab3176148566bde05d450 Mon Sep 17 00:00:00 2001 From: ChristinaDara Date: Thu, 29 Feb 2024 11:35:17 +0200 Subject: [PATCH 1/3] fix: oci multi platform support --- package-lock.json | 12 ++++++------ package.json | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 59d362470..d03a4ecc9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,7 @@ "packageurl-js": "^1.2.1", "sleep-promise": "^9.1.0", "snyk-config": "5.3.0", - "snyk-docker-plugin": "^6.10.2", + "snyk-docker-plugin": "6.10.3", "source-map-support": "^0.5.21", "tunnel": "0.0.6", "typescript": "4.7.4", @@ -10208,12 +10208,12 @@ } }, "node_modules/snyk-docker-plugin": { - "version": "6.10.2", - "resolved": "https://registry.npmjs.org/snyk-docker-plugin/-/snyk-docker-plugin-6.10.2.tgz", - "integrity": "sha512-PiDtLp8VhIunwwe3lwoxPLQ940LhE8ZysDZJ1Xuy4Gbgv2gdXmH3O/iAQgCTsZFK80PNVzpAEt2oyz+LG7lqfA==", + "version": "6.10.3", + "resolved": "https://registry.npmjs.org/snyk-docker-plugin/-/snyk-docker-plugin-6.10.3.tgz", + "integrity": "sha512-q5urrXWck+tNkmzWhLY0mvHam6sIDEOZPTv+Om4MgYziR/4kuxgZfV9lgQMjblk0sAzRjC0MmpE1iOzU6vWNkw==", "dependencies": { "@snyk/composer-lockfile-parser": "^1.4.1", - "@snyk/dep-graph": "^2.8.0", + "@snyk/dep-graph": "^2.8.1", "@snyk/docker-registry-v2-client": "^2.11.0", "@snyk/rpm-parser": "3.1.0", "@snyk/snyk-docker-pull": "^3.11.0", @@ -10230,7 +10230,7 @@ "packageurl-js": "1.2.0", "semver": "^7.5.4", "shescape": "^1.7.4", - "snyk-nodejs-lockfile-parser": "^1.52.10", + "snyk-nodejs-lockfile-parser": "^1.52.11", "snyk-poetry-lockfile-parser": "^1.4.0", "tar-stream": "^2.1.0", "tmp": "^0.2.1", diff --git a/package.json b/package.json index 02679f274..da598baaf 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "packageurl-js": "^1.2.1", "sleep-promise": "^9.1.0", "snyk-config": "5.3.0", - "snyk-docker-plugin": "^6.10.2", + "snyk-docker-plugin": "6.10.3", "source-map-support": "^0.5.21", "tunnel": "0.0.6", "typescript": "4.7.4", From 2858fe5408b44bdba729e1178bb07f35baf9e2b7 Mon Sep 17 00:00:00 2001 From: Jonny Powell Date: Mon, 4 Mar 2024 08:54:34 +0000 Subject: [PATCH 2/3] fix: avoid sending empty runtime images payloads --- src/data-scraper/index.ts | 6 +- src/data-scraper/scraping-v1.ts | 6 +- src/transmitter/payload.ts | 7 ++- test/unit/data-scraper/scrape-data.spec.ts | 72 ++++++++++++++++++++++ 4 files changed, 86 insertions(+), 5 deletions(-) diff --git a/src/data-scraper/index.ts b/src/data-scraper/index.ts index 202cab477..edfe137c2 100644 --- a/src/data-scraper/index.ts +++ b/src/data-scraper/index.ts @@ -101,8 +101,10 @@ export async function scrapeData(): Promise { 2, ); - logger.info({}, 'sending runtime data upstream'); - await sendRuntimeData(runtimeDataPayload); + if (runtimeDataPayload) { + logger.info({}, 'sending runtime data upstream'); + await sendRuntimeData(runtimeDataPayload); + } cursor = responseBody?.page.next || ''; if (!cursor) { diff --git a/src/data-scraper/scraping-v1.ts b/src/data-scraper/scraping-v1.ts index bda1872a8..c55cdc4c5 100644 --- a/src/data-scraper/scraping-v1.ts +++ b/src/data-scraper/scraping-v1.ts @@ -91,8 +91,10 @@ export async function scrapeDataV1(): Promise { 1, ); - logger.info({}, 'sending runtime data upstream'); - await sendRuntimeData(runtimeDataPayload); + if (runtimeDataPayload) { + logger.info({}, 'sending runtime data upstream'); + await sendRuntimeData(runtimeDataPayload); + } cursor = responseBody?.page.next || ''; if (!cursor) { diff --git a/src/transmitter/payload.ts b/src/transmitter/payload.ts index b0de5fdde..943b1cb67 100644 --- a/src/transmitter/payload.ts +++ b/src/transmitter/payload.ts @@ -152,10 +152,11 @@ const workloadKindMap = { pod: 'Pod', rollout: 'Rollout', }; + export function constructRuntimeData( runtimeResults: IRuntimeImage[], sysdigVersion: number, -): IRuntimeDataPayload { +): IRuntimeDataPayload | undefined { const filteredRuntimeResults = runtimeResults.reduce((acc, runtimeResult) => { if (!isExcludedNamespace(runtimeResult.namespace)) { const mappedWorkloadKind = @@ -178,6 +179,10 @@ export function constructRuntimeData( return acc; }, [] as IRuntimeImage[]); + if (filteredRuntimeResults.length === 0) { + return; + } + const dataFact: IRuntimeDataFact = { type: 'loadedPackages', data: filteredRuntimeResults, diff --git a/test/unit/data-scraper/scrape-data.spec.ts b/test/unit/data-scraper/scrape-data.spec.ts index c39f43a6a..122a9d19a 100644 --- a/test/unit/data-scraper/scrape-data.spec.ts +++ b/test/unit/data-scraper/scrape-data.spec.ts @@ -132,6 +132,78 @@ describe('dataScraper()', () => { throw err; } }); + + it('correctly skips pages of data without relevant runtime images', async () => { + const runtimeImageTemplate = { + imageID: 'something', + namespace: 'sysdig', + workloadName: 'workload', + workloadKind: 'Deployment', + container: 'box', + packages: [], + }; + const page1 = { + data: Array(10).fill({ + ...runtimeImageTemplate, + namespace: 'kube-system', + }), + page: { + returned: 10, + next: 'xxx', + }, + }; + const page2 = { + data: [runtimeImageTemplate], + page: { + returned: 1, + }, + }; + nock('https://sysdig') + .get( + '/api/scanning/eveintegration/v2/runtimeimages?clusterName=test-sysdig-cluster&limit=10', + ) + .times(1) + .reply(200, page1) + .get( + '/api/scanning/eveintegration/v2/runtimeimages?clusterName=test-sysdig-cluster&limit=10&cursor=xxx', + ) + .times(1) + .reply(200, page2); + + nock('https://api.snyk.io') + .post( + '/v2/kubernetes-upstream/api/v1/runtime-results?version=2023-02-10', + ) + .times(1) + .reply(200, (_, requestBody: transmitterTypes.IRuntimeDataPayload) => { + expect(requestBody).toEqual({ + identity: { + type: 'sysdig', + sysdigVersion: 2, + }, + target: { + userLocator: expect.any(String), + cluster: expect.any(String), + agentId: expect.any(String), + }, + facts: [ + { + type: 'loadedPackages', + data: page2.data, + }, + ], + }); + }); + + await scrapeData(); + + try { + expect(nock.isDone()).toBeTruthy(); + } catch (err) { + console.error(`nock pending mocks: ${nock.pendingMocks()}`); + throw err; + } + }); }); describe('when sysdig v1 and v2 env vars configured, should use v2', () => { beforeAll(() => { From e918a0612a86c6dc85fe2785fe32a104e6814c2d Mon Sep 17 00:00:00 2001 From: Jonny Powell Date: Mon, 4 Mar 2024 08:59:24 +0000 Subject: [PATCH 3/3] fix: update circleci images --- .circleci/config.yml | 155 ++++++++++++++++++------------------------- 1 file changed, 63 insertions(+), 92 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 17dbe3bb8..03482b3a4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,27 +16,16 @@ commands: sudo apt update sudo apt install python3-requests when: always - setup_node: - description: Setup Node 18 - steps: - - run: - command: | - export NVM_DIR="/opt/circleci/.nvm" - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" - nvm install v18 - npm ci - echo 'export NVM_DIR="/opt/circleci/.nvm"' >> $BASH_ENV - echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"' >> $BASH_ENV - echo 'nvm alias default v18' >> $BASH_ENV jobs: aks_integration_tests: - machine: - docker_layer_caching: true - image: ubuntu-2004:202111-01 + docker: + - image: cimg/node:18.19.1 resource_class: large steps: - checkout - - setup_node + - setup_remote_docker + - run: + command: npm ci - install_python_requests - azure-cli/install - run: @@ -44,7 +33,6 @@ jobs: name: Create temp dir for logs - run: command: | - npm ci && export KUBERNETES_MONITOR_IMAGE_NAME_AND_TAG=$(./scripts/circleci-jobs/setup-integration-tests.py) .circleci/do-exclusively --branch staging --job ${CIRCLE_JOB} npm run test:integration:aks:yaml name: Integration tests AKS @@ -58,10 +46,7 @@ jobs: working_directory: ~/kubernetes-monitor build_and_upload_operator: docker: - - auth: - password: $DOCKERHUB_PASSWORD - username: $DOCKERHUB_USER - image: cimg/python:3.10 + - image: cimg/python:3.10 steps: - checkout - setup_remote_docker @@ -106,10 +91,11 @@ jobs: when: on_fail working_directory: ~/kubernetes-monitor build_image: - machine: - image: ubuntu-2004:202111-02 + docker: + - image: cimg/base:current steps: - checkout + - setup_remote_docker - install_python_requests - run: command: | @@ -149,12 +135,12 @@ jobs: when: on_fail working_directory: ~/kubernetes-monitor code_formatter: - machine: - docker_layer_caching: true - image: ubuntu-2004:202111-01 + docker: + - image: cimg/node:18.19.1 steps: - checkout - - setup_node + - run: + command: npm ci - install_python_requests - run: command: | @@ -168,10 +154,7 @@ jobs: working_directory: ~/kubernetes-monitor prepare_to_deploy: docker: - - auth: - password: $DOCKERHUB_PASSWORD - username: $DOCKERHUB_USER - image: cimg/node:16.13 + - image: cimg/base:current steps: - checkout - install_python_requests @@ -185,10 +168,7 @@ jobs: working_directory: ~/kubernetes-monitor deploy_to_prod: docker: - - auth: - password: $DOCKERHUB_PASSWORD - username: $DOCKERHUB_USER - image: cimg/node:16.13 + - image: cimg/base:current steps: - checkout - install_python_requests @@ -202,10 +182,7 @@ jobs: working_directory: ~/kubernetes-monitor deploy_to_sysdig_integration_cluster: docker: - - auth: - password: $DOCKERHUB_PASSWORD - username: $DOCKERHUB_USER - image: cimg/base:stable + - image: cimg/base:stable steps: - checkout - run: @@ -242,14 +219,15 @@ jobs: when: on_fail working_directory: ~/kubernetes-monitor eks_integration_tests: - machine: - docker_layer_caching: true - image: ubuntu-2004:202111-01 + docker: + - image: cimg/node:18.19.1 resource_class: large steps: - checkout + - setup_remote_docker + - run: + command: npm ci - install_python_requests - - setup_node - aws-cli/install: override-installed: true - run: @@ -257,7 +235,6 @@ jobs: name: Create temp dir for logs - run: command: | - npm ci && export KUBERNETES_MONITOR_IMAGE_NAME_AND_TAG=$(./scripts/circleci-jobs/setup-integration-tests.py) .circleci/do-exclusively --branch staging --job ${CIRCLE_JOB} npm run test:integration:eks:yaml name: Integration tests EKS @@ -270,13 +247,14 @@ jobs: path: /tmp/logs/test/integration/eks working_directory: ~/kubernetes-monitor integration_tests: - machine: - docker_layer_caching: true - image: ubuntu-2004:202111-01 + docker: + - image: cimg/node:18.19.1 resource_class: large steps: - checkout - - setup_node + - setup_remote_docker + - run: + command: npm ci - install_python_requests - run: command: mkdir -p /tmp/logs/test/integration/kind @@ -295,13 +273,14 @@ jobs: path: /tmp/logs/test/integration/kind working_directory: ~/kubernetes-monitor integration_tests_helm: - machine: - docker_layer_caching: true - image: ubuntu-2004:202111-01 + docker: + - image: cimg/node:18.19.1 resource_class: large steps: - checkout - - setup_node + - setup_remote_docker + - run: + command: npm ci - install_python_requests - run: command: mkdir -p /tmp/logs/test/integration/kind-helm @@ -320,13 +299,14 @@ jobs: path: /tmp/logs/test/integration/kind-helm working_directory: ~/kubernetes-monitor integration_tests_operator_on_k8s: - machine: - docker_layer_caching: true - image: ubuntu-2004:202111-01 - resource_class: large + docker: + - image: cimg/node:18.19.1 + resource_class: large steps: - checkout - - setup_node + - setup_remote_docker + - run: + command: npm ci - install_python_requests - run: command: mkdir -p /tmp/logs/test/integration/kind-olm-operator @@ -347,13 +327,14 @@ jobs: path: /tmp/logs/test/integration/kind-olm-operator working_directory: ~/kubernetes-monitor integration_tests_proxy: - machine: - docker_layer_caching: true - image: ubuntu-2004:202111-01 + docker: + - image: cimg/node:18.19.1 resource_class: large steps: - checkout - - setup_node + - setup_remote_docker + - run: + command: npm ci - install_python_requests - run: command: mkdir -p /tmp/logs/test/integration/proxy @@ -372,12 +353,12 @@ jobs: path: /tmp/logs/test/integration/proxy working_directory: ~/kubernetes-monitor lint: - machine: - docker_layer_caching: true - image: ubuntu-2004:202111-01 + docker: + - image: cimg/node:18.19.1 steps: - checkout - - setup_node + - run: + command: npm ci - install_python_requests - run: command: | @@ -390,13 +371,14 @@ jobs: when: on_fail working_directory: ~/kubernetes-monitor openshift4_integration_tests: - machine: - docker_layer_caching: true - image: ubuntu-2204:current - resource_class: large + docker: + - image: cimg/node:18.19.1 + resource_class: large steps: - checkout - - setup_node + - setup_remote_docker + - run: + command: npm ci - install_python_requests - run: command: mkdir -p /tmp/logs/test/integration/openshift4 @@ -650,10 +632,7 @@ jobs: working_directory: ~/kubernetes-monitor publish: docker: - - auth: - password: $DOCKERHUB_PASSWORD - username: $DOCKERHUB_USER - image: cimg/node:16.13 + - image: cimg/node:18.19.1 steps: - checkout - setup_remote_docker @@ -877,10 +856,7 @@ jobs: working_directory: ~/kubernetes-monitor sync_community_operators_with_snyk_fork: docker: - - auth: - password: $DOCKERHUB_PASSWORD - username: $DOCKERHUB_USER - image: cimg/python:3.10 + - image: cimg/python:3.10 steps: - checkout - install_python_requests @@ -909,10 +885,7 @@ jobs: working_directory: ~/kubernetes-monitor sync_embedded_community_operators_with_snyk_fork: docker: - - auth: - password: $DOCKERHUB_PASSWORD - username: $DOCKERHUB_USER - image: cimg/python:3.10 + - image: cimg/python:3.10 steps: - checkout - install_python_requests @@ -942,10 +915,11 @@ jobs: system_tests: machine: docker_layer_caching: true - image: ubuntu-2204:2022.04.1 + image: default steps: - checkout - - setup_node + - run: + command: npm ci - install_python_requests - run: command: | @@ -966,10 +940,7 @@ jobs: working_directory: ~/kubernetes-monitor tag_and_push: docker: - - auth: - password: $DOCKERHUB_PASSWORD - username: $DOCKERHUB_USER - image: cimg/node:16.13 + - image: cimg/node:18.19.1 steps: - checkout - setup_remote_docker @@ -992,12 +963,12 @@ jobs: when: on_fail working_directory: ~/kubernetes-monitor unit_tests: - machine: - docker_layer_caching: true - image: ubuntu-2004:202111-01 + docker: + - image: cimg/node:18.19.1 steps: - checkout - - setup_node + - run: + command: npm ci - install_python_requests - snyk/scan: additional-arguments: --all-projects --exclude=test