From 2058465a90e7ebfe5486f3f1a290293aa9671123 Mon Sep 17 00:00:00 2001 From: Pavel Sorokin <60606414+pavel-snyk@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:50:01 +0200 Subject: [PATCH] Revert universtal broker support changes Following commits are reverted: - 514abf2923e80882f4a6c74c50634300dec3451d - 7779ac5f7f480415a9aae92d4fb42e84c6a21ae1 --- config.default.json | 199 +------ config.universal.json | 12 - config.universal.jsontemplate | 285 ---------- lib/client/index.js | 17 +- lib/config.js | 41 +- lib/filter-rules-loading.js | 194 +++---- lib/filters.js | 318 ++++++----- lib/log.js | 41 +- lib/relay.js | 20 +- lib/server/broker-middleware.ts | 23 + lib/server/index.js | 39 +- lib/utils.ts | 71 --- package.json | 1 - test/fixtures/client/universalFilters.json | 215 -------- .../server-client-universal.test.ts | 80 +++ .../server-universal-client.test.ts | 518 ------------------ .../universal-server-legacy-client.test.ts | 509 ----------------- test/unit/config-universal-broker.test.ts | 13 - test/unit/log-universal.test.ts | 154 ------ test/unit/log.test.ts | 4 +- 20 files changed, 387 insertions(+), 2367 deletions(-) delete mode 100644 config.universal.json delete mode 100644 config.universal.jsontemplate create mode 100644 lib/server/broker-middleware.ts delete mode 100644 lib/utils.ts delete mode 100644 test/fixtures/client/universalFilters.json create mode 100644 test/functional/server-client-universal.test.ts delete mode 100644 test/functional/server-universal-client.test.ts delete mode 100644 test/functional/universal-server-legacy-client.test.ts delete mode 100644 test/unit/config-universal-broker.test.ts delete mode 100644 test/unit/log-universal.test.ts diff --git a/config.default.json b/config.default.json index 331f23c2b..1cc6bf75d 100644 --- a/config.default.json +++ b/config.default.json @@ -1,200 +1,3 @@ { - "BROKER_BOOT_MODE": "classic", - "BROKER_SERVER_URL": "https://broker.snyk.io", - "BROKER_HEALTHCHECK_PATH": "/healthcheck", - "ACCEPT_LARGE_MANIFESTS": "false", - "ARTIFACTORY": { - "BROKER_CLIENT_VALIDATION_URL":"https://$ARTIFACTORY_URL/api/system/ping", - "BROKER_CLIENT_VALIDATION_JSON_DISABLED":"true" - }, - "AZURE-REPOS": { - "AZURE_REPOS_HOST":"dev.azure.com", - "BROKER_CLIENT_VALIDATION_BASIC_AUTH":"PAT:$AZURE_REPOS_TOKEN", - "BROKER_CLIENT_VALIDATION_URL":"https://$AZURE_REPOS_HOST/$AZURE_REPOS_ORG/_apis/git/repositories", - "GIT_URL":"$AZURE_REPOS_HOST/$AZURE_REPOS_ORG", - "GIT_USERNAME":"PAT", - "GIT_PASSWORD":"$AZURE_REPOS_TOKEN" - }, - "BITBUCKET-SERVER": { - "BITBUCKET_API":"$BITBUCKET/rest/api/1.0", - - "BROKER_CLIENT_VALIDATION_URL":"https://$BITBUCKET/rest/api/1.0/projects", - "BROKER_CLIENT_VALIDATION_BASIC_AUTH":"$BITBUCKET_USERNAME:$BITBUCKET_PASSWORD", - - "GIT_URL":"$BITBUCKET", - "GIT_USERNAME":"$BITBUCKET_USERNAME", - "GIT_PASSWORD":"$BITBUCKET_PASSWORD" - }, - "CR-AGENT": { - "CR_AGENT_URL":"https://:" - }, - "GITHUB": { - "GITHUB":"github.com", - "GITHUB_RAW":"raw.githubusercontent.com", - "GITHUB_API":"api.github.com", - "GITHUB_GRAPHQL":"api.github.com", - - "BROKER_CLIENT_VALIDATION_URL":"https://$GITHUB_API/user", - "BROKER_CLIENT_VALIDATION_AUTHORIZATION_HEADER":"token $GITHUB_TOKEN", - "GIT_URL":"$GITHUB", - "GIT_USERNAME":"x-access-token", - "GIT_PASSWORD":"$GITHUB_TOKEN" - }, - "GITHUB-ENTERPRISE": { - - "GITHUB_API":"$GITHUB/api/v3", - "GITHUB_GRAPHQL":"$GITHUB/api", - "BROKER_CLIENT_VALIDATION_URL":"https://$GITHUB_API/user", - "BROKER_CLIENT_VALIDATION_AUTHORIZATION_HEADER":"token $GITHUB_TOKEN", - - "GIT_URL":"$GITHUB", - "GIT_USERNAME":"x-access-token", - "GIT_PASSWORD":"$GITHUB_TOKEN" - }, - "GITLAB": { - "BROKER_CLIENT_VALIDATION_URL":"https://$GITLAB/api/v3/user?private_token:$GITLAB_TOKEN", - - "GIT_URL":"$GITLAB", - - "GIT_USERNAME":"oauth2", - "GIT_PASSWORD":"$GITLAB_TOKEN" - - }, - "JIRA": { - "BROKER_CLIENT_VALIDATION_URL":"https://$JIRA_HOSTNAME/rest/api/2/myself", - "BROKER_CLIENT_VALIDATION_BASIC_AUTH":"$JIRA_USERNAME:$JIRA_PASSWORD" - }, - "NEXUS": { - "NEXUS_URL":"$BASE_NEXUS_URL/repository", - "BROKER_CLIENT_VALIDATION_URL":"$BASE_NEXUS_URL/service/rest/v1/status/check", - - "BROKER_CLIENT_VALIDATION_JSON_DISABLED":"true", - "REMOVE_X_FORWARDED_HEADERS":"true" - }, - "NEXUS2": { - "NEXUS_URL":"$BASE_NEXUS_URL/nexus/content", - "BROKER_CLIENT_VALIDATION_URL":"$BASE_NEXUS_URL/nexus/service/local/status", - "BROKER_CLIENT_VALIDATION_JSON_DISABLED":"true" - }, - "FILTER_RULES_PATHS": {}, - "SOURCE_TYPES": { - "github": { - "publicId": "9a3e5d90-b782-468a-a042-9a2073736f0b", - "name": "GitHub", - "type": "github", - "brokerType": "github" - }, - "bitbucket-server": { - "publicId": "87c79724-9555-4959-b811-08aa22635614", - "name": "Bitbucket Server", - "type": "bitbucket-server", - "brokerType": "bitbucket-server" - }, - "github-enterprise": { - "publicId": "219fbe9f-2fb3-4919-b69c-afd6fab25d54", - "name": "GitHub Enterprise", - "type": "github-enterprise", - "brokerType": "github-enterprise" - }, - "gitlab": { - "publicId": "6F37586C-ED9C-47BB-9D95-A2C93698836D", - "name": "GitLab", - "type": "gitlab", - "brokerType": "gitlab" - }, - "docker-hub": { - "publicId": "7a4b3f39-eb1c-4a6e-9960-025f783b45c9", - "name": "Docker Hub", - "type": "docker-hub", - "brokerType": "container-registry-agent" - }, - "ecr": { - "publicId": "850ff1ce-8bad-f00d-fee1-d00d00caca00", - "name": "ECR", - "type": "ecr", - "brokerType": "container-registry-agent" - }, - "acr": { - "publicId": "75831964-2f5d-4c74-a798-0f3839b6ac00", - "name": "ACR", - "type": "acr", - "brokerType": "container-registry-agent" - }, - "gcr": { - "publicId": "8a2cf306-50ae-4b3f-b98e-5a819c0ba9b6", - "name": "GCR", - "type": "gcr", - "brokerType": "container-registry-agent" - }, - "artifactory-cr": { - "publicId": "66831964-2f5d-4c74-a798-0f3839b6ac00", - "name": "Artifactory", - "type": "artifactory-cr", - "brokerType": "container-registry-agent" - }, - "harbor-cr": { - "publicId": "62b98e6e-0c70-47c1-9bc3-f6e7005d1069", - "name": "Harbor", - "type": "harbor-cr", - "brokerType": "container-registry-agent" - }, - "quay-cr": { - "publicId": "3e82698b-2563-461f-921a-e41c483e3daf", - "name": "Quay", - "type": "quay-cr", - "brokerType": "container-registry-agent" - }, - "github-cr": { - "publicId": "86e781e7-a8f9-40ed-b5b5-e442cc3037de", - "name": "GitHub Container Registry", - "type": "github-cr", - "brokerType": "container-registry-agent" - }, - "nexus-cr": { - "publicId": "bc5dc4b4-530e-4dff-b494-b927dbab568f", - "name": "Nexus", - "type": "nexus-cr", - "brokerType": "container-registry-agent" - }, - "digitalocean-cr": { - "publicId": "465c35cf-7155-4173-bf59-b2c404ad4867", - "name": "DigitalOcean", - "type": "digitalocean-cr", - "brokerType": "container-registry-agent" - }, - "gitlab-cr": { - "publicId": "872fe239-fdc8-4d39-9e88-4fb99a2d5f7e", - "name": "GitLab Container Registry", - "type": "gitlab-cr", - "brokerType": "container-registry-agent" - }, - "google-artifact-cr": { - "publicId": "481b389f-e23b-4255-a045-f245d425cef0", - "uri": "http://docker-registry-agent", - "name": "Google Artifact Registry", - "type": "google-artifact-cr", - "brokerType": "container-registry-agent" - }, - "azure-repos": { - "publicId": "92ff53a7-366d-4639-b75d-d762b5e840b4", - "name": "Azure Repos", - "type": "azure-repos", - "brokerType": "azure-repos" - }, - "artifactory": { - "name": "Artifactory Private Registry", - "type": "artifactory", - "brokerType": "artifactory" - }, - "nexus": { - "name": "Nexus Private Registry", - "type": "nexus", - "brokerType": "nexus" - }, - "nexus2": { - "name": "Nexus2 Private Registry", - "type": "nexus2", - "brokerType": "nexus2" - } - } + "BROKER_SERVER_UNIVERSAL_CONFIG_ENABLED": false } \ No newline at end of file diff --git a/config.universal.json b/config.universal.json deleted file mode 100644 index 08f973617..000000000 --- a/config.universal.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - - "BROKER_CLIENT_URL": "http://my.client:myport", - "BROKER_TYPES": "github,github-enterprise", - "GITHUB": { - "GITHUB_TOKEN":"GITHUB_123" - }, - "GITHUB-ENTERPRISE": { - "GITHUB_TOKEN":"GITHUB_ENTERPRISE_123", - "GITHUB": "ghe.mycompany.com" - } -} \ No newline at end of file diff --git a/config.universal.jsontemplate b/config.universal.jsontemplate deleted file mode 100644 index a33751f09..000000000 --- a/config.universal.jsontemplate +++ /dev/null @@ -1,285 +0,0 @@ -{ - "SERVICE_ENV": "universal", - // Select only the types wanted/needed - "BROKER_TYPES": "github,github-enterprise,bitbucket-server,gitlab,container-registry-agent,azure-repos,artifactory,nexus,nexus2", - "ARTIFACTORY": { - // The URL to your artifactory - // If not using basic auth this will only be /artifactory - "ARTIFACTORY_URL":":@/artifactory", - - - - - // Provide RES_BODY_URL_SUB with the URL of the artifactory without credentials and http protocol - // This URL substitution is required for NPM integration - // RES_BODY_URL_SUB":"http:///artifactory", - - // Artifactory validation url, checked by broker client systemcheck endpoint - "BROKER_CLIENT_VALIDATION_URL":"https://$ARTIFACTORY_URL/api/system/ping", - "BROKER_CLIENT_VALIDATION_JSON_DISABLED":"true" - }, - "AZURE_REPOS": { - // Guide how to get/create the token https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view":"azure-devops&tabs":"preview-page", - // Scopes: Ensure Custom defined is selected and under Code select Read & write - "AZURE_REPOS_TOKEN":"", - - // https://dev.azure.com/ - "AZURE_REPOS_ORG":"", - - // the host excluding scheme - "AZURE_REPOS_HOST":"dev.azure.com", - - // the url of your broker client (including scheme and port) - "BROKER_CLIENT_URL":"https://:", - - // auth header with special format - "BROKER_CLIENT_VALIDATION_BASIC_AUTH":"PAT:$AZURE_REPOS_TOKEN", - - // Azure validation url, checked by broker client systemcheck endpoint - "BROKER_CLIENT_VALIDATION_URL":"https://$AZURE_REPOS_HOST/$AZURE_REPOS_ORG/_apis/git/repositories", - - // the host where the git server resides - "GIT_URL":"$AZURE_REPOS_HOST/$AZURE_REPOS_ORG", - - // git credentials for cloning repos - "GIT_USERNAME":"PAT", - "GIT_PASSWORD":"$AZURE_REPOS_TOKEN" - }, - "BITBUCKET_SERVER": { - // your personal username to your bitbucket server account - "BITBUCKET_USERNAME":"", - - // your personal password to your bitbucket server account - "BITBUCKET_PASSWORD":"", - - // the host where your Bitbucket Server is running, excluding scheme. - // for bitbucket.yourdomain.com - // this should be "bitbucket.yourdomain.com" - "BITBUCKET":"bitbucket.yourdomain.com", - - // the url that the Bitbucket server API should be accessed at. - // for bitbucket.yourdomain.com this should be - // changed to "bitbucket.yourdomain.com/rest/api/1.0" - "BITBUCKET_API":"$BITBUCKET/rest/api/1.0", - - // the url of your broker client (including scheme and port) - // BROKER_CLIENT_URL":" - - // Bitbucket server validation url, checked by broker client systemcheck endpoint - "BROKER_CLIENT_VALIDATION_URL":"https://$BITBUCKET/rest/api/1.0/projects", - - // Bitbucket server basic auth creds", - "BROKER_CLIENT_VALIDATION_BASIC_AUTH":"$BITBUCKET_USERNAME:$BITBUCKET_PASSWORD", - - // the host where the git server resides", - "GIT_URL":"$BITBUCKET", - - // git credentials for cloning repos", - "GIT_USERNAME":"$BITBUCKET_USERNAME", - "GIT_PASSWORD":"$BITBUCKET_PASSWORD" - - // the url of your snyk git client (including scheme and port). - // GIT_CLIENT_URL":"https://: - }, - "CONTAINER_REGISTRY_AGENT": { - // The URL of your broker client (including scheme and port), used by container", - // registry agent to call back to Snyk through brokered connection", - "BROKER_CLIENT_URL":"https://:", - - // The URL of your container registry agent", - "CR_AGENT_URL":"https://:" - }, - "GITHUB": { - // your personal access token to your github.com account - "GITHUB_TOKEN":"", - - // the host for GitHub, excluding scheme - "GITHUB":"github.com", - - // the host for GitHub's raw content, excluding scheme - "GITHUB_RAW":"raw.githubusercontent.com", - - // the GitHub REST API url, excluding scheme - "GITHUB_API":"api.github.com", - - // the GitHub GraphQL API url, excluding scheme - "GITHUB_GRAPHQL":"api.github.com", - - // the url of your broker client (including scheme and port) - // BROKER_CLIENT_URL":" - - // GitHub validation url, checked by broker client systemcheck endpoint - "BROKER_CLIENT_VALIDATION_URL":"https://$GITHUB_API/user", - - // GitHub validation request Authorization header - "BROKER_CLIENT_VALIDATION_AUTHORIZATION_HEADER":"token $GITHUB_TOKEN", - - // the host where the git server resides - "GIT_URL":"$GITHUB", - - // git credentials for cloning repos - "GIT_USERNAME":"x-access-token", - "GIT_PASSWORD":"$GITHUB_TOKEN" - - // the url of your snyk git client (including scheme and port). - // GIT_CLIENT_URL":"https://: - }, - "GITHUB_COM": { - // your personal access token to your github.com account", - "GITHUB_TOKEN":"", - - // the host for GitHub, excluding scheme", - "GITHUB":"github.com", - - // the host for GitHub's raw content, excluding scheme", - "GITHUB_RAW":"raw.githubusercontent.com", - - // the GitHub REST API url, excluding scheme", - "GITHUB_API":"api.github.com", - - // the GitHub GraphQL API url, excluding scheme", - "GITHUB_GRAPHQL":"api.github.com", - - // the url of your broker client (including scheme and port) - // BROKER_CLIENT_URL":" - - // GitHub validation url, checked by broker client systemcheck endpoint", - "BROKER_CLIENT_VALIDATION_URL":"https://$GITHUB_API/user", - - // GitHub validation request Authorization header", - "BROKER_CLIENT_VALIDATION_AUTHORIZATION_HEADER":"token $GITHUB_TOKEN", - - - // the host where the git server resides", - "GIT_URL":"$GITHUB", - - // git credentials for cloning repos", - "GIT_USERNAME":"x-access-token", - "GIT_PASSWORD":"$GITHUB_TOKEN" - - // the url of your snyk git client (including scheme and port). - // GIT_CLIENT_URL":"https://: - }, - "GITHUB_ENTERPRISE": { - // your personal access token to your github enterprise account", - "GITHUB_TOKEN":"", - - // the host for your GitHub Enterprise deployment, excluding scheme", - "GITHUB":"ghe.yourdomain.com", - - // the GitHub Enterprise REST API url, excluding scheme", - "GITHUB_API":"$GITHUB/api/v3", - - // the GitHub Enterprise GraphQL API url, excluding scheme", - "GITHUB_GRAPHQL":"$GITHUB/api", - - // the url of your broker client (including scheme and port) - // BROKER_CLIENT_URL":" - - // GitHub Enterprise validation url, checked by broker client systemcheck endpoint", - "BROKER_CLIENT_VALIDATION_URL":"https://$GITHUB_API/user", - - // GitHub Enterprise validation request Authorization header", - "BROKER_CLIENT_VALIDATION_AUTHORIZATION_HEADER":"token $GITHUB_TOKEN", - - // The path for the broker's internal healthcheck URL. Must start with a '/'. - "BROKER_HEALTHCHECK_PATH":"/healthcheck", - - // the host where the git server resides", - "GIT_URL":"$GITHUB", - - // git credentials for cloning repos", - "GIT_USERNAME":"x-access-token", - "GIT_PASSWORD":"$GITHUB_TOKEN" - - // the url of your snyk git client (including scheme and port). - // GIT_CLIENT_URL":"https://: - }, - "GITLAB": { - // your personal token to your Gitlab server account - "GITLAB_TOKEN":"", - - // the host where your Gitlab Server is running, excluding scheme. - // i.e. gitlab.yourdomain.com", - "GITLAB":"gitlab.yourdomain.com", - - // the url of your broker client (including scheme and port) - // BROKER_CLIENT_URL":" - - // GitLab validation url, checked by broker client systemcheck endpoint", - "BROKER_CLIENT_VALIDATION_URL":"https://$GITLAB/api/v3/user?private_token:$GITLAB_TOKEN", - - // the host where the git server resides", - "GIT_URL":"$GITLAB", - - // git credentials for cloning repos", - "GIT_USERNAME":"oauth2", - "GIT_PASSWORD":"$GITLAB_TOKEN" - - // the url of your snyk git client (including scheme and port). - // GIT_CLIENT_URL":"https://: - }, - "JIRA": { - // your personal username to your Jira Server account", - "JIRA_USERNAME":"", - - // your personal password or API token to your Jira Server account", - "JIRA_PASSWORD":"", - - // your Jira Server hostname, i.e. jira.yourdomain.com", - "JIRA_HOSTNAME":"jira.yourdomain.com", - - // the url of your broker client (including scheme and port) - // BROKER_CLIENT_URL":" - - // Jira validation url, checked by broker client systemcheck endpoint", - "BROKER_CLIENT_VALIDATION_URL":"https://$JIRA_HOSTNAME/rest/api/2/myself", - - // Jira basic auth creds", - "BROKER_CLIENT_VALIDATION_BASIC_AUTH":"$JIRA_USERNAME:$JIRA_PASSWORD", - }, - "NEXUS": { - // The Base URL for your Nexus Repository Manager", - // If not using basic auth this will only be "https://" - "BASE_NEXUS_URL":"https://:@", - - // The URL to your Nexus Repository Manager", - "NEXUS_URL":"$BASE_NEXUS_URL/repository", - // Provide RES_BODY_URL_SUB with the URL of the Nexus without credentials", - // This URL substitution is required for NPM integration", - // RES_BODY_URL_SUB":"https:///repository", - - // Nexus validation url, checked by broker client systemcheck endpoint", - // users must have 'nx-metrics-all' privilege enabled to access", - // https://support.sonatype.com/hc/en-us/articles/226254487-System-Status-and-Metrics-REST-API", - // NOTE! If NOT using username and password in BASE_NEXUS_URL, because your Nexus server is open on a private network, then use '/service/rest/v1/status' for BROKER_CLIENT_VALIDATION_URL. - // '/service/rest/v1/status/check' requires an authenticated connection while '/service/rest/v1/status' does not. - //BROKER_CLIENT_VALIDATION_URL":"$BASE_NEXUS_URL/service/rest/v1/status", - "BROKER_CLIENT_VALIDATION_URL":"$BASE_NEXUS_URL/service/rest/v1/status/check", - - "BROKER_CLIENT_VALIDATION_JSON_DISABLED":"true", - - // Disable X-Forwarded-For headers so Nexus doesn't return npm tarball uri pointing to the broker server", - "REMOVE_X_FORWARDED_HEADERS":"true" - }, - "NEXUS2": { - // The Base URL for your Nexus Repository Manager", - // If not using basic auth this will only be "https://" - "BASE_NEXUS_URL":"https://:@", - - // The URL to your Nexus Repository Manager", - "NEXUS_URL":"$BASE_NEXUS_URL/nexus/content", - - - // Provide RES_BODY_URL_SUB with the URL of the Nexus without credentials", - // This URL substitution is required for NPM integration", - // RES_BODY_URL_SUB":"https:///nexus/content", - - // Nexus validation url, checked by broker client systemcheck endpoint", - // works with / without auth", - "BROKER_CLIENT_VALIDATION_URL":"$BASE_NEXUS_URL/nexus/service/local/status", - - "BROKER_CLIENT_VALIDATION_JSON_DISABLED":"true" - - } -} \ No newline at end of file diff --git a/lib/client/index.js b/lib/client/index.js index 7f72d9a21..146d8db1d 100644 --- a/lib/client/index.js +++ b/lib/client/index.js @@ -12,7 +12,6 @@ const { handleCheckIdsRoutes, } = require('./checks/api/checks-handler'); const { commitSigningEnabled, commitSigningFilterRules } = require('./scm'); -const utils = require('../utils'); module.exports = async ({ port = null, config = {}, filters = {} }) => { logger.info({ version }, 'running in client mode'); @@ -45,12 +44,7 @@ module.exports = async ({ port = null, config = {}, filters = {} }) => { if (commitSigningEnabled(config)) { const commitSigningRules = commitSigningFilterRules(); - if (config.BROKER_BOOT_MODE == 'universal') { - filters['github'].private?.push(...commitSigningRules); - filters['github-enterprise'].github.private?.push(...commitSigningRules); - } else { - filters.private?.push(...commitSigningRules); - } + filters.private?.push(...commitSigningRules); logger.info( { enabled: true, rulesCount: commitSigningRules.length }, 'loading commit signing rules', @@ -68,7 +62,7 @@ module.exports = async ({ port = null, config = {}, filters = {} }) => { const io = socket({ token: config.brokerToken, url: config.brokerServerUrl, - filters: config.BROKER_BOOT_MODE == 'universal' ? filters : filters.private, + filters: filters.private, config, identifyingMetadata, serverId, @@ -166,12 +160,7 @@ module.exports = async ({ port = null, config = {}, filters = {} }) => { res.locals.io = io; next(); }, - relay.request( - // Universal broker must have all webhook filters together since we don't know the type of system we get the webhook from. - config.BROKER_BOOT_MODE == 'universal' - ? utils.concatAllPublicFiltersIntoArray(filters) - : filters.public, - ), + relay.request(filters.public), ); return { diff --git a/lib/config.js b/lib/config.js index ddd31412f..ab27b9876 100644 --- a/lib/config.js +++ b/lib/config.js @@ -3,7 +3,7 @@ const path = require('path'); const camelcase = require('camelcase'); const { loadConfig } = require('snyk-config'); -let config = loadConfig(process.cwd()); +const config = loadConfig(__dirname + '/..'); function camelify(res) { return Object.keys(res).reduce((acc, _) => { @@ -16,9 +16,6 @@ function camelify(res) { function expandValue(obj, value) { let poolFound = undefined; let keyWithPool = undefined; - if (typeof value == 'boolean') { - return; - } const variableRegex = /(\\?\$.+?\b)/g; const variableMatcher = value.match(variableRegex); if (variableMatcher) { @@ -70,17 +67,12 @@ function expand(obj) { const keys = Object.keys(obj); for (const key of keys) { - // Expand nested objects - if (typeof obj[key] == 'object') { - expand(obj[key]); - } else { - const value = expandValue(obj, obj[key]); - if (value && Array.isArray(value)) { - // This will get camel-cased later on - obj[key + '_POOL'] = value; - } else if (value !== obj[key]) { - obj[key] = value; - } + const value = expandValue(obj, obj[key]); + if (value && Array.isArray(value)) { + // This will get camel-cased later on + obj[key + '_POOL'] = value; + } else if (value !== obj[key]) { + obj[key] = value; } } @@ -88,21 +80,16 @@ function expand(obj) { } // allow the user to define their own configuration -if (config.BROKER_BOOT_MODE != 'universal') { - // Classic broker - const dotenv = require('dotenv'); - - dotenv.config({ - silent: true, - path: process.cwd() + '/.env', - }); -} else { - expand(config); -} +const dotenv = require('dotenv'); + +dotenv.config({ + silent: true, + path: process.cwd() + '/.env', +}); expand(process.env); -const res = Object.assign({}, config, camelify(process.env)); +const res = Object.assign({}, camelify(config), camelify(process.env)); if (res.caCert) { res.caCert = fs.readFileSync(path.resolve(process.cwd(), res.caCert)); } diff --git a/lib/filter-rules-loading.js b/lib/filter-rules-loading.js index 18cd4a5b3..7510cee29 100644 --- a/lib/filter-rules-loading.js +++ b/lib/filter-rules-loading.js @@ -1,7 +1,7 @@ const path = require('path'); const yaml = require('js-yaml'); const fs = require('fs'); -const config = require('./config'); + const logger = require('./log'); const SUPPORTED_IAC_EXTENSIONS = ['tf', 'yaml', 'yml', 'tpl', 'json']; @@ -24,14 +24,12 @@ function nestedCopy(array) { } function injectRulesAtRuntime(filters) { - const ACCEPT_IAC = process.env.ACCEPT_IAC || config.ACCEPT_IAC; - if (ACCEPT_IAC) { + if (process.env.ACCEPT_IAC) { logger.info( - { accept: ACCEPT_IAC }, + { accept: process.env.ACCEPT_IAC }, 'Injecting Accept rules for IAC extensions - Possible values tf, yaml, yml, json, tpl', ); - - const extensions = ACCEPT_IAC.replace(/\s/g, '') + const extensions = process.env.ACCEPT_IAC.replace(/\s/g, '') .split(',') .filter((extension) => SUPPORTED_IAC_EXTENSIONS.includes(extension)); if (extensions.length <= 0) { @@ -89,16 +87,12 @@ function injectRulesAtRuntime(filters) { } } else { logger.error( - { accept: ACCEPT_IAC }, + { accept: process.env.ACCEPT_IAC }, 'Error Unexpected error at rules injection time. Remove ACCEPT_IAC env var to skip injection logic.', ); } } - - const ACCEPT_LARGE_MANIFESTS = - process.env.ACCEPT_LARGE_MANIFESTS || - config.ACCEPT_LARGE_MANIFESTS == 'true'; - if (ACCEPT_LARGE_MANIFESTS) { + if (process.env.ACCEPT_LARGE_MANIFESTS) { const scmType = CODE_SCM_ORIGINS.find((element) => { if (filters.private[0].origin.includes(element)) { return true; @@ -106,8 +100,8 @@ function injectRulesAtRuntime(filters) { }); if (scmType === 'GITHUB') { logger.info( - { accept: ACCEPT_LARGE_MANIFESTS }, - 'Injecting Accept rules for Manifest files', + { accept: process.env.ACCEPT_LARGE_MANIFESTS }, + 'Injecting Accept rules for Large Manifest files', ); const largeManifestRule = { '//': 'used to get given manifest file', @@ -116,16 +110,18 @@ function injectRulesAtRuntime(filters) { origin: 'https://${GITHUB_TOKEN}@${GITHUB_API}', }; filters.private.push(...[largeManifestRule]); + } else { + logger.error( + { accept: process.env.ACCEPT_LARGE_MANIFESTS }, + 'Large Manifest files Rules is only applicable to Github systems', + ); } } - const ACCEPT_CODE = - process.env.ACCEPT_CODE || - process.env.ACCEPT_GIT || - config.ACCEPT_CODE || - config.ACCEPT_GIT; - if (ACCEPT_CODE) { - logger.info({ accept: ACCEPT_CODE }, 'Injecting Accept rules for Code/Git'); - + if (process.env.ACCEPT_CODE || process.env.ACCEPT_GIT) { + logger.info( + { accept: process.env.ACCEPT_CODE || process.env.ACCEPT_GIT }, + 'Injecting Accept rules for Code/Git', + ); let templateGET = nestedCopy( filters.private.filter( (entry) => @@ -133,76 +129,71 @@ function injectRulesAtRuntime(filters) { CODE_SCM_ORIGINS.filter((origin) => entry.origin.includes(`{${origin}}`), ).length > 0, - )[0] || [], + )[0], ); - if (!Array.isArray(templateGET)) { - // skipping all non SCM types - const scmType = CODE_SCM_ORIGINS.find((element) => { - if (templateGET.origin.includes(element)) { - return true; - } - }); - templateGET['//'] = 'allow info refs (for git clone)'; - templateGET.origin = templateGET.origin - .replace('https://${GITHUB_TOKEN}', 'https://pat:${GITHUB_TOKEN}') - .replace( - 'https://${GITLAB}', - 'https://oauth2:${GITLAB_TOKEN}@${GITLAB}', - ); - let templatePOST = nestedCopy(templateGET); + const scmType = CODE_SCM_ORIGINS.find((element) => { + if (templateGET.origin.includes(element)) { + return true; + } + }); - templatePOST.method = 'POST'; + templateGET['//'] = 'allow info refs (for git clone)'; + templateGET.origin = templateGET.origin + .replace('https://${GITHUB_TOKEN}', 'https://pat:${GITHUB_TOKEN}') + .replace('https://${GITLAB}', 'https://oauth2:${GITLAB_TOKEN}@${GITLAB}'); + let templatePOST = nestedCopy(templateGET); - templatePOST['//'] = 'allow git-upload-pack (for git clone)'; + templatePOST.method = 'POST'; - // Code snippets rules - let templateGETForSnippets = nestedCopy( - filters.private.filter( - (entry) => - entry.method === 'GET' && - SNIPPETS_CODE_SCM_ORIGINS.filter((origin) => - entry.origin.includes(`{${origin}}`), - ).length > 0, - )[0], - ); - templateGETForSnippets['//'] = 'needed to load code snippets'; + templatePOST['//'] = 'allow git-upload-pack (for git clone)'; - switch (scmType) { - case 'AZURE_REPOS_HOST': - templateGET.path = '*/info/refs*'; - templateGETForSnippets.path = - '/:owner/_apis/git/repositories/:repo/items'; - templatePOST.path = '*/git-upload-pack'; - break; - case 'GITHUB': - templateGET.path = '*/info/refs*'; - templateGETForSnippets.path = '/repos/:name/:repo/contents/:path'; - templatePOST.path = '*/git-upload-pack'; - break; - case 'GITLAB': - templateGET.path = '*/info/refs*'; - templateGETForSnippets.path = - '/api/v4/projects/:project/repository/files/:path'; - templatePOST.path = '*/git-upload-pack'; - break; - case 'BITBUCKET': - templateGET.path = '*/info/refs*'; - templateGETForSnippets.path = - '/projects/:project/repos/:repo/browse*/:file'; - templatePOST.path = '*/git-upload-pack'; - break; - default: - logger.error( - {}, - 'Error writing Code specific rules - Cannot determine SCM type', - ); - } + // Code snippets rules + let templateGETForSnippets = nestedCopy( + filters.private.filter( + (entry) => + entry.method === 'GET' && + SNIPPETS_CODE_SCM_ORIGINS.filter((origin) => + entry.origin.includes(`{${origin}}`), + ).length > 0, + )[0], + ); + templateGETForSnippets['//'] = 'needed to load code snippets'; - filters.private.push( - ...[templateGET, templatePOST, templateGETForSnippets], - ); + switch (scmType) { + case 'AZURE_REPOS_HOST': + templateGET.path = '*/info/refs*'; + templateGETForSnippets.path = + '/:owner/_apis/git/repositories/:repo/items'; + templatePOST.path = '*/git-upload-pack'; + break; + case 'GITHUB': + templateGET.path = '*/info/refs*'; + templateGETForSnippets.path = '/repos/:name/:repo/contents/:path'; + templatePOST.path = '*/git-upload-pack'; + break; + case 'GITLAB': + templateGET.path = '*/info/refs*'; + templateGETForSnippets.path = + '/api/v4/projects/:project/repository/files/:path'; + templatePOST.path = '*/git-upload-pack'; + break; + case 'BITBUCKET': + templateGET.path = '*/info/refs*'; + templateGETForSnippets.path = + '/projects/:project/repos/:repo/browse*/:file'; + templatePOST.path = '*/git-upload-pack'; + break; + default: + logger.error( + {}, + 'Error writing Code specific rules - Cannot determine SCM type', + ); } + + filters.private.push( + ...[templateGET, templatePOST, templateGETForSnippets], + ); } return filters; } @@ -217,33 +208,16 @@ module.exports = (acceptFilename = '', folderLocation = '') => { filters = yaml.safeLoad(fs.readFileSync(acceptLocation, 'utf8')); } - if (config.BROKER_BOOT_MODE != 'universal') { - // If user brings an accept json, skip the IAC|CODE rules injection logic - // If going through the effort of loading a separate file, add your rules there - if (process.env.ACCEPT === 'accept.json') { - filters = injectRulesAtRuntime(filters); - } else { - logger.info( - { accept: process.env.ACCEPT }, - 'Custom accept json, skipping filter rule injection', - ); - } + + // If user brings an accept json, skip the IAC|CODE rules injection logic + // If going through the effort of loading a separate file, add your rules there + if (process.env.ACCEPT === 'accept.json') { + filters = injectRulesAtRuntime(filters); } else { - const brokerTypes = config.BROKER_TYPES.split(','); - brokerTypes.forEach((type) => { - filters[type] = yaml.safeLoad( - fs.readFileSync( - `${path.resolve( - process.cwd(), - // can put the paths to the default accept json under filter_rules_path: { github: /path/to/acceptjson } - config['FILTER_RULES_PATHS'][`${type.toUpperCase()}`] || - `client-templates/${type}/accept.json.sample`, - )}`, - 'utf8', - ), - ); - filters[type] = injectRulesAtRuntime(filters[type]); - }); + logger.info( + { accept: process.env.ACCEPT }, + 'Custom accept json, skipping filter rule injection', + ); } return filters; }; diff --git a/lib/filters.js b/lib/filters.js index f57a2d6d4..842c5f177 100644 --- a/lib/filters.js +++ b/lib/filters.js @@ -25,17 +25,16 @@ const validateHeaders = (headerFilters, requestHeaders = []) => { }; // reads config that defines -module.exports = (ruleSource, filterType) => { - let rules = {}; +module.exports = (ruleSource) => { + let rules = []; const config = require('./config'); // polymorphic support if (Array.isArray(ruleSource)) { - rules['default'] = {}; - rules['default'][filterType] = ruleSource; + rules = ruleSource; } else if (ruleSource) { try { - rules = ruleSource; + rules = require(ruleSource); } catch (error) { logger.warn( { ruleSource, error }, @@ -43,191 +42,180 @@ module.exports = (ruleSource, filterType) => { ); } } - Object.keys(rules).forEach((key) => { - if (!Array.isArray(rules[key][filterType])) { - throw new Error( - `Expected array of filter rules, got ${typeof rules} instead.`, - ); - } - }); + + if (!Array.isArray(rules)) { + throw new Error( + `Expected array of filter rules, got '${typeof rules}' instead.`, + ); + } logger.info({ rulesCount: rules.length }, 'loading new rules'); // array of entries with - const tests = []; - Object.keys(rules).forEach((ruleSystemType) => { - const testsForKey = rules[ruleSystemType][filterType].map((entry) => { - const keys = []; - let { - method, - origin, - path: entryPath, - valid, - requiredCapabilities, - } = entry; - const baseOrigin = origin; - const { stream } = entry; - method = (method || 'get').toLowerCase(); - valid = valid || []; - - const bodyFilters = valid.filter((v) => !!v.path && !v.regex); - const bodyRegexFilters = valid.filter((v) => !!v.path && !!v.regex); - const queryFilters = valid.filter((v) => !!v.queryParam); - const headerFilters = valid.filter((v) => !!v.header); - - // now track if there's any values that we need to interpolate later - const fromConfig = {}; - - // slightly bespoke version of replace-vars.js - entryPath = (entryPath || '').replace(/(\${.*?})/g, (_, match) => { - const key = match.slice(2, -1); // ditch the wrappers - fromConfig[key] = config[key] || ''; - return ':' + key; - }); - - if (entryPath[0] !== '/') { - entryPath = '/' + entryPath; - } + const tests = rules.map((entry) => { + const keys = []; + let { + method, + origin, + path: entryPath, + valid, + requiredCapabilities, + } = entry; + const baseOrigin = origin; + const { stream } = entry; + method = (method || 'get').toLowerCase(); + valid = valid || []; + + const bodyFilters = valid.filter((v) => !!v.path && !v.regex); + const bodyRegexFilters = valid.filter((v) => !!v.path && !!v.regex); + const queryFilters = valid.filter((v) => !!v.queryParam); + const headerFilters = valid.filter((v) => !!v.header); + + // now track if there's any values that we need to interpolate later + const fromConfig = {}; + + // slightly bespoke version of replace-vars.js + entryPath = (entryPath || '').replace(/(\${.*?})/g, (_, match) => { + const key = match.slice(2, -1); // ditch the wrappers + fromConfig[key] = config[key] || ''; + return ':' + key; + }); - logger.debug({ method, path: entryPath }, 'adding new filter rule'); - const regexp = pathRegexp(entryPath, keys); + if (entryPath[0] !== '/') { + entryPath = '/' + entryPath; + } - return (req) => { - if ( - ruleSystemType != 'default' && - req.headers['x-snyk-broker-type'] != ruleSystemType // this is always false for now, header is uuid, rulesystemtype is github|gitlab|etc - ) { - return false; - } - // check the request method - if (req.method.toLowerCase() !== method && method !== 'any') { - return false; - } + logger.debug({ method, path: entryPath }, 'adding new filter rule'); + const regexp = pathRegexp(entryPath, keys); - // Do not allow directory traversal - if (path.normalize(req.url) !== req.url) { - return false; - } + return (req) => { + // check the request method + if (req.method.toLowerCase() !== method && method !== 'any') { + return false; + } - // Discard any fragments before further processing - const mainURI = req.url.split('#')[0]; + // Do not allow directory traversal + if (path.normalize(req.url) !== req.url) { + return false; + } - // query params might contain additional "?"s, only split on the 1st one - const parts = mainURI.split('?'); - let [url, querystring] = [parts[0], parts.slice(1).join('?')]; - const res = regexp.exec(url); - if (!res) { - // no url match - return false; - } + // Discard any fragments before further processing + const mainURI = req.url.split('#')[0]; - // reconstruct the url from the user config - for (let i = 1; i < res.length; i++) { - const val = fromConfig[keys[i - 1].name]; - if (val) { - url = url.replace(res[i], val); - } + // query params might contain additional "?"s, only split on the 1st one + const parts = mainURI.split('?'); + let [url, querystring] = [parts[0], parts.slice(1).join('?')]; + const res = regexp.exec(url); + if (!res) { + // no url match + return false; + } + + // reconstruct the url from the user config + for (let i = 1; i < res.length; i++) { + const val = fromConfig[keys[i - 1].name]; + if (val) { + url = url.replace(res[i], val); } + } - // if validity filters are present, at least one must be satisfied - if ( - bodyFilters.length || - bodyRegexFilters.length || - queryFilters.length - ) { - let isValid; - - let parsedBody; - if (bodyFilters.length) { - parsedBody = tryJSONParse(req.body); - - // validate against the body - isValid = bodyFilters.some(({ path: filterPath, value }) => { - return undefsafe(parsedBody, filterPath, value); - }); - } + // if validity filters are present, at least one must be satisfied + if ( + bodyFilters.length || + bodyRegexFilters.length || + queryFilters.length + ) { + let isValid; + + let parsedBody; + if (bodyFilters.length) { + parsedBody = tryJSONParse(req.body); + + // validate against the body + isValid = bodyFilters.some(({ path: filterPath, value }) => { + return undefsafe(parsedBody, filterPath, value); + }); + } - if (!isValid && bodyRegexFilters.length) { - parsedBody = parsedBody || tryJSONParse(req.body); - - // validate against the body by regex - isValid = bodyRegexFilters.some(({ path: filterPath, regex }) => { - try { - const re = new RegExp(regex); - return re.test(undefsafe(parsedBody, filterPath)); - } catch (error) { - logger.error( - { error, path: filterPath, regex }, - 'failed to test regex rule', - ); - return false; - } - }); - } + if (!isValid && bodyRegexFilters.length) { + parsedBody = parsedBody || tryJSONParse(req.body); + + // validate against the body by regex + isValid = bodyRegexFilters.some(({ path: filterPath, regex }) => { + try { + const re = new RegExp(regex); + return re.test(undefsafe(parsedBody, filterPath)); + } catch (error) { + logger.error( + { error, path: filterPath, regex }, + 'failed to test regex rule', + ); + return false; + } + }); + } - // no need to check query filters if the request is already valid - if (!isValid && queryFilters.length) { - const parsedQuerystring = qs.parse(querystring); + // no need to check query filters if the request is already valid + if (!isValid && queryFilters.length) { + const parsedQuerystring = qs.parse(querystring); - // validate against the querystring - isValid = queryFilters.every(({ queryParam, values }) => { - return values.some((value) => - minimatch(parsedQuerystring[queryParam] || '', value, { - dot: true, - }), - ); - }); - } + // validate against the querystring + isValid = queryFilters.every(({ queryParam, values }) => { + return values.some((value) => + minimatch(parsedQuerystring[queryParam] || '', value, { + dot: true, + }), + ); + }); + } - if (!isValid) { - return false; - } + if (!isValid) { + return false; } + } - if (headerFilters.length) { - if (!validateHeaders(headerFilters, req.headers)) { - return false; - } + if (headerFilters.length) { + if (!validateHeaders(headerFilters, req.headers)) { + return false; } + } - if (requiredCapabilities) { - let matchedAll = true; - for (const c of requiredCapabilities) { - if (!req?.locals?.capabilities.includes(c)) { - matchedAll = false; - logger.warn( - { - path: entryPath, - capability: c, - clientCapabilities: req?.locals?.capabilities, - }, - 'client does not report support for capability', - ); - } - } - if (!matchedAll) { - // We have to throw to avoid it getting approved by a generic matcher later on - throw new Error( - 'client does not support all required capabilities for endpoint', + if (requiredCapabilities) { + let matchedAll = true; + for (const c of requiredCapabilities) { + if (!req?.locals?.capabilities.includes(c)) { + matchedAll = false; + logger.warn( + { + path: entryPath, + capability: c, + clientCapabilities: req?.locals?.capabilities, + }, + 'client does not report support for capability', ); } } + if (!matchedAll) { + // We have to throw to avoid it getting approved by a generic matcher later on + throw new Error( + 'client does not support all required capabilities for endpoint', + ); + } + } - const origin = replace(baseOrigin, config); - logger.debug( - { path: entryPath, origin, url, querystring }, - 'rule matched', - ); - - querystring = querystring ? `?${querystring}` : ''; - return { - url: origin + url + querystring, - auth: entry.auth && authHeader(entry.auth), - stream, - }; + const origin = replace(baseOrigin, config); + logger.debug( + { path: entryPath, origin, url, querystring }, + 'rule matched', + ); + + querystring = querystring ? `?${querystring}` : ''; + return { + url: origin + url + querystring, + auth: entry.auth && authHeader(entry.auth), + stream, }; - }); - tests.push(...testsForKey); + }; }); return (payload, callback) => { diff --git a/lib/log.js b/lib/log.js index 0be350055..cff37efe2 100644 --- a/lib/log.js +++ b/lib/log.js @@ -2,18 +2,24 @@ const bunyan = require('bunyan'); const escapeRegExp = require('lodash.escaperegexp'); const mapValues = require('lodash.mapvalues'); const config = require('./config'); -const utils = require('./utils'); -const get = require('lodash.get'); -const sanitiseConfigVariable = (raw, variable, coordinate = []) => +const sanitiseConfigVariable = (raw, variable) => raw.replace( - new RegExp( - escapeRegExp(coordinate ? get(config, coordinate) : config[variable]), - 'igm', - ), + new RegExp(escapeRegExp(config[variable]), 'igm'), '${' + variable + '}', ); +const sanitiseConfigVariables = (raw, variable) => { + for (const pool of config[variable]) { + raw = raw.replace( + new RegExp(escapeRegExp(pool), 'igm'), + '${' + variable + '}', + ); + } + + return raw; +}; + // sanitises sensitive values, replacing all occurences with label function sanitise(raw) { if (!raw || typeof raw !== 'string') { @@ -23,6 +29,7 @@ function sanitise(raw) { const variables = [ 'BROKER_TOKEN', 'GITHUB_TOKEN', + 'GITHUB_TOKEN_POOL', 'BITBUCKET_USERNAME', 'BITBUCKET_PASSWORD', 'GITLAB_TOKEN', @@ -50,18 +57,14 @@ function sanitise(raw) { // Copies original `raw`, doesn't mutate it. // Regexp is case-insensitive, global and multiline matching, // this way all occurences are replaced. - - const variableCoordinates = utils.findMemberCoordinates(variable, config); - variableCoordinates.forEach((coordinate) => { - raw = sanitiseConfigVariable(raw, variable, coordinate.path); - }); - const variablePoolCoordinates = utils.findMemberCoordinates( - `${variable}_POOL`, - config, - ); - variablePoolCoordinates.forEach((coordinate) => { - raw = sanitiseConfigVariable(raw, `${variable}_POOL`, coordinate.path); - }); + if (config[variable]) { + raw = sanitiseConfigVariable(raw, variable); + } + + const pool = `${variable}_POOL`; + if (config[pool]) { + raw = sanitiseConfigVariables(raw, pool); + } } return raw; diff --git a/lib/relay.js b/lib/relay.js index 34842b27d..bc10b5a52 100644 --- a/lib/relay.js +++ b/lib/relay.js @@ -14,7 +14,6 @@ const NodeCache = require('node-cache'); const metrics = require('./metrics'); const config = require('./config'); const { BrokerServerPostResponseHandler } = require('./stream-posts'); -const utils = require('./utils'); const { gitHubCommitSigningEnabled, gitHubTreeCheckNeeded, @@ -139,7 +138,7 @@ function streamResponseHandler(token) { // 4. Get response over websocket conn (logged) // 5. Send response over HTTP conn function forwardHttpRequest(filterRules) { - const filters = Filters(filterRules, 'public'); // public only here + const filters = Filters(filterRules); return (req, res) => { // If this is the server, we should receive a Snyk-Request-Id header from upstream @@ -152,7 +151,6 @@ function forwardHttpRequest(filterRules) { requestId: req.headers['snyk-request-id'], maskedToken: req.maskedToken, hashedToken: req.hashedToken, - type: req.headers['x-snyk-broker-type'] || '', }; logger.info(logContext, 'received request over HTTP connection'); @@ -342,24 +340,14 @@ function legacyStreaming(logContext, rqst, config, io, streamingID) { // 3. Relay over HTTP conn (logged) // 4. Get response over HTTP conn (logged) // 5. Send response over websocket conn -function forwardWebSocketRequest(rawFilterRules, config, io, serverId) { +function forwardWebSocketRequest(filterRules, config, io, serverId) { + const filters = Filters(filterRules); + return (brokerToken) => ( { url, headers = {}, method, body = null, streamingID = '' } = {}, emit, ) => { - const targetedServiceType = - config.BROKER_BOOT_MODE == 'universal' && headers['x-snyk-broker-type'] - ? utils.getRequestTypeFromTypeId( - config, - headers['x-snyk-broker-type'], - ) - : null; - const filterRules = targetedServiceType - ? rawFilterRules[targetedServiceType].private - : rawFilterRules; - - const filters = Filters(filterRules, 'private'); // expected private only here const requestId = headers['snyk-request-id']; const logContext = { url, diff --git a/lib/server/broker-middleware.ts b/lib/server/broker-middleware.ts new file mode 100644 index 000000000..b44127fb3 --- /dev/null +++ b/lib/server/broker-middleware.ts @@ -0,0 +1,23 @@ +import { Request, Response, NextFunction } from 'express'; +import * as logger from '../log'; +import * as config from '../config'; +export const validateBrokerTypeMiddleware = ( + req: Request, + res: Response, + next: NextFunction, +) => { + const localConfig = config as unknown as Record; + if ( + localConfig.brokerServerUniversalConfigEnabled && + !req?.headers['x-snyk-broker-type'] + ) { + const logContext = { url: req.url, headers: req.headers }; + logger.warn( + { logContext }, + 'Error: Request does not contain the x-snyk-broker-type header', + ); + // Will eventually return an error when all services will have this enabled + // return res.status(400).json({ error: 'Missing x-broker-type header' }); + } + next(); // Passes the request to the next middleware +}; diff --git a/lib/server/index.js b/lib/server/index.js index cb5d7d621..a64682558 100644 --- a/lib/server/index.js +++ b/lib/server/index.js @@ -5,6 +5,7 @@ const version = require('../version'); const { maskToken, hashToken } = require('../token'); const metrics = require('../metrics'); const { applyPrometheusMiddleware } = require('./prometheus-middleware'); +const { validateBrokerTypeMiddleware } = require('./broker-middleware'); module.exports = async ({ config = {}, port = null, filters = {} }) => { logger.info({ version }, 'running in server mode'); @@ -48,43 +49,6 @@ module.exports = async ({ config = {}, port = null, filters = {} }) => { return res.status(404).json({ ok: false }); }); - app.all( - '/broker/universal/:typeId/:token/*', - (req, res, next) => { - const token = req.params.token; - const typeId = req.params.typeId; - const maskedToken = maskToken(token); - const hashedToken = hashToken(token); - req.maskedToken = maskedToken; - req.hashedToken = hashedToken; - // check if we have this broker in the connections - if (!connections.has(token)) { - metrics.incrementHttpRequestsTotal(false); - logger.warn( - { maskedToken, hashedToken }, - 'no matching connection found', - ); - return res.status(404).json({ ok: false }); - } - - // Grab a first (newest) client from the pool - // This is really silly... - res.locals.io = connections.get(token)[0].socket; - res.locals.socketVersion = connections.get(token)[0].socketVersion; - res.locals.capabilities = connections.get(token)[0].metadata.capabilities; - req.locals = {}; - req.locals.capabilities = connections.get(token)[0].metadata.capabilities; - - // strip the leading url - req.url = req.url.slice(`/broker/universal/${typeId}/${token}`.length); - req.headers['x-snyk-broker-type'] = typeId; - logger.debug({ url: req.url, typeId: typeId }, 'request'); - - next(); - }, - relay.request(filters.public), - ); - app.all( '/broker/:token/*', (req, res, next) => { @@ -118,6 +82,7 @@ module.exports = async ({ config = {}, port = null, filters = {} }) => { next(); }, + validateBrokerTypeMiddleware, relay.request(filters.public), ); diff --git a/lib/utils.ts b/lib/utils.ts deleted file mode 100644 index 2d4ba54e1..000000000 --- a/lib/utils.ts +++ /dev/null @@ -1,71 +0,0 @@ -interface SourceTypes { - [key: string]: SourceType; -} - -export interface Coordinates { - path: string[]; - value: any; -} - -interface SourceType { - publicId: string; - brokerType: string; -} -const findTypeByPublicId = ( - sourceTypes: SourceTypes, - publicId: string, -): string | null => { - const typesArray = Object.keys(sourceTypes); - for (let i = 0; i < typesArray.length; i++) { - if (sourceTypes[typesArray[i]].publicId == publicId) { - return sourceTypes[typesArray[i]].brokerType; - } - } - return null; -}; - -export const getRequestTypeFromTypeId = ( - config, - typeId: string, -): string | null => { - const result = findTypeByPublicId(config['SOURCE_TYPES'], typeId); - return result || null; -}; - -export const concatAllPublicFiltersIntoArray = (allFilters: Object): Object => { - const output: Object[] = []; - const filtersKeys = Object.keys(allFilters); - for (let i = 0; i < filtersKeys.length; i++) { - if (allFilters[filtersKeys[i]].hasOwnProperty('public')) { - output.push(...allFilters[filtersKeys[i]].public); - } - } - - return output; -}; - -export const findMemberCoordinates = ( - memberName: string, - obj: any, - path: string[] = [], -): Coordinates[] => { - const coordinates: Coordinates[] = []; - - for (const key in obj) { - if (obj.hasOwnProperty(key)) { - const newPath = [...path, key]; - - if (key === memberName) { - coordinates.push({ path: newPath, value: obj[key] }); - } - - if (typeof obj[key] === 'object' && obj[key] !== null) { - coordinates.push( - ...findMemberCoordinates(memberName, obj[key], newPath), - ); - } - } - } - - return coordinates; -}; diff --git a/package.json b/package.json index 33e298930..7e9c7ae01 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,6 @@ "express-prom-bundle": "^5.1.5", "js-yaml": "^3.13.1", "lodash.escaperegexp": "^4.1.2", - "lodash.get": "^4.4.2", "lodash.mapvalues": "^4.6.0", "lodash.merge": "^4.6.2", "minimatch": "^3.0.4", diff --git a/test/fixtures/client/universalFilters.json b/test/fixtures/client/universalFilters.json deleted file mode 100644 index 4732453b0..000000000 --- a/test/fixtures/client/universalFilters.json +++ /dev/null @@ -1,215 +0,0 @@ -{ - "private": [ - { - "path": "/basic-auth", - "method": "GET", - "origin": "http://localhost:9000", - "auth": { - "scheme": "basic", - "username": "user", - "password": "pass" - } - }, - - { - "path": "/echo-param/:param", - "method": "GET", - "origin": "http://localhost:9000" - }, - - { - "path": "/echo-body", - "method": "POST", - "origin": "http://localhost:9000" - }, - - { - "path": "/echo-body/filtered", - "method": "POST", - "origin": "http://localhost:9000", - "valid": [ - { - "//": "accept requests with 'proxy.*: please' in their body", - "path": "proxy.*", - "value": "please" - } - ] - }, - - { - "path": "/echo-headers/github", - "method": "POST", - "origin": "http://githubToken@localhost:9000" - }, - - { - "path": "/echo-headers/github-token-in-origin", - "method": "POST", - "origin": "http://githubToken1@localhost:9000" - }, - - { - "path": "/echo-headers/bitbucket", - "method": "POST", - "origin": "http://bitbucketUser:bitbucketPassword@localhost:9000" - }, - - { - "path": "/echo-headers", - "method": "POST", - "origin": "http://localhost:9000" - }, - - { - "path": "/echo-query", - "method": "GET", - "origin": "http://localhost:9000" - }, - - { - "path": "/echo-query/filtered", - "method": "GET", - "origin": "http://localhost:9000", - "valid": [ - { - "queryParam": "proxyMe", - "values": ["please"] - } - ] - }, - - { - "path": "/test-blob/*", - "method": "GET", - "origin": "http://localhost:9000" - }, - - { - "path": "/test-blob-param/*", - "method": "GET", - "origin": "http://localhost:9000" - }, - - { - "path": "/repos/:repo/:owner/contents/folder*/package.json", - "method": "GET", - "origin": "http://localhost:9000" - }, - - { - "path": "/long/nested/path/to/file.ext", - "method": "GET", - "origin": "http://localhost:9000" - }, - - { - "path": "/long/nested/partially/encoded%2Fpath%2Fto%2Ffile.ext", - "method": "GET", - "origin": "http://localhost:9000" - }, - - { - "path": "/huge-file", - "method": "GET", - "origin": "http://localhost:9000" - }, - - { - "path": "/echo-param-protected/:param", - "method": "GET", - "origin": "http://localhost:9000", - "valid": [ - { - "header": "accept", - "values": ["valid.accept.header"] - } - ] - }, - { - "path": "/client-not-capable", - "method": "GET", - "origin": "http://localhost:9000" - }, - { - "//": "used to redirect requests to snyk git client", - "method": "any", - "path": "/snykgit/*", - "origin": "http://localhost:9000" - } - ], - "public": [ - { - "path": "/basic-auth", - "method": "GET" - }, - - { - "path": "/echo-param/:param", - "method": "GET" - }, - - { - "path": "/echo-body", - "method": "POST" - }, - - { - "path": "/echo-body/filtered", - "method": "POST", - "valid": [ - { - "//": "accept requests with 'proxy.*: please' in their body", - "path": "proxy.*", - "value": "please" - } - ] - }, - - { - "path": "/echo-headers", - "method": "POST" - }, - - { - "path": "/echo-query", - "method": "GET" - }, - - { - "path": "/huge-file", - "method": "GET" - }, - - { - "path": "/echo-query/filtered", - "method": "GET", - "valid": [ - { - "queryParam": "proxyMe", - "values": ["please"] - } - ] - }, - { - "path": "/echo-param-protected/:param", - "method": "GET", - "valid": [ - { - "header": "accept", - "values": ["valid.accept.header"] - } - ] - }, - { - "method": "any", - "path": "/server-side-blocked", - "origin": "http://localhost:9000" - }, - { - "method": "any", - "path": "/server-side-blocked-streaming", - "origin": "http://localhost:9000", - "stream": true - } - ] -} diff --git a/test/functional/server-client-universal.test.ts b/test/functional/server-client-universal.test.ts new file mode 100644 index 000000000..51ddeedac --- /dev/null +++ b/test/functional/server-client-universal.test.ts @@ -0,0 +1,80 @@ +process.env.SNYK_BROKER_SERVER_UNIVERSAL_CONFIG_ENABLED = 'true'; +import * as path from 'path'; +import { axiosClient } from '../setup/axios-client'; +import { + BrokerClient, + closeBrokerClient, + createBrokerClient, +} from '../setup/broker-client'; +import { + BrokerServer, + closeBrokerServer, + createBrokerServer, + waitForBrokerClientConnection, +} from '../setup/broker-server'; +import { TestWebServer, createTestWebServer } from '../setup/test-web-server'; + +const fixtures = path.resolve(__dirname, '..', 'fixtures'); +const serverAccept = path.join(fixtures, 'server', 'filters.json'); +const clientAccept = path.join(fixtures, 'client', 'filters.json'); + +describe('proxy requests originating from behind the broker server', () => { + let tws: TestWebServer; + let bs: BrokerServer; + let bc: BrokerClient; + let brokerToken: string; + + const spyLogWarn = jest + .spyOn(require('bunyan').prototype, 'warn') + .mockImplementation((value) => { + return value; + }); + + beforeAll(async () => { + tws = await createTestWebServer(); + + bs = await createBrokerServer({ filters: serverAccept, port: 8100 }); + + bc = await createBrokerClient({ + brokerServerUrl: `http://localhost:${bs.port}`, + brokerToken: 'broker-token-12345', + filters: clientAccept, + type: 'client', + }); + ({ brokerToken } = await waitForBrokerClientConnection(bs)); + }); + + afterEach(async () => { + spyLogWarn.mockReset(); + }); + afterAll(async () => { + spyLogWarn.mockReset(); + await tws.server.close(); + await closeBrokerClient(bc); + await closeBrokerServer(bs); + }); + + it('successfully broker GET', async () => { + const response = await axiosClient.get( + `http://localhost:${bs.port}/broker/${brokerToken}/echo-param/xyz`, + ); + + expect(response.status).toEqual(200); + expect(response.data).toEqual('xyz'); + }); + + it('successfully warn logs requests without x-snyk-broker-type header', async () => { + const response = await axiosClient.get( + `http://localhost:${bs.port}/broker/${brokerToken}/echo-param/xyz`, + ); + + expect(response.status).toEqual(200); + expect(response.data).toEqual('xyz'); + + expect(spyLogWarn).toHaveBeenCalledTimes(1); + expect(spyLogWarn).toHaveBeenCalledWith( + expect.any(Object), + 'Error: Request does not contain the x-snyk-broker-type header', + ); + }); +}); diff --git a/test/functional/server-universal-client.test.ts b/test/functional/server-universal-client.test.ts deleted file mode 100644 index ebe0b533e..000000000 --- a/test/functional/server-universal-client.test.ts +++ /dev/null @@ -1,518 +0,0 @@ -process.env.SNYK_BROKER_BOOT_MODE = 'universal'; -process.env.SNYK_BROKER_TYPES = 'testtype'; -process.env.SNYK_SOURCE_TYPES__testtype__publicId = - '9a3e5d90-b782-468a-a042-9a2073736f00'; -process.env.SNYK_SOURCE_TYPES__testtype__brokerType = 'testtype'; - -import * as path from 'path'; -import * as version from '../../lib/version'; -import { axiosClient } from '../setup/axios-client'; - -const fixtures = path.resolve(__dirname, '..', 'fixtures'); -process.env.SNYK_FILTER_RULES_PATHS__TESTTYPE = `${path.join( - fixtures, - 'client', - 'universalFilters.json', -)}`; - -import { - BrokerClient, - closeBrokerClient, - createBrokerClient, -} from '../setup/broker-client'; -import { - BrokerServer, - closeBrokerServer, - createBrokerServer, - waitForBrokerClientConnection, -} from '../setup/broker-server'; -import { TestWebServer, createTestWebServer } from '../setup/test-web-server'; - -const serverAccept = path.join(fixtures, 'server', 'filters.json'); -// const clientAccept = path.join(fixtures, 'client', 'filters.json'); - -describe('proxy requests originating from behind the broker server', () => { - let tws: TestWebServer; - let bs: BrokerServer; - let bc: BrokerClient; - let brokerToken: string; - let metadata: unknown; - - beforeAll(async () => { - tws = await createTestWebServer(); - - bs = await createBrokerServer({ filters: serverAccept }); - - bc = await createBrokerClient({ - brokerServerUrl: `http://localhost:${bs.port}`, - brokerToken: 'broker-token-12345', - // filters: clientAccept, - type: 'client', - }); - ({ brokerToken, metadata } = await waitForBrokerClientConnection(bs)); - }); - - afterAll(async () => { - await tws.server.close(); - await closeBrokerClient(bc); - await closeBrokerServer(bs); - }); - - it('server identifies self to client', async () => { - expect(brokerToken).toEqual('broker-token-12345'); - expect(metadata).toStrictEqual({ - capabilities: ['post-streams'], - filters: expect.any(Object), - clientId: expect.any(String), - preflightChecks: expect.any(Array), - version: version, - }); - }); - - it('successfully broker POST', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-body`, - { some: { example: 'json' } }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toStrictEqual({ - some: { example: 'json' }, - }); - }); - - it('successfully broker exact bytes of POST body', async () => { - // stringify the JSON unusually to ensure an unusual exact body - const body = Buffer.from( - JSON.stringify({ some: { example: 'json' } }, null, 5), - ); - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-body`, - body, - { - headers: { - 'content-type': 'application/json', - }, - - transformResponse: (r) => r, - }, - ); - - expect(response.status).toEqual(200); - expect(Buffer.from(response.data)).toEqual(body); - }); - - it('successfully broker GET', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-param/xyz`, - ); - - expect(response.status).toEqual(200); - expect(response.data).toEqual('xyz'); - }); - - it('variable substitution', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-body`, - { - BROKER_VAR_SUB: ['swap.me'], - swap: { me: '${BROKER_TYPE}:${BROKER_TOKEN}' }, - }, - { - headers: { - 'content-type': 'application/json', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toStrictEqual({ - BROKER_VAR_SUB: ['swap.me'], - swap: { me: 'client:broker-token-12345' }, - }); - }); - - it('block request for non-whitelisted url', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/not-allowed`, - {}, - { - headers: { - 'content-type': 'application/json', - }, - }, - ); - - expect(response.status).toEqual(401); - expect(response.data).toStrictEqual({ - message: 'blocked', - reason: - 'Response does not match any accept rule, blocking websocket request', - url: '/not-allowed', - }); - }); - - it('allow request for valid url with valid body', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-body/filtered`, - { proxy: { me: 'please' } }, - { - headers: { - 'content-type': 'application/json', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toStrictEqual({ - proxy: { me: 'please' }, - }); - }); - - it('block request for valid url with invalid body', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-body/filtered`, - { proxy: { me: 'now!' } }, - { - headers: { - 'content-type': 'application/json', - }, - }, - ); - - expect(response.status).toEqual(401); - expect(response.data).toStrictEqual({ - message: 'blocked', - reason: - 'Response does not match any accept rule, blocking websocket request', - url: '/echo-body/filtered', - }); - }); - - it('allow request for valid url with valid query param', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-query/filtered`, - { - params: { proxyMe: 'please' }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toStrictEqual({ proxyMe: 'please' }); - }); - - it('block request for valid url with invalid query param', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-query/filtered`, - { - params: { proxyMe: 'now!' }, - }, - ); - - expect(response.status).toEqual(401); - expect(response.data).toStrictEqual({ - message: 'blocked', - reason: - 'Response does not match any accept rule, blocking websocket request', - url: '/echo-query/filtered?proxyMe=now!', - }); - }); - - it('block request for valid url with missing query param', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-query/filtered`, - ); - - expect(response.status).toEqual(401); - expect(response.data).toStrictEqual({ - message: 'blocked', - reason: - 'Response does not match any accept rule, blocking websocket request', - url: '/echo-query/filtered', - }); - }); - - it('bad broker id', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}XXX/echo-body`, - {}, - ); - - expect(response.status).toEqual(404); - }); - - it('broker token is not included in headers from client to private', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-headers`, - {}, - ); - - expect(response.status).toEqual(200); - expect(response.data).not.toHaveProperty('x-broker-token'); - }); - - it('x-forwarded-* headers are stripped from the request before being forwarded', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-headers`, - {}, - { - headers: { - 'x-forwarded-proto': 'https', - 'x-forwarded-for': '127.0.0.1', - 'x-forwarded-port': '8080', - 'X-Forwarded-Port': '8080', - 'x-forwarded-host': 'banana', - forwarded: - 'by=broker;for=127.0.0.1;host=banana;port=8080;proto=https', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).not.toHaveProperty('x-forwarded-proto'); - expect(response.data).not.toHaveProperty('x-forwarded-for'); - expect(response.data).not.toHaveProperty('x-forwarded-port'); - expect(response.data).not.toHaveProperty('X-Forwarded-Port'); - expect(response.data).not.toHaveProperty('x-forwarded-host'); - expect(response.data).not.toHaveProperty('forwarded'); - }); - - it('querystring parameters are brokered', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-query`, - { - params: { - shape: 'square', - colour: 'yellow', - url_as_param: 'https://clojars.org/search?q=btc', - one_more_top_level_param: 'true', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toStrictEqual({ - shape: 'square', - colour: 'yellow', - url_as_param: 'https://clojars.org/search?q=btc', - one_more_top_level_param: 'true', - }); - }); - - it('approved URLs are blocked when escaped', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/long/nested%2Fpath%2Fto%2Ffile.ext`, - ); - - expect(response.status).toEqual(401); - expect(response.data).toStrictEqual({ - message: 'blocked', - reason: - 'Response does not match any accept rule, blocking websocket request', - url: '/long/nested%2Fpath%2Fto%2Ffile.ext', - }); - }); - - it('block request for url where client does not support required capability', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/client-not-capable`, - ); - - expect(response.status).toEqual(401); - expect(response.data).toStrictEqual({ - message: 'blocked', - reason: 'Request does not match any accept rule, blocking HTTP request', - url: '/client-not-capable', - }); - }); - - it('approved URLs are brokered when escaped as expected', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/long/nested/partially/encoded%2Fpath%2Fto%2Ffile.ext`, - ); - - expect(response.status).toEqual(200); - expect(response.data).toEqual( - '/long/nested/partially/encoded%2Fpath%2Fto%2Ffile.ext', - ); - }); - - it('content-length is not set when using chunked http', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-headers`, - { - headers: { - 'Transfer-Encoding': 'chunked', - }, - }, - ); - - expect(response.headers).not.toHaveProperty('content-length'); - }); - - it('content-length is set without chunked http', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-headers`, - {}, - ); - - expect(response.headers).toHaveProperty('content-length'); - }); - - it('auth header is replaced when url contains token', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-headers/github`, - {}, - { - headers: { - Authorization: 'broker auth', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toHaveProperty('authorization'); - expect(response.data.authorization).toEqual('token githubToken'); - }); - - it('auth header is is replaced when url contains basic auth', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-headers/bitbucket`, - {}, - ); - const auth = response.data.authorization?.replace('Basic ', ''); - const encodedAuth = Buffer.from(auth, 'base64').toString('utf-8'); - - expect(response.status).toEqual(200); - expect(response.data).toHaveProperty('authorization'); - expect(encodedAuth).toEqual('bitbucketUser:bitbucketPassword'); - }); - - it('successfully broker on endpoint that forwards requests with basic auth', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/basic-auth`, - ); - const auth = response.data.replace('Basic ', ''); - const encodedAuth = Buffer.from(auth, 'base64').toString('utf-8'); - - expect(response.status).toEqual(200); - expect(encodedAuth).toEqual('user:pass'); - }); - - it('ignores accept-encoding (gzip)', async () => { - const paramRequiringCompression = 'hello-'.repeat(200); - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-param/${paramRequiringCompression}`, - { - headers: { - 'Accept-Encoding': 'gzip', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toEqual(paramRequiringCompression); - }); - - it('ignores accept-encoding (deflate)', async () => { - const paramRequiringCompression = 'hello-'.repeat(200); - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-param/${paramRequiringCompression}`, - { - headers: { - 'Accept-Encoding': 'deflate', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toEqual(paramRequiringCompression); - }); - - it('successfully stream data', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/test-blob/1`, - { - headers: {}, - }, - ); - const buf = Buffer.alloc(500); - for (let i = 0; i < 500; i++) { - buf.writeUInt8(i & 0xff, i); - } - - expect(response.status).toEqual(299); - expect(response.headers).toHaveProperty('test-orig-url'); - expect(response.headers['test-orig-url']).toEqual('/test-blob/1'); - expect(response.data).toEqual(String(buf)); - }); - - it('successfully redirect POST request to git client', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/snykgit/echo-body`, - { some: { example: 'json' } }, - { - headers: { - 'content-type': 'application/json', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toStrictEqual({ - some: { example: 'json' }, - }); - }); - - it('successfully redirect exact bytes of POST body to git client', async () => { - const body = Buffer.from( - JSON.stringify({ some: { example: 'json' } }, null, 5), - ); - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/snykgit/echo-body`, - body, - { - headers: { - 'content-type': 'application/json', - }, - - transformResponse: (r) => r, - }, - ); - - expect(response.status).toEqual(200); - expect(Buffer.from(response.data)).toEqual(body); - }); - - it('successfully redirect exact bytes of POST body to git client', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/snykgit/echo-param/xyz`, - ); - - expect(response.status).toEqual(200); - expect(response.data).toEqual('xyz'); - }); - - it('accept large responses', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/huge-file`, - ); - - expect(response.status).toEqual(200); - expect(response.data.data.length).toEqual(20971522); - }); - - it('allow request to git client with valid param', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/snykgit/echo-query`, - { - params: { - proxyMe: 'please', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toStrictEqual({ proxyMe: 'please' }); - }); -}); diff --git a/test/functional/universal-server-legacy-client.test.ts b/test/functional/universal-server-legacy-client.test.ts deleted file mode 100644 index 064a4d791..000000000 --- a/test/functional/universal-server-legacy-client.test.ts +++ /dev/null @@ -1,509 +0,0 @@ -process.env.SNYK_SOURCE_TYPES__testtype__publicId = - '9a3e5d90-b782-468a-a042-9a2073736f00'; - -import * as path from 'path'; -import * as version from '../../lib/version'; -import { axiosClient } from '../setup/axios-client'; - -import { - BrokerClient, - closeBrokerClient, - createBrokerClient, -} from '../setup/broker-client'; -import { - BrokerServer, - closeBrokerServer, - createBrokerServer, - waitForBrokerClientConnection, -} from '../setup/broker-server'; -import { TestWebServer, createTestWebServer } from '../setup/test-web-server'; - -const fixtures = path.resolve(__dirname, '..', 'fixtures'); -const serverAccept = path.join(fixtures, 'server', 'filters.json'); -const clientAccept = path.join(fixtures, 'client', 'filters.json'); - -describe('proxy requests originating from behind the broker server', () => { - let tws: TestWebServer; - let bs: BrokerServer; - let bc: BrokerClient; - let brokerToken: string; - let metadata: unknown; - - beforeAll(async () => { - tws = await createTestWebServer(); - - bs = await createBrokerServer({ filters: serverAccept }); - - bc = await createBrokerClient({ - brokerServerUrl: `http://localhost:${bs.port}`, - brokerToken: 'broker-token-12345', - filters: clientAccept, - type: 'client', - }); - ({ brokerToken, metadata } = await waitForBrokerClientConnection(bs)); - }); - - afterAll(async () => { - await tws.server.close(); - await closeBrokerClient(bc); - await closeBrokerServer(bs); - }); - - it('server identifies self to client', async () => { - expect(brokerToken).toEqual('broker-token-12345'); - expect(metadata).toStrictEqual({ - capabilities: ['post-streams'], - filters: expect.any(Object), - clientId: expect.any(String), - preflightChecks: expect.any(Array), - version: version, - }); - }); - - it('successfully broker POST', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-body`, - { some: { example: 'json' } }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toStrictEqual({ - some: { example: 'json' }, - }); - }); - - it('successfully broker exact bytes of POST body', async () => { - // stringify the JSON unusually to ensure an unusual exact body - const body = Buffer.from( - JSON.stringify({ some: { example: 'json' } }, null, 5), - ); - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-body`, - body, - { - headers: { - 'content-type': 'application/json', - }, - - transformResponse: (r) => r, - }, - ); - - expect(response.status).toEqual(200); - expect(Buffer.from(response.data)).toEqual(body); - }); - - it('successfully broker GET', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-param/xyz`, - ); - - expect(response.status).toEqual(200); - expect(response.data).toEqual('xyz'); - }); - - it('variable substitution', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-body`, - { - BROKER_VAR_SUB: ['swap.me'], - swap: { me: '${BROKER_TYPE}:${BROKER_TOKEN}' }, - }, - { - headers: { - 'content-type': 'application/json', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toStrictEqual({ - BROKER_VAR_SUB: ['swap.me'], - swap: { me: 'client:broker-token-12345' }, - }); - }); - - it('block request for non-whitelisted url', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/not-allowed`, - {}, - { - headers: { - 'content-type': 'application/json', - }, - }, - ); - - expect(response.status).toEqual(401); - expect(response.data).toStrictEqual({ - message: 'blocked', - reason: - 'Response does not match any accept rule, blocking websocket request', - url: '/not-allowed', - }); - }); - - it('allow request for valid url with valid body', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-body/filtered`, - { proxy: { me: 'please' } }, - { - headers: { - 'content-type': 'application/json', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toStrictEqual({ - proxy: { me: 'please' }, - }); - }); - - it('block request for valid url with invalid body', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-body/filtered`, - { proxy: { me: 'now!' } }, - { - headers: { - 'content-type': 'application/json', - }, - }, - ); - - expect(response.status).toEqual(401); - expect(response.data).toStrictEqual({ - message: 'blocked', - reason: - 'Response does not match any accept rule, blocking websocket request', - url: '/echo-body/filtered', - }); - }); - - it('allow request for valid url with valid query param', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-query/filtered`, - { - params: { proxyMe: 'please' }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toStrictEqual({ proxyMe: 'please' }); - }); - - it('block request for valid url with invalid query param', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-query/filtered`, - { - params: { proxyMe: 'now!' }, - }, - ); - - expect(response.status).toEqual(401); - expect(response.data).toStrictEqual({ - message: 'blocked', - reason: - 'Response does not match any accept rule, blocking websocket request', - url: '/echo-query/filtered?proxyMe=now!', - }); - }); - - it('block request for valid url with missing query param', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-query/filtered`, - ); - - expect(response.status).toEqual(401); - expect(response.data).toStrictEqual({ - message: 'blocked', - reason: - 'Response does not match any accept rule, blocking websocket request', - url: '/echo-query/filtered', - }); - }); - - it('bad broker id', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}XXX/echo-body`, - {}, - ); - - expect(response.status).toEqual(404); - }); - - it('broker token is not included in headers from client to private', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-headers`, - {}, - ); - - expect(response.status).toEqual(200); - expect(response.data).not.toHaveProperty('x-broker-token'); - }); - - it('x-forwarded-* headers are stripped from the request before being forwarded', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-headers`, - {}, - { - headers: { - 'x-forwarded-proto': 'https', - 'x-forwarded-for': '127.0.0.1', - 'x-forwarded-port': '8080', - 'X-Forwarded-Port': '8080', - 'x-forwarded-host': 'banana', - forwarded: - 'by=broker;for=127.0.0.1;host=banana;port=8080;proto=https', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).not.toHaveProperty('x-forwarded-proto'); - expect(response.data).not.toHaveProperty('x-forwarded-for'); - expect(response.data).not.toHaveProperty('x-forwarded-port'); - expect(response.data).not.toHaveProperty('X-Forwarded-Port'); - expect(response.data).not.toHaveProperty('x-forwarded-host'); - expect(response.data).not.toHaveProperty('forwarded'); - }); - - it('querystring parameters are brokered', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-query`, - { - params: { - shape: 'square', - colour: 'yellow', - url_as_param: 'https://clojars.org/search?q=btc', - one_more_top_level_param: 'true', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toStrictEqual({ - shape: 'square', - colour: 'yellow', - url_as_param: 'https://clojars.org/search?q=btc', - one_more_top_level_param: 'true', - }); - }); - - it('approved URLs are blocked when escaped', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/long/nested%2Fpath%2Fto%2Ffile.ext`, - ); - - expect(response.status).toEqual(401); - expect(response.data).toStrictEqual({ - message: 'blocked', - reason: - 'Response does not match any accept rule, blocking websocket request', - url: '/long/nested%2Fpath%2Fto%2Ffile.ext', - }); - }); - - it('block request for url where client does not support required capability', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/client-not-capable`, - ); - - expect(response.status).toEqual(401); - expect(response.data).toStrictEqual({ - message: 'blocked', - reason: 'Request does not match any accept rule, blocking HTTP request', - url: '/client-not-capable', - }); - }); - - it('approved URLs are brokered when escaped as expected', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/long/nested/partially/encoded%2Fpath%2Fto%2Ffile.ext`, - ); - - expect(response.status).toEqual(200); - expect(response.data).toEqual( - '/long/nested/partially/encoded%2Fpath%2Fto%2Ffile.ext', - ); - }); - - it('content-length is not set when using chunked http', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-headers`, - { - headers: { - 'Transfer-Encoding': 'chunked', - }, - }, - ); - - expect(response.headers).not.toHaveProperty('content-length'); - }); - - it('content-length is set without chunked http', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-headers`, - {}, - ); - - expect(response.headers).toHaveProperty('content-length'); - }); - - it('auth header is replaced when url contains token', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-headers/github`, - {}, - { - headers: { - Authorization: 'broker auth', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toHaveProperty('authorization'); - expect(response.data.authorization).toEqual('token githubToken'); - }); - - it('auth header is is replaced when url contains basic auth', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-headers/bitbucket`, - {}, - ); - const auth = response.data.authorization?.replace('Basic ', ''); - const encodedAuth = Buffer.from(auth, 'base64').toString('utf-8'); - - expect(response.status).toEqual(200); - expect(response.data).toHaveProperty('authorization'); - expect(encodedAuth).toEqual('bitbucketUser:bitbucketPassword'); - }); - - it('successfully broker on endpoint that forwards requests with basic auth', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/basic-auth`, - ); - const auth = response.data.replace('Basic ', ''); - const encodedAuth = Buffer.from(auth, 'base64').toString('utf-8'); - - expect(response.status).toEqual(200); - expect(encodedAuth).toEqual('user:pass'); - }); - - it('ignores accept-encoding (gzip)', async () => { - const paramRequiringCompression = 'hello-'.repeat(200); - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-param/${paramRequiringCompression}`, - { - headers: { - 'Accept-Encoding': 'gzip', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toEqual(paramRequiringCompression); - }); - - it('ignores accept-encoding (deflate)', async () => { - const paramRequiringCompression = 'hello-'.repeat(200); - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/echo-param/${paramRequiringCompression}`, - { - headers: { - 'Accept-Encoding': 'deflate', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toEqual(paramRequiringCompression); - }); - - it('successfully stream data', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/test-blob/1`, - { - headers: {}, - }, - ); - const buf = Buffer.alloc(500); - for (let i = 0; i < 500; i++) { - buf.writeUInt8(i & 0xff, i); - } - - expect(response.status).toEqual(299); - expect(response.headers).toHaveProperty('test-orig-url'); - expect(response.headers['test-orig-url']).toEqual('/test-blob/1'); - expect(response.data).toEqual(String(buf)); - }); - - it('successfully redirect POST request to git client', async () => { - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/snykgit/echo-body`, - { some: { example: 'json' } }, - { - headers: { - 'content-type': 'application/json', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toStrictEqual({ - some: { example: 'json' }, - }); - }); - - it('successfully redirect exact bytes of POST body to git client', async () => { - const body = Buffer.from( - JSON.stringify({ some: { example: 'json' } }, null, 5), - ); - const response = await axiosClient.post( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/snykgit/echo-body`, - body, - { - headers: { - 'content-type': 'application/json', - }, - - transformResponse: (r) => r, - }, - ); - - expect(response.status).toEqual(200); - expect(Buffer.from(response.data)).toEqual(body); - }); - - it('successfully redirect exact bytes of POST body to git client', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/snykgit/echo-param/xyz`, - ); - - expect(response.status).toEqual(200); - expect(response.data).toEqual('xyz'); - }); - - it('accept large responses', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/huge-file`, - ); - - expect(response.status).toEqual(200); - expect(response.data.data.length).toEqual(20971522); - }); - - it('allow request to git client with valid param', async () => { - const response = await axiosClient.get( - `http://localhost:${bs.port}/broker/universal/${process.env.SNYK_SOURCE_TYPES__testtype__publicId}/${brokerToken}/snykgit/echo-query`, - { - params: { - proxyMe: 'please', - }, - }, - ); - - expect(response.status).toEqual(200); - expect(response.data).toStrictEqual({ proxyMe: 'please' }); - }); -}); diff --git a/test/unit/config-universal-broker.test.ts b/test/unit/config-universal-broker.test.ts deleted file mode 100644 index ad3de0af0..000000000 --- a/test/unit/config-universal-broker.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -process.env.SNYK_BROKER_BOOT_MODE = 'universal'; -process.env.SERVICE_ENV = 'universal'; -process.env.SNYK_BROKER_TYPES = 'testtype'; -process.env.SNYK_SOURCE_TYPES__testtype__publicId = - '9a3e5d90-b782-468a-a042-9a2073736f00'; - -describe('config', () => { - it('contain default config', () => { - const config = require('../../lib/config'); - - expect(config).not.toBeNull(); - }); -}); diff --git a/test/unit/log-universal.test.ts b/test/unit/log-universal.test.ts deleted file mode 100644 index e0f1b0dbd..000000000 --- a/test/unit/log-universal.test.ts +++ /dev/null @@ -1,154 +0,0 @@ -process.env.SNYK_BROKER_BOOT_MODE = 'universal'; -process.env.SERVICE_ENV = 'universal'; -process.env.SNYK_BROKER_TYPES = 'testtype'; -process.env.SNYK_SOURCE_TYPES__testtype__publicId = - '9a3e5d90-b782-468a-a042-9a2073736f00'; - -import { Writable } from 'stream'; - -describe('log', () => { - it('sanitizes log data', () => { - const brokerTk = (process.env.SNYK_BROKER_TOKEN = 'BROKER_123'); - const githubTk = (process.env.SNYK_GITHUB__GITHUB_TOKEN = 'GITHUB_123'); - const githubTkPool = [ - (process.env.SNYK_GITHUB__GITHUB_TOKEN_POOL = 'GITHUB_456,GITHUB_789'), - ]; - const gitlabTk = (process.env.SNYK_GITLAB__GITLAB_TOKEN = 'GITLAB_123'); - const bbUser = (process.env['SNYK_BITBUCKET-SERVER__BITBUCKET_USERNAME'] = - 'BB_USER'); - const bbPass = (process.env['SNYK_BITBUCKET-SERVER__BITBUCKET_PASSWORD'] = - 'BB_PASS'); - const jiraUser = (process.env.SNYK_JIRA__JIRA_USERNAME = 'JRA_USER'); - const jiraPass = (process.env.SNYK_JIRA__JIRA_PASSWORD = 'JRA_PASS'); - const jiraPassPool = [ - (process.env.SNYK_JIRA__JIRA_PASSWORD_POOL = 'JRA_POOL_PASS'), - ]; - const azureReposToken = (process.env[ - 'SNYK_AZURE-REPOS__AZURE_REPOS_TOKEN' - ] = 'AZURE_TOKEN'); - const artifactoryUrl = (process.env.SNYK_ARTIFACTORY__ARTIFACTORY_URL = - 'http://basic:auth@artifactory.com'); - const nexusUrl = (process.env.SNYK_NEXUS__BASE_NEXUS_URL = - 'http://basic:auth@nexus.com'); - const crAgentUrl = (process.env['SNYK_CR-AGENT__CR_AGENT_URL'] = - 'CONTAINER_AGENT_URL'); - const crCredentials = (process.env['SNYK_CR-AGENT__CR_CREDENTIALS'] = - 'CR_CREDS'); - const crUsername = (process.env['SNYK_CR-AGENT__CR_USERNAME'] = - 'CONTAINER_USERNAME'); - const crPassword = (process.env['SNYK_CR-AGENT__CR_PASSWORD'] = - 'CONTAINER_PASSWORD'); - const crToken = (process.env['SNYK_CR-AGENT__CR_TOKEN'] = - 'CONTAINER_TOKEN'); - const crRoleArn = (process.env['SNYK_CR-AGENT__CR_ROLE_ARN'] = - 'CONTAINER_ROLE_ARN'); - const crExternalId = (process.env['SNYK_CR-AGENT__CR_EXTERNAL_ID'] = - 'CONTAINER_EXTERNAL_ID'); - const crRegion = (process.env['SNYK_CR-AGENT__CR_REGION'] = - 'CONTAINER_REGION'); - const crBase = (process.env['SNYK_CR-AGENT__CR_BASE'] = 'CONTAINER_BASE'); - const gitUsername = (process.env.SNYK_GIT_USERNAME = 'G_USER'); - const gitPassword = (process.env.SNYK_GIT_PASSWORD = 'G_PASS'); - const gitClientUrl = (process.env.SNYK_GIT_CLIENT_URL = - 'http://git-client-url.com'); - - const log = require('../../lib/log'); - - const sensitiveInfo = [ - brokerTk, - githubTk, - githubTkPool[0], - gitlabTk, - azureReposToken, - bbUser, - bbPass, - jiraUser, - jiraPass, - jiraPassPool, - artifactoryUrl, - nexusUrl, - crAgentUrl, - crCredentials, - crUsername, - crPassword, - crToken, - crRoleArn, - crExternalId, - crRegion, - crBase, - gitUsername, - gitPassword, - gitClientUrl, - ].join(); - const sanitizedTokens = - '${BROKER_TOKEN},${GITHUB_TOKEN},${GITHUB_TOKEN_POOL},${GITLAB_TOKEN},${AZURE_REPOS_TOKEN}'; - const sanitizedBitBucket = '${BITBUCKET_USERNAME},${BITBUCKET_PASSWORD}'; - const sanitizedJira = - '${JIRA_USERNAME},${JIRA_PASSWORD},${JIRA_PASSWORD_POOL}'; - const sanitizedArtifactory = '${ARTIFACTORY_URL}'; - const sanitizedNexus = '${BASE_NEXUS_URL}'; - const sanitizedCRData = - '${CR_AGENT_URL},${CR_CREDENTIALS},${CR_USERNAME},${CR_PASSWORD},${CR_TOKEN},${CR_ROLE_ARN},${CR_EXTERNAL_ID},${CR_REGION},${CR_BASE}'; - const sanitizedGitUsername = '${GIT_USERNAME}'; - const sanitizedGitPassword = '${GIT_PASSWORD}'; - const sanitizedGitClientUrl = '${GIT_CLIENT_URL}'; - - // setup logger output capturing - const logs: string[] = []; - const testStream = new Writable(); - testStream._write = function (chunk, encoding, done) { - logs.push(chunk.toString()); - done(); - }; - log.addStream({ stream: testStream }); - - // try to log sensitive information - log.info({ - token: sensitiveInfo, - result: sensitiveInfo, - origin: sensitiveInfo, - url: sensitiveInfo, - httpUrl: sensitiveInfo, - ioUrl: sensitiveInfo, - }); - - const logged = logs[0]; - expect(logs).toHaveLength(1); - - // assert no sensitive data is logged - expect(logged).not.toMatch(brokerTk); - expect(logged).not.toMatch(githubTk); - expect(logged).not.toMatch(githubTkPool[0]); - expect(logged).not.toMatch(gitlabTk); - expect(logged).not.toMatch(bbUser); - expect(logged).not.toMatch(bbPass); - expect(logged).not.toMatch(jiraUser); - expect(logged).not.toMatch(jiraPass); - expect(logged).not.toMatch(jiraPassPool[0]); - expect(logged).not.toMatch(azureReposToken); - expect(logged).not.toMatch(artifactoryUrl); - expect(logged).not.toMatch(crAgentUrl); - expect(logged).not.toMatch(crCredentials); - expect(logged).not.toMatch(crUsername); - expect(logged).not.toMatch(crPassword); - expect(logged).not.toMatch(crToken); - expect(logged).not.toMatch(crRoleArn); - expect(logged).not.toMatch(crExternalId); - expect(logged).not.toMatch(crRegion); - expect(logged).not.toMatch(crBase); - expect(logged).not.toMatch(gitUsername); - expect(logged).not.toMatch(gitPassword); - expect(logged).not.toMatch(gitClientUrl); - - // assert sensitive data is masked - expect(logged).toMatch(sanitizedBitBucket); - expect(logged).toMatch(sanitizedTokens); - expect(logged).toMatch(sanitizedJira); - expect(logged).toMatch(sanitizedArtifactory); - expect(logged).toMatch(sanitizedNexus); - expect(logged).toMatch(sanitizedCRData); - expect(logged).toMatch(sanitizedGitUsername); - expect(logged).toMatch(sanitizedGitPassword); - expect(logged).toMatch(sanitizedGitClientUrl); - }); -}); diff --git a/test/unit/log.test.ts b/test/unit/log.test.ts index 358e5425e..f16774aa8 100644 --- a/test/unit/log.test.ts +++ b/test/unit/log.test.ts @@ -4,9 +4,7 @@ describe('log', () => { it('sanitizes log data', () => { const brokerTk = (process.env.BROKER_TOKEN = 'BROKER_123'); const githubTk = (process.env.GITHUB_TOKEN = 'GITHUB_123'); - const githubTkPool = [ - (process.env.GITHUB_TOKEN_POOL = 'GITHUB_456,GITHUB_789'), - ]; + const githubTkPool = [(process.env.GITHUB_TOKEN_POOL = 'GITHUB_456')]; const gitlabTk = (process.env.GITLAB_TOKEN = 'GITLAB_123'); const bbUser = (process.env.BITBUCKET_USERNAME = 'BB_USER'); const bbPass = (process.env.BITBUCKET_PASSWORD = 'BB_PASS');