diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index 9ec0237f16b..b5169b7edd9 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -316,11 +316,12 @@ functions: SKIP_DEPS=${SKIP_DEPS|1} \ bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh - "install package": + install package: - command: shell.exec params: working_dir: src script: | + source ${DRIVERS_TOOLS}/.evergreen/init-node-and-npm-env.sh npm install --force ${PACKAGE} "run lint checks": @@ -772,8 +773,8 @@ functions: ${PREPARE_SHELL} ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-aws-test.sh - "run aws auth test with aws credentials and session token as environment variables": - - command: shell.exec + ? "run aws auth test with aws credentials and session token as environment variables" + : - command: shell.exec type: test params: working_dir: "src" @@ -829,8 +830,8 @@ functions: ${PREPARE_SHELL} ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-aws-test.sh - "run aws auth test AssumeRoleWithWebIdentity with AWS_ROLE_SESSION_NAME unset": - - command: shell.exec + ? "run aws auth test AssumeRoleWithWebIdentity with AWS_ROLE_SESSION_NAME unset" + : - command: shell.exec type: test params: working_dir: "src" @@ -1143,12 +1144,12 @@ functions: - ${PROJECT_DIRECTORY}/.evergreen/install-mongodb-client-encryption.sh tasks: - - name: 'test-atlas-data-lake' + - name: "test-atlas-data-lake" tags: ["datalake", "mongohouse"] commands: - - func: 'install dependencies' - - func: 'bootstrap mongohoused' - - func: 'run data lake tests' + - func: "install dependencies" + - func: "bootstrap mongohoused" + - func: "run data lake tests" - name: "test-serverless" tags: ["serverless"] @@ -1333,7 +1334,8 @@ tasks: ENVIRONMENT: k8s K8S_VARIANT: eks SCRIPT: run-oidc-prose-tests.sh - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + include_expansions_in_env: + ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: - .evergreen/run-oidc-tests-k8s.sh @@ -1351,7 +1353,8 @@ tasks: ENVIRONMENT: k8s K8S_VARIANT: gke SCRIPT: run-oidc-prose-tests.sh - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + include_expansions_in_env: + ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: - .evergreen/run-oidc-tests-k8s.sh @@ -1369,7 +1372,8 @@ tasks: ENVIRONMENT: k8s K8S_VARIANT: aks SCRIPT: run-oidc-prose-tests.sh - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + include_expansions_in_env: + ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: - .evergreen/run-oidc-tests-k8s.sh @@ -1658,11 +1662,12 @@ task_groups: - command: subprocess.exec params: binary: bash - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + include_expansions_in_env: + ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] env: MONGODB_VERSION: "8.0" args: - - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/setup.sh + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/setup.sh setup_group_can_fail_task: true setup_group_timeout_secs: 1800 tasks: diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 238fbdd25d8..a4233889269 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -287,6 +287,7 @@ functions: params: working_dir: src script: | + source ${DRIVERS_TOOLS}/.evergreen/init-node-and-npm-env.sh npm install --force ${PACKAGE} run lint checks: - command: subprocess.exec @@ -2105,7 +2106,23 @@ tasks: - func: install dependencies - func: bootstrap mongo-orchestration - func: run socks5 tests - - name: test-zstd-compression + - name: test-snappy-compression + tags: + - latest + - snappy + commands: + - command: expansions.update + type: setup + params: + updates: + - {key: VERSION, value: latest} + - {key: TOPOLOGY, value: replica_set} + - {key: AUTH, value: auth} + - {key: COMPRESSOR, value: snappy} + - func: install dependencies + - func: bootstrap mongo-orchestration + - func: run-compression-tests + - name: test-zstd-1.x-compression tags: - latest - zstd @@ -2120,11 +2137,14 @@ tasks: - {key: COMPRESSOR, value: zstd} - func: install dependencies - func: bootstrap mongo-orchestration + - func: install package + vars: + PACKAGE: '@mongodb-js/zstd@1.x' - func: run-compression-tests - - name: test-snappy-compression + - name: test-zstd-2.x-compression tags: - latest - - snappy + - zstd commands: - command: expansions.update type: setup @@ -2133,7 +2153,7 @@ tasks: - {key: VERSION, value: latest} - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: auth} - - {key: COMPRESSOR, value: snappy} + - {key: COMPRESSOR, value: zstd} - func: install dependencies - func: bootstrap mongo-orchestration - func: run-compression-tests @@ -4785,6 +4805,9 @@ buildvariants: - test-socks5 - test-socks5-csfle - test-socks5-tls + - test-snappy-compression + - test-zstd-1.x-compression + - test-zstd-2.x-compression - test-tls-support-latest - test-tls-support-8.0 - test-tls-support-7.0 @@ -4841,6 +4864,9 @@ buildvariants: - test-socks5 - test-socks5-csfle - test-socks5-tls + - test-snappy-compression + - test-zstd-1.x-compression + - test-zstd-2.x-compression - test-tls-support-latest - test-tls-support-8.0 - test-tls-support-7.0 @@ -4897,6 +4923,9 @@ buildvariants: - test-socks5 - test-socks5-csfle - test-socks5-tls + - test-snappy-compression + - test-zstd-1.x-compression + - test-zstd-2.x-compression - test-tls-support-latest - test-tls-support-8.0 - test-tls-support-7.0 @@ -4953,8 +4982,9 @@ buildvariants: - test-socks5 - test-socks5-csfle - test-socks5-tls - - test-zstd-compression - test-snappy-compression + - test-zstd-1.x-compression + - test-zstd-2.x-compression - test-tls-support-latest - test-tls-support-8.0 - test-tls-support-7.0 @@ -5009,8 +5039,9 @@ buildvariants: - test-auth-ldap - test-socks5-csfle - test-socks5-tls - - test-zstd-compression - test-snappy-compression + - test-zstd-1.x-compression + - test-zstd-2.x-compression - test-tls-support-latest - test-tls-support-8.0 - test-tls-support-7.0 @@ -5055,6 +5086,9 @@ buildvariants: - test-latest-server-v1-api - test-socks5 - test-socks5-tls + - test-snappy-compression + - test-zstd-1.x-compression + - test-zstd-2.x-compression - test-tls-support-latest - test-tls-support-8.0 - test-tls-support-7.0 @@ -5099,6 +5133,9 @@ buildvariants: - test-latest-server-v1-api - test-socks5 - test-socks5-tls + - test-snappy-compression + - test-zstd-1.x-compression + - test-zstd-2.x-compression - test-tls-support-latest - test-tls-support-8.0 - test-tls-support-7.0 @@ -5143,6 +5180,9 @@ buildvariants: - test-latest-server-v1-api - test-socks5 - test-socks5-tls + - test-snappy-compression + - test-zstd-1.x-compression + - test-zstd-2.x-compression - test-tls-support-latest - test-tls-support-8.0 - test-tls-support-7.0 @@ -5187,8 +5227,9 @@ buildvariants: - test-latest-server-v1-api - test-socks5 - test-socks5-tls - - test-zstd-compression - test-snappy-compression + - test-zstd-1.x-compression + - test-zstd-2.x-compression - test-tls-support-latest - test-tls-support-8.0 - test-tls-support-7.0 diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index e51f519b7b6..fc986108644 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -211,23 +211,60 @@ TASKS.push( ] ); -for (const compressor of ['zstd', 'snappy']) { - TASKS.push({ - name: `test-${compressor}-compression`, - tags: ['latest', compressor], - commands: [ - updateExpansions({ - VERSION: 'latest', - TOPOLOGY: 'replica_set', - AUTH: 'auth', - COMPRESSOR: compressor - }), - { func: 'install dependencies' }, - { func: 'bootstrap mongo-orchestration' }, - { func: 'run-compression-tests' } - ] - }); -} +TASKS.push({ + name: `test-snappy-compression`, + tags: ['latest', 'snappy'], + commands: [ + updateExpansions({ + VERSION: 'latest', + TOPOLOGY: 'replica_set', + AUTH: 'auth', + COMPRESSOR: 'snappy' + }), + { func: 'install dependencies' }, + { func: 'bootstrap mongo-orchestration' }, + { func: 'run-compression-tests' } + ] +}); + +TASKS.push({ + name: `test-zstd-1.x-compression`, + tags: ['latest', 'zstd'], + commands: [ + updateExpansions({ + VERSION: 'latest', + TOPOLOGY: 'replica_set', + AUTH: 'auth', + COMPRESSOR: 'zstd' + }), + { func: 'install dependencies' }, + { func: 'bootstrap mongo-orchestration' }, + { + func: 'install package', + vars: { + PACKAGE: '@mongodb-js/zstd@1.x' + } + }, + { func: 'run-compression-tests' } + ] +}); + +TASKS.push({ + name: `test-zstd-2.x-compression`, + tags: ['latest', 'zstd'], + commands: [ + updateExpansions({ + VERSION: 'latest', + TOPOLOGY: 'replica_set', + AUTH: 'auth', + COMPRESSOR: 'zstd' + }), + { func: 'install dependencies' }, + { func: 'bootstrap mongo-orchestration' }, + // no need to manually install zstd - we specify 2.x as a dev dependency in package.json + { func: 'run-compression-tests' } + ] +}); const AWS_LAMBDA_HANDLER_TASKS = []; // Add task for testing lambda example without aws auth. @@ -766,15 +803,6 @@ BUILD_VARIANTS.push({ tasks: ['.resource-management'] }); -// TODO(NODE-4575): unskip zstd and snappy on node 16 -for (const variant of BUILD_VARIANTS.filter( - variant => variant.expansions && [16, 18, 20].includes(variant.expansions.NODE_LTS_VERSION) -)) { - variant.tasks = variant.tasks.filter( - name => !['test-zstd-compression', 'test-snappy-compression'].includes(name) - ); -} - // TODO(NODE-4897): Debug socks5 tests on node latest for (const variant of BUILD_VARIANTS.filter( variant => variant.expansions && ['latest'].includes(variant.expansions.NODE_LTS_VERSION) diff --git a/package-lock.json b/package-lock.json index b4d5e414602..0fd61c25b58 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@istanbuljs/nyc-config-typescript": "^1.0.2", "@microsoft/api-extractor": "^7.47.11", "@microsoft/tsdoc-config": "^0.17.0", - "@mongodb-js/zstd": "^1.2.2", + "@mongodb-js/zstd": "^2.0.0-alpha.3", "@types/chai": "^4.3.17", "@types/chai-subset": "^1.3.5", "@types/express": "^4.17.21", @@ -70,7 +70,7 @@ }, "peerDependencies": { "@aws-sdk/credential-providers": "^3.188.0", - "@mongodb-js/zstd": "^1.1.0", + "@mongodb-js/zstd": "^1.1.0 || ^2.0.0-alpha.3", "gcp-metadata": "^5.2.0", "kerberos": "^2.0.1", "mongodb-client-encryption": ">=6.0.0 <7", @@ -1639,21 +1639,18 @@ } }, "node_modules/@mongodb-js/zstd": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@mongodb-js/zstd/-/zstd-1.2.2.tgz", - "integrity": "sha512-NRXiFhk2Nl8UMuIZ4pviKkGVZY/e5P37Opam1u0OtgXjEE0kO1HLapA9heTcZ1PUomArnKS426XbiRFr5iaWvw==", + "version": "2.0.0-alpha.3", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd/-/zstd-2.0.0-alpha.3.tgz", + "integrity": "sha512-KgcwwzzhgYOuRGzaXwpNtOOuNTyrV+0b0wPyGVk439u6WHvZwFyXFA7u75G4imGNmqLPTOztSkzk4G2qgYvCjA==", "dev": true, - "engines": { - "node": ">= 10" + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "node-addon-api": "^4.3.0", + "prebuild-install": "^7.1.2" }, - "optionalDependencies": { - "@mongodb-js/zstd-darwin-arm64": "1.2.2", - "@mongodb-js/zstd-darwin-x64": "1.2.2", - "@mongodb-js/zstd-linux-arm64-gnu": "1.2.2", - "@mongodb-js/zstd-linux-arm64-musl": "1.2.2", - "@mongodb-js/zstd-linux-x64-gnu": "1.2.2", - "@mongodb-js/zstd-linux-x64-musl": "1.2.2", - "@mongodb-js/zstd-win32-x64-msvc": "1.2.2" + "engines": { + "node": ">= 16.20.1" } }, "node_modules/@mongodb-js/zstd-darwin-arm64": { @@ -1664,10 +1661,12 @@ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "darwin" ], + "peer": true, "engines": { "node": ">= 10" } @@ -1680,10 +1679,12 @@ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "darwin" ], + "peer": true, "engines": { "node": ">= 10" } @@ -1696,10 +1697,12 @@ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 10" } @@ -1712,10 +1715,12 @@ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 10" } @@ -1728,10 +1733,12 @@ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 10" } @@ -1744,10 +1751,12 @@ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], + "peer": true, "engines": { "node": ">= 10" } @@ -1760,10 +1769,12 @@ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" ], + "peer": true, "engines": { "node": ">= 10" } @@ -6484,6 +6495,27 @@ "node": ">=16.20.1" } }, + "node_modules/mongodb-legacy/node_modules/@mongodb-js/zstd": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd/-/zstd-1.2.2.tgz", + "integrity": "sha512-NRXiFhk2Nl8UMuIZ4pviKkGVZY/e5P37Opam1u0OtgXjEE0kO1HLapA9heTcZ1PUomArnKS426XbiRFr5iaWvw==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "peer": true, + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@mongodb-js/zstd-darwin-arm64": "1.2.2", + "@mongodb-js/zstd-darwin-x64": "1.2.2", + "@mongodb-js/zstd-linux-arm64-gnu": "1.2.2", + "@mongodb-js/zstd-linux-arm64-musl": "1.2.2", + "@mongodb-js/zstd-linux-x64-gnu": "1.2.2", + "@mongodb-js/zstd-linux-x64-musl": "1.2.2", + "@mongodb-js/zstd-win32-x64-msvc": "1.2.2" + } + }, "node_modules/mongodb-legacy/node_modules/mongodb": { "version": "6.7.0", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.7.0.tgz", diff --git a/package.json b/package.json index dbfa1afc6f8..4fc22de5eb3 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ }, "peerDependencies": { "@aws-sdk/credential-providers": "^3.188.0", - "@mongodb-js/zstd": "^1.1.0", + "@mongodb-js/zstd": "^1.1.0 || ^2.0.0-alpha.3", "gcp-metadata": "^5.2.0", "kerberos": "^2.0.1", "mongodb-client-encryption": ">=6.0.0 <7", @@ -67,7 +67,7 @@ "@istanbuljs/nyc-config-typescript": "^1.0.2", "@microsoft/api-extractor": "^7.47.11", "@microsoft/tsdoc-config": "^0.17.0", - "@mongodb-js/zstd": "^1.2.2", + "@mongodb-js/zstd": "^2.0.0-alpha.3", "@types/chai": "^4.3.17", "@types/chai-subset": "^1.3.5", "@types/express": "^4.17.21", diff --git a/test/tools/runner/hooks/configuration.ts b/test/tools/runner/hooks/configuration.ts index 8ad9c2412cc..063c6453dbd 100644 --- a/test/tools/runner/hooks/configuration.ts +++ b/test/tools/runner/hooks/configuration.ts @@ -160,6 +160,9 @@ const testConfigBeforeHook = async function () { await client.close(); + // eslint-disable-next-line @typescript-eslint/no-require-imports + const zstdVersion = require('@mongodb-js/zstd/package.json').version; + const currentEnv = { // TODO(NODE-3714): Improve environment detection topology: this.configuration.topologyType, @@ -188,7 +191,8 @@ const testConfigBeforeHook = async function () { ocsp: process.env.OCSP_TLS_SHOULD_SUCCEED != null && process.env.CA_FILE != null, socks5: MONGODB_URI.includes('proxyHost='), compressor: process.env.COMPRESSOR, - cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH + cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH, + zstdVersion }; console.error(inspect(currentEnv, { colors: true })); diff --git a/test/unit/cmap/wire_protocol/compression.test.ts b/test/unit/cmap/wire_protocol/compression.test.ts index a860fa39733..e67fa0afb7b 100644 --- a/test/unit/cmap/wire_protocol/compression.test.ts +++ b/test/unit/cmap/wire_protocol/compression.test.ts @@ -1,3 +1,4 @@ +import * as zstd from '@mongodb-js/zstd'; import { expect } from 'chai'; import { compress, Compressor, decompress } from '../../../mongodb'; @@ -12,20 +13,8 @@ describe('compression', function () { it('compresses the data', async function () { const data = await compress(options, buffer); - const zstdMagicNumber = data.reverse().toString('hex').substring(16, 26); - // Zstd magic number first set of bytes is is 0xFD2FB528 - expect(zstdMagicNumber).to.equal('00fd2fb528'); - }); - }); - - context('when a level is provided', function () { - const options = { agreedCompressor: 'zstd' as const, zlibCompressionLevel: 2 }; - - it('compresses the data', async function () { - const data = await compress(options, buffer); - const zstdMagicNumber = data.reverse().toString('hex').substring(16, 26); - // Zstd magic number first set of bytes is is 0xFD2FB528 - expect(zstdMagicNumber).to.equal('00fd2fb528'); + // decompress throws if the message is not zstd compresed + expect(await zstd.decompress(data)).to.deep.equal(buffer); }); }); });