From f7a442d05291f8f4b1479672de521ffc3a09e6ac Mon Sep 17 00:00:00 2001 From: Timo Stamm Date: Thu, 22 Aug 2024 18:11:54 +0200 Subject: [PATCH] Update conformance testee for @connectrpc/connect-web --- .github/workflows/web-conformance.yaml | 19 +++++ .gitignore | 5 +- Makefile | 19 +++-- .../connect-conformance/src/conformance.ts | 53 ++++++------ .../connect-conformance/src/promise-client.ts | 1 - packages/connect-web/conformance/README.md | 29 +++---- .../connect-web/conformance/browserscript.ts | 19 ++--- packages/connect-web/conformance/client.ts | 83 +++++++++---------- packages/connect-web/package.json | 13 +-- 9 files changed, 129 insertions(+), 112 deletions(-) diff --git a/.github/workflows/web-conformance.yaml b/.github/workflows/web-conformance.yaml index c4a119b5e..0dd28b076 100644 --- a/.github/workflows/web-conformance.yaml +++ b/.github/workflows/web-conformance.yaml @@ -57,3 +57,22 @@ jobs: ${{ runner.os }}-connect-web-safari-conformance- - name: testwebsafariconformance run: make testwebsafariconformance + node: + runs-on: ubuntu-22.04 + steps: + - name: checkout + uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + - name: cache + uses: actions/cache@v4 + with: + path: | + ~/.tmp + .tmp + key: ${{ runner.os }}-connect-web-node-conformance-${{ hashFiles('Makefile') }} + restore-keys: | + ${{ runner.os }}-connect-web-node-conformance- + - name: testwebnodeconformance + run: make testwebnodeconformance diff --git a/.gitignore b/.gitignore index 23369f101..2892551d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ /.tmp /packages/*/dist -/packages/connect-web-test/local.log -/packages/connect-node-test/.connect-node-h1-server.* +/packages/connect-web/conformance/logs/* node_modules -.wrangler \ No newline at end of file +.wrangler diff --git a/Makefile b/Makefile index 2d7ad6fd6..a8a525dc6 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,6 @@ NODE18_VERSION ?= v18.16.0 NODE16_VERSION ?= v16.20.0 NODE_OS = $(subst Linux,linux,$(subst Darwin,darwin,$(shell uname -s))) NODE_ARCH = $(subst x86_64,x64,$(subst aarch64,arm64,$(shell uname -m))) -CONFORMANCE_BROWSER ?= chrome node_modules: package-lock.json npm ci @@ -205,23 +204,27 @@ testconnectfastifyconformance: $(BUILD)/connect-fastify npm run -w packages/connect-fastify conformance .PHONY: testwebconformance -testwebconformance: testwebchromeconformance testwebfirefoxconformance testwebsafariconformance +testwebconformance: testwebchromeconformance testwebfirefoxconformance testwebsafariconformance testwebnodeconformance .PHONY: testwebchromeconformance testwebchromeconformance: $(BUILD)/connect-web - npm run -w packages/connect-web conformance:client:chrome + npm run -w packages/connect-web conformance:client:chrome:promise + npm run -w packages/connect-web conformance:client:chrome:callback .PHONY: testwebfirefoxconformance testwebfirefoxconformance: $(BUILD)/connect-web - npm run -w packages/connect-web conformance:client:firefox + npm run -w packages/connect-web conformance:client:firefox:promise + npm run -w packages/connect-web conformance:client:firefox:callback .PHONY: testwebsafariconformance testwebsafariconformance: $(BUILD)/connect-web - npm run -w packages/connect-web conformance:client:safari + npm run -w packages/connect-web conformance:client:safari:promise + npm run -w packages/connect-web conformance:client:safari:callback -.PHONY: testwebconformancelocal -testwebconformancelocal: $(BUILD)/connect-conformance - npm run -w packages/connect-web conformance:client:browser -- --browser $(CONFORMANCE_BROWSER) +.PHONY: testwebnodeconformance +testwebnodeconformance: $(BUILD)/connect-web + npm run -w packages/connect-web conformance:client:node:promise + npm run -w packages/connect-web conformance:client:node:callback .PHONY: testcloudflareconformance testcloudflareconformance: $(BUILD)/connect-conformance $(BUILD)/connect-node diff --git a/packages/connect-conformance/src/conformance.ts b/packages/connect-conformance/src/conformance.ts index 5915bf9f9..4df4ce74a 100644 --- a/packages/connect-conformance/src/conformance.ts +++ b/packages/connect-conformance/src/conformance.ts @@ -20,7 +20,7 @@ import { chmodSync, mkdirSync, } from "node:fs"; -import { join as joinPath } from "node:path"; +import { join as joinPath, basename } from "node:path"; import { unzipSync, gunzipSync } from "fflate"; import * as tar from "tar-stream"; import { pipeline } from "node:stream/promises"; @@ -34,15 +34,18 @@ const [, version] = /conformance:(v\d+\.\d+\.\d+)/.exec(scripts.generate) ?? [ "?", ]; -const name = "connectconformance"; const downloadUrl = `https://github.com/connectrpc/conformance/releases/download/${version}`; export async function run() { + const { archive, bin } = getArtifactNameForEnv(); const tempDir = getTempDir(); - const artifactName = getArtifactNameForEnv(); - const assetPath = joinPath(tempDir, artifactName); - await download(`${downloadUrl}/${artifactName}`, assetPath); - execFileSync(await extractBin(assetPath), process.argv.slice(2), { + const binPath = joinPath(tempDir, bin); + if (!existsSync(binPath)) { + const archivePath = joinPath(tempDir, archive); + await download(`${downloadUrl}/${archive}`, archivePath); + await extractBin(archivePath, binPath); + } + execFileSync(binPath, process.argv.slice(2), { stdio: "inherit", }); } @@ -58,31 +61,28 @@ async function download(url: string, path: string) { writeFileSync(path, new Uint8Array(await res.arrayBuffer())); } -async function extractBin(path: string) { - if (path.endsWith(".zip")) { - const unzipped = unzipSync(readFileSync(path), { +async function extractBin(archivePath: string, binPath: string) { + const binName = basename(binPath); + if (archivePath.endsWith(".zip")) { + const unzipped = unzipSync(readFileSync(archivePath), { filter(file) { - return file.name === "connectconformance.exe"; + return file.name === binName; }, }); - const binBytes = unzipped["connectconformance.exe"] as - | Uint8Array - | undefined; + const binBytes = unzipped[binName] as Uint8Array | undefined; if (binBytes === undefined) { - throw new Error("Failed to extract connectconformance.exe"); + throw new Error(`Failed to extract ${binName}`); } - const bin = joinPath(getTempDir(), "connectconformance.exe"); - writeFileSync(bin, binBytes); - return bin; + writeFileSync(binPath, binBytes); } - const bin = joinPath(getTempDir(), "connectconformance"); const extract = tar.extract(); extract.on("entry", (header, stream, next) => { - if (header.name === "connectconformance") { + if (header.name === binName) { const chunks: Buffer[] = []; stream.on("data", (chunk: Buffer) => chunks.push(chunk)); stream.on("end", () => { - writeFileSync(bin, Buffer.concat(chunks)); + writeFileSync(binPath, Buffer.concat(chunks)); + chmodSync(binPath, 0o755); next(); }); } else { @@ -93,14 +93,12 @@ async function extractBin(path: string) { await pipeline( new Readable({ read() { - this.push(gunzipSync(readFileSync(path))); + this.push(gunzipSync(readFileSync(archivePath))); this.push(null); }, }), extract, ); - chmodSync(bin, 755); - return bin; } function getTempDir() { @@ -111,9 +109,10 @@ function getTempDir() { return tempDir; } -function getArtifactNameForEnv() { +function getArtifactNameForEnv(): { archive: string; bin: string } { let build = ""; let ext = ".tar.gz"; + let bin = "connectconformance"; switch (os.platform()) { case "darwin": switch (os.arch()) { @@ -141,6 +140,7 @@ function getArtifactNameForEnv() { break; case "win32": ext = ".zip"; + bin = "connectconformance.exe"; switch (os.arch()) { case "arm64": build = "Windows-arm64"; @@ -155,5 +155,8 @@ function getArtifactNameForEnv() { default: throw new Error(`Unsupported platform: ${os.platform()}`); } - return `${name}-${version}-${build}${ext}`; + return { + archive: `connectconformance-${version}-${build}${ext}`, + bin, + }; } diff --git a/packages/connect-conformance/src/promise-client.ts b/packages/connect-conformance/src/promise-client.ts index aa6e5002e..d437b8378 100644 --- a/packages/connect-conformance/src/promise-client.ts +++ b/packages/connect-conformance/src/promise-client.ts @@ -73,7 +73,6 @@ async function unary( if (req.requestMessages.length !== 1) { throw new Error("Unary method requires exactly one request message"); } - req.cancel; const msg = req.requestMessages[0]; const uReq = idempotent ? new IdempotentUnaryRequest() : new UnaryRequest(); if (!msg.unpackTo(uReq)) { diff --git a/packages/connect-web/conformance/README.md b/packages/connect-web/conformance/README.md index 290cf3ded..311a72da4 100644 --- a/packages/connect-web/conformance/README.md +++ b/packages/connect-web/conformance/README.md @@ -6,26 +6,27 @@ It uses the [conformance runner](https://github.com/connectrpc/conformance/relea ## Running conformance tests -## Using a headless browser - -Run `make testwebconformance` to run all conformance tests in the following headless browsers / environments: +Tests run in the following environments: * Chrome * Firefox -* Node -* Safari (only if running in OSX. Safari requires users to enable the `Allow Remote Automation` option in Safari's Develop menu) +* Safari (only if running in OSX. Safari requires users to enable the "Allow Remote Automation" option in Safari's Develop menu) +* Node.js -The individual tests can also be run via npm: +For every environment, two client flavors are available: +* Promise (using `createPromiseClient`) +* Callback (using `createCallbackClient`) -`npm run conformance:client:chrome` -`npm run conformance:client:firefox` -`npm run conformance:client:safari` -`npm run conformance:client:node` +For every combination, an npm script is available: -## Using a local browser +`npm run conformance:client::` -Run `make testwebconformancelocal` to run the tests in a local browser. This will open a Chrome browser and run the tests. If you want to run the tests in a different browser, set the `CONFORMANCE_BROWSER` environment variable. +Before you run npm scripts, make sure to build dependencies with `make .tmp/build/connect-web`. + +## Using a local browser -Also available as an npm script: +To launch a browser window with access to the browser's network inspector, append the `--openBrowser` flag to the npm script: -`npm run conformance:client:browser` +``` +npm run conformance:client:chrome:promise -- --openBrowser +``` diff --git a/packages/connect-web/conformance/browserscript.ts b/packages/connect-web/conformance/browserscript.ts index 2a3dd2c63..a58427dd7 100644 --- a/packages/connect-web/conformance/browserscript.ts +++ b/packages/connect-web/conformance/browserscript.ts @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { ConnectError } from "@connectrpc/connect"; import { invokeWithCallbackClient, invokeWithPromiseClient, @@ -31,8 +30,7 @@ declare global { } } -// The main entry point into the browser code running in Puppeteer/headless Chrome. -// This function is invoked by the page.evaluate call in grpcwebclient. +// This function is invoked by the browser.executeAsync call in client.ts async function runTestCase( data: number[], useCallbackClient: boolean, @@ -42,18 +40,17 @@ async function runTestCase( testName: req.testName, }); try { - let invoke; - if (useCallbackClient) { - invoke = invokeWithCallbackClient; - } else { - invoke = invokeWithPromiseClient; - } - const result = await invoke(createTransport(req), req); + const transport = createTransport(req); + const result = useCallbackClient + ? await invokeWithCallbackClient(transport, req) + : await invokeWithPromiseClient(transport, req); res.result = { case: "response", value: result }; } catch (err) { res.result = { case: "error", - value: new ClientErrorResult({ message: ConnectError.from(err).message }), + value: new ClientErrorResult({ + message: `Failed to run test case: ${String(err)}`, + }), }; } return Array.from(res.toBinary()); diff --git a/packages/connect-web/conformance/client.ts b/packages/connect-web/conformance/client.ts index a2a03e6c2..659cb2932 100755 --- a/packages/connect-web/conformance/client.ts +++ b/packages/connect-web/conformance/client.ts @@ -15,7 +15,6 @@ // limitations under the License. import { remote } from "webdriverio"; -import type { RemoteOptions } from "webdriverio"; import * as esbuild from "esbuild"; import { parseArgs } from "node:util"; import { @@ -34,11 +33,13 @@ const { values: flags } = parseArgs({ options: { browser: { type: "string", default: "chrome" }, headless: { type: "boolean" }, + openBrowser: { type: "boolean" }, useCallbackClient: { type: "boolean" }, }, }); void main(); + /** * This program implements a client under test for the connect conformance test * runner. It reads ClientCompatRequest messages from stdin. For each request, @@ -68,10 +69,12 @@ async function main() { try { const invokeResult = await invoke(createTransport(req), req); res.result = { case: "response", value: invokeResult }; - } catch (e) { + } catch (err) { res.result = { case: "error", - value: new ClientErrorResult({ message: (e as Error).message }), + value: new ClientErrorResult({ + message: `Failed to run test case: ${String(err)}`, + }), }; } process.stdout.write(writeSizeDelimitedBuffer(res.toBinary())); @@ -79,78 +82,68 @@ async function main() { } async function runBrowser() { - let capabilities: RemoteOptions["capabilities"] = { - acceptInsecureCerts: true, - }; + let browserName: string; switch (flags.browser) { case "chrome": case undefined: - capabilities = { - ...capabilities, - browserName: "chrome", - "goog:chromeOptions": { - args: [ - "--disable-gpu", - flags.headless === true - ? "--headless" - : "--auto-open-devtools-for-tabs", - ], - }, - }; + browserName = "chrome"; break; case "firefox": - capabilities = { - ...capabilities, - browserName: "firefox", - "moz:firefoxOptions": { - args: [flags.headless === true ? "-headless" : "--devtools"], - }, - }; - break; case "safari": - capabilities = { - ...capabilities, - browserName: "safari", - // Safari does not support headless mode - }; + browserName = flags.browser; break; default: throw new Error(`Unsupported browser: ${flags.browser}`); } const browser = await remote({ - // webdriverio prints all the logs to stdout, this will interfere with the conformance test output - // so we set the log level to silent - // - // TODO: look for a way to redirect the logs to a file/stderr - logLevel: "silent", - capabilities, + capabilities: { + browserName, + acceptInsecureCerts: true, + "goog:chromeOptions": { + args: [ + "--disable-gpu", + flags.openBrowser === true + ? "--auto-open-devtools-for-tabs" + : "--headless", + ], + }, + "moz:firefoxOptions": { + args: [flags.openBrowser === true ? "--devtools" : "-headless"], + }, + // Safari does not support headless mode + }, + // Directory to store all testrunner log files (including reporter logs and wdio logs). + // If not set, all logs are streamed to stdout, which conflicts with the conformance runner I/O. + outputDir: new URL("logs", import.meta.url).pathname, }); await browser.executeScript(await buildBrowserScript(), []); for await (const next of readSizeDelimitedBuffers(process.stdin)) { const invokeResult = await browser.executeAsync( - (reqBytes, useCallbackClient, done: (res: number[]) => void) => { - void window.runTestCase(reqBytes, useCallbackClient).then(done); + (data, useCallbackClient, done: (res: number[]) => void) => { + void window.runTestCase(data, useCallbackClient).then(done); }, Array.from(next), - flags.useCallbackClient, + flags.useCallbackClient === true, ); process.stdout.write( writeSizeDelimitedBuffer(new Uint8Array(invokeResult)), ); } - if (flags.headless === true) { - await browser.deleteSession(); - } else { + if (flags.openBrowser == true) { await browser.executeScript( - `document.write("Tests done. You can inspect requests in the network explorer.")`, + `const p = document.createElement("p"); + p.innerText = "Tests done. You can inspect requests in the network explorer." + document.body.append(p);`, [], ); + } else { + await browser.deleteSession(); } } async function buildBrowserScript() { const buildResult = await esbuild.build({ - entryPoints: ["./conformance/browserscript.ts"], + entryPoints: [new URL("browserscript.ts", import.meta.url).pathname], bundle: true, write: false, }); diff --git a/packages/connect-web/package.json b/packages/connect-web/package.json index 12ee53a95..f22a0ce2c 100644 --- a/packages/connect-web/package.json +++ b/packages/connect-web/package.json @@ -13,11 +13,14 @@ "build:cjs": "tsc --project tsconfig.build.json --module commonjs --verbatimModuleSyntax false --moduleResolution node10 --outDir ./dist/cjs --declaration --declarationDir ./dist/cjs && echo >./dist/cjs/package.json '{\"type\":\"commonjs\"}'", "build:esm": "tsc --project tsconfig.build.json --outDir ./dist/esm --declaration --declarationDir ./dist/esm", "attw": "attw --pack", - "conformance:client:chrome": "connectconformance --mode client --conf ./conformance/conformance-web.yaml -v -- ./conformance/client.ts --browser chrome --headless && connectconformance --mode client --conf ./conformance/conformance-web.yaml -v --known-failing @./conformance/known-failing-callback-client.txt -- ./conformance/client.ts --browser chrome --headless --useCallbackClient", - "conformance:client:firefox": "connectconformance --mode client --conf ./conformance/conformance-web.yaml -v --known-failing @./conformance/known-failing-promise-client-firefox.txt -- ./conformance/client.ts --browser firefox --headless && connectconformance --mode client --conf ./conformance/conformance-web.yaml -v --known-failing @./conformance/known-failing-callback-client.txt -- ./conformance/client.ts --browser firefox --headless --useCallbackClient", - "conformance:client:safari": "connectconformance --mode client --conf ./conformance/conformance-web.yaml -v -- ./conformance/client.ts --browser safari --headless && connectconformance --mode client --conf ./conformance/conformance-web.yaml -v --known-failing @./conformance/known-failing-callback-client.txt -- ./conformance/client.ts --browser safari --headless --useCallbackClient", - "conformance:client:node": "connectconformance --mode client --conf ./conformance/conformance-web-node.yaml -v -- ./conformance/client.ts --browser node && connectconformance --mode client --conf ./conformance/conformance-web-node.yaml -v --known-failing @./conformance/known-failing-callback-client.txt -- ./conformance/client.ts --browser node --useCallbackClient", - "conformance:client:browser": "connectconformance --mode client --conf ./conformance/conformance-web.yaml -v -- ./conformance/client.ts && connectconformance --mode client --conf ./conformance/conformance-web.yaml -v --known-failing @./conformance/known-failing-callback-client.txt -- ./conformance/client.ts --useCallbackClient", + "conformance:client:chrome:promise": "connectconformance --mode client --conf ./conformance/conformance-web.yaml -v -- ./conformance/client.ts --browser chrome", + "conformance:client:chrome:callback": "connectconformance --mode client --conf ./conformance/conformance-web.yaml -v --known-failing @./conformance/known-failing-callback-client.txt -- ./conformance/client.ts --browser chrome --useCallbackClient", + "conformance:client:firefox:promise": "connectconformance --mode client --conf ./conformance/conformance-web.yaml -v --known-failing @./conformance/known-failing-promise-client-firefox.txt -- ./conformance/client.ts --browser firefox", + "conformance:client:firefox:callback": "connectconformance --mode client --conf ./conformance/conformance-web.yaml -v --known-failing @./conformance/known-failing-callback-client.txt -- ./conformance/client.ts --browser firefox --useCallbackClient", + "conformance:client:safari:promise": "connectconformance --mode client --conf ./conformance/conformance-web.yaml -v -- ./conformance/client.ts --browser safari", + "conformance:client:safari:callback": "connectconformance --mode client --conf ./conformance/conformance-web.yaml -v --known-failing @./conformance/known-failing-callback-client.txt -- ./conformance/client.ts --browser safari --useCallbackClient", + "conformance:client:node:promise": "connectconformance --mode client --conf ./conformance/conformance-web-node.yaml -v -- ./conformance/client.ts --browser node", + "conformance:client:node:callback": "connectconformance --mode client --conf ./conformance/conformance-web-node.yaml -v --known-failing @./conformance/known-failing-callback-client.txt -- ./conformance/client.ts --browser node --useCallbackClient", "jasmine": "jasmine --config=jasmine.json", "pregenerate": "rm -rf browserstack/gen/*", "generate": "buf generate buf.build/connectrpc/eliza --template browserstack/buf.gen.yaml",