pipeline refactor test

apps/go-test-caching/src/build-tests.ts

@@ -2,26 +2,40 @@ import { readdirSync, createWriteStream } from "fs";
 import * as path from "path";
 
 import * as core from "@actions/core";
-import { execa } from "execa";
+import { execa, ExecaError, Result } from "execa";
 
 import { logSection } from "./log.js";
 import { Inputs } from "./index.js";
+import { GoPackage } from "./filter-tests.js";
+import { CompiledTestPackages, FilteredTestPackages } from "./types.js";
+
+export type CompilationResult = CompilationSuccess | CompilationFailure;
+export type CompilationSuccess = {
+  output: {
+    binary: string;
+    log: string;
+  }
+  pkg: GoPackage;
+  execution: Result,
+}
 
-type CompilationResult = {
-  outputFile: string;
-  logOutputFile: string;
-  stdout: string;
-  stderr: string;
-};
+export type CompilationFailure = {
+  output: {
+    binary: string;
+    log: string;
+  }
+  pkg: GoPackage;
+  error: ExecaError;
+}
 
 async function compileTestBinary(
   workingDir: string,
   outputDir: string,
-  pkg: string,
+  pkg: GoPackage,
 ): Promise<CompilationResult> {
   core.info(`Compiling test for package ${pkg}`);
 
-  const filename = pkg.replace(/\//g, "-") + "-test";
+  const filename = pkg.importPath.replace(/\//g, "-") + "-test";
   const outputFile = path.join(outputDir, filename);
 
   const logOutputFile = path.join(outputDir, filename + ".compile.log");
@@ -32,7 +46,7 @@ async function compileTestBinary(
 
   try {
     const cmd = "go";
-    const flags = ["test", "-c", "-o", outputFile, "-vet=off", pkg];
+    const flags = ["test", "-c", "-o", outputFile, "-vet=off", pkg.importPath];
     core.info(`Exec: ${cmd} ${flags.join(" ")}`);
 
     const subProcess = execa(cmd, flags, {
@@ -56,33 +70,40 @@ async function compileTestBinary(
     }
 
     const execution = await subProcess;
-    const { stdout, stderr } = execution;
-
     if (core.isDebug()) {
       core.debug(
         `${execution.command} exited with ${execution.exitCode} (cwd: ${execution.cwd})`,
       );
     }
 
-    return { outputFile, logOutputFile, stdout, stderr };
+    return {
+      output: {
+        binary: outputFile,
+        log: logOutputFile,
+      },
+      pkg,
+      execution,
+    } as CompilationSuccess;
   } catch (error) {
-    const stdout = "";
-    let stderr = "";
-
-    if (error instanceof Error && "stderr" in error) {
-      stderr = "" + error.stderr;
-    } else {
-      stderr = "unknown error: " + error;
+    if (!(error instanceof ExecaError)) {
+      throw error;
     }
 
-    return { outputFile, logOutputFile, stderr, stdout };
+    return {
+      output: {
+        binary: outputFile,
+        log: logOutputFile,
+      },
+      pkg,
+      error,
+    } as CompilationFailure;
   }
 }
 
 async function compileTestBinariesConcurrent(
   workingDir: string,
   outputDir: string,
-  packages: string[],
+  packages: FilteredTestPackages,
   maxConcurrency: number,
 ) {
   core.info(
@@ -91,7 +112,7 @@ async function compileTestBinariesConcurrent(
 
   const finished: CompilationResult[] = [];
   const executing: Promise<CompilationResult>[] = [];
-  for (const pkg of packages) {
+  for (const pkg of Object.values(packages)) {
     const task = compileTestBinary(workingDir, outputDir, pkg);
     executing.push(task);
 
@@ -114,8 +135,8 @@ async function compileTestBinariesConcurrent(
   return finished;
 }
 
-export async function buildTestBinaries(packages: string[], inputs: Inputs) {
-  logSection("Build Tests");
+logSection("Build Tests");
+export async function buildTestBinaries(inputs: Inputs, packages: FilteredTestPackages): Promise<CompiledTestPackages> {
   const maxBuildConcurrency = parseInt(core.getInput("build-concurrency")) || 4;
 
   try {
@@ -126,12 +147,64 @@ export async function buildTestBinaries(packages: string[], inputs: Inputs) {
       maxBuildConcurrency,
     );
 
+    const failures = compilationResults.filter(isCompilationFailure);
+    if (failures.length > 0) {
+      failures.forEach((failure) => {
+        core.setFailed(
+          `Failed to compile test for package ${failure.pkg}: ${failure.error.message}`,
+        );
+      });
+      throw new Error("Failed to compile test binaries");
+    }
+
     const binaries = readdirSync(inputs.buildDirectory).filter((file) =>
       file.endsWith("-test"),
     );
     core.info(`Built ${binaries.length} test binaries.`);
+
+    const successes = compilationResults.filter(isCompilationSuccess);
+    return collectCompilationSuccesses(successes);
   } catch (error) {
-    core.error("" + error);
+    core.error( "Error building test binaries: " + error);
     process.exit(1);
   }
 }
+
+function isCompilationSuccess(
+  result: CompilationResult
+): result is CompilationSuccess {
+  return 'execution' in result;
+}
+
+function isCompilationFailure(
+  result: CompilationResult
+): result is CompilationFailure {
+  return 'error' in result;
+}
+
+// function filterCompilationSuccesses(binaries: string[], successes: CompilationSuccess[]) {
+//   core.debug(`Filtering built binaries: ${binaries.join(", ")}`);
+//   core.debug(`Compilation successes: ${successes.map((s) => s.output.binary).join(", ")}`);
+
+//   return successes.filter((success) => {
+//     return binaries.some((binary) => success.output.binary === binary);
+//   });
+// }
+
+function collectCompilationSuccesses(successes: CompilationSuccess[]) {
+  return successes.reduce((acc, success) => {
+    if (acc[success.pkg.importPath]) {
+      core.warning(`Duplicate package found: ${success.pkg.importPath}`);
+      return acc;
+    }
+    acc[success.pkg.importPath] = {
+      ...success.pkg,
+      compile: {
+        binary: success.output.binary,
+        log: success.output.log,
+        execution: success.execution,
+      }
+    }
+    return acc;
+  }, {} as CompiledTestPackages);
+}

apps/go-test-caching/src/filter-tests.ts

@@ -5,30 +5,49 @@ import { execa, ExecaError } from "execa";
 
 import { logSection } from "./log.js";
 import { Inputs } from "./index.js";
+import { FilteredTestPackages } from "./types.js";
+
+/**
+ * A Go package with an import path and directory.
+ * The import path being the package's name, and the directory being the path to the package within the repository.
+ */
+export type GoPackage = { importPath: string; directory: string }
 
 /**
  * Lists all packages in the given path. Defaults to the current directory (./).
  * @param path The path to list packages for
  * @returns A list of packages in the given path
  */
-async function listPackages(moduleDirectory: string, path = "./") {
-  try {
-    const cmd = "go";
-    const flags = ["list", `${path}...`];
-    core.info(`Exec: ${cmd} ${flags.join(" ")}`);
-
-    const { stdout } = await execa(cmd, flags, {
-      stdout: "pipe",
-      lines: true,
-      cwd: moduleDirectory,
-    });
-    return stdout.filter((pkg) => pkg.trim() !== "");
-  } catch (error) {
-    if (error instanceof ExecaError) {
-      console.error(error);
+async function listPackages(moduleDirectory: string) {
+  const cmd = "go";
+  const flags = ["list", "-f", "'{{.importPath}}:{{.Dir}}'", `./...`];
+  core.info(`Exec: ${cmd} ${flags.join(" ")}`);
+  core.debug(`Listing packages in ${moduleDirectory}`);
+
+  const { stdout } = await execa(cmd, flags, {
+    stdout: "pipe",
+    lines: true,
+    cwd: moduleDirectory,
+  });
+
+  return stdout.filter((line) => line.trim() !== "").reduce((acc, line) => {
+    const [importPath, directory] = line.split(":");
+
+    if (acc[importPath]) {
+      core.info(`Duplicate package found`);
+      core.debug(`Existing: ${acc[importPath].importPath} - ${acc[importPath].directory}`);
+      core.debug(`Duplicate: ${importPath} - ${directory}`);
+      return acc;
     }
-    throw new Error(`Error listing packages: ${(error as Error).message}`);
-  }
+
+    return {
+      ...acc,
+      [importPath]: {
+        importPath,
+        directory,
+      },
+    }
+  }, {} as FilteredTestPackages);
 }
 
 /**
@@ -41,46 +60,40 @@ async function listPackages(moduleDirectory: string, path = "./") {
 async function findTaggedTestPackages(moduleDirectory: string, tag: string) {
   core.debug(`Finding packages with build tag ${tag}`);
 
-  try {
-    const cmd = "find";
-    const flags = [
-      `.`,
-      "-name",
-      "*_test.go",
-      "-exec",
-      "grep",
-      "-l",
-      `//go:build ${tag}`,
-      "{}",
-      "+",
-    ];
-    core.info(`Exec: ${cmd} ${flags.join(" ")}`);
-
-    const { stdout } = await execa(cmd, flags, {
-      stdout: "pipe",
-      lines: true,
-      cwd: moduleDirectory,
-    });
+  const cmd = "find";
+  const flags = [
+    `.`,
+    "-name",
+    "*_test.go",
+    "-exec",
+    "grep",
+    "-l",
+    `//go:build ${tag}`,
+    "{}",
+    "+",
+  ];
+  core.info(`Exec: ${cmd} ${flags.join(" ")}`);
 
-    if (stdout.length === 0) {
-      core.debug(`No packages found with build tag ${tag}`);
-      return [];
-    }
+  const { stdout } = await execa(cmd, flags, {
+    stdout: "pipe",
+    lines: true,
+    cwd: moduleDirectory,
+  });
 
-    const directories = stdout
-      .filter((pkg) => pkg.trim() !== "")
-      .map((file) => {
-        return path.dirname(file);
-      });
+  if (stdout.length === 0) {
+    core.debug(`No packages found with build tag ${tag}`);
+    return {};
+  }
 
-    const packagePromises = directories.map((dir) => listPackages(dir));
-    const packages = await Promise.all(packagePromises);
+  const directories = stdout
+    .filter((pkg) => pkg.trim() !== "")
+    .map((file) => {
+      return path.dirname(file);
+    });
 
-    // Flatten and remove duplicates
-    return [...new Set(packages.flat())];
-  } catch (error) {
-    throw new Error(`Error finding tagged tests: ${(error as Error).message}`);
-  }
+  const packagePromises = directories.map((dir) => listPackages(dir));
+  const packages = await Promise.all(packagePromises);
+  return flatten(packages);
 }
 
 export async function getTestPackages(inputs: Inputs) {
@@ -89,6 +102,9 @@ export async function getTestPackages(inputs: Inputs) {
     return findTaggedTestPackages(inputs.moduleDirectory, inputs.tagFilter);
   }
 
-  // TODO: Could optimize this by checking for _test.go files for each package
   return listPackages(inputs.moduleDirectory);
 }
+
+function flatten(packages: FilteredTestPackages[]) {
+  return packages.reduce((acc, obj) => ({ ...acc, ...obj }), {});
+}

apps/go-test-caching/src/hash-tests.ts

@@ -6,27 +6,29 @@ import * as path from "path";
 import * as core from "@actions/core";
 
 import { logSection } from "./log.js";
+import { CompiledTestPackages } from "./types.js";
 
 async function hashFile(filePath: string): Promise<string> {
   const hash = createHash("sha256");
   const input = createReadStream(filePath);
   await pipeline(input, hash);
   return hash.digest("hex");
 }
-
 export async function generateHashes(
-  outputDirectory: string,
+  buildDir: string,
+  compiledPackages: CompiledTestPackages,
 ): Promise<Record<string, string>> {
   logSection("Hashing Test Binaries");
   try {
-    core.info(`Hashing files in ${outputDirectory}`);
-    const files: string[] = readdirSync(outputDirectory);
+    core.info(`Hashing files in ${buildDir}`);
+
+    const files: string[] = readdirSync(buildDir);
     const testFiles: string[] = files.filter((file) => file.endsWith("-test"));
     core.debug(`Test files: ${testFiles.join(", ")}`);
 
     const hashes: Record<string, string> = {};
     const hashPromises = testFiles.map(async (file) => {
-      const filePath = path.join(outputDirectory, file);
+      const filePath = path.join(buildDir, file);
       const hash = await hashFile(filePath);
       hashes[file] = hash;
     });