feat(fluid-build): Release group root script support (microsoft#17835)

Allow specifying task in the release group root and allow root script to
be invoked.

- Change the mono repo to load the `package.json` file as if it is a
package.
- Depends on all the package in the release group for the root package
in the build graph.
- Add default task definitions to for release group root if the name
doesn't exist as a script, or the script starts with "fluid-build" to
just trigger the task of the same name within the release group.
- Add incremental support for `flub list`, `flub check layer` and `flub
check policy`
- More argument support for `copyfile` task.
- See microsoft#17837 on enable root script for the client release group (can't
do it in one go because we need to merge and release a new version
first)

build-tools/packages/build-tools/README.md

@@ -122,8 +122,8 @@ Note that --symlink\* changes any symlink, the tool will run the clean script fo
 ### Task and dependency definition
 
 `fluid-build` uses task and dependency definitions to construct a build graph. It is used to determine which task and
-the order to run in. The default definitions are located in at the root `fluidBuild.config.cjs` file under the `tasks` property.
-This definitions applies to all packages in the repo. Script tasks and dependencies specified in this default definitions
+the order to run in. The default definitions for packages are located in at the root `fluidBuild.config.cjs` file under the `tasks` property.
+This definitions applies to all packages in the repo (but not release group root). Script tasks and dependencies specified in this default definitions
 doesn't have to appear on every package and will be ignored if it is not found.
 
 The task definitions is an object with task names as keys, the task dependencies and config to define the action of the task.
@@ -170,6 +170,14 @@ For example:
 }
 ```
 
+When building release group, by default, it will trigger the task on all the packages within the release group. That also mean
+that scripts at the release group root are not considered.
+
+Release group root scripts support can be enabled by adding `fluidBuild.tasks` to the release group's `package.json`. `fluid-build`
+will follow the definition if specified for the task, or it will trigger the root script if the script doesn't invoke `fluid-build`.
+If the script doesn't exist or if it starts with `fluid-build`, then fluid-build will fall back to the default behavior of triggering the task
+on all the packages within the release group. There is no support for "global definitions." Task definitions in `fluidBuild.config.cjs` only apply to packages, not release group roots. Release group root scripts must be defined in the `fluidBuild.tasks` section of the root's `package.json`.
+
 ### Concurrency
 
 `fluid-build` will run task in parallel based on the dependencies information from the build graph. Task are queued

build-tools/packages/build-tools/src/common/fluidTaskDefinitions.ts

@@ -137,19 +137,25 @@ export function getTaskDefinitions(
 		for (const name in packageTaskDefinitions) {
 			const config = packageTaskDefinitions[name];
 			const full = getFullTaskConfig(config);
-			if (full.script && json.scripts?.[name] === undefined) {
-				throw new Error(`Script not found for task definition '${name}'`);
+			if (full.script) {
+				const script = json.scripts?.[name];
+				if (script === undefined) {
+					throw new Error(`Script not found for task definition '${name}'`);
+				} else if (script.startsWith("fluid-build ")) {
+					throw new Error(`Script task should not invoke 'fluid-build' in '${name}'`);
+				}
 			}
 
+			const currentTaskConfig = taskConfig[name];
 			const dependsOn = full.dependsOn.filter((value) => value !== "...");
-			if (taskConfig[name] !== undefined && dependsOn.length !== full.dependsOn.length) {
-				full.dependsOn = dependsOn.concat(taskConfig[name].dependsOn);
+			if (currentTaskConfig !== undefined && dependsOn.length !== full.dependsOn.length) {
+				full.dependsOn = dependsOn.concat(currentTaskConfig.dependsOn);
 			} else {
 				full.dependsOn = dependsOn;
 			}
 			const before = full.before.filter((value) => value !== "...");
-			if (taskConfig[name] !== undefined && before.length !== full.before.length) {
-				full.before = before.concat(taskConfig[name].before);
+			if (currentTaskConfig !== undefined && before.length !== full.before.length) {
+				full.before = before.concat(currentTaskConfig.before);
 			} else {
 				full.before = before;
 			}

build-tools/packages/build-tools/src/common/gitRepo.ts

@@ -208,7 +208,10 @@ export class GitRepo {
 	 * Returns an array containing all the modified files in the repo.
 	 */
 	public async getModifiedFiles(): Promise<string[]> {
-		const results = await this.exec(`ls-files -m --deduplicate`, `get modified files`);
+		const results = await this.exec(
+			`ls-files -mo --exclude-standard --deduplicate`,
+			`get modified files`,
+		);
 		return results.split("\n").filter((t) => t !== undefined && t !== "" && t !== null);
 	}
 

build-tools/packages/build-tools/src/common/monoRepo.ts

@@ -10,7 +10,7 @@ import YAML from "yaml";
 
 import { IFluidBuildConfig, IFluidRepoPackage } from "./fluidRepo";
 import { Logger, defaultLogger } from "./logging";
-import { Package, PackageJson } from "./npmPackage";
+import { Package } from "./npmPackage";
 import { execWithErrorAsync, existsSync, rimrafWithErrorAsync } from "./utils";
 
 import registerDebug from "debug";
@@ -80,6 +80,7 @@ export class MonoRepo {
 	public readonly packages: Package[] = [];
 	public readonly version: string;
 	public readonly workspaceGlobs: string[];
+	public readonly pkg: Package;
 
 	public get name(): string {
 		return this.kind;
@@ -96,8 +97,6 @@ export class MonoRepo {
 		return this.kind as "build-tools" | "client" | "server" | "gitrest" | "historian";
 	}
 
-	private _packageJson: PackageJson;
-
 	static load(group: string, repoPackage: IFluidRepoPackage) {
 		const { directory, ignoredDirs, defaultInterdependencyRange } = repoPackage;
 		let packageManager: PackageManager;
@@ -167,33 +166,33 @@ export class MonoRepo {
 	) {
 		this.version = "";
 		this.workspaceGlobs = [];
-		const pnpmWorkspace = path.join(repoPath, "pnpm-workspace.yaml");
-		const lernaPath = path.join(repoPath, "lerna.json");
-		const yarnLockPath = path.join(repoPath, "yarn.lock");
+
 		const packagePath = path.join(repoPath, "package.json");
 		let versionFromLerna = false;
 
 		if (!existsSync(packagePath)) {
 			throw new Error(`ERROR: package.json not found in ${repoPath}`);
 		}
 
-		this._packageJson = readJsonSync(packagePath);
-
-		const validatePackageManager = existsSync(pnpmWorkspace)
-			? "pnpm"
-			: existsSync(yarnLockPath)
-			? "yarn"
-			: "npm";
+		this.pkg = Package.load(packagePath, kind, this);
 
-		if (this.packageManager !== validatePackageManager) {
+		if (this.packageManager !== this.pkg.packageManager) {
 			throw new Error(
-				`Package manager mismatch between ${packageManager} and ${validatePackageManager}`,
+				`Package manager mismatch between ${packageManager} and ${this.pkg.packageManager}`,
 			);
 		}
 
+		for (const pkgDir of packageDirs) {
+			traceInit(`${kind}: Loading packages from ${pkgDir}`);
+			this.packages.push(Package.load(path.join(pkgDir, "package.json"), kind, this));
+		}
+
+		// only needed for bump tools
+		const lernaPath = path.join(repoPath, "lerna.json");
 		if (existsSync(lernaPath)) {
 			const lerna = readJsonSync(lernaPath);
 			if (packageManager === "pnpm") {
+				const pnpmWorkspace = path.join(repoPath, "pnpm-workspace.yaml");
 				const workspaceString = readFileSync(pnpmWorkspace, "utf-8");
 				this.workspaceGlobs = YAML.parse(workspaceString).packages;
 			} else if (lerna.packages !== undefined) {
@@ -207,25 +206,19 @@ export class MonoRepo {
 			}
 		} else {
 			// Load globs from package.json directly
-			if (this._packageJson.workspaces instanceof Array) {
-				this.workspaceGlobs = this._packageJson.workspaces;
+			if (this.pkg.packageJson.workspaces instanceof Array) {
+				this.workspaceGlobs = this.pkg.packageJson.workspaces;
 			} else {
-				this.workspaceGlobs = (this._packageJson.workspaces as any).packages;
+				this.workspaceGlobs = (this.pkg.packageJson.workspaces as any).packages;
 			}
 		}
 
 		if (!versionFromLerna) {
-			this.version = this._packageJson.version;
+			this.version = this.pkg.packageJson.version;
 			traceInit(
-				`${kind}: Loading version (${this._packageJson.version}) from ${packagePath}`,
+				`${kind}: Loading version (${this.pkg.packageJson.version}) from ${packagePath}`,
 			);
 		}
-
-		traceInit(`${kind}: Loading packages from ${packageManager}`);
-		for (const pkgDir of packageDirs) {
-			this.packages.push(Package.load(path.join(pkgDir, "package.json"), kind, this));
-		}
-		return;
 	}
 
 	public static isSame(a: MonoRepo | undefined, b: MonoRepo | undefined) {
@@ -241,7 +234,7 @@ export class MonoRepo {
 	}
 
 	public get fluidBuildConfig(): IFluidBuildConfig | undefined {
-		return this._packageJson.fluidBuild;
+		return this.pkg.packageJson.fluidBuild;
 	}
 
 	public getNodeModulePath() {

build-tools/packages/build-tools/src/fluidBuild/buildGraph.ts

@@ -60,24 +60,29 @@ class BuildContext {
 	public readonly fileHashCache = new FileHashCache();
 	public readonly taskStats = new TaskStats();
 	public readonly failedTaskLines: string[] = [];
-	constructor(public readonly workerPool?: WorkerPool) {}
+	constructor(
+		public readonly repoPackageMap: Map<string, Package>,
+		public readonly workerPool?: WorkerPool,
+	) {}
 }
 
 export class BuildPackage {
 	private tasks = new Map<string, Task>();
 	public readonly dependentPackages = new Array<BuildPackage>();
 	public level: number = -1;
 	private buildP?: Promise<BuildResult>;
-	private readonly taskDefinitions: TaskDefinitions;
+
+	// This field shouldn't be used directly, use getTaskDefinition instead
+	private readonly _taskDefinitions: TaskDefinitions;
 
 	constructor(
 		public readonly buildContext: BuildContext,
 		public readonly pkg: Package,
 		globalTaskDefinitions: TaskDefinitionsOnDisk | undefined,
 	) {
-		this.taskDefinitions = getTaskDefinitions(this.pkg.packageJson, globalTaskDefinitions);
+		this._taskDefinitions = getTaskDefinitions(this.pkg.packageJson, globalTaskDefinitions);
 		traceTaskDef(
-			`${pkg.nameColored}: Task def: ${JSON.stringify(this.taskDefinitions, undefined, 2)}`,
+			`${pkg.nameColored}: Task def: ${JSON.stringify(this._taskDefinitions, undefined, 2)}`,
 		);
 	}
 
@@ -105,8 +110,25 @@ export class BuildPackage {
 		return tasks.length !== 0;
 	}
 
+	private getTaskDefinition(taskName: string) {
+		let taskDefinition = this._taskDefinitions[taskName];
+		if (taskDefinition === undefined && this.pkg.isReleaseGroupRoot) {
+			// Only enable release group root script if it is explicitly defined, for places that don't use it yet
+			const script = this.pkg.getScript(taskName);
+			if (
+				this.pkg.packageJson.fluidBuild?.tasks === undefined ||
+				script === undefined ||
+				script.startsWith("fluid-build ")
+			) {
+				// default for release group root is to depend on the task of all packages in the release group
+				taskDefinition = { dependsOn: [`^${taskName}`], script: false, before: [] };
+			}
+		}
+		return taskDefinition;
+	}
+
 	private createTask(taskName: string, pendingInitDep: Task[]) {
-		const config = this.taskDefinitions[taskName];
+		const config = this.getTaskDefinition(taskName);
 		if (config?.script === false) {
 			const task = TaskFactory.CreateTargetTask(this, taskName);
 			pendingInitDep.push(task);
@@ -117,7 +139,7 @@ export class BuildPackage {
 
 	private createScriptTask(taskName: string, pendingInitDep: Task[]) {
 		const command = this.pkg.getScript(taskName);
-		if (command !== undefined) {
+		if (command !== undefined && !command.startsWith("fluid-build ")) {
 			const task = TaskFactory.Create(this, command, pendingInitDep, taskName);
 			pendingInitDep.push(task);
 			return task;
@@ -139,7 +161,7 @@ export class BuildPackage {
 	}
 
 	public getScriptTask(taskName: string, pendingInitDep: Task[]): Task | undefined {
-		const config = this.taskDefinitions[taskName];
+		const config = this.getTaskDefinition(taskName);
 		if (config?.script === false) {
 			// it is not a script task
 			return undefined;
@@ -158,7 +180,7 @@ export class BuildPackage {
 
 	public getDependentTasks(task: Task, taskName: string, pendingInitDep: Task[]) {
 		const dependentTasks: Task[] = [];
-		const taskConfig = this.taskDefinitions[taskName];
+		const taskConfig = this.getTaskDefinition(taskName);
 		if (taskConfig === undefined) {
 			return dependentTasks;
 		}
@@ -214,14 +236,17 @@ export class BuildPackage {
 			if (task.taskName === undefined) {
 				return;
 			}
-			const taskConfig = this.taskDefinitions[task.taskName];
+			const taskConfig = this.getTaskDefinition(task.taskName);
 			if (taskConfig === undefined) {
 				return;
 			}
 			if (taskConfig.before.includes("*")) {
 				this.tasks.forEach((depTask) => {
-					if (depTask !== task) {
+					// initializeDependentTask should have been called on all the task already
+					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+					if (depTask !== task && !task.dependentTasks!.includes(depTask)) {
 						traceTaskDepTask(`${depTask.nameColored} -> ${task.nameColored}`);
+						// initializeDependentTask should have been called on all the task already
 						// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 						depTask.dependentTasks!.push(task);
 					}
@@ -233,6 +258,7 @@ export class BuildPackage {
 						continue;
 					}
 					traceTaskDepTask(`${depTask.nameColored} -> ${task.nameColored}`);
+					// initializeDependentTask should have been called on all the task already
 					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 					depTask.dependentTasks!.push(task);
 				}
@@ -294,19 +320,27 @@ export class BuildPackage {
 export class BuildGraph {
 	private matchedPackages = 0;
 	private readonly buildPackages = new Map<Package, BuildPackage>();
-	private readonly buildContext = new BuildContext(
-		options.worker
-			? new WorkerPool(options.workerThreads, options.workerMemoryLimit)
-			: undefined,
-	);
+	private readonly buildContext;
 
 	public constructor(
 		packages: Map<string, Package>,
+		releaseGroupPackages: Package[],
 		private readonly buildTaskNames: string[],
 		globalTaskDefinitions: TaskDefinitionsOnDisk | undefined,
 		getDepFilter: (pkg: Package) => (dep: Package) => boolean,
 	) {
-		this.initializePackages(packages, globalTaskDefinitions, getDepFilter);
+		this.buildContext = new BuildContext(
+			packages,
+			options.worker
+				? new WorkerPool(options.workerThreads, options.workerMemoryLimit)
+				: undefined,
+		);
+		this.initializePackages(
+			packages,
+			releaseGroupPackages,
+			globalTaskDefinitions,
+			getDepFilter,
+		);
 		this.populateLevel();
 		this.initializeTasks(buildTaskNames);
 	}
@@ -422,6 +456,7 @@ export class BuildGraph {
 
 	private initializePackages(
 		packages: Map<string, Package>,
+		releaseGroupPackages: Package[],
 		globalTaskDefinitions: TaskDefinitionsOnDisk | undefined,
 		getDepFilter: (pkg: Package) => (dep: Package) => boolean,
 	) {
@@ -433,6 +468,13 @@ export class BuildGraph {
 			}
 		}
 
+		for (const releaseGroupPackage of releaseGroupPackages) {
+			// Start with only matched packages
+			if (releaseGroupPackage.matched) {
+				this.getBuildPackage(releaseGroupPackage, {}, pendingInitDep);
+			}
+		}
+
 		traceGraph("package created");
 
 		// Create all the dependent packages
@@ -442,6 +484,16 @@ export class BuildGraph {
 			if (node === undefined) {
 				break;
 			}
+			if (node.pkg.isReleaseGroupRoot) {
+				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+				for (const dep of node.pkg.monoRepo!.packages) {
+					traceGraph(`Package dependency: ${node.pkg.nameColored} => ${dep.nameColored}`);
+					node.dependentPackages.push(
+						this.getBuildPackage(dep, globalTaskDefinitions, pendingInitDep),
+					);
+				}
+				continue;
+			}
 			const depFilter = getDepFilter(node.pkg);
 			for (const { name, version } of node.pkg.combinedDependencies) {
 				const dep = packages.get(name);