diff --git a/build-tools/packages/build-cli/docs/check.md b/build-tools/packages/build-cli/docs/check.md index 3de84b9d2e70..a495e27b86b6 100644 --- a/build-tools/packages/build-cli/docs/check.md +++ b/build-tools/packages/build-cli/docs/check.md @@ -4,6 +4,7 @@ Check commands are used to verify repo state, apply policy, etc. * [`flub check buildVersion`](#flub-check-buildversion) +* [`flub check changedPackages`](#flub-check-changedpackages) * [`flub check changeset`](#flub-check-changeset) * [`flub check latestVersions VERSION PACKAGE_OR_RELEASE_GROUP`](#flub-check-latestversions-version-package_or_release_group) * [`flub check layers`](#flub-check-layers) @@ -66,6 +67,35 @@ DESCRIPTION _See code: [src/commands/check/buildVersion.ts](https://github.com/microsoft/FluidFramework/blob/main/build-tools/packages/build-cli/src/commands/check/buildVersion.ts)_ +## `flub check changedPackages` + +Computes Azure DevOps output variables used by pipelines to conditionally skip tests. + +``` +USAGE + $ flub check changedPackages [--json] [-v | --quiet] [--targetBranch ] [--searchPath ] + +FLAGS + --searchPath= Path used to locate the build project. Defaults to the current working directory. + --targetBranch= [env: TARGET_BRANCH] Target branch to compare against. Defaults to the TARGET_BRANCH + environment variable. + +LOGGING FLAGS + -v, --verbose Enable verbose logging. + --quiet Disable all logging. + +GLOBAL FLAGS + --json Format output as json. + +DESCRIPTION + Computes Azure DevOps output variables used by pipelines to conditionally skip tests. + + Compares the current PR branch to the merge base with a target branch, then emits 'shouldRunTests' and + 'scopedPnpmFilter' as Azure DevOps output variables. Unexpected errors conservatively fall back to a full test run. +``` + +_See code: [src/commands/check/changedPackages.ts](https://github.com/microsoft/FluidFramework/blob/main/build-tools/packages/build-cli/src/commands/check/changedPackages.ts)_ + ## `flub check changeset` Checks if a changeset was added when compared against a branch. This is used in CI to enforce that changesets are present for a PR. diff --git a/build-tools/packages/build-cli/src/commands/check/changedPackages.ts b/build-tools/packages/build-cli/src/commands/check/changedPackages.ts new file mode 100644 index 000000000000..aec244c0a0d6 --- /dev/null +++ b/build-tools/packages/build-cli/src/commands/check/changedPackages.ts @@ -0,0 +1,241 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import path from "node:path"; + +import { + getChangedSinceRef, + getMergeBaseRemote, + getPackageDirsAtRef, + getRemote, + isFileInPackageDir, +} from "@fluid-tools/build-infrastructure"; +import { Flags } from "@oclif/core"; + +import { + formatLogIssue, + formatSetVariable, +} from "../../library/azureDevops/pipelineCommands.js"; +import { normalizeTargetBranch } from "../../library/branches.js"; +import { BaseCommandWithBuildProject } from "../../library/commands/base.js"; + +/** + * Full-run trigger patterns. A diff touching any of these paths forces running + * every package's tests. Keep this conservative since these files can affect + * dependency resolution, build behavior, or pipeline behavior across packages. + */ +const fullRunPatterns: readonly RegExp[] = [ + /^package\.json$/, + /^pnpm-lock\.yaml$/, + /^pnpm-workspace\.yaml$/, + /^\.pnpmfile\.cjs$/, + /^\.npmrc$/, + /^\.nvmrc$/, + /^fluidBuild\.config\.cjs$/, + /^tsconfig[^/]*\.json$/, + /^biome\./, + /^tools\//, + /^common\//, + /^scripts\//, + /^\.changeset\/config\.json$/, +]; + +/** + * Result of computing which packages have changed since the target branch. + * + * The same shape is returned both for full-run fallbacks (e.g. unexpected errors or trigger-pattern + * matches) and for the normal scoped-filter case. Consumers can inspect {@link forcedFullRunPattern} + * to disambiguate. + */ +export interface ChangedPackagesResult { + /** Whether any tests should run at all. `false` only when no changed file maps to a workspace package. */ + shouldRunTests: boolean; + /** The computed `pnpm --filter` expression, or an empty string for full / no-op runs. */ + scopedPnpmFilter: string; + /** The (normalized) target branch the comparison was performed against. */ + targetBranch: string; + /** The merge-base commit between HEAD and the target branch, when it could be determined. */ + mergeBase?: string; + /** The list of files that changed since the merge base. Empty on the error fallback path. */ + changedFiles: string[]; + /** The source of the first {@link fullRunPatterns} entry that matched a changed file, if any. */ + forcedFullRunPattern?: string; + /** Number of workspace packages reported as changed by `getChangedSinceRef`. */ + changedPackageCount: number; +} + +/** + * Returns the first pattern in `patterns` that matches any path in `files`, or `undefined` if + * none match. + */ +function findFullRunPatternMatch( + files: readonly string[], + patterns: readonly RegExp[], +): RegExp | undefined { + for (const pattern of patterns) { + if (files.some((file) => pattern.test(file))) { + return pattern; + } + } + return undefined; +} + +export default class CheckChangedPackagesCommand extends BaseCommandWithBuildProject< + typeof CheckChangedPackagesCommand +> { + static readonly summary = + "Computes Azure DevOps output variables used by pipelines to conditionally skip tests."; + + static readonly description = + "Compares the current PR branch to the merge base with a target branch, then emits 'shouldRunTests' and 'scopedPnpmFilter' as Azure DevOps output variables. Unexpected errors conservatively fall back to a full test run."; + + static readonly enableJsonFlag = true; + + static readonly flags = { + targetBranch: Flags.string({ + description: + "Target branch to compare against. Defaults to the TARGET_BRANCH environment variable.", + env: "TARGET_BRANCH", + }), + searchPath: Flags.directory({ + description: + "Path used to locate the build project. Defaults to the current working directory.", + exists: true, + }), + ...BaseCommandWithBuildProject.flags, + } as const; + + public async run(): Promise { + const targetBranch = normalizeTargetBranch( + this.flags.targetBranch ?? process.env.TARGET_BRANCH ?? "", + ); + + if (targetBranch === "") { + return this.fallbackFullRun("TARGET_BRANCH not set;", targetBranch); + } + + this.info(`Target branch: ${targetBranch}`); + + try { + const buildProject = this.getBuildProject( + path.resolve(this.flags.searchPath ?? process.cwd()), + ); + const git = await buildProject.getGitRepository(); + const remote = await getRemote(git, buildProject.upstreamRemotePartialUrl); + if (remote === undefined) { + return this.fallbackFullRun( + `Could not find upstream remote for ${buildProject.upstreamRemotePartialUrl};`, + targetBranch, + ); + } + + await git.fetch([remote, targetBranch]); + const mergeBase = await getMergeBaseRemote( + git, + targetBranch, + remote, + "HEAD", + (message) => this.info(message), + ); + this.info(`Merge base: ${mergeBase}`); + + const changed = await getChangedSinceRef(buildProject, targetBranch, remote); + const changedFiles = changed.files; + this.info(`Changed files (${changedFiles.length}):`); + for (const file of changedFiles.slice(0, 30)) { + this.info(file); + } + if (changedFiles.length > 30) { + this.info(`... and ${changedFiles.length - 30} more`); + } + + const match = findFullRunPatternMatch(changedFiles, fullRunPatterns); + if (match !== undefined) { + this.info(`Match for full-run pattern '${match.source}' - forcing full test run.`); + this.emitVsoOutputs(true, ""); + return { + shouldRunTests: true, + scopedPnpmFilter: "", + targetBranch, + mergeBase, + changedFiles, + forcedFullRunPattern: match.source, + changedPackageCount: changed.packages.length, + }; + } + + // Union of package directories at the merge-base tree and the current working tree so + // that packages added, removed, or moved between the two refs are all considered. + const historicalDirs = await getPackageDirsAtRef(git, mergeBase); + const currentDirs = await getPackageDirsAtRef(git); + const packageDirs = new Set([...historicalDirs, ...currentDirs]); + + if (!changedFiles.some((file) => isFileInPackageDir(file, packageDirs))) { + this.logWarning( + `No changed files mapped to a workspace package - skipping all test execution. Files considered (${changedFiles.length}):`, + ); + for (const file of changedFiles) { + this.info(` ${file}`); + } + this.emitVsoOutputs(false, ""); + return { + shouldRunTests: false, + scopedPnpmFilter: "", + targetBranch, + mergeBase, + changedFiles, + changedPackageCount: 0, + }; + } + + const scopedPnpmFilter = `...[${mergeBase}]`; + this.info(`Computed pnpm filter: ${scopedPnpmFilter}`); + this.emitVsoOutputs(true, scopedPnpmFilter); + return { + shouldRunTests: true, + scopedPnpmFilter, + targetBranch, + mergeBase, + changedFiles, + changedPackageCount: changed.packages.length, + }; + } catch (error) { + return this.fallbackFullRun( + error instanceof Error ? `${error.message};` : `${String(error)};`, + targetBranch, + ); + } + } + + private emitVsoOutputs(shouldRunTests: boolean, scopedPnpmFilter: string): void { + if (this.jsonEnabled()) { + return; + } + + const flag = shouldRunTests ? "true" : "false"; + this.log(`shouldRunTests=${flag}`); + this.log(`scopedPnpmFilter=${scopedPnpmFilter}`); + this.log(formatSetVariable("shouldRunTests", flag, { isOutput: true })); + this.log(formatSetVariable("scopedPnpmFilter", scopedPnpmFilter, { isOutput: true })); + } + + private logWarning(message: string): void { + if (!this.jsonEnabled()) { + this.log(formatLogIssue("warning", message)); + } + } + + private fallbackFullRun(reason: string, targetBranch: string): ChangedPackagesResult { + this.logWarning(`${reason} Falling back to full test run.`); + this.emitVsoOutputs(true, ""); + return { + shouldRunTests: true, + scopedPnpmFilter: "", + targetBranch, + changedFiles: [], + changedPackageCount: 0, + }; + } +} diff --git a/build-tools/packages/build-cli/src/commands/check/latestVersions.ts b/build-tools/packages/build-cli/src/commands/check/latestVersions.ts index 6d5af6db80c5..11ac071de8d2 100644 --- a/build-tools/packages/build-cli/src/commands/check/latestVersions.ts +++ b/build-tools/packages/build-cli/src/commands/check/latestVersions.ts @@ -4,6 +4,7 @@ */ import { findPackageOrReleaseGroup, packageOrReleaseGroupArg, semverArg } from "../../args.js"; +import { formatSetVariable } from "../../library/azureDevops/pipelineCommands.js"; import { BaseCommand } from "../../library/commands/base.js"; import { isLatestInMajor } from "../../library/latestVersions.js"; @@ -52,9 +53,9 @@ export default class LatestVersionsCommand extends BaseCommand { + it("strips refs/heads prefix", () => { + expect(normalizeTargetBranch("refs/heads/main")).to.equal("main"); + }); + + it("passes plain branch names through", () => { + expect(normalizeTargetBranch("next")).to.equal("next"); + }); + + it("preserves slashes after the prefix", () => { + expect(normalizeTargetBranch("refs/heads/release/2.x")).to.equal("release/2.x"); + }); + + it("returns empty string for empty input", () => { + expect(normalizeTargetBranch("")).to.equal(""); + }); +}); + +describe("flub check changedPackages", () => { + it("falls back to a full test run when target branch is missing", async () => { + const { stdout } = await runCommand( + ["check changedPackages", "--searchPath", testRepoRoot, "--quiet"], + { root: import.meta.url }, + ); + + expect(stdout).to.contain("shouldRunTests=true"); + expect(stdout).to.contain("scopedPnpmFilter="); + expect(stdout).to.contain( + "##vso[task.setvariable variable=shouldRunTests;isOutput=true]true", + ); + expect(stdout).to.contain( + "##vso[task.setvariable variable=scopedPnpmFilter;isOutput=true]", + ); + }); + + it("returns structured JSON without throwing on safe fallback", async () => { + const { stdout, error } = await runCommand( + ["check changedPackages", "--searchPath", testRepoRoot, "--json", "--quiet"], + { root: import.meta.url }, + ); + + expect(error).to.equal(undefined); + const output = JSON.parse(stdout) as { + shouldRunTests: boolean; + scopedPnpmFilter: string; + changedPackageCount: number; + }; + expect(output.shouldRunTests).to.equal(true); + expect(output.scopedPnpmFilter).to.equal(""); + expect(output.changedPackageCount).to.equal(0); + }); +}); diff --git a/build-tools/packages/build-cli/src/test/commands/vnext/check/latestVersions.test.ts b/build-tools/packages/build-cli/src/test/commands/vnext/check/latestVersions.test.ts index f5efd9d2123c..650f086b03d5 100644 --- a/build-tools/packages/build-cli/src/test/commands/vnext/check/latestVersions.test.ts +++ b/build-tools/packages/build-cli/src/test/commands/vnext/check/latestVersions.test.ts @@ -63,7 +63,7 @@ describe("vnext:check:latestVersions", () => { stdoutLineEquals( stdout, 1, - "##vso[task.setvariable variable=shouldDeploy;isoutput=true]true", + "##vso[task.setvariable variable=shouldDeploy;isOutput=true]true", ); }); @@ -96,7 +96,7 @@ describe("vnext:check:latestVersions", () => { stdoutLineEquals( stdout, 1, - "##vso[task.setvariable variable=shouldDeploy;isoutput=true]false", + "##vso[task.setvariable variable=shouldDeploy;isOutput=true]false", ); }); @@ -129,7 +129,7 @@ describe("vnext:check:latestVersions", () => { stdoutLineEquals( stdout, 1, - "##vso[task.setvariable variable=shouldDeploy;isoutput=true]false", + "##vso[task.setvariable variable=shouldDeploy;isOutput=true]false", ); }); }); diff --git a/build-tools/packages/build-infrastructure/api-report/build-infrastructure.api.md b/build-tools/packages/build-infrastructure/api-report/build-infrastructure.api.md index c6274fdc3613..2eeca932d1db 100644 --- a/build-tools/packages/build-infrastructure/api-report/build-infrastructure.api.md +++ b/build-tools/packages/build-infrastructure/api-report/build-infrastructure.api.md @@ -67,7 +67,10 @@ export function getChangedSinceRef

(buildProject: IBuildProje export function getFiles(git: SimpleGit, directory: string): Promise; // @public -export function getMergeBaseRemote(git: SimpleGit, branch: string, remote?: string, localRef?: string): Promise; +export function getMergeBaseRemote(git: SimpleGit, branch: string, remote?: string, localRef?: string, onDeepen?: (message: string) => void): Promise; + +// @public +export function getPackageDirsAtRef(git: SimpleGit, ref?: string): Promise>; // @public export function getRemote(git: SimpleGit, partialUrl: string | undefined): Promise; @@ -148,6 +151,9 @@ export interface IReleaseGroup extends Reloadable { readonly workspace: IWorkspace; } +// @public +export function isFileInPackageDir(file: string, packageDirs: ReadonlySet): boolean; + // @public export function isIPackage(pkg: any): pkg is IPackage; @@ -166,6 +172,9 @@ export interface IWorkspace extends Installable, Reloadable { toString(): string; } +// @public +export function listPackageJsonPaths(git: SimpleGit, ref?: string): Promise; + // @public export function loadBuildProject

(searchPath: string, upstreamRemotePartialUrl?: string): IBuildProject

; diff --git a/build-tools/packages/build-infrastructure/src/git.ts b/build-tools/packages/build-infrastructure/src/git.ts index 4cc8faa33dcb..ab56552a23de 100644 --- a/build-tools/packages/build-infrastructure/src/git.ts +++ b/build-tools/packages/build-infrastructure/src/git.ts @@ -59,9 +59,13 @@ export function findGitRootSync(cwd = process.cwd()): string { /** * Get the merge base between the current HEAD and a remote branch. * + * If the repository is a shallow clone and no merge base is found, the clone will be deepened + * and the merge base computation retried once. + * * @param branch - The branch to compare against. * @param remote - The remote to compare against. If this is undefined, then the local branch is compared with. * @param localRef - The local ref to compare against. Defaults to HEAD. + * @param onDeepen - Optional callback invoked with a status message if a shallow-clone deepen is performed. * @returns The ref of the merge base between the current HEAD and the remote branch. */ export async function getMergeBaseRemote( @@ -69,6 +73,7 @@ export async function getMergeBaseRemote( branch: string, remote?: string, localRef = "HEAD", + onDeepen?: (message: string) => void, ): Promise { if (remote !== undefined) { // make sure we have the latest remote refs @@ -76,8 +81,22 @@ export async function getMergeBaseRemote( } const compareRef = remote === undefined ? branch : `refs/remotes/${remote}/${branch}`; - const base = await git.raw("merge-base", compareRef, localRef); - return base; + try { + const base = await git.raw("merge-base", compareRef, localRef); + return base.trim(); + } catch (error) { + const isShallow = (await git.raw("rev-parse", "--is-shallow-repository")).trim(); + if (isShallow !== "true" || remote === undefined) { + throw error; + } + + onDeepen?.( + `Merge-base with ${compareRef} not found in shallow clone; deepening and retrying.`, + ); + await git.fetch(["--deepen", "1000", remote, branch]); + const base = await git.raw("merge-base", compareRef, localRef); + return base.trim(); + } } /** @@ -121,6 +140,63 @@ function filePathsToDirectories(files: string[]): string[] { return [...dirs]; } +/** + * Matches paths that end in `package.json` (either at the root or under a directory). + */ +const packageJsonPathPattern = /(^|\/)package\.json$/; + +/** + * Lists all `package.json` file paths tracked at the given ref, or in the current working tree + * when no ref is provided. Paths are repo-relative and use POSIX separators (as returned by git). + * + * @param git - The git instance. + * @param ref - Optional ref. When provided, uses `git ls-tree -r --name-only ` to enumerate + * tracked files at that historical snapshot. When omitted, uses `git ls-files` against the + * current working tree. + */ +export async function listPackageJsonPaths(git: SimpleGit, ref?: string): Promise { + const raw = + ref === undefined + ? await git.raw("ls-files", "--", "package.json", "*/package.json") + : await git.raw("ls-tree", "-r", "--name-only", ref); + return raw.split("\n").filter((file) => packageJsonPathPattern.test(file)); +} + +/** + * Returns the set of repo-relative directories that contain a `package.json` at the given ref + * (or in the current working tree when `ref` is omitted). + * + * Useful for attributing changed files to packages when the set of packages may have changed + * between two refs (e.g. packages added, removed, or moved between a merge base and HEAD). + */ +export async function getPackageDirsAtRef(git: SimpleGit, ref?: string): Promise> { + const files = await listPackageJsonPaths(git, ref); + return new Set(files.map((file) => path.posix.dirname(file))); +} + +/** + * Returns `true` if `file` lives inside any directory in `packageDirs` (or any nested + * subdirectory of one). + * + * Walks `file`'s POSIX-style ancestor directories upward toward the repo root, returning `true` + * on the first hit. Empty strings return `false`. The repo-root pseudo-directory (`"."`) is not + * walked into, so a root-level file does not match `packageDirs` containing `"."`. + */ +export function isFileInPackageDir(file: string, packageDirs: ReadonlySet): boolean { + if (file === "") { + return false; + } + + let dir = path.posix.dirname(file); + while (dir !== "." && dir !== "/") { + if (packageDirs.has(dir)) { + return true; + } + dir = path.posix.dirname(dir); + } + return false; +} + /** * Gets the changed files, directories, release groups, and packages since the given ref. * diff --git a/build-tools/packages/build-infrastructure/src/index.ts b/build-tools/packages/build-infrastructure/src/index.ts index 62046f8e801a..456a5220c815 100644 --- a/build-tools/packages/build-infrastructure/src/index.ts +++ b/build-tools/packages/build-infrastructure/src/index.ts @@ -34,7 +34,10 @@ export { getChangedSinceRef, getFiles, getMergeBaseRemote, + getPackageDirsAtRef, getRemote, + isFileInPackageDir, + listPackageJsonPaths, } from "./git.js"; export { PackageBase } from "./package.js"; export { updatePackageJsonFile, updatePackageJsonFileAsync } from "./packageJsonUtils.js"; diff --git a/build-tools/packages/build-infrastructure/src/test/git.test.ts b/build-tools/packages/build-infrastructure/src/test/git.test.ts index 3b3dd6c31673..94894e3b9bfb 100644 --- a/build-tools/packages/build-infrastructure/src/test/git.test.ts +++ b/build-tools/packages/build-infrastructure/src/test/git.test.ts @@ -15,7 +15,13 @@ import { CleanOptions, simpleGit } from "simple-git"; import { loadBuildProject } from "../buildProject.js"; import { NotInGitRepository } from "../errors.js"; -import { findGitRootSync, getChangedSinceRef, getFiles, getRemote } from "../git.js"; +import { + findGitRootSync, + getChangedSinceRef, + getFiles, + getRemote, + isFileInPackageDir, +} from "../git.js"; import type { PackageJson } from "../types.js"; import { packageRootPath, testRepoRoot } from "./init.js"; @@ -161,3 +167,35 @@ describe("getFiles", () => { ); }); }); + +describe("isFileInPackageDir", () => { + const packageDirs = new Set(["packages/alive"]); + + it("detects file inside known package dir", () => { + expect(isFileInPackageDir("packages/alive/src/x.ts", packageDirs)).to.equal(true); + }); + + it("walks up from deeply nested paths", () => { + expect(isFileInPackageDir("packages/alive/src/deep/nested/x.ts", packageDirs)).to.equal( + true, + ); + }); + + it("returns false for root-only changes", () => { + expect(isFileInPackageDir("README.md", packageDirs)).to.equal(false); + }); + + it("returns false for unrelated sibling directory", () => { + expect(isFileInPackageDir("packages/other/src.ts", packageDirs)).to.equal(false); + }); + + it("returns false for empty input", () => { + expect(isFileInPackageDir("", packageDirs)).to.equal(false); + }); + + it("does not treat root pseudo-dir as a per-package hit", () => { + expect(isFileInPackageDir("some-root-file.md", new Set([".", "packages/alive"]))).to.equal( + false, + ); + }); +}); diff --git a/package.json b/package.json index 5cb6eeedb2fa..ffdc9f6436aa 100644 --- a/package.json +++ b/package.json @@ -105,9 +105,9 @@ "test:copyresults": "copyfiles --exclude \"**/node_modules/**\" \"**/nyc/**\" nyc", "test:coverage": "c8 npm test", "test:fromroot": "mocha \"packages/**/dist/test/**/*.spec.*js\" --exit", - "test:jest": "assign-test-ports && pnpm puppeteer browsers install chrome-headless-shell && pnpm -r --no-sort --stream --no-bail test:jest --color", - "test:jest:bail": "assign-test-ports && pnpm puppeteer browsers install chrome-headless-shell && pnpm -r --no-sort --stream test:jest", - "test:jest:report": "assign-test-ports && pnpm puppeteer browsers install chrome-headless-shell && pnpm -r --no-sort --stream --no-bail --workspace-concurrency=4 test:jest", + "test:jest": "assign-test-ports && cross-env npm_config_filter= pnpm puppeteer browsers install chrome-headless-shell && pnpm -r --no-sort --stream --no-bail test:jest --color", + "test:jest:bail": "assign-test-ports && cross-env npm_config_filter= pnpm puppeteer browsers install chrome-headless-shell && pnpm -r --no-sort --stream test:jest", + "test:jest:report": "assign-test-ports && cross-env npm_config_filter= pnpm puppeteer browsers install chrome-headless-shell && pnpm -r --no-sort --stream --no-bail --workspace-concurrency=4 test:jest", "test:mocha": "pnpm run -r --no-sort --stream --no-bail test:mocha --color", "test:mocha:bail": "pnpm run -r --no-sort --stream test:mocha", "test:realsvc": "pnpm run -r --no-sort --stream --no-bail test:realsvc", diff --git a/tools/pipelines/templates/build-npm-client-package.yml b/tools/pipelines/templates/build-npm-client-package.yml index 29e99f671238..2c15b7ef4920 100644 --- a/tools/pipelines/templates/build-npm-client-package.yml +++ b/tools/pipelines/templates/build-npm-client-package.yml @@ -199,6 +199,40 @@ extends: displayName: Build Stage dependsOn: [] # this stage doesn't depend on preceding stage jobs: + # Detect packages changed by this PR; publishes output variables that + # scope downstream test jobs. Activated only for PR builds whose + # source branch name contains 'test/filtered-ci/' — a rollout opt-in + # to be broadened once the feature has soaked. See + # include-detect-changed-packages.yml for the full semantics. + - job: detect_changes + displayName: Detect changed packages + condition: >- + and( + eq(variables['Build.Reason'], 'PullRequest'), + contains(variables['System.PullRequest.SourceBranch'], 'test/filtered-ci/') + ) + variables: + - name: targetBranchName + value: $(System.PullRequest.TargetBranch) + steps: + - checkout: self + path: $(FluidFrameworkDirectory) + clean: true + # Shallow clone; `flub check changedPackages` deepens on demand + # if the merge-base falls outside this depth. Most PRs + # merge-base within a few hundred commits. + fetchDepth: 200 + + - template: /tools/pipelines/templates/include-install-build-tools.yml@self + parameters: + buildDirectory: ${{ parameters.buildDirectory }} + buildToolsVersionToInstall: repo + + - template: /tools/pipelines/templates/include-detect-changed-packages.yml@self + parameters: + buildDirectory: ${{ parameters.buildDirectory }} + targetBranchName: $(targetBranchName) + # Job - Build - job: build displayName: Build @@ -623,7 +657,13 @@ extends: - job: Coverage_tests displayName: "Coverage tests" - dependsOn: build + dependsOn: + - build + - detect_changes + # Skip when detect_changes explicitly returned shouldRunTests=false. + # Use `build.result` (not `succeeded()`) so a Skipped detect_changes + # on non-opt-in PR builds doesn't false-skip this job. + condition: and(in(dependencies.build.result, 'Succeeded', 'SucceededWithIssues'), ne(dependencies.detect_changes.outputs['setChangedPackages.shouldRunTests'], 'false')) variables: - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - name: targetBranchName @@ -639,6 +679,9 @@ extends: value: 'false' - name: COMMIT_SHA value: $[ dependencies.build.outputs['setCommitSHA.COMMIT_SHA'] ] + # Empty when detect_changes was skipped — pnpm treats that as no filter. + - name: scopedPnpmFilter + value: $[ dependencies.detect_changes.outputs['setChangedPackages.scopedPnpmFilter'] ] steps: # Setup @@ -691,6 +734,7 @@ extends: taskTestStep: '${{ test.name }}' buildDirectory: '${{ parameters.buildDirectory }}' testCoverage: '${{ parameters.testCoverage }}' + pnpmFilter: $(scopedPnpmFilter) - task: Npm@1 displayName: 'npm run test:copyresults' @@ -788,7 +832,18 @@ extends: - ${{ each test in parameters.taskTest }}: - job: Test_${{ test.jobName }} displayName: "Run Task Test ${{ test.jobName }}" - dependsOn: build + dependsOn: + - build + - detect_changes + # Skip the job entirely when detect_changes explicitly concluded no + # packages were affected — saves agent allocation, checkout, and + # install on scoped PRs. Empty (detect_changes was skipped) still + # runs. See include-detect-changed-packages.yml for the invariants. + # + # We check `build.result` directly instead of using `succeeded()` + # because the latter treats a Skipped `detect_changes` dependency + # as non-success and false-skips this job on non-opt-in PR builds. + condition: and(in(dependencies.build.result, 'Succeeded', 'SucceededWithIssues'), ne(dependencies.detect_changes.outputs['setChangedPackages.shouldRunTests'], 'false')) variables: - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - name: targetBranchName @@ -804,6 +859,9 @@ extends: value: 'false' - name: COMMIT_SHA value: $[ dependencies.build.outputs['setCommitSHA.COMMIT_SHA'] ] + # Empty when detect_changes was skipped — pnpm treats that as no filter. + - name: scopedPnpmFilter + value: $[ dependencies.detect_changes.outputs['setChangedPackages.scopedPnpmFilter'] ] steps: # Setup - checkout: self @@ -849,6 +907,7 @@ extends: taskTestStep: '${{ test.name }}' buildDirectory: '${{ parameters.buildDirectory }}' testCoverage: 'false' + pnpmFilter: $(scopedPnpmFilter) - task: Npm@1 displayName: 'npm run test:copyresults' diff --git a/tools/pipelines/templates/include-detect-changed-packages.yml b/tools/pipelines/templates/include-detect-changed-packages.yml new file mode 100644 index 000000000000..fcb40046d444 --- /dev/null +++ b/tools/pipelines/templates/include-detect-changed-packages.yml @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft Corporation and contributors. All rights reserved. +# Licensed under the MIT License. + +# Pipeline wrapper around `flub check changedPackages`. +# +# This template intentionally has no checkout or build-tools installation steps. +# Callers own those prerequisites so this reusable template has no hidden side +# effects: +# - the repository must already be checked out at +# `$(Pipeline.Workspace)/${{ parameters.buildDirectory }}`; +# - `flub` must already be available on PATH, usually via +# include-install-build-tools.yml; +# - callers choose checkout depth/clean behavior. A shallow checkout is OK; the +# command deepens on demand when the merge-base is outside the current depth. +# +# Keep the task name `setChangedPackages`: downstream jobs read its ADO output +# variables (`shouldRunTests` and `scopedPnpmFilter`) by that name. + +parameters: +- name: buildDirectory + type: string + +- name: targetBranchName + type: string + +steps: + - task: Bash@3 + name: setChangedPackages + displayName: Detect changed packages + env: + TARGET_BRANCH: ${{ parameters.targetBranchName }} + inputs: + targetType: inline + workingDirectory: '$(Pipeline.Workspace)/${{ parameters.buildDirectory }}' + script: | + set -eu -o pipefail + flub check changedPackages --targetBranch "${TARGET_BRANCH}" diff --git a/tools/pipelines/templates/include-test-task.yml b/tools/pipelines/templates/include-test-task.yml index 6901553a5d7b..9de0e1e8e6b8 100644 --- a/tools/pipelines/templates/include-test-task.yml +++ b/tools/pipelines/templates/include-test-task.yml @@ -14,6 +14,23 @@ parameters: type: boolean default: false +# Optional pnpm filter expression (e.g. "...[]"). When non-empty, it is +# set as `npm_config_filter` so pnpm scopes the recursive run to the listed +# packages plus their dependents. Empty means "no filter" — pnpm falls back +# to running across every workspace package (the historical behavior). +# +# IMPORTANT: `npm_config_filter` propagates to every `pnpm` invocation the +# npm script transitively makes, not just `pnpm -r`. Non-recursive `pnpm` +# calls (e.g. `pnpm puppeteer ...`, `pnpm exec ...`) will fail with +# ERR_PNPM_RECURSIVE_EXEC_NO_PACKAGE when a filter is set. Any root-level +# script that chains a non-recursive pnpm call before a recursive one must +# clear the filter for the non-recursive portion via +# `cross-env npm_config_filter= pnpm `. See test:jest in the +# root package.json for the pattern. +- name: pnpmFilter + type: string + default: '' + steps: # Test - With coverage - ${{ if and(parameters.testCoverage, startsWith(parameters.taskTestStep, 'ci:test')) }}: @@ -25,6 +42,8 @@ steps: customCommand: 'run ${{ parameters.taskTestStep }}:coverage' condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) env: + ${{ if ne(parameters.pnpmFilter, '') }}: + npm_config_filter: ${{ parameters.pnpmFilter }} # Tests can use this environment variable to behave differently when running from a test branch ${{ if contains(parameters.taskTestStep, 'tinylicious') }}: # Disable colorization for tinylicious logs (not useful when printing to a file) @@ -41,6 +60,8 @@ steps: customCommand: 'run ${{ parameters.taskTestStep }}' condition: and(succeededOrFailed(), eq(variables['startTest'], 'true')) env: + ${{ if ne(parameters.pnpmFilter, '') }}: + npm_config_filter: ${{ parameters.pnpmFilter }} # Tests can use this environment variable to behave differently when running from a test branch ${{ if contains(parameters.taskTestStep, 'tinylicious') }}: # Disable colorization for tinylicious logs (not useful when printing to a file)