diff --git a/.github/workflows/devcontainer-release.yml b/.github/workflows/devcontainer-release.yml index ccde6cb3f..3f178ba0b 100644 --- a/.github/workflows/devcontainer-release.yml +++ b/.github/workflows/devcontainer-release.yml @@ -225,19 +225,17 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Setup uv - uses: astral-sh/setup-uv@v6 - - - name: Download wheels + - name: Download artifacts uses: actions/download-artifact@v4 with: path: dist merge-multiple: true - - name: Publish wheels to PyPI + - name: Collect PyPI wheels shell: bash run: | set -euo pipefail + mkdir -p pypi-dist mapfile -t wheels < <(find dist -type f -name '*.whl' | sort) if [[ "${#wheels[@]}" -eq 0 ]]; then @@ -245,7 +243,13 @@ jobs: exit 1 fi - uv publish "${wheels[@]}" + cp "${wheels[@]}" pypi-dist/ + + - name: Publish wheels to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: pypi-dist + skip-existing: true npm: needs: [prepare, build, release] @@ -324,7 +328,7 @@ jobs: shell: bash run: | set -euo pipefail - node build/publish-npm-packages.js \ + node build/publish-npm-packages.js --skip-unregistered \ dist/npm/devcontainer-rs-devcontainer-darwin-x64 \ dist/npm/devcontainer-rs-devcontainer-darwin-arm64 \ dist/npm/devcontainer-rs-devcontainer-linux-x64-gnu \ diff --git a/build/check-npm-publish-workflow.js b/build/check-npm-publish-workflow.js index 21937f5bc..2b23541d9 100644 --- a/build/check-npm-publish-workflow.js +++ b/build/check-npm-publish-workflow.js @@ -12,6 +12,7 @@ const workflowPath = path.join( const workflow = fs.readFileSync(workflowPath, "utf8"); const buildJobMatch = workflow.match(/^ build:\n([\s\S]+?)^ release:/m); +const pypiJobMatch = workflow.match(/^ pypi:\n([\s\S]+?)^ npm:/m); const npmJobMatch = workflow.match(/^ npm:\n([\s\S]+)$/m); assert.ok( @@ -19,8 +20,10 @@ assert.ok( "expected build release job in devcontainer-release workflow", ); assert.ok(npmJobMatch, "expected npm release job in devcontainer-release workflow"); +assert.ok(pypiJobMatch, "expected PyPI release job in devcontainer-release workflow"); const buildJob = buildJobMatch[0]; +const pypiJob = pypiJobMatch[0]; const npmJob = npmJobMatch[0]; assert.match( @@ -70,7 +73,7 @@ assert.match( ); assert.match( npmJob, - /node build\/publish-npm-packages\.js /, + /node build\/publish-npm-packages\.js --skip-unregistered /, "npm publish job should use the repo-owned idempotent npm publish helper", ); assert.match( @@ -98,3 +101,33 @@ assert.doesNotMatch( /\bnpm publish --access public\b/, "npm publish job should not inline raw npm publish commands", ); +assert.match( + pypiJob, + /uses:\s*pypa\/gh-action-pypi-publish@release\/v1\b/, + "PyPI publish job should use PyPA's trusted-publishing action", +); +assert.match( + pypiJob, + /find dist -type f -name '\*\.whl'/, + "PyPI publish job should collect wheel files from the mixed artifact download", +); +assert.match( + pypiJob, + /packages-dir:\s*pypi-dist\b/, + "PyPI publish job should upload from a wheel-only directory", +); +assert.doesNotMatch( + pypiJob, + /packages-dir:\s*dist\b/, + "PyPI publish job should not upload the mixed artifact download directory", +); +assert.match( + pypiJob, + /skip-existing:\s*true\b/, + "PyPI publish job should skip already uploaded files after a partial release", +); +assert.doesNotMatch( + pypiJob, + /\buv publish\b/, + "PyPI publish job should not use uv for the final upload path", +); diff --git a/build/publish-npm-packages.js b/build/publish-npm-packages.js index 3bbaf1598..f69a7950a 100644 --- a/build/publish-npm-packages.js +++ b/build/publish-npm-packages.js @@ -4,6 +4,8 @@ const { execFileSync } = require("node:child_process"); const CONFLICT_PATTERN = /You cannot publish over the previously published versions|cannot publish over existing version|EPUBLISHCONFLICT/; +const NPM_NOT_FOUND_PATTERN = + /\bE404\b|404 Not Found|is not in this registry|No match found for version/i; function readPackageInfo(packageDir) { const manifestPath = path.join(packageDir, "package.json"); @@ -27,11 +29,32 @@ function packageVersionExists(packageInfo, runNpm = defaultRunNpm) { try { runNpm(["view", versionSpec(packageInfo), "version"]); return true; - } catch { - return false; + } catch (error) { + if (isNpmNotFoundError(error)) { + return false; + } + throw error; } } +function packageNameExists(packageInfo, runNpm = defaultRunNpm) { + try { + runNpm(["view", packageInfo.name, "name"]); + return true; + } catch (error) { + if (isNpmNotFoundError(error)) { + return false; + } + throw error; + } +} + +function isNpmNotFoundError(error) { + return NPM_NOT_FOUND_PATTERN.test( + `${error.stdout || ""}${error.stderr || ""}${error.message || ""}`, + ); +} + function defaultRunNpm(args) { return execFileSync("npm", args, { encoding: "utf8", @@ -70,17 +93,94 @@ function publishPackage(packageDir, options = {}) { } } +function pruneOptionalDependencies(packageInfo, availablePackageNames, log = () => {}) { + const manifest = JSON.parse(fs.readFileSync(packageInfo.manifestPath, "utf8")); + if (!manifest.optionalDependencies) { + return; + } + + const pruned = Object.fromEntries( + Object.entries(manifest.optionalDependencies).filter(([name]) => + availablePackageNames.has(name), + ), + ); + const removed = Object.keys(manifest.optionalDependencies).filter( + (name) => !availablePackageNames.has(name), + ); + + if (removed.length === 0) { + return; + } + + if (Object.keys(pruned).length === 0) { + delete manifest.optionalDependencies; + } else { + manifest.optionalDependencies = pruned; + } + + fs.writeFileSync(packageInfo.manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); + log( + `pruned unavailable optional dependencies from ${versionSpec(packageInfo)}: ${removed.join(", ")}`, + ); +} + function publishPackageDirs(packageDirs, options = {}) { - for (const packageDir of packageDirs) { - publishPackage(packageDir, options); + const runNpm = options.runNpm || defaultRunNpm; + const log = options.log || (() => {}); + const packageInfos = packageDirs.map(readPackageInfo); + let publishablePackageNames = null; + + if (options.skipUnregistered) { + publishablePackageNames = new Set(); + for (const packageInfo of packageInfos) { + if (packageNameExists(packageInfo, runNpm)) { + publishablePackageNames.add(packageInfo.name); + } else { + log(`skipping ${versionSpec(packageInfo)} (package is not registered on npm)`); + } + } + + for (const packageInfo of packageInfos) { + if (publishablePackageNames.has(packageInfo.name)) { + pruneOptionalDependencies(packageInfo, publishablePackageNames, log); + } + } } + + for (const packageInfo of packageInfos) { + if (publishablePackageNames && !publishablePackageNames.has(packageInfo.name)) { + continue; + } + publishPackage(packageInfo.packageDir, options); + } +} + +function parseArgs(argv) { + const options = { + packageDirs: [], + skipUnregistered: false, + }; + + for (const arg of argv) { + if (arg === "--skip-unregistered") { + options.skipUnregistered = true; + } else if (arg.startsWith("--")) { + throw new Error(`Unknown option: ${arg}`); + } else { + options.packageDirs.push(arg); + } + } + + return options; } function main(argv = process.argv.slice(2)) { - if (argv.length === 0) { + const options = parseArgs(argv); + if (options.packageDirs.length === 0) { throw new Error("Expected one or more package directories"); } - publishPackageDirs(argv, { + publishPackageDirs(options.packageDirs, { + skipUnregistered: options.skipUnregistered, log(message) { process.stdout.write(`${message}\n`); }, @@ -92,6 +192,7 @@ module.exports = { publishPackageDirs, readPackageInfo, packageVersionExists, + packageNameExists, }; if (require.main === module) { diff --git a/build/test-publish-npm-packages.js b/build/test-publish-npm-packages.js index 35423d5af..f073c12e3 100644 --- a/build/test-publish-npm-packages.js +++ b/build/test-publish-npm-packages.js @@ -21,6 +21,10 @@ function writePackage(dir, name, version) { ); } +function readPackage(dir) { + return JSON.parse(fs.readFileSync(path.join(dir, "package.json"), "utf8")); +} + function makeConflictError(message) { const error = new Error(message); error.stdout = ""; @@ -28,6 +32,13 @@ function makeConflictError(message) { return error; } +function makeNotFoundError(message = "npm error code E404") { + const error = new Error(message); + error.stdout = ""; + error.stderr = message; + return error; +} + test("skips publish when the package version already exists", () => { const packageDir = mkTempDir("devcontainer-rs-publish-skip-"); writePackage(packageDir, "@devcontainer-rs/example", "1.2.3"); @@ -56,7 +67,7 @@ test("continues when npm publish reports a publish conflict", () => { runNpm(args) { calls.push(args); if (args[0] === "view") { - throw new Error("not found"); + throw makeNotFoundError(); } if (args[0] === "publish") { throw makeConflictError( @@ -74,6 +85,67 @@ test("continues when npm publish reports a publish conflict", () => { ]); }); +test("skips unregistered packages and prunes unavailable optional dependencies", () => { + const existingNativeDir = mkTempDir("devcontainer-rs-publish-existing-native-"); + const missingNativeDir = mkTempDir("devcontainer-rs-publish-missing-native-"); + const wrapperDir = mkTempDir("devcontainer-rs-publish-wrapper-"); + + writePackage(existingNativeDir, "@devcontainer-rs/devcontainer-linux-x64-gnu", "1.2.3"); + writePackage(missingNativeDir, "@devcontainer-rs/devcontainer-linux-arm64-gnu", "1.2.3"); + fs.mkdirSync(wrapperDir, { recursive: true }); + fs.writeFileSync( + path.join(wrapperDir, "package.json"), + `${JSON.stringify( + { + name: "@devcontainer-rs/cli", + version: "1.2.3", + optionalDependencies: { + "@devcontainer-rs/devcontainer-linux-x64-gnu": "1.2.3", + "@devcontainer-rs/devcontainer-linux-arm64-gnu": "1.2.3", + }, + }, + null, + 2, + )}\n`, + "utf8", + ); + + const calls = []; + publishPackageDirs([existingNativeDir, missingNativeDir, wrapperDir], { + skipUnregistered: true, + runNpm(args) { + calls.push(args); + if (args[0] === "view" && args.length === 3 && args[2] === "name") { + if (args[1] === "@devcontainer-rs/devcontainer-linux-arm64-gnu") { + throw makeNotFoundError(); + } + return args[1]; + } + if (args[0] === "view" && args.length === 3 && args[2] === "version") { + throw makeNotFoundError("npm error code E404: new version not published yet"); + } + if (args[0] === "publish") { + return "published"; + } + throw new Error(`unexpected npm args: ${args.join(" ")}`); + }, + log() {}, + }); + + assert.deepEqual(readPackage(wrapperDir).optionalDependencies, { + "@devcontainer-rs/devcontainer-linux-x64-gnu": "1.2.3", + }); + assert.deepEqual(calls, [ + ["view", "@devcontainer-rs/devcontainer-linux-x64-gnu", "name"], + ["view", "@devcontainer-rs/devcontainer-linux-arm64-gnu", "name"], + ["view", "@devcontainer-rs/cli", "name"], + ["view", "@devcontainer-rs/devcontainer-linux-x64-gnu@1.2.3", "version"], + ["publish", "--access", "public", existingNativeDir], + ["view", "@devcontainer-rs/cli@1.2.3", "version"], + ["publish", "--access", "public", wrapperDir], + ]); +}); + test("publishes packages in the given order", () => { const firstDir = mkTempDir("devcontainer-rs-publish-order-"); const secondDir = mkTempDir("devcontainer-rs-publish-order-"); @@ -85,7 +157,7 @@ test("publishes packages in the given order", () => { runNpm(args) { calls.push(args); if (args[0] === "view") { - throw new Error("not found"); + throw makeNotFoundError(); } if (args[0] === "publish") { return "published"; @@ -102,3 +174,27 @@ test("publishes packages in the given order", () => { ["publish", "--access", "public", secondDir], ]); }); + +test("fails skip-unregistered publishing when npm name lookup fails for a reason other than not found", () => { + const packageDir = mkTempDir("devcontainer-rs-publish-lookup-failure-"); + writePackage(packageDir, "@devcontainer-rs/example", "1.2.3"); + + const error = new Error("npm error code E503"); + error.stdout = ""; + error.stderr = "npm error code E503\nnpm error registry unavailable"; + + assert.throws( + () => + publishPackageDirs([packageDir], { + skipUnregistered: true, + runNpm(args) { + if (args[0] === "view" && args.length === 3 && args[2] === "name") { + throw error; + } + throw new Error(`unexpected npm args: ${args.join(" ")}`); + }, + log() {}, + }), + /E503/, + ); +}); diff --git a/docs/standalone/distribution.md b/docs/standalone/distribution.md index 8415b3709..6c33e2d30 100644 --- a/docs/standalone/distribution.md +++ b/docs/standalone/distribution.md @@ -74,6 +74,7 @@ Homebrew maps the backing repository `jooh/homebrew-tap` to the tap shorthand `j - The repository no longer ships or maintains the old bundled-Node installer path. - Release automation does not currently sign artifacts or notarize macOS builds. - PyPI publication requires a PyPI Trusted Publisher configured for `.github/workflows/devcontainer-release.yml` and the `pypi` GitHub environment. -- npm publication requires npm Trusted Publisher entries for each npm package, pointing at the same `.github/workflows/devcontainer-release.yml` workflow. +- PyPI publication uses PyPA's trusted publishing action with duplicate-file skipping so reruns can recover from partial releases. +- npm publication requires npm Trusted Publisher entries for each npm package, pointing at the same `.github/workflows/devcontainer-release.yml` workflow. The release job skips native npm package names that have not been registered yet and prunes them from wrapper optional dependencies for that publish. - Homebrew formula publication is tap-owned and scheduled, so a source release can be available on GitHub/PyPI/npm before the tap cron commits the formula update. - Compatibility tooling in `package.json` is not part of the runtime distribution path.