[codex] Harden release publishing retries

.github/workflows/devcontainer-release.yml

@@ -225,27 +225,31 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Setup uv
-        uses: astral-sh/setup-uv@v6
-
-      - name: Download wheels
+      - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
           path: dist
           merge-multiple: true
 
-      - name: Publish wheels to PyPI
+      - name: Collect PyPI wheels
         shell: bash
         run: |
           set -euo pipefail
+          mkdir -p pypi-dist
           mapfile -t wheels < <(find dist -type f -name '*.whl' | sort)
 
           if [[ "${#wheels[@]}" -eq 0 ]]; then
             echo "no PyPI wheels found" >&2
             exit 1
           fi
 
-          uv publish "${wheels[@]}"
+          cp "${wheels[@]}" pypi-dist/
+
+      - name: Publish wheels to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: pypi-dist
+          skip-existing: true
 
   npm:
     needs: [prepare, build, release]
@@ -324,7 +328,7 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
-          node build/publish-npm-packages.js \
+          node build/publish-npm-packages.js --skip-unregistered \
             dist/npm/devcontainer-rs-devcontainer-darwin-x64 \
             dist/npm/devcontainer-rs-devcontainer-darwin-arm64 \
             dist/npm/devcontainer-rs-devcontainer-linux-x64-gnu \

build/check-npm-publish-workflow.js

@@ -12,15 +12,18 @@ const workflowPath = path.join(
 
 const workflow = fs.readFileSync(workflowPath, "utf8");
 const buildJobMatch = workflow.match(/^  build:\n([\s\S]+?)^  release:/m);
+const pypiJobMatch = workflow.match(/^  pypi:\n([\s\S]+?)^  npm:/m);
 const npmJobMatch = workflow.match(/^  npm:\n([\s\S]+)$/m);
 
 assert.ok(
   buildJobMatch,
   "expected build release job in devcontainer-release workflow",
 );
 assert.ok(npmJobMatch, "expected npm release job in devcontainer-release workflow");
+assert.ok(pypiJobMatch, "expected PyPI release job in devcontainer-release workflow");
 
 const buildJob = buildJobMatch[0];
+const pypiJob = pypiJobMatch[0];
 const npmJob = npmJobMatch[0];
 
 assert.match(
@@ -70,7 +73,7 @@ assert.match(
 );
 assert.match(
   npmJob,
-  /node build\/publish-npm-packages\.js /,
+  /node build\/publish-npm-packages\.js --skip-unregistered /,
   "npm publish job should use the repo-owned idempotent npm publish helper",
 );
 assert.match(
@@ -98,3 +101,33 @@ assert.doesNotMatch(
   /\bnpm publish --access public\b/,
   "npm publish job should not inline raw npm publish commands",
 );
+assert.match(
+  pypiJob,
+  /uses:\s*pypa\/gh-action-pypi-publish@release\/v1\b/,
+  "PyPI publish job should use PyPA's trusted-publishing action",
+);
+assert.match(
+  pypiJob,
+  /find dist -type f -name '\*\.whl'/,
+  "PyPI publish job should collect wheel files from the mixed artifact download",
+);
+assert.match(
+  pypiJob,
+  /packages-dir:\s*pypi-dist\b/,
+  "PyPI publish job should upload from a wheel-only directory",
+);
+assert.doesNotMatch(
+  pypiJob,
+  /packages-dir:\s*dist\b/,
+  "PyPI publish job should not upload the mixed artifact download directory",
+);
+assert.match(
+  pypiJob,
+  /skip-existing:\s*true\b/,
+  "PyPI publish job should skip already uploaded files after a partial release",
+);
+assert.doesNotMatch(
+  pypiJob,
+  /\buv publish\b/,
+  "PyPI publish job should not use uv for the final upload path",
+);

build/publish-npm-packages.js

@@ -4,6 +4,8 @@ const { execFileSync } = require("node:child_process");
 
 const CONFLICT_PATTERN =
   /You cannot publish over the previously published versions|cannot publish over existing version|EPUBLISHCONFLICT/;
+const NPM_NOT_FOUND_PATTERN =
+  /\bE404\b|404 Not Found|is not in this registry|No match found for version/i;
 
 function readPackageInfo(packageDir) {
   const manifestPath = path.join(packageDir, "package.json");
@@ -27,11 +29,32 @@ function packageVersionExists(packageInfo, runNpm = defaultRunNpm) {
   try {
     runNpm(["view", versionSpec(packageInfo), "version"]);
     return true;
-  } catch {
-    return false;
+  } catch (error) {
+    if (isNpmNotFoundError(error)) {
+      return false;
+    }
+    throw error;
   }
 }
 
+function packageNameExists(packageInfo, runNpm = defaultRunNpm) {
+  try {
+    runNpm(["view", packageInfo.name, "name"]);
+    return true;
+  } catch (error) {
+    if (isNpmNotFoundError(error)) {
+      return false;
+    }
+    throw error;
+  }
+}
+
+function isNpmNotFoundError(error) {
+  return NPM_NOT_FOUND_PATTERN.test(
+    `${error.stdout || ""}${error.stderr || ""}${error.message || ""}`,
+  );
+}
+
 function defaultRunNpm(args) {
   return execFileSync("npm", args, {
     encoding: "utf8",
@@ -70,17 +93,94 @@ function publishPackage(packageDir, options = {}) {
   }
 }
 
+function pruneOptionalDependencies(packageInfo, availablePackageNames, log = () => {}) {
+  const manifest = JSON.parse(fs.readFileSync(packageInfo.manifestPath, "utf8"));
+  if (!manifest.optionalDependencies) {
+    return;
+  }
+
+  const pruned = Object.fromEntries(
+    Object.entries(manifest.optionalDependencies).filter(([name]) =>
+      availablePackageNames.has(name),
+    ),
+  );
+  const removed = Object.keys(manifest.optionalDependencies).filter(
+    (name) => !availablePackageNames.has(name),
+  );
+
+  if (removed.length === 0) {
+    return;
+  }
+
+  if (Object.keys(pruned).length === 0) {
+    delete manifest.optionalDependencies;
+  } else {
+    manifest.optionalDependencies = pruned;
+  }
+
+  fs.writeFileSync(packageInfo.manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
+  log(
+    `pruned unavailable optional dependencies from ${versionSpec(packageInfo)}: ${removed.join(", ")}`,
+  );
+}
+
 function publishPackageDirs(packageDirs, options = {}) {
-  for (const packageDir of packageDirs) {
-    publishPackage(packageDir, options);
+  const runNpm = options.runNpm || defaultRunNpm;
+  const log = options.log || (() => {});
+  const packageInfos = packageDirs.map(readPackageInfo);
+  let publishablePackageNames = null;
+
+  if (options.skipUnregistered) {
+    publishablePackageNames = new Set();
+    for (const packageInfo of packageInfos) {
+      if (packageNameExists(packageInfo, runNpm)) {
+        publishablePackageNames.add(packageInfo.name);
+      } else {
+        log(`skipping ${versionSpec(packageInfo)} (package is not registered on npm)`);
+      }
+    }
+
+    for (const packageInfo of packageInfos) {
+      if (publishablePackageNames.has(packageInfo.name)) {
+        pruneOptionalDependencies(packageInfo, publishablePackageNames, log);
+      }
+    }
   }
+
+  for (const packageInfo of packageInfos) {
+    if (publishablePackageNames && !publishablePackageNames.has(packageInfo.name)) {
+      continue;
+    }
+    publishPackage(packageInfo.packageDir, options);
+  }
+}
+
+function parseArgs(argv) {
+  const options = {
+    packageDirs: [],
+    skipUnregistered: false,
+  };
+
+  for (const arg of argv) {
+    if (arg === "--skip-unregistered") {
+      options.skipUnregistered = true;
+    } else if (arg.startsWith("--")) {
+      throw new Error(`Unknown option: ${arg}`);
+    } else {
+      options.packageDirs.push(arg);
+    }
+  }
+
+  return options;
 }
 
 function main(argv = process.argv.slice(2)) {
-  if (argv.length === 0) {
+  const options = parseArgs(argv);
+  if (options.packageDirs.length === 0) {
     throw new Error("Expected one or more package directories");
   }
-  publishPackageDirs(argv, {
+  publishPackageDirs(options.packageDirs, {
+    skipUnregistered: options.skipUnregistered,
     log(message) {
       process.stdout.write(`${message}\n`);
     },
@@ -92,6 +192,7 @@ module.exports = {
   publishPackageDirs,
   readPackageInfo,
   packageVersionExists,
+  packageNameExists,
 };
 
 if (require.main === module) {

build/test-publish-npm-packages.js

@@ -21,13 +21,24 @@ function writePackage(dir, name, version) {
   );
 }
 
+function readPackage(dir) {
+  return JSON.parse(fs.readFileSync(path.join(dir, "package.json"), "utf8"));
+}
+
 function makeConflictError(message) {
   const error = new Error(message);
   error.stdout = "";
   error.stderr = message;
   return error;
 }
 
+function makeNotFoundError(message = "npm error code E404") {
+  const error = new Error(message);
+  error.stdout = "";
+  error.stderr = message;
+  return error;
+}
+
 test("skips publish when the package version already exists", () => {
   const packageDir = mkTempDir("devcontainer-rs-publish-skip-");
   writePackage(packageDir, "@devcontainer-rs/example", "1.2.3");
@@ -56,7 +67,7 @@ test("continues when npm publish reports a publish conflict", () => {
     runNpm(args) {
       calls.push(args);
       if (args[0] === "view") {
-        throw new Error("not found");
+        throw makeNotFoundError();
       }
       if (args[0] === "publish") {
         throw makeConflictError(
@@ -74,6 +85,67 @@ test("continues when npm publish reports a publish conflict", () => {
   ]);
 });
 
+test("skips unregistered packages and prunes unavailable optional dependencies", () => {
+  const existingNativeDir = mkTempDir("devcontainer-rs-publish-existing-native-");
+  const missingNativeDir = mkTempDir("devcontainer-rs-publish-missing-native-");
+  const wrapperDir = mkTempDir("devcontainer-rs-publish-wrapper-");
+
+  writePackage(existingNativeDir, "@devcontainer-rs/devcontainer-linux-x64-gnu", "1.2.3");
+  writePackage(missingNativeDir, "@devcontainer-rs/devcontainer-linux-arm64-gnu", "1.2.3");
+  fs.mkdirSync(wrapperDir, { recursive: true });
+  fs.writeFileSync(
+    path.join(wrapperDir, "package.json"),
+    `${JSON.stringify(
+      {
+        name: "@devcontainer-rs/cli",
+        version: "1.2.3",
+        optionalDependencies: {
+          "@devcontainer-rs/devcontainer-linux-x64-gnu": "1.2.3",
+          "@devcontainer-rs/devcontainer-linux-arm64-gnu": "1.2.3",
+        },
+      },
+      null,
+      2,
+    )}\n`,
+    "utf8",
+  );
+
+  const calls = [];
+  publishPackageDirs([existingNativeDir, missingNativeDir, wrapperDir], {
+    skipUnregistered: true,
+    runNpm(args) {
+      calls.push(args);
+      if (args[0] === "view" && args.length === 3 && args[2] === "name") {
+        if (args[1] === "@devcontainer-rs/devcontainer-linux-arm64-gnu") {
+          throw makeNotFoundError();
+        }
+        return args[1];
+      }
+      if (args[0] === "view" && args.length === 3 && args[2] === "version") {
+        throw makeNotFoundError("npm error code E404: new version not published yet");
+      }
+      if (args[0] === "publish") {
+        return "published";
+      }
+      throw new Error(`unexpected npm args: ${args.join(" ")}`);
+    },
+    log() {},
+  });
+
+  assert.deepEqual(readPackage(wrapperDir).optionalDependencies, {
+    "@devcontainer-rs/devcontainer-linux-x64-gnu": "1.2.3",
+  });
+  assert.deepEqual(calls, [
+    ["view", "@devcontainer-rs/devcontainer-linux-x64-gnu", "name"],
+    ["view", "@devcontainer-rs/devcontainer-linux-arm64-gnu", "name"],
+    ["view", "@devcontainer-rs/cli", "name"],
+    ["view", "@devcontainer-rs/devcontainer-linux-x64-gnu@1.2.3", "version"],
+    ["publish", "--access", "public", existingNativeDir],
+    ["view", "@devcontainer-rs/cli@1.2.3", "version"],
+    ["publish", "--access", "public", wrapperDir],
+  ]);
+});
+
 test("publishes packages in the given order", () => {
   const firstDir = mkTempDir("devcontainer-rs-publish-order-");
   const secondDir = mkTempDir("devcontainer-rs-publish-order-");
@@ -85,7 +157,7 @@ test("publishes packages in the given order", () => {
     runNpm(args) {
       calls.push(args);
       if (args[0] === "view") {
-        throw new Error("not found");
+        throw makeNotFoundError();
       }
       if (args[0] === "publish") {
         return "published";
@@ -102,3 +174,27 @@ test("publishes packages in the given order", () => {
     ["publish", "--access", "public", secondDir],
   ]);
 });
+
+test("fails skip-unregistered publishing when npm name lookup fails for a reason other than not found", () => {
+  const packageDir = mkTempDir("devcontainer-rs-publish-lookup-failure-");
+  writePackage(packageDir, "@devcontainer-rs/example", "1.2.3");
+
+  const error = new Error("npm error code E503");
+  error.stdout = "";
+  error.stderr = "npm error code E503\nnpm error registry unavailable";
+
+  assert.throws(
+    () =>
+      publishPackageDirs([packageDir], {
+        skipUnregistered: true,
+        runNpm(args) {
+          if (args[0] === "view" && args.length === 3 && args[2] === "name") {
+            throw error;
+          }
+          throw new Error(`unexpected npm args: ${args.join(" ")}`);
+        },
+        log() {},
+      }),
+    /E503/,
+  );
+});