fix(webapp): retry on version collision when initializing a deployment (#3610)

d-cs · claude · web-flow · commit 5022769f7001 · 2026-05-14T16:23:32.000+01:00
Concurrent `POST /api/v1/deployments` requests for the same environment
race on the `WorkerDeployment(environmentId, version)` unique
constraint. Both requests read the same latest deployment via
`findFirst`, compute the same next version via
`calculateNextBuildVersion`, and both attempt
`prisma.workerDeployment.create()` — one wins, the other crashes with
Prisma `P2002`. The bug is a classic TOCTOU between the version read and
the version write; it's been latent since the version-assignment logic
was first added but only fires when two deploys land within milliseconds
of each other (CI matrices, retried CLI calls, webhook-triggered
redeploys).

## Approach

Extracts the version assignment + create into a small helper
`createDeploymentWithNextVersion`
(`apps/webapp/app/v3/services/initializeDeployment/createDeploymentWithNextVersion.server.ts`).
The helper retries on `P2002 (environmentId, version)` up to 5 times
with randomised 5–50ms jitter so N concurrent racers don't loop in
lockstep. Each attempt re-reads the latest version, recomputes via
`calculateNextBuildVersion`, and re-runs the caller's `buildData`
callback so version-dependent fields (image ref tag, friendlyId) are
always consistent with the version actually persisted. A `logger.warn`
fires per collision so the retry rate is observable in production logs.

When retries are exhausted, the helper throws a dedicated
`DeploymentVersionCollisionError` carrying `environmentId`, `attempts`,
and `lastAttemptedVersion`, with the original
`PrismaClientKnownRequestError` attached as `cause`. Sentry walks the
`cause` chain natively, so contention exhaustion shows up as a
distinguishable wrapper exception linked to the underlying `P2002`
rather than a generic unique-constraint violation that looks identical
to every other duplicate-key bug.

The behavioural change is limited to "catch P2002 and retry instead of
crashing." The image ref computation stays inside the builder callback
(same call site as before the refactor), so ECR / non-ECR behaviour, S2
stream creation order, and all downstream side effects are unchanged.

## Non-goals

- No new database migrations, no schema changes, no isolation-level /
locking changes. A serialisable transaction or advisory lock would also
fix this; retry-on-conflict is the smaller change that keeps the
existing version-allocation logic intact.
- Does not touch the analogous `calculateNextBuildVersion` call in
`createBackgroundWorker.server.ts`, which likely has the same race shape
against `BackgroundWorker`'s unique constraint — flagged as a follow-up.

## Test plan

- [x] `pnpm run typecheck --filter webapp` passes (no new errors in the
modified files).
- [x] Three real-Postgres tests in
`apps/webapp/test/createDeploymentWithNextVersion.test.ts` via
`containerTest`:
- 5 concurrent calls all produce distinct, persistable versions
(`Set(versions).size === concurrency`). The naive read-then-create
version of the helper fails this test with the exact same `P2002` seen
in production; the retry version passes.
- Non-`P2002` errors raised from the `buildData` callback propagate
immediately without retry, builder invoked exactly once.
- With `maxRetries: 0`, concurrent racers surface the wrapped
`DeploymentVersionCollisionError` (not a raw `P2002`); `environmentId`,
`attempts`, `lastAttemptedVersion` are populated and `error.cause.code
=== "P2002"`.
- [x] Existing `apps/webapp/test/getDeploymentImageRef.test.ts` still
green (the file was untouched in the final diff).

## Follow-ups (not in this PR)

- `createBackgroundWorker.server.ts` likely has the same TOCTOU shape
against its background-worker version unique constraint — should use the
same helper.
- Sentry visibility check: confirm `error.cause` chain renders as a
linked exception in the Sentry UI when the wrapped error fires (requires
a sandboxed triggering of the exhaustion path).

Co-authored-by: Claude Opus 4.7 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/.server-changes/fix-worker-deployment-version-race.md b/.server-changes/fix-worker-deployment-version-race.md
@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+Retry on unique-constraint collisions when assigning the next worker deployment version so concurrent deploys to the same environment no longer fail with P2002.
diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts
@@ -9,13 +9,13 @@ import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { generateFriendlyId } from "../friendlyIdentifiers";
 import { createRemoteImageBuild, remoteBuildsEnabled } from "../remoteImageBuilder.server";
-import { calculateNextBuildVersion } from "../utils/calculateNextBuildVersion";
 import { BaseService, ServiceValidationError } from "./baseService.server";
 import { TimeoutDeploymentService } from "./timeoutDeployment.server";
 import { getDeploymentImageRef } from "../getDeploymentImageRef.server";
 import { tryCatch } from "@trigger.dev/core";
 import { getRegistryConfig } from "../registryConfig.server";
 import { DeploymentService } from "./deployment.server";
+import { createDeploymentWithNextVersion } from "./initializeDeployment/createDeploymentWithNextVersion.server";
 import { errAsync } from "neverthrow";
 
 const nanoid = customAlphabet("1234567890abcdefghijklmnopqrstuvwxyz", 8);
@@ -97,18 +97,6 @@ export class InitializeDeploymentService extends BaseService {
         });
       }
 
-      const latestDeployment = await this._prisma.workerDeployment.findFirst({
-        where: {
-          environmentId: environment.id,
-        },
-        orderBy: {
-          createdAt: "desc",
-        },
-        take: 1,
-      });
-
-      const nextVersion = calculateNextBuildVersion(latestDeployment?.version);
-
       if (payload.selfHosted && remoteBuildsEnabled()) {
         throw new ServiceValidationError(
           "Self-hosted deployments are not supported on this instance"
@@ -146,30 +134,6 @@ export class InitializeDeploymentService extends BaseService {
 
       const deploymentShortCode = nanoid(8);
 
-      const [imageRefError, imageRefResult] = await tryCatch(
-        getDeploymentImageRef({
-          registry: registryConfig,
-          projectRef: environment.project.externalRef,
-          nextVersion,
-          environmentType: environment.type,
-          deploymentShortCode,
-        })
-      );
-
-      if (imageRefError) {
-        logger.error("Failed to get deployment image ref", {
-          environmentId: environment.id,
-          projectId: environment.projectId,
-          version: nextVersion,
-          triggeredById: triggeredBy?.id,
-          type: payload.type,
-          cause: imageRefError.message,
-        });
-        throw new ServiceValidationError("Failed to get deployment image ref");
-      }
-
-      const { imageRef, isEcr, repoCreated } = imageRefResult;
-
       // We keep using `BUILDING` as the initial status if not explicitly set
       // to avoid changing the behavior for deployments not created in the build server.
       // Native builds always start in the `PENDING` status.
@@ -208,20 +172,6 @@ export class InitializeDeploymentService extends BaseService {
           }
         : undefined;
 
-      logger.debug("Creating deployment", {
-        environmentId: environment.id,
-        projectId: environment.projectId,
-        version: nextVersion,
-        triggeredById: triggeredBy?.id,
-        type: payload.type,
-        imageRef,
-        isEcr,
-        repoCreated,
-        initialStatus,
-        artifactKey: payload.isNativeBuild ? payload.artifactKey : undefined,
-        isNativeBuild: payload.isNativeBuild,
-      });
-
       const buildServerMetadata: BuildServerMetadata | undefined =
         payload.isNativeBuild || payload.buildId
           ? {
@@ -238,28 +188,77 @@ export class InitializeDeploymentService extends BaseService {
             }
           : undefined;
 
-      const deployment = await this._prisma.workerDeployment.create({
-        data: {
-          friendlyId: generateFriendlyId("deployment"),
-          contentHash: payload.contentHash,
-          shortCode: deploymentShortCode,
-          version: nextVersion,
-          status: initialStatus,
-          environmentId: environment.id,
-          projectId: environment.projectId,
-          externalBuildData,
-          buildServerMetadata,
-          triggeredById: triggeredBy?.id,
-          type: payload.type,
-          imageReference: imageRef,
-          imagePlatform: env.DEPLOY_IMAGE_PLATFORM,
-          git: payload.gitMeta ?? undefined,
-          commitSHA: payload.gitMeta?.commitSha ?? undefined,
-          runtime: payload.runtime ?? undefined,
-          triggeredVia: payload.triggeredVia ?? undefined,
-          startedAt: initialStatus === "BUILDING" ? new Date() : undefined,
-        },
-      });
+      // Concurrent deploys to the same environment race on the
+      // `(environmentId, version)` unique constraint. The helper retries on
+      // P2002, recomputing the version (and re-running the image ref call so
+      // the persisted imageReference always matches the persisted version)
+      // each attempt.
+      const deployment = await createDeploymentWithNextVersion(
+        this._prisma,
+        environment.id,
+        async (nextVersion) => {
+          const [imageRefError, imageRefResult] = await tryCatch(
+            getDeploymentImageRef({
+              registry: registryConfig,
+              projectRef: environment.project.externalRef,
+              nextVersion,
+              environmentType: environment.type,
+              deploymentShortCode,
+            })
+          );
+
+          if (imageRefError) {
+            logger.error("Failed to get deployment image ref", {
+              environmentId: environment.id,
+              projectId: environment.projectId,
+              version: nextVersion,
+              triggeredById: triggeredBy?.id,
+              type: payload.type,
+              cause: imageRefError.message,
+            });
+            throw new ServiceValidationError("Failed to get deployment image ref");
+          }
+
+          const { imageRef, isEcr, repoCreated } = imageRefResult;
+
+          logger.debug("Creating deployment", {
+            environmentId: environment.id,
+            projectId: environment.projectId,
+            version: nextVersion,
+            triggeredById: triggeredBy?.id,
+            type: payload.type,
+            imageRef,
+            isEcr,
+            repoCreated,
+            initialStatus,
+            artifactKey: payload.isNativeBuild ? payload.artifactKey : undefined,
+            isNativeBuild: payload.isNativeBuild,
+          });
+
+          return {
+            // Regenerated per attempt: each attempt is a fresh `create` that
+            // must satisfy `WorkerDeployment.friendlyId @unique`, so reusing a
+            // friendlyId across retries would risk a spurious P2002 on
+            // friendlyId instead of the version collision we're retrying.
+            friendlyId: generateFriendlyId("deployment"),
+            contentHash: payload.contentHash,
+            shortCode: deploymentShortCode,
+            status: initialStatus,
+            projectId: environment.projectId,
+            externalBuildData,
+            buildServerMetadata,
+            triggeredById: triggeredBy?.id,
+            type: payload.type,
+            imageReference: imageRef,
+            imagePlatform: env.DEPLOY_IMAGE_PLATFORM,
+            git: payload.gitMeta ?? undefined,
+            commitSHA: payload.gitMeta?.commitSha ?? undefined,
+            runtime: payload.runtime ?? undefined,
+            triggeredVia: payload.triggeredVia ?? undefined,
+            startedAt: initialStatus === "BUILDING" ? new Date() : undefined,
+          };
+        }
+      );
 
       const timeoutMs =
         deployment.status === "PENDING" ? env.DEPLOY_QUEUE_TIMEOUT_MS : env.DEPLOY_TIMEOUT_MS;
@@ -309,7 +308,7 @@ export class InitializeDeploymentService extends BaseService {
 
       return {
         deployment,
-        imageRef,
+        imageRef: deployment.imageReference ?? "",
         eventStream,
       };
     });
diff --git a/apps/webapp/app/v3/services/initializeDeployment/createDeploymentWithNextVersion.server.ts b/apps/webapp/app/v3/services/initializeDeployment/createDeploymentWithNextVersion.server.ts
@@ -0,0 +1,99 @@
+import {
+  isUniqueConstraintError,
+  type Prisma,
+  type PrismaClientOrTransaction,
+  type WorkerDeployment,
+} from "@trigger.dev/database";
+import { setTimeout as sleep } from "node:timers/promises";
+import { logger } from "~/services/logger.server";
+import { calculateNextBuildVersion } from "../../utils/calculateNextBuildVersion";
+
+export type CreateDeploymentData = Omit<
+  Prisma.WorkerDeploymentUncheckedCreateInput,
+  "version" | "environmentId"
+>;
+
+export type CreateDeploymentWithNextVersionOptions = {
+  maxRetries?: number;
+  jitterMs?: { min: number; max: number };
+};
+
+const DEFAULT_MAX_RETRIES = 5;
+const DEFAULT_JITTER_MS = { min: 5, max: 50 };
+
+export class DeploymentVersionCollisionError extends Error {
+  readonly name = "DeploymentVersionCollisionError";
+  readonly environmentId: string;
+  readonly attempts: number;
+  readonly lastAttemptedVersion: string;
+
+  constructor(args: {
+    environmentId: string;
+    attempts: number;
+    lastAttemptedVersion: string;
+    cause: unknown;
+  }) {
+    super(
+      `Failed to allocate a unique worker deployment version for environment ${args.environmentId} after ${args.attempts} attempt(s); last tried "${args.lastAttemptedVersion}"`,
+      { cause: args.cause }
+    );
+    this.environmentId = args.environmentId;
+    this.attempts = args.attempts;
+    this.lastAttemptedVersion = args.lastAttemptedVersion;
+  }
+}
+
+export async function createDeploymentWithNextVersion(
+  prisma: PrismaClientOrTransaction,
+  environmentId: string,
+  buildData: (nextVersion: string) => CreateDeploymentData | Promise<CreateDeploymentData>,
+  options: CreateDeploymentWithNextVersionOptions = {}
+): Promise<WorkerDeployment> {
+  const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
+  const jitterMs = options.jitterMs ?? DEFAULT_JITTER_MS;
+
+  let lastError: unknown;
+  let lastVersion = "";
+
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    const latest = await prisma.workerDeployment.findFirst({
+      where: { environmentId },
+      orderBy: { createdAt: "desc" },
+      take: 1,
+    });
+
+    const version = calculateNextBuildVersion(latest?.version);
+    lastVersion = version;
+    const data = await buildData(version);
+
+    try {
+      return await prisma.workerDeployment.create({
+        data: { ...data, environmentId, version },
+      });
+    } catch (error) {
+      if (!isUniqueConstraintError(error, ["environmentId", "version"])) {
+        throw error;
+      }
+
+      lastError = error;
+      logger.warn("Worker deployment version collided, retrying", {
+        environmentId,
+        attempt: attempt + 1,
+        maxRetries,
+        attemptedVersion: version,
+      });
+
+      // Randomised backoff so N concurrent racers don't loop in lockstep into the
+      // same collision again.
+      const delay = jitterMs.min + Math.random() * (jitterMs.max - jitterMs.min);
+      await sleep(delay);
+    }
+  }
+
+  throw new DeploymentVersionCollisionError({
+    environmentId,
+    attempts: maxRetries + 1,
+    lastAttemptedVersion: lastVersion,
+    cause: lastError,
+  });
+}
diff --git a/apps/webapp/test/createDeploymentWithNextVersion.test.ts b/apps/webapp/test/createDeploymentWithNextVersion.test.ts
