ci: Add Lighthouse Test as Nightly CI run

.github/workflows/build.yml

@@ -8,6 +8,7 @@ on:
       - v8
       - release/**
   pull_request:
+    types: [opened, synchronize, reopened]
   merge_group:
     types: [checks_requested]
   workflow_dispatch:

.github/workflows/lighthouse.yml

@@ -0,0 +1,143 @@
+name: 'Nightly: Lighthouse'
+
+# Builds the instrumented Sentry test apps (default-browser, react-19, nextjs-16)
+# in three Sentry feature modes (no-sentry, init-only, tracing-replay), tars each
+# of the 9 cells, and uploads them to the Sentry Lighthouse lab at
+# https://lighthouse.sentry.gg.
+#
+# The lab runs Lighthouse on dedicated hardware (no shared-runner flake), saves
+# results to its own DB + dashboard, and ships metrics to Sentry. See the wire
+# protocol in ~/Projects/sentry-lhci/docs/sentry-javascript-handoff.md.
+#
+# This workflow only builds + uploads — no Lighthouse runs on GitHub Actions.
+# Never blocks merges (not in build.yml's required-jobs gate).
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+# Independent of build.yml so nothing here can cancel or queue against main CI.
+concurrency:
+  group: lighthouse-${{ github.run_id }}
+  cancel-in-progress: false
+
+env:
+  CACHED_DEPENDENCY_PATHS: |
+    ${{ github.workspace }}/node_modules
+    ${{ github.workspace }}/packages/*/node_modules
+    ${{ github.workspace }}/dev-packages/*/node_modules
+    ~/.cache/mongodb-binaries/
+  TARBALL_ARTIFACT_GLOB: packages/*/*.tgz
+  TARBALL_ARTIFACT_DOWNLOAD_PATH: ${{ github.workspace }}/packages
+  DISABLE_V8_COMPILE_CACHE: '1'
+
+jobs:
+  job_build:
+    name: Build SDK + Tarballs
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    outputs:
+      dependency_cache_key: ${{ steps.install_dependencies.outputs.cache_key }}
+    steps:
+      - name: Check out current commit
+        uses: actions/checkout@v6
+
+      - name: Set up Node
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: 'package.json'
+
+      - name: Install Dependencies
+        uses: ./.github/actions/install-dependencies
+        id: install_dependencies
+
+      - name: Build packages
+        run: yarn build:ci
+
+      - name: Compute build artifact paths from Nx
+        id: nx_build_paths
+        run: |
+          {
+            echo 'paths<<EOF'
+            yarn --silent ci:print-build-artifact-paths
+            echo 'EOF'
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Upload build artifacts
+        # Required by .github/actions/restore-cache in the downstream job — mirrors
+        # the `build-output` artifact that build.yml's job_build produces. Without
+        # it, the bundle job fails on restore-cache with "Artifact not found".
+        uses: actions/upload-artifact@v7
+        with:
+          name: build-output
+          path: ${{ steps.nx_build_paths.outputs.paths }}
+          retention-days: 4
+          compression-level: 6
+          overwrite: true
+
+      - name: Build tarballs
+        run: yarn build:tarball
+
+      - name: Upload tarball artifacts
+        uses: actions/upload-artifact@v7
+        with:
+          name: lighthouse-build-tarball-output
+          path: ${{ env.TARBALL_ARTIFACT_GLOB }}
+          if-no-files-found: error
+          retention-days: 4
+          compression-level: 6
+          overwrite: true
+
+  job_bundle_and_upload:
+    name: Bundle test apps and upload to Lighthouse lab
+    needs: [job_build]
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    env:
+      LIGHTHOUSE_LAB_URL: ${{ secrets.LIGHTHOUSE_LAB_URL }}
+      LIGHTHOUSE_UPLOAD_TOKEN: ${{ secrets.LIGHTHOUSE_UPLOAD_TOKEN }}
+      # Synthetic DSNs — the apps only need *something* parseable at build time;
+      # nothing actually phones home (the Lighthouse run is what we care about).
+      E2E_TEST_DSN: 'https://username@domain/123'
+      NEXT_PUBLIC_E2E_TEST_DSN: 'https://username@domain/123'
+      PUBLIC_E2E_TEST_DSN: 'https://username@domain/123'
+      REACT_APP_E2E_TEST_DSN: 'https://username@domain/123'
+    steps:
+      - name: Check out current commit
+        uses: actions/checkout@v6
+
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
+        with:
+          version: 9.15.9
+
+      - name: Set up Node
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: 'package.json'
+
+      - name: Restore caches
+        uses: ./.github/actions/restore-cache
+        with:
+          dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }}
+
+      - name: Restore tarball artifacts
+        uses: actions/download-artifact@v7
+        with:
+          name: lighthouse-build-tarball-output
+          path: ${{ env.TARBALL_ARTIFACT_DOWNLOAD_PATH }}
+
+      - name: Prepare E2E tests (copy packed tgzs into dev-packages/e2e-tests/packed)
+        run: yarn test:prepare
+        working-directory: dev-packages/e2e-tests
+
+      - name: Validate E2E tests setup
+        run: yarn test:validate
+        working-directory: dev-packages/e2e-tests
+
+      - name: Bundle every cell and upload to the lab
+        # bundle-and-upload.mjs handles the full 3 × 3 matrix in one process:
+        # copies each app to a temp dir, applies pnpm overrides, builds with the
+        # right SENTRY_LIGHTHOUSE_MODE env var, tars, POSTs all 9 bundles to the
+        # lab, polls until terminal, writes a Job Summary.
+        run: node dev-packages/lighthouse-bundle/bundle-and-upload.mjs

dev-packages/e2e-tests/test-applications/default-browser/build.mjs

@@ -17,8 +17,15 @@ webpack(
       minimize: true,
       minimizer: [new TerserPlugin()],
     },
+    // The Lighthouse-CI baseline mode (SENTRY_LIGHTHOUSE_MODE=no-sentry) emits the
+    // @sentry/browser chunk as a separately code-split asset (~265 KiB raw). Webpack's
+    // default performance.hints='warning' fires AssetsOverSizeLimitWarning for that
+    // chunk, and build.mjs's `if (stats.hasWarnings()) process.exit(1)` would then
+    // abort the build. This is a test app, not a size-tracked production bundle, so
+    // hints are disabled — size is tracked separately via .size-limit.js at the repo root.
+    performance: { hints: false },
     plugins: [
-      new webpack.EnvironmentPlugin(['E2E_TEST_DSN']),
+      new webpack.EnvironmentPlugin({ E2E_TEST_DSN: '', SENTRY_LIGHTHOUSE_MODE: '' }),
       new HtmlWebpackPlugin({
         template: path.join(__dirname, 'public/index.html'),
       }),

dev-packages/e2e-tests/test-applications/default-browser/src/index.js

@@ -1,18 +1,49 @@
-import * as Sentry from '@sentry/browser';
-
-Sentry.init({
-  dsn: process.env.E2E_TEST_DSN,
-  integrations: [Sentry.browserTracingIntegration()],
-  tracesSampleRate: 1.0,
-  release: 'e2e-test',
-  environment: 'qa',
-  tunnel: 'http://localhost:3031',
-});
+const lighthouseMode = process.env.SENTRY_LIGHTHOUSE_MODE;
 
+// Event listeners are attached synchronously at module top-level so E2E tests that do
+// `page.goto('/')` followed immediately by `button.click()` cannot race the dynamic
+// `import('@sentry/browser')` below. The handlers don't depend on Sentry being
+// initialized — Sentry's global error/transaction handlers attach via window-level
+// listeners installed by `Sentry.init()` and pick up the thrown error regardless.
 document.getElementById('exception-button').addEventListener('click', () => {
   throw new Error('I am an error!');
 });
 
 document.getElementById('navigation-link').addEventListener('click', () => {
   document.getElementById('navigation-target').scrollIntoView({ behavior: 'smooth' });
 });
+
+// Sentry is loaded via dynamic `import()` so the `no-sentry` Lighthouse build can
+// tree-shake the SDK out completely. Wrapped in an async IIFE because top-level await
+// isn't supported by the webpack target used for this app's bundle.
+if (lighthouseMode !== 'no-sentry') {
+  void (async () => {
+    const Sentry = await import('@sentry/browser');
+
+    const integrations = [];
+
+    // Existing E2E behavior (empty string) and 'tracing-replay' mode both include tracing.
+    // 'init-only' mode omits all integrations so we can measure SDK-core overhead.
+    if (lighthouseMode !== 'init-only') {
+      integrations.push(Sentry.browserTracingIntegration());
+    }
+
+    // Replay is gated to 'tracing-replay' so we can measure its overhead independently
+    // from tracing. Existing E2E behavior (unset env var) did not include replay, so we
+    // preserve that.
+    if (lighthouseMode === 'tracing-replay') {
+      integrations.push(Sentry.replayIntegration());
+    }
+
+    Sentry.init({
+      dsn: process.env.E2E_TEST_DSN,
+      integrations,
+      tracesSampleRate: 1.0,
+      replaysSessionSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : 0,
+      replaysOnErrorSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : 0,
+      release: 'e2e-test',
+      environment: 'qa',
+      tunnel: 'http://localhost:3031',
+    });
+  })();
+}

dev-packages/e2e-tests/test-applications/nextjs-16/instrumentation-client.ts

@@ -1,22 +1,42 @@
 import * as Sentry from '@sentry/nextjs';
 import type { Log } from '@sentry/nextjs';
 
-Sentry.init({
-  environment: 'qa', // dynamic sampling bias to keep transactions
-  dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN,
-  tunnel: `http://localhost:3031/`, // proxy server
-  tracesSampleRate: 1.0,
-  sendDefaultPii: true,
-  integrations: [
-    Sentry.thirdPartyErrorFilterIntegration({
-      filterKeys: ['nextjs-16-e2e'],
-      behaviour: 'apply-tag-if-contains-third-party-frames',
-    }),
-  ],
-  // Verify Log type is available
-  beforeSendLog(log: Log) {
-    return log;
-  },
-});
+// SENTRY_LIGHTHOUSE_MODE values:
+//   ''/undefined   - existing E2E behavior (no tracing/replay; third-party-error-filter on)
+//   'no-sentry'    - Sentry.init() is skipped entirely (SDK is still in the bundle because
+//                    Next.js doesn't reliably treeshake @sentry/nextjs; see plan Risk 5)
+//   'init-only'    - Sentry.init() with no integrations (measures SDK-core overhead)
+//   'tracing-replay' - Sentry.init() with browserTracing + replay (measures feature overhead)
+const lighthouseMode = process.env.NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE;
 
-export const onRouterTransitionStart = Sentry.captureRouterTransitionStart;
+if (lighthouseMode !== 'no-sentry') {
+  Sentry.init({
+    environment: 'qa', // dynamic sampling bias to keep transactions
+    dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN,
+    tunnel: `http://localhost:3031/`, // proxy server
+    tracesSampleRate: 1.0,
+    replaysSessionSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : 0,
+    replaysOnErrorSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : 0,
+    sendDefaultPii: true,
+    // Existing E2E behavior (mode unset/'') keeps third-party-error-filter on.
+    // init-only / tracing-replay drop it so we measure SDK overhead without app-specific noise.
+    // tracing-replay additionally enables browserTracing + replay.
+    integrations: [
+      ...(lighthouseMode === undefined || lighthouseMode === ''
+        ? [
+            Sentry.thirdPartyErrorFilterIntegration({
+              filterKeys: ['nextjs-16-e2e'],
+              behaviour: 'apply-tag-if-contains-third-party-frames',
+            }),
+          ]
+        : []),
+      ...(lighthouseMode === 'tracing-replay' ? [Sentry.browserTracingIntegration(), Sentry.replayIntegration()] : []),
+    ],
+    // Verify Log type is available
+    beforeSendLog(log: Log) {
+      return log;
+    },
+  });
+}
+
+export const onRouterTransitionStart = lighthouseMode !== 'no-sentry' ? Sentry.captureRouterTransitionStart : undefined;

dev-packages/e2e-tests/test-applications/react-19/src/index.tsx

@@ -1,26 +1,58 @@
-import * as Sentry from '@sentry/react';
 import React from 'react';
 import ReactDOM from 'react-dom/client';
 import Index from './pages/Index';
 
-Sentry.init({
-  environment: 'qa', // dynamic sampling bias to keep transactions
-  dsn: process.env.REACT_APP_E2E_TEST_DSN,
-  release: 'e2e-test',
-  tunnel: 'http://localhost:3031/', // proxy server
-});
+const lighthouseMode = process.env.REACT_APP_SENTRY_LIGHTHOUSE_MODE;
 
-const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement, {
-  onUncaughtError: Sentry.reactErrorHandler((error, errorInfo) => {
-    console.warn(error, errorInfo);
-  }),
-  onCaughtError: Sentry.reactErrorHandler((error, errorInfo) => {
-    console.warn(error, errorInfo);
-  }),
-});
+if (lighthouseMode === 'no-sentry') {
+  // No Sentry at all — sync render so webpack/CRA can fully dead-code-eliminate the
+  // @sentry/react import below. CRA inlines `process.env.REACT_APP_*` at build time,
+  // so this branch becomes the only one in the bundle when the env var is set.
+  const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement);
+  root.render(
+    <div>
+      <Index />
+    </div>,
+  );
+} else {
+  // Dynamic-import Sentry so it lives in a separate chunk that webpack/CRA drops
+  // entirely from the no-sentry build above. Preserves existing E2E behavior when
+  // the env var is unset (init + error handlers, no tracing/replay).
+  void (async () => {
+    const Sentry = await import('@sentry/react');
 
-root.render(
-  <div>
-    <Index />
-  </div>,
-);
+    const integrations: unknown[] = [];
+    if (lighthouseMode === 'tracing-replay') {
+      integrations.push(Sentry.browserTracingIntegration());
+      integrations.push(Sentry.replayIntegration());
+    }
+
+    Sentry.init({
+      environment: 'qa', // dynamic sampling bias to keep transactions
+      dsn: process.env.REACT_APP_E2E_TEST_DSN,
+      release: 'e2e-test',
+      tunnel: 'http://localhost:3031/', // proxy server
+      integrations: integrations as Parameters<typeof Sentry.init>[0]['integrations'],
+      tracesSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : undefined,
+      replaysSessionSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : 0,
+      replaysOnErrorSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : 0,
+    });
+
+    const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement, {
+      onUncaughtError: Sentry.reactErrorHandler((error, errorInfo) => {
+        // oxlint-disable-next-line no-console
+        console.warn(error, errorInfo);
+      }),
+      onCaughtError: Sentry.reactErrorHandler((error, errorInfo) => {
+        // oxlint-disable-next-line no-console
+        console.warn(error, errorInfo);
+      }),
+    });
+
+    root.render(
+      <div>
+        <Index />
+      </div>,
+    );
+  })();
+}