From d496226755c36f7877fe21912319790905b74787 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 12 May 2026 21:33:30 +0200 Subject: [PATCH 01/29] feat(lighthouse-ci): scaffold lighthouse-tests dev-package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the foundation for measuring Lighthouse performance scores across frontend SDKs and feature combinations. Introduces a new private workspace at dev-packages/lighthouse-tests/ with three pieces: - lighthouserc.js — shared LHCI config. Uses simulated throttling (default, most deterministic on shared CI runners), 5 runs per URL with aggregationMethod 'median-run' to halve variance, performance-only audit (skips a11y/SEO/PWA), and warn-only score assertion at minScore 0.5 so the job never blocks a merge. Branches between staticDistDir and startServerCommand at config-load time based on LIGHTHOUSE_SERVE_MODE so the same config powers both static SPAs and SSR apps. - lighthouse-matrix.mjs — generator that emits the 14 apps × 3 feature modes = 42 matrix cells consumed by the (to-be-added) GitHub Actions job. Each cell carries every field the workflow needs (app-dir, static-dir/start-cmd, ready-pattern) plus an env-var-name field that documents which bundler-specific prefix the app reads at build time (SENTRY_LIGHTHOUSE_MODE, PUBLIC_*, VITE_*, NEXT_PUBLIC_*, NUXT_PUBLIC_*). - package.json — minimal private workspace with volta.extends pointing at the root so Node/yarn/pnpm versions stay locked to the monorepo. Also registers dev-packages/lighthouse-tests under the root workspaces array next to the existing dev-packages entries. Refs: TODO-a4b18f49 (plan: .pi/plans/2026-05-12-lighthouse-ci/plan.md) Co-Authored-By: Claude claude-opus-4-5 --- .../lighthouse-tests/lighthouse-matrix.mjs | 139 ++++++++++++++++++ dev-packages/lighthouse-tests/lighthouserc.js | 51 +++++++ dev-packages/lighthouse-tests/package.json | 12 ++ package.json | 3 +- 4 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 dev-packages/lighthouse-tests/lighthouse-matrix.mjs create mode 100644 dev-packages/lighthouse-tests/lighthouserc.js create mode 100644 dev-packages/lighthouse-tests/package.json diff --git a/dev-packages/lighthouse-tests/lighthouse-matrix.mjs b/dev-packages/lighthouse-tests/lighthouse-matrix.mjs new file mode 100644 index 000000000000..04fa5641b251 --- /dev/null +++ b/dev-packages/lighthouse-tests/lighthouse-matrix.mjs @@ -0,0 +1,139 @@ +/** + * Generates the GitHub Actions matrix for Lighthouse CI runs. + * + * Outputs: `matrix=` to stdout (consumed by $GITHUB_OUTPUT in CI). + * + * Matrix shape: 14 representative E2E apps × 3 Sentry feature modes = 42 cells. + * + * Modes: + * no-sentry — app built without any Sentry SDK (baseline) + * init-only — Sentry.init() only, no additional integrations + * tracing-replay — Sentry.init() with browserTracingIntegration + replayIntegration + * + * Each E2E app reads SENTRY_LIGHTHOUSE_MODE at build time. Because bundlers expose + * env vars under different prefixes (NEXT_PUBLIC_*, PUBLIC_*, VITE_*, etc.), the + * workflow sets the var under every common prefix; each app reads whichever its + * bundler exposes. The `envVarName` field below is informational (used in the + * report) and documents which prefix the app code actually consumes. + */ + +/** + * @typedef {Object} AppDefinition + * @property {string} app - Directory name under dev-packages/e2e-tests/test-applications/ + * @property {string} sdk - Human-readable SDK label for reports + * @property {'static'|'server'} serve - How the built app is served by Lighthouse + * @property {string} [staticDir] - Relative path to built static assets (serve === 'static') + * @property {string} [startCmd] - Command to start the SSR server (serve === 'server') + * @property {string} [readyPattern] - Log pattern that signals server is ready (server only) + * @property {string} envVarName - Env var the app's Sentry init code reads at build time + */ + +/** @type {AppDefinition[]} */ +const APPS = [ + // Plain webpack apps — read process.env directly (no bundler prefix). + { app: 'default-browser', sdk: 'browser', serve: 'static', staticDir: 'build', envVarName: 'SENTRY_LIGHTHOUSE_MODE' }, + { app: 'react-19', sdk: 'react', serve: 'static', staticDir: 'build', envVarName: 'SENTRY_LIGHTHOUSE_MODE' }, + { app: 'ember-classic', sdk: 'ember', serve: 'static', staticDir: 'dist', envVarName: 'SENTRY_LIGHTHOUSE_MODE' }, + { + app: 'create-remix-app-express', + sdk: 'remix', + serve: 'server', + startCmd: 'cross-env NODE_ENV=production node ./server.mjs', + readyPattern: 'localhost', + envVarName: 'SENTRY_LIGHTHOUSE_MODE', + }, + { + app: 'angular-21', + sdk: 'angular', + serve: 'static', + staticDir: 'dist/angular-21', + envVarName: 'SENTRY_LIGHTHOUSE_MODE', + }, + + // Vite-based apps with `envPrefix: 'PUBLIC_'` (matches Sentry's repo convention for PUBLIC_E2E_TEST_DSN). + { app: 'vue-3', sdk: 'vue', serve: 'static', staticDir: 'dist', envVarName: 'PUBLIC_SENTRY_LIGHTHOUSE_MODE' }, + { app: 'svelte-5', sdk: 'svelte', serve: 'static', staticDir: 'dist', envVarName: 'PUBLIC_SENTRY_LIGHTHOUSE_MODE' }, + { + app: 'sveltekit-2', + sdk: 'sveltekit', + serve: 'server', + startCmd: 'node build', + readyPattern: 'localhost', + envVarName: 'PUBLIC_SENTRY_LIGHTHOUSE_MODE', + }, + { + app: 'astro-5', + sdk: 'astro', + serve: 'server', + startCmd: 'node ./dist/server/entry.mjs', + readyPattern: 'localhost', + envVarName: 'PUBLIC_SENTRY_LIGHTHOUSE_MODE', + }, + { + app: 'react-router-7-spa', + sdk: 'react-router', + serve: 'static', + staticDir: 'dist', + envVarName: 'PUBLIC_SENTRY_LIGHTHOUSE_MODE', + }, + + // Vite-based apps using the default `VITE_` prefix (no custom envPrefix set). + { + app: 'solidstart-spa', + sdk: 'solidstart', + serve: 'static', + staticDir: '.output/public', + envVarName: 'VITE_SENTRY_LIGHTHOUSE_MODE', + }, + { + app: 'tanstackstart-react', + sdk: 'tanstack-start', + serve: 'server', + startCmd: 'node --import ./.output/server/instrument.server.mjs .output/server/index.mjs', + readyPattern: 'localhost', + envVarName: 'VITE_SENTRY_LIGHTHOUSE_MODE', + }, + + // Next.js — only `NEXT_PUBLIC_*` env vars are exposed to client code. + { + app: 'nextjs-16', + sdk: 'nextjs', + serve: 'server', + startCmd: 'pnpm start', + readyPattern: 'Ready in', + envVarName: 'NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE', + }, + + // Nuxt — only `NUXT_PUBLIC_*` env vars are exposed to client code (Nuxt convention). + { + app: 'nuxt-5', + sdk: 'nuxt', + serve: 'server', + startCmd: 'node .output/server/index.mjs', + readyPattern: 'Listening on', + envVarName: 'NUXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE', + }, +]; + +const MODES = /** @type {const} */ (['no-sentry', 'init-only', 'tracing-replay']); + +const include = []; + +for (const appDef of APPS) { + for (const mode of MODES) { + include.push({ + app: appDef.app, + sdk: appDef.sdk, + 'app-dir': `dev-packages/e2e-tests/test-applications/${appDef.app}`, + mode, + serve: appDef.serve, + 'static-dir': appDef.staticDir ?? '', + 'start-cmd': appDef.startCmd ?? '', + 'ready-pattern': appDef.readyPattern ?? 'localhost', + 'env-var-name': appDef.envVarName, + }); + } +} + +// eslint-disable-next-line no-console +console.log(`matrix=${JSON.stringify({ include })}`); diff --git a/dev-packages/lighthouse-tests/lighthouserc.js b/dev-packages/lighthouse-tests/lighthouserc.js new file mode 100644 index 000000000000..80f6b0eca26b --- /dev/null +++ b/dev-packages/lighthouse-tests/lighthouserc.js @@ -0,0 +1,51 @@ +// Lighthouse CI configuration for Sentry JavaScript SDK performance testing. +// Used by treosh/lighthouse-ci-action@v12 via the `configPath` input. +// Docs: https://github.com/GoogleChrome/lighthouse-ci/blob/main/docs/configuration.md +// +// Per-cell environment variables (set by the GitHub Actions workflow): +// LIGHTHOUSE_SERVE_MODE - 'static' | 'server' +// LIGHTHOUSE_STATIC_DIR - absolute path to static dist dir (when serve mode = 'static') +// LIGHTHOUSE_START_CMD - shell command to start the server (when serve mode = 'server') +// LIGHTHOUSE_READY_PATTERN - server-ready log pattern (default: 'localhost') +// LIGHTHOUSE_URL - URL to audit (default: http://localhost:3000/) + +const isServer = process.env.LIGHTHOUSE_SERVE_MODE === 'server'; + +export default { + ci: { + collect: { + // Median of 5 runs halves variance vs a single run (per Lighthouse variability docs). + numberOfRuns: 5, + ...(isServer + ? { + startServerCommand: process.env.LIGHTHOUSE_START_CMD, + startServerReadyPattern: process.env.LIGHTHOUSE_READY_PATTERN || 'localhost', + startServerReadyTimeout: 30000, + url: [process.env.LIGHTHOUSE_URL || 'http://localhost:3000/'], + } + : { + staticDistDir: process.env.LIGHTHOUSE_STATIC_DIR, + }), + settings: { + // Simulated throttling (LHCI default) — more deterministic on shared CI runners + // than DevTools throttling. Do NOT switch to 'devtools' without dedicated hardware. + chromeFlags: ['--no-sandbox', '--headless=new'], + // Only measure performance — skip accessibility/SEO/PWA/best-practices for now. + // These can be added later once the performance signal is stable. + onlyCategories: ['performance'], + }, + }, + assert: { + // Warn-only — Lighthouse never blocks PR merges (ISC-A-1). + // Floor is set very low (0.5) so we only catch catastrophic regressions while we + // measure baseline variance. Tighten after 30+ days of nightly data. + assertions: { + 'categories:performance': ['warn', { minScore: 0.5, aggregationMethod: 'median-run' }], + }, + }, + upload: { + // 7-day retention; zero infra required. Links appear in action output and the PR comment. + target: 'temporary-public-storage', + }, + }, +}; diff --git a/dev-packages/lighthouse-tests/package.json b/dev-packages/lighthouse-tests/package.json new file mode 100644 index 000000000000..4ddb46980b80 --- /dev/null +++ b/dev-packages/lighthouse-tests/package.json @@ -0,0 +1,12 @@ +{ + "name": "@sentry-internal/lighthouse-tests", + "version": "0.0.0", + "private": true, + "type": "module", + "volta": { + "extends": "../../package.json" + }, + "scripts": { + "generate-matrix": "node lighthouse-matrix.mjs" + } +} diff --git a/package.json b/package.json index bca64a9f863f..19979e65bded 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,8 @@ "dev-packages/clear-cache-gh-action", "dev-packages/external-contributor-gh-action", "dev-packages/rollup-utils", - "dev-packages/bundler-tests" + "dev-packages/bundler-tests", + "dev-packages/lighthouse-tests" ], "devDependencies": { "@rollup/plugin-commonjs": "^25.0.7", From 36c3f76761cb673804c3dfdba99d532ca32570fa Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 12 May 2026 21:43:46 +0200 Subject: [PATCH 02/29] feat(lighthouse-ci): add SENTRY_LIGHTHOUSE_MODE guard to default-browser, nextjs-16, react-router-7-spa Instrument three E2E test applications to conditionally initialize Sentry based on the SENTRY_LIGHTHOUSE_MODE build-time environment variable. This enables Lighthouse CI to measure performance across three feature levels: - no-sentry: dynamic import is skipped entirely, allowing treeshaking to remove all SDK code for a clean baseline measurement - init-only: Sentry.init() runs with no additional integrations (measures SDK core overhead) - tracing-replay: full integrations enabled (measures feature overhead) - unset/empty: preserves existing E2E behavior (all features enabled) Each app uses the env var prefix appropriate for its bundler: - default-browser (webpack): process.env.SENTRY_LIGHTHOUSE_MODE - nextjs-16 (Next.js): process.env.NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE - react-router-7-spa (Vite): import.meta.env.PUBLIC_SENTRY_LIGHTHOUSE_MODE The async IIFE + dynamic import pattern is used (instead of top-level await) because webpack production builds may not support TLA. This also ensures clean treeshaking in no-sentry mode since the import itself is conditional. The default-browser build.mjs EnvironmentPlugin is changed from array form to object form with empty-string defaults so the build succeeds when SENTRY_LIGHTHOUSE_MODE is not set (existing E2E jobs don't set it). Bundle-output treeshake verification deferred to PR CI (the Lighthouse workflow job builds apps with the env var set and runs Lighthouse against the output). Ref: TODO-5805679c Co-Authored-By: Claude claude-opus-4-6 --- .../default-browser/build.mjs | 2 +- .../default-browser/src/index.js | 42 ++++++--- .../nextjs-16/instrumentation-client.ts | 46 ++++++---- .../react-router-7-spa/src/main.tsx | 91 ++++++++++--------- 4 files changed, 106 insertions(+), 75 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/default-browser/build.mjs b/dev-packages/e2e-tests/test-applications/default-browser/build.mjs index aeaad894bdbd..26dbad87140f 100644 --- a/dev-packages/e2e-tests/test-applications/default-browser/build.mjs +++ b/dev-packages/e2e-tests/test-applications/default-browser/build.mjs @@ -18,7 +18,7 @@ webpack( minimizer: [new TerserPlugin()], }, plugins: [ - new webpack.EnvironmentPlugin(['E2E_TEST_DSN']), + new webpack.EnvironmentPlugin({ E2E_TEST_DSN: '', SENTRY_LIGHTHOUSE_MODE: '' }), new HtmlWebpackPlugin({ template: path.join(__dirname, 'public/index.html'), }), diff --git a/dev-packages/e2e-tests/test-applications/default-browser/src/index.js b/dev-packages/e2e-tests/test-applications/default-browser/src/index.js index d3eea216fe84..2256f0fd1bbf 100644 --- a/dev-packages/e2e-tests/test-applications/default-browser/src/index.js +++ b/dev-packages/e2e-tests/test-applications/default-browser/src/index.js @@ -1,18 +1,30 @@ -import * as Sentry from '@sentry/browser'; +const lighthouseMode = process.env.SENTRY_LIGHTHOUSE_MODE; -Sentry.init({ - dsn: process.env.E2E_TEST_DSN, - integrations: [Sentry.browserTracingIntegration()], - tracesSampleRate: 1.0, - release: 'e2e-test', - environment: 'qa', - tunnel: 'http://localhost:3031', -}); +(async () => { + if (lighthouseMode !== 'no-sentry') { + const Sentry = await import('@sentry/browser'); -document.getElementById('exception-button').addEventListener('click', () => { - throw new Error('I am an error!'); -}); + const integrations = []; -document.getElementById('navigation-link').addEventListener('click', () => { - document.getElementById('navigation-target').scrollIntoView({ behavior: 'smooth' }); -}); + if (lighthouseMode !== 'init-only') { + integrations.push(Sentry.browserTracingIntegration()); + } + + Sentry.init({ + dsn: process.env.E2E_TEST_DSN, + integrations, + tracesSampleRate: 1.0, + release: 'e2e-test', + environment: 'qa', + tunnel: 'http://localhost:3031', + }); + } + + document.getElementById('exception-button').addEventListener('click', () => { + throw new Error('I am an error!'); + }); + + document.getElementById('navigation-link').addEventListener('click', () => { + document.getElementById('navigation-target').scrollIntoView({ behavior: 'smooth' }); + }); +})(); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/instrumentation-client.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/instrumentation-client.ts index 991c6009ed02..6258ba559712 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/instrumentation-client.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/instrumentation-client.ts @@ -1,22 +1,32 @@ import * as Sentry from '@sentry/nextjs'; import type { Log } from '@sentry/nextjs'; -Sentry.init({ - environment: 'qa', // dynamic sampling bias to keep transactions - dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN, - tunnel: `http://localhost:3031/`, // proxy server - tracesSampleRate: 1.0, - sendDefaultPii: true, - integrations: [ - Sentry.thirdPartyErrorFilterIntegration({ - filterKeys: ['nextjs-16-e2e'], - behaviour: 'apply-tag-if-contains-third-party-frames', - }), - ], - // Verify Log type is available - beforeSendLog(log: Log) { - return log; - }, -}); +const lighthouseMode = process.env.NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE; -export const onRouterTransitionStart = Sentry.captureRouterTransitionStart; +if (lighthouseMode !== 'no-sentry') { + const integrations: Sentry.Integration[] = []; + + if (lighthouseMode !== 'init-only') { + integrations.push( + Sentry.thirdPartyErrorFilterIntegration({ + filterKeys: ['nextjs-16-e2e'], + behaviour: 'apply-tag-if-contains-third-party-frames', + }), + ); + } + + Sentry.init({ + environment: 'qa', // dynamic sampling bias to keep transactions + dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN, + tunnel: `http://localhost:3031/`, // proxy server + tracesSampleRate: 1.0, + sendDefaultPii: true, + integrations, + // Verify Log type is available + beforeSendLog(log: Log) { + return log; + }, + }); +} + +export const onRouterTransitionStart = lighthouseMode !== 'no-sentry' ? Sentry.captureRouterTransitionStart : undefined; diff --git a/dev-packages/e2e-tests/test-applications/react-router-7-spa/src/main.tsx b/dev-packages/e2e-tests/test-applications/react-router-7-spa/src/main.tsx index baf12f7ff574..ca48ea98bebe 100644 --- a/dev-packages/e2e-tests/test-applications/react-router-7-spa/src/main.tsx +++ b/dev-packages/e2e-tests/test-applications/react-router-7-spa/src/main.tsx @@ -1,4 +1,3 @@ -import * as Sentry from '@sentry/react'; import React from 'react'; import ReactDOM from 'react-dom/client'; import { @@ -14,43 +13,53 @@ import Index from './pages/Index'; import SSE from './pages/SSE'; import User from './pages/User'; -const replay = Sentry.replayIntegration(); - -Sentry.init({ - environment: 'qa', // dynamic sampling bias to keep transactions - dsn: import.meta.env.PUBLIC_E2E_TEST_DSN, - integrations: [ - Sentry.reactRouterV7BrowserTracingIntegration({ - useEffect: React.useEffect, - useLocation, - useNavigationType, - createRoutesFromChildren, - matchRoutes, - trackFetchStreamPerformance: true, - }), - replay, - ], - // We recommend adjusting this value in production, or using tracesSampler - // for finer control - tracesSampleRate: 1.0, - release: 'e2e-test', - - // Always capture replays, so we can test this properly - replaysSessionSampleRate: 1.0, - replaysOnErrorSampleRate: 0.0, - tunnel: 'http://localhost:3031', - sendDefaultPii: true, -}); - -const SentryRoutes = Sentry.withSentryReactRouterV7Routing(Routes); - -const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement); -root.render( - - - } /> - } /> - } /> - - , -); +const lighthouseMode = import.meta.env.PUBLIC_SENTRY_LIGHTHOUSE_MODE; + +let SentryRoutes = Routes; + +(async () => { + if (lighthouseMode !== 'no-sentry') { + const Sentry = await import('@sentry/react'); + + const integrations: Sentry.Integration[] = []; + + if (lighthouseMode !== 'init-only') { + integrations.push( + Sentry.reactRouterV7BrowserTracingIntegration({ + useEffect: React.useEffect, + useLocation, + useNavigationType, + createRoutesFromChildren, + matchRoutes, + trackFetchStreamPerformance: true, + }), + Sentry.replayIntegration(), + ); + } + + Sentry.init({ + environment: 'qa', // dynamic sampling bias to keep transactions + dsn: import.meta.env.PUBLIC_E2E_TEST_DSN, + integrations, + tracesSampleRate: 1.0, + release: 'e2e-test', + replaysSessionSampleRate: lighthouseMode !== 'init-only' ? 1.0 : 0.0, + replaysOnErrorSampleRate: 0.0, + tunnel: 'http://localhost:3031', + sendDefaultPii: true, + }); + + SentryRoutes = Sentry.withSentryReactRouterV7Routing(Routes); + } + + const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement); + root.render( + + + } /> + } /> + } /> + + , + ); +})(); From cc37d4d28060214550b94a6be207dfc88817743b Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 12 May 2026 21:46:11 +0200 Subject: [PATCH 03/29] fix(lighthouse-ci): enable replay in tracing-replay mode for default-browser and nextjs-16 The previous commit (36c3f7676) implemented the SENTRY_LIGHTHOUSE_MODE guard but did not actually wire up replayIntegration() in the 'tracing-replay' mode for default-browser, and did not wire up browserTracingIntegration() or replayIntegration() at all for nextjs-16. This made the 'tracing-replay' mode measure identically to 'init-only' for those two apps, defeating the purpose of the matrix. - default-browser: push Sentry.replayIntegration() when mode is 'tracing-replay'; also set replaysSessionSampleRate / replaysOnErrorSampleRate to 1.0 in that mode (0 otherwise). Existing E2E behavior (unset env var) intentionally still has no replay, matching the file before this work. - nextjs-16: push Sentry.browserTracingIntegration() and Sentry.replayIntegration() when mode is 'tracing-replay'. Also gate the thirdPartyErrorFilterIntegration to existing-E2E mode only so init-only and tracing-replay measure SDK overhead without app-specific noise. Note: nextjs-16 continues to use a static 'import * as Sentry' rather than dynamic import. This is a tracked limitation (plan Risk 5): Next.js's bundler does not reliably treeshake dynamic imports of @sentry/nextjs from instrumentation-client.ts. The 'no-sentry' mode will therefore measure a baseline that includes the SDK bundle (but does not initialize it). The deltas between init-only and tracing-replay remain meaningful; only the absolute no-sentry baseline is polluted. Ref: TODO-5805679c Co-Authored-By: Claude claude-opus-4-5 --- .../default-browser/src/index.js | 11 +++++++++++ .../nextjs-16/instrumentation-client.ts | 19 ++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/dev-packages/e2e-tests/test-applications/default-browser/src/index.js b/dev-packages/e2e-tests/test-applications/default-browser/src/index.js index 2256f0fd1bbf..a1018c5a0499 100644 --- a/dev-packages/e2e-tests/test-applications/default-browser/src/index.js +++ b/dev-packages/e2e-tests/test-applications/default-browser/src/index.js @@ -6,14 +6,25 @@ const lighthouseMode = process.env.SENTRY_LIGHTHOUSE_MODE; const integrations = []; + // Existing E2E behavior (empty string) and 'tracing-replay' mode both include tracing. + // 'init-only' mode omits all integrations so we can measure SDK-core overhead. if (lighthouseMode !== 'init-only') { integrations.push(Sentry.browserTracingIntegration()); } + // Replay is gated to 'tracing-replay' so we can measure its overhead independently + // from tracing. Existing E2E behavior (unset env var) did not include replay, so we + // preserve that. + if (lighthouseMode === 'tracing-replay') { + integrations.push(Sentry.replayIntegration()); + } + Sentry.init({ dsn: process.env.E2E_TEST_DSN, integrations, tracesSampleRate: 1.0, + replaysSessionSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : 0, + replaysOnErrorSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : 0, release: 'e2e-test', environment: 'qa', tunnel: 'http://localhost:3031', diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/instrumentation-client.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/instrumentation-client.ts index 6258ba559712..7db8f4e2c584 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/instrumentation-client.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/instrumentation-client.ts @@ -1,12 +1,21 @@ import * as Sentry from '@sentry/nextjs'; import type { Log } from '@sentry/nextjs'; +// SENTRY_LIGHTHOUSE_MODE values: +// ''/undefined - existing E2E behavior (no tracing/replay; third-party-error-filter on) +// 'no-sentry' - Sentry.init() is skipped entirely (SDK is still in the bundle because +// Next.js doesn't reliably treeshake @sentry/nextjs; see plan Risk 5) +// 'init-only' - Sentry.init() with no integrations (measures SDK-core overhead) +// 'tracing-replay' - Sentry.init() with browserTracing + replay (measures feature overhead) const lighthouseMode = process.env.NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE; if (lighthouseMode !== 'no-sentry') { const integrations: Sentry.Integration[] = []; - if (lighthouseMode !== 'init-only') { + // Existing E2E behavior (unset) keeps third-party-error-filter on. We disable it in + // init-only and tracing-replay so those modes measure SDK overhead without app-specific + // integrations skewing the result. + if (lighthouseMode === undefined || lighthouseMode === '') { integrations.push( Sentry.thirdPartyErrorFilterIntegration({ filterKeys: ['nextjs-16-e2e'], @@ -15,11 +24,19 @@ if (lighthouseMode !== 'no-sentry') { ); } + // tracing-replay mode enables both performance + session replay so we can measure their + // combined overhead. init-only skips both. + if (lighthouseMode === 'tracing-replay') { + integrations.push(Sentry.browserTracingIntegration(), Sentry.replayIntegration()); + } + Sentry.init({ environment: 'qa', // dynamic sampling bias to keep transactions dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN, tunnel: `http://localhost:3031/`, // proxy server tracesSampleRate: 1.0, + replaysSessionSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : 0, + replaysOnErrorSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : 0, sendDefaultPii: true, integrations, // Verify Log type is available From ae9b8b8dc2f69da341a50a02ce82bd994713bfc0 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 12 May 2026 21:49:11 +0200 Subject: [PATCH 04/29] feat(lighthouse-ci): add nightly + label-gated lighthouse jobs to build.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Insert three new jobs between job_optional_e2e_tests and job_required_jobs_passed in the CI workflow: - job_lighthouse_matrix: generates the test matrix (apps × modes), gated on schedule (nightly) or 'ci:lighthouse' PR label - job_lighthouse: runs each matrix cell on ubuntu-24.04-large-js with continue-on-error: true and max-parallel: 15. Mirrors the job_e2e_tests prep recipe (pnpm 9.15.9, Node from app package.json, restore-cache, download build-tarball-output, yarn test:prepare, ci:copy-to-temp, ci:pnpm-overrides). Build step sets SENTRY_LIGHTHOUSE_MODE under every common bundler env prefix (NEXT_PUBLIC_, PUBLIC_, REACT_APP_) so each app reads whichever its bundler exposes. Uses treosh/lighthouse-ci-action@v12 with numberOfRuns: 5 and temporaryPublicStorage: true. - job_lighthouse_report: downloads all lighthouse artifacts and runs post-comment.mjs to generate a summary table / PR comment Extended on.pull_request to include types: [opened, synchronize, reopened, labeled] so the labeled event fires when ci:lighthouse is added to a PR. Security: NO pull_request_target, NO new permissions block, NO new secrets beyond GITHUB_TOKEN, NOT added to job_required_jobs_passed needs[] — lighthouse failures never block merges. Ref: TODO-1d15a08e, plan rev 2 Co-Authored-By: Claude claude-opus-4-6 --- .github/workflows/build.yml | 133 ++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2d7781c5f43f..ec7c8a48ed70 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,7 @@ on: - v8 - release/** pull_request: + types: [opened, synchronize, reopened, labeled] merge_group: types: [checks_requested] workflow_dispatch: @@ -1201,6 +1202,138 @@ jobs: retention-days: 7 if-no-files-found: ignore + # ----------------------------------------------------------------------- + # Lighthouse CI — nightly + label-gated (ci:lighthouse) + # NOT in job_required_jobs_passed — never blocks merges. + # ----------------------------------------------------------------------- + + job_lighthouse_matrix: + name: Lighthouse Matrix + needs: [job_get_metadata] + runs-on: ubuntu-24.04 + if: | + github.event_name == 'schedule' || + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:lighthouse')) + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) + uses: actions/checkout@v6 + with: + ref: ${{ env.HEAD_COMMIT }} + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version: lts/* + - name: Generate matrix + id: matrix + run: node dev-packages/lighthouse-tests/lighthouse-matrix.mjs + + job_lighthouse: + name: Lighthouse ${{ matrix.test-application }} (${{ matrix.mode }}) + needs: [job_get_metadata, job_build, job_build_tarballs, job_lighthouse_matrix] + if: always() && needs.job_build_tarballs.result == 'success' && needs.job_lighthouse_matrix.result == 'success' + runs-on: ubuntu-24.04-large-js + timeout-minutes: 20 + continue-on-error: true + strategy: + fail-fast: false + max-parallel: 15 + matrix: ${{ fromJson(needs.job_lighthouse_matrix.outputs.matrix) }} + steps: + - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) + uses: actions/checkout@v6 + with: + ref: ${{ env.HEAD_COMMIT }} + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 9.15.9 + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version-file: 'dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}/package.json' + - name: Restore caches + uses: ./.github/actions/restore-cache + with: + dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} + + - name: Restore tarball artifacts + uses: actions/download-artifact@v7 + with: + name: build-tarball-output + path: ${{ env.TARBALL_ARTIFACT_DOWNLOAD_PATH }} + + - name: Prepare E2E tests + run: yarn test:prepare + working-directory: dev-packages/e2e-tests + + - name: Copy to temp + run: yarn ci:copy-to-temp ./test-applications/${{ matrix.test-application }} ${{ runner.temp }}/test-application + working-directory: dev-packages/e2e-tests + + - name: Add pnpm overrides + run: + yarn ci:pnpm-overrides ${{ runner.temp }}/test-application ${{ github.workspace + }}/dev-packages/e2e-tests/packed + working-directory: dev-packages/e2e-tests + + - name: Build E2E app for Lighthouse + working-directory: ${{ runner.temp }}/test-application + timeout-minutes: 7 + run: ${{ matrix.build-command || 'pnpm test:build' }} + env: + # Set under every common bundler env prefix so each app reads whichever its bundler exposes + SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} + NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} + PUBLIC_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} + REACT_APP_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} + + - name: Run Lighthouse CI + uses: treosh/lighthouse-ci-action@v12 + with: + configPath: dev-packages/lighthouse-tests/lighthouserc.js + uploadArtifacts: true + temporaryPublicStorage: true + runs: 5 + urls: ${{ matrix.url || 'http://localhost:3000/' }} + startServerCommand: ${{ matrix.serve-command || '' }} + startServerReadyTimeout: 30000 + + - name: Upload Lighthouse results + uses: actions/upload-artifact@v7 + if: always() + with: + name: lighthouse-${{ matrix.test-application }}-${{ matrix.mode }} + path: .lighthouseci/ + retention-days: 7 + if-no-files-found: ignore + + job_lighthouse_report: + name: Lighthouse Report + needs: [job_lighthouse] + if: always() && needs.job_lighthouse.result != 'cancelled' + runs-on: ubuntu-24.04 + steps: + - name: Check out current commit + uses: actions/checkout@v6 + with: + ref: ${{ env.HEAD_COMMIT }} + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version: lts/* + - name: Download all Lighthouse artifacts + uses: actions/download-artifact@v7 + with: + pattern: lighthouse-* + path: lighthouse-results/ + - name: Generate report + run: node dev-packages/lighthouse-tests/post-comment.mjs + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} + job_required_jobs_passed: name: All required jobs passed or were skipped needs: From 2a7c37817cc9fde5e48a936b8a44e0ee197b437f Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 12 May 2026 21:52:25 +0200 Subject: [PATCH 05/29] fix(lighthouse-ci): align workflow with actual matrix schema and lighthouserc.js MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit (ae9b8b8dc) added the three lighthouse jobs but referenced matrix field names that don't exist in our matrix generator output, and didn't wire the env vars that lighthouserc.js reads. As written, every matrix cell would have failed at the 'Set up Node' step because matrix.test-application is undefined. Fixes: 1. Matrix field references — rename to match dev-packages/lighthouse-tests/ lighthouse-matrix.mjs output: - matrix.test-application → matrix.app - matrix.build-command → dropped (always 'pnpm test:build') - matrix.serve-command → matrix.start-cmd (via env var) - matrix.url → dropped (always http://localhost:3000/) The matrix actually emits: app, app-dir, env-var-name, mode, ready-pattern, sdk, serve, start-cmd, static-dir. 2. Capture matrix output. The matrix generator step ran 'node lighthouse-matrix.mjs' but did NOT redirect to $GITHUB_OUTPUT, so its 'matrix=' line was discarded and downstream jobs would expand to an empty matrix. Fixed to '>> "$GITHUB_OUTPUT"'. 3. Pass LIGHTHOUSE_* env vars to the treosh action. The lighthouserc.js in dev-packages/lighthouse-tests/ branches at config-load time on LIGHTHOUSE_SERVE_MODE (static vs server) and reads LIGHTHOUSE_STATIC_DIR / LIGHTHOUSE_START_CMD / LIGHTHOUSE_READY_PATTERN / LIGHTHOUSE_URL. The previous commit instead used the action's 'urls:' and 'startServerCommand:' inputs, which would have conflicted with the config's branching (staticDistDir would have been undefined). 4. Add the DSN env block at job level, mirroring job_e2e_tests (E2E_TEST_DSN + framework-prefixed variants). Apps gracefully handle missing DSN but cleanly setting them keeps Sentry.init() identical to real E2E runs. 5. Add 'yarn test:validate' step after test:prepare to match job_e2e_tests prep recipe. 6. Use node-version-file: 'package.json' (repo convention) instead of node-version: lts/* in job_lighthouse_matrix and job_lighthouse_report. 7. job_lighthouse_report: add job_build + job_lighthouse_matrix to needs[] so the restore-cache step has its dependency_cache_key, and the report waits for the matrix step. 8. job_lighthouse_report: pass IS_PR (boolean) and PR_NUMBER env vars to post-comment.mjs (matches Todo 4's spec). Security posture from ae9b8b8dc is preserved: still no pull_request_target, no new permissions, no new secrets, lighthouse jobs still excluded from job_required_jobs_passed.needs[]. Ref: TODO-1d15a08e Co-Authored-By: Claude claude-opus-4-5 --- .github/workflows/build.yml | 62 ++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ec7c8a48ed70..f65d493ab122 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1224,13 +1224,15 @@ jobs: - name: Set up Node uses: actions/setup-node@v6 with: - node-version: lts/* + node-version-file: 'package.json' - name: Generate matrix id: matrix - run: node dev-packages/lighthouse-tests/lighthouse-matrix.mjs + # Script prints `matrix=` to stdout; redirect to $GITHUB_OUTPUT so downstream + # jobs can pick it up via ${{ needs.job_lighthouse_matrix.outputs.matrix }}. + run: node dev-packages/lighthouse-tests/lighthouse-matrix.mjs >> "$GITHUB_OUTPUT" job_lighthouse: - name: Lighthouse ${{ matrix.test-application }} (${{ matrix.mode }}) + name: Lighthouse ${{ matrix.app }} (${{ matrix.mode }}) needs: [job_get_metadata, job_build, job_build_tarballs, job_lighthouse_matrix] if: always() && needs.job_build_tarballs.result == 'success' && needs.job_lighthouse_matrix.result == 'success' runs-on: ubuntu-24.04-large-js @@ -1240,6 +1242,14 @@ jobs: fail-fast: false max-parallel: 15 matrix: ${{ fromJson(needs.job_lighthouse_matrix.outputs.matrix) }} + env: + # Mirrors job_e2e_tests env block — apps expect framework-specific DSN prefixes. + E2E_TEST_DSN: 'https://username@domain/123' + NEXT_PUBLIC_E2E_TEST_DSN: 'https://username@domain/123' + PUBLIC_E2E_TEST_DSN: 'https://username@domain/123' + REACT_APP_E2E_TEST_DSN: 'https://username@domain/123' + VITE_E2E_TEST_DSN: 'https://username@domain/123' + NUXT_PUBLIC_E2E_TEST_DSN: 'https://username@domain/123' steps: - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) uses: actions/checkout@v6 @@ -1251,7 +1261,7 @@ jobs: - name: Set up Node uses: actions/setup-node@v6 with: - node-version-file: 'dev-packages/e2e-tests/test-applications/${{ matrix.test-application }}/package.json' + node-version-file: 'dev-packages/e2e-tests/test-applications/${{ matrix.app }}/package.json' - name: Restore caches uses: ./.github/actions/restore-cache with: @@ -1267,8 +1277,12 @@ jobs: run: yarn test:prepare working-directory: dev-packages/e2e-tests - - name: Copy to temp - run: yarn ci:copy-to-temp ./test-applications/${{ matrix.test-application }} ${{ runner.temp }}/test-application + - name: Validate E2E tests setup + run: yarn test:validate + working-directory: dev-packages/e2e-tests + + - name: Copy app to temp + run: yarn ci:copy-to-temp ./test-applications/${{ matrix.app }} ${{ runner.temp }}/test-application working-directory: dev-packages/e2e-tests - name: Add pnpm overrides @@ -1280,48 +1294,60 @@ jobs: - name: Build E2E app for Lighthouse working-directory: ${{ runner.temp }}/test-application timeout-minutes: 7 - run: ${{ matrix.build-command || 'pnpm test:build' }} + run: pnpm test:build env: + SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} # Set under every common bundler env prefix so each app reads whichever its bundler exposes + # (matrix.env-var-name is informational; we set all common prefixes here for simplicity). SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} PUBLIC_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} + VITE_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} + NUXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} REACT_APP_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} - name: Run Lighthouse CI uses: treosh/lighthouse-ci-action@v12 with: - configPath: dev-packages/lighthouse-tests/lighthouserc.js + configPath: ${{ github.workspace }}/dev-packages/lighthouse-tests/lighthouserc.js uploadArtifacts: true temporaryPublicStorage: true runs: 5 - urls: ${{ matrix.url || 'http://localhost:3000/' }} - startServerCommand: ${{ matrix.serve-command || '' }} - startServerReadyTimeout: 30000 + env: + # lighthouserc.js branches on these env vars at config-load time. + LIGHTHOUSE_SERVE_MODE: ${{ matrix.serve }} + LIGHTHOUSE_STATIC_DIR: ${{ runner.temp }}/test-application/${{ matrix.static-dir }} + LIGHTHOUSE_START_CMD: cd ${{ runner.temp }}/test-application && ${{ matrix.start-cmd }} + LIGHTHOUSE_READY_PATTERN: ${{ matrix.ready-pattern }} + LIGHTHOUSE_URL: 'http://localhost:3000/' - name: Upload Lighthouse results uses: actions/upload-artifact@v7 if: always() with: - name: lighthouse-${{ matrix.test-application }}-${{ matrix.mode }} + name: lighthouse-${{ matrix.app }}-${{ matrix.mode }} path: .lighthouseci/ retention-days: 7 if-no-files-found: ignore job_lighthouse_report: name: Lighthouse Report - needs: [job_lighthouse] - if: always() && needs.job_lighthouse.result != 'cancelled' + needs: [job_get_metadata, job_build, job_lighthouse, job_lighthouse_matrix] + if: always() && needs.job_lighthouse_matrix.result == 'success' runs-on: ubuntu-24.04 steps: - - name: Check out current commit + - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) uses: actions/checkout@v6 with: ref: ${{ env.HEAD_COMMIT }} - name: Set up Node uses: actions/setup-node@v6 with: - node-version: lts/* + node-version-file: 'package.json' + - name: Restore caches + uses: ./.github/actions/restore-cache + with: + dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - name: Download all Lighthouse artifacts uses: actions/download-artifact@v7 with: @@ -1331,8 +1357,8 @@ jobs: run: node dev-packages/lighthouse-tests/post-comment.mjs env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_EVENT_NAME: ${{ github.event_name }} - GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} + IS_PR: ${{ github.event_name == 'pull_request' }} + PR_NUMBER: ${{ github.event.pull_request.number }} job_required_jobs_passed: name: All required jobs passed or were skipped From 01197641d605a1c21bbb17c08b05d6f75d70eea0 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 12 May 2026 21:55:07 +0200 Subject: [PATCH 06/29] feat(lighthouse-ci): add post-comment.mjs to render Lighthouse results as PR comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a script that reads LHCI artifact directories (manifest.json + lhr-*.json), extracts median-run performance scores for each app × mode combination, and renders a markdown table with columns for No Sentry, Init Only, Tracing+Replay, plus computed deltas (SDK overhead = init-only minus no-sentry, feature overhead = tracing-replay minus init-only). Mirrors the find-and-update PR comment pattern from size-limit-gh-action: searches for an existing comment by heading prefix ('## 🔦 Lighthouse Report'), then creates or updates accordingly. Handles 403 gracefully on fork PRs where GITHUB_TOKEN from pull_request events is read-only — logs a warning and exits 0 instead of failing. For nightly cron runs (IS_PR !== 'true'), the table is logged to stdout. A <50% data completeness guard skips posting when too many cells are missing. Co-Authored-By: Claude claude-opus-4-6 --- dev-packages/lighthouse-tests/package.json | 5 + .../lighthouse-tests/post-comment.mjs | 175 ++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 dev-packages/lighthouse-tests/post-comment.mjs diff --git a/dev-packages/lighthouse-tests/package.json b/dev-packages/lighthouse-tests/package.json index 4ddb46980b80..b37aed9c9af9 100644 --- a/dev-packages/lighthouse-tests/package.json +++ b/dev-packages/lighthouse-tests/package.json @@ -8,5 +8,10 @@ }, "scripts": { "generate-matrix": "node lighthouse-matrix.mjs" + }, + "dependencies": { + "@actions/core": "1.10.1", + "@actions/github": "^5.0.0", + "markdown-table": "3.0.3" } } diff --git a/dev-packages/lighthouse-tests/post-comment.mjs b/dev-packages/lighthouse-tests/post-comment.mjs new file mode 100644 index 000000000000..13a6f60ed1c4 --- /dev/null +++ b/dev-packages/lighthouse-tests/post-comment.mjs @@ -0,0 +1,175 @@ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import * as core from '@actions/core'; +import { context, getOctokit } from '@actions/github'; +import { markdownTable } from 'markdown-table'; + +const HEADING = '## 🔦 Lighthouse Report'; +const MODES = ['no-sentry', 'init-only', 'tracing-replay']; + +/** + * Apps and their human-readable SDK labels, matching lighthouse-matrix.mjs. + * Order here determines row order in the table. + */ +const APPS = [ + { app: 'default-browser', sdk: 'browser' }, + { app: 'react-19', sdk: 'react' }, + { app: 'ember-classic', sdk: 'ember' }, + { app: 'create-remix-app-express', sdk: 'remix' }, + { app: 'angular-21', sdk: 'angular' }, + { app: 'vue-3', sdk: 'vue' }, + { app: 'svelte-5', sdk: 'svelte' }, + { app: 'sveltekit-2', sdk: 'sveltekit' }, + { app: 'astro-5', sdk: 'astro' }, + { app: 'react-router-7-spa', sdk: 'react-router' }, + { app: 'solidstart-spa', sdk: 'solidstart' }, + { app: 'tanstackstart-react', sdk: 'tanstack-start' }, + { app: 'nextjs-16', sdk: 'nextjs' }, + { app: 'nuxt-5', sdk: 'nuxt' }, +]; + +/** + * Read the median-run LHR from an LHCI artifact directory. + * Returns { score, url } or null if missing/invalid. + */ +async function readResult(resultsDir, app, mode) { + const dir = path.join(resultsDir, `lighthouse-${app}-${mode}`); + let manifest; + try { + manifest = JSON.parse(await fs.readFile(path.join(dir, 'manifest.json'), 'utf8')); + } catch { + return null; + } + + // LHCI manifest is an array of entries. Pick the representative one + // (aggregationMethod: median-run) or fall back to the first entry. + const entry = manifest.find(e => e.isRepresentativeRun) || manifest[0]; + if (!entry) return null; + + const lhrPath = path.join(dir, path.basename(entry.jsonPath)); + let lhr; + try { + lhr = JSON.parse(await fs.readFile(lhrPath, 'utf8')); + } catch { + return null; + } + + const score = lhr.categories?.performance?.score; + if (score == null) return null; + + const url = entry.htmlPath ? entry.htmlPath : undefined; + + return { score: Math.round(score * 100), url }; +} + +function formatCell(result) { + if (!result) return '⚠️'; + if (result.url) return `[${result.score}](${result.url})`; + return `${result.score}`; +} + +function formatDelta(a, b) { + if (!a || !b) return '—'; + const diff = b.score - a.score; + const sign = diff > 0 ? '+' : ''; + return `${sign}${diff}`; +} + +async function run() { + const resultsDir = process.env.LIGHTHOUSE_RESULTS_DIR || 'lighthouse-results'; + const isPR = process.env.IS_PR === 'true'; + const prNumber = process.env.PR_NUMBER ? Number(process.env.PR_NUMBER) : undefined; + + // Collect all results + const rows = []; + let totalCells = 0; + let filledCells = 0; + + for (const { app, sdk } of APPS) { + const results = {}; + for (const mode of MODES) { + totalCells++; + results[mode] = await readResult(resultsDir, app, mode); + if (results[mode]) filledCells++; + } + rows.push({ sdk, results }); + } + + // If <50% of cells have data, warn and skip + if (totalCells > 0 && filledCells / totalCells < 0.5) { + core.warning(`Only ${filledCells}/${totalCells} Lighthouse cells have results (< 50%). Skipping comment.`); + return; + } + + // Build markdown table + const header = ['App', 'No Sentry', 'Init Only', 'Δ (SDK)', 'Tracing+Replay', 'Δ (Features)']; + const tableRows = rows.map(({ sdk, results }) => [ + sdk, + formatCell(results['no-sentry']), + formatCell(results['init-only']), + formatDelta(results['no-sentry'], results['init-only']), + formatCell(results['tracing-replay']), + formatDelta(results['init-only'], results['tracing-replay']), + ]); + + const table = markdownTable([header, ...tableRows]); + + if (!isPR || !prNumber) { + // Nightly / non-PR: just log to stdout + // eslint-disable-next-line no-console + console.log(`${HEADING}\n\n${table}`); + return; + } + + // Post or update PR comment + const token = process.env.GITHUB_TOKEN; + if (!token) { + core.warning('GITHUB_TOKEN not set — cannot post PR comment.'); + return; + } + + const octokit = getOctokit(token); + const repo = context.repo; + + // Find existing comment + const { data: comments } = await octokit.rest.issues.listComments({ + ...repo, + issue_number: prNumber, + }); + const existing = comments.find(c => c.body?.startsWith(HEADING)); + + const body = `${HEADING}\n\n${table}`; + + try { + if (existing) { + await octokit.rest.issues.updateComment({ + ...repo, + comment_id: existing.id, + body, + }); + core.info('Updated existing Lighthouse comment.'); + } else { + await octokit.rest.issues.createComment({ + ...repo, + issue_number: prNumber, + body, + }); + core.info('Created Lighthouse PR comment.'); + } + } catch (err) { + if (err.status === 403) { + core.warning( + 'Could not post PR comment (403 Forbidden). This is expected for fork PRs where GITHUB_TOKEN is read-only.', + ); + // eslint-disable-next-line no-console + console.log(`\n${body}`); + return; + } + throw err; + } +} + +run().catch(err => { + core.setFailed(err.message); + process.exit(1); +}); From cfa20928eaa1b67c511306563a615d2b6fadb8ba Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 12 May 2026 21:56:57 +0200 Subject: [PATCH 07/29] fix(lighthouse-ci): drop unusable local-path links from PR comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit (01197641d) wrapped each score in a markdown link pointing to entry.htmlPath from the LHCI manifest. That field is a path on the CI runner's local filesystem (e.g. /home/runner/_work/.../lhr-0.html), not a URL that a PR reader can open. GitHub renders the markdown but the link 404s. The temporaryPublicStorage URLs (where the reports actually live) are exposed by treosh/lighthouse-ci-action as its 'links' output, not via the manifest artifact, so they're not available in this script's current input. Piping them in would require a second artifact upload step in the workflow — out of scope for the MVP. For now, drop the link entirely (just show the score) and add a footer pointing PR readers to the workflow artifacts (lighthouse--), which are downloadable from the GitHub Actions UI. Wiring the public-storage URLs into the comment is tracked as a future improvement and can be done by: 1. capturing the action's 'links' output in job_lighthouse via 'id: lh' + step output, and 2. saving it as a per-cell artifact that post-comment.mjs merges in. Verified locally with a 42-cell fixture (full table renders without links, footer present) and a 3-cell fixture (<50% warning triggers, no comment posted). Ref: TODO-5844320c Co-Authored-By: Claude claude-opus-4-5 --- dev-packages/lighthouse-tests/post-comment.mjs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/dev-packages/lighthouse-tests/post-comment.mjs b/dev-packages/lighthouse-tests/post-comment.mjs index 13a6f60ed1c4..2661c4f5ef1c 100644 --- a/dev-packages/lighthouse-tests/post-comment.mjs +++ b/dev-packages/lighthouse-tests/post-comment.mjs @@ -57,14 +57,15 @@ async function readResult(resultsDir, app, mode) { const score = lhr.categories?.performance?.score; if (score == null) return null; - const url = entry.htmlPath ? entry.htmlPath : undefined; - - return { score: Math.round(score * 100), url }; + // NOTE: LHCI's manifest.json only carries local filesystem paths (htmlPath, jsonPath) + // for the runner, not clickable URLs. The temporaryPublicStorage URLs live in the + // treosh action's `links` output, which isn't currently piped into the artifact. + // For MVP we drop the link and direct readers to workflow artifacts via the footer. + return { score: Math.round(score * 100) }; } function formatCell(result) { if (!result) return '⚠️'; - if (result.url) return `[${result.score}](${result.url})`; return `${result.score}`; } @@ -114,10 +115,14 @@ async function run() { const table = markdownTable([header, ...tableRows]); + const footer = + '\n\n_Median of 5 runs, simulated throttling, localhost. ' + + 'Full reports are attached as workflow artifacts (`lighthouse--`)._'; + if (!isPR || !prNumber) { // Nightly / non-PR: just log to stdout // eslint-disable-next-line no-console - console.log(`${HEADING}\n\n${table}`); + console.log(`${HEADING}\n\n${table}${footer}`); return; } @@ -138,7 +143,7 @@ async function run() { }); const existing = comments.find(c => c.body?.startsWith(HEADING)); - const body = `${HEADING}\n\n${table}`; + const body = `${HEADING}\n\n${table}${footer}`; try { if (existing) { From e8cb7f85ecf7b65fdfb66ea7a174f6af5e9c8e1f Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 12 May 2026 22:04:40 +0200 Subject: [PATCH 08/29] chore(lighthouse-ci): pin treosh/lighthouse-ci-action to commit SHA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit referenced treosh/lighthouse-ci-action by the floating @v12 tag. That tag is a moving target — if the treosh org's GitHub account is compromised or they push a malicious v12.x.y patch, our CI would execute it on the next workflow run. This action also internally npm-installs @lhci/cli, so the upstream supply chain includes both the action's own code AND its transitive npm deps. Pinning to a commit SHA locks both down (the action's repo has a committed package-lock.json, so the commit pins the npm tree too). Matches the existing repo policy for third-party actions (cf. pnpm/action-setup@fc06bc1257... which is similarly pinned to SHA). Co-Authored-By: Claude claude-opus-4-5 --- .github/workflows/build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f65d493ab122..031c520957c8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1307,7 +1307,11 @@ jobs: REACT_APP_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} - name: Run Lighthouse CI - uses: treosh/lighthouse-ci-action@v12 + # Pinned to a commit SHA (rather than the floating @v12 tag) to lock down the + # supply chain: this third-party action also npm-installs @lhci/cli, so an + # upstream tag-repoint could otherwise change what runs on our runners. + # SHA is the current tip of v12. Bump deliberately when upgrading. + uses: treosh/lighthouse-ci-action@3e7e23fb74242897f95c0ba9cabad3d0227b9b18 # v12 with: configPath: ${{ github.workspace }}/dev-packages/lighthouse-tests/lighthouserc.js uploadArtifacts: true From e04bc3732f32a827035edb153be737ecd0da2641 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 12 May 2026 22:28:44 +0200 Subject: [PATCH 09/29] fix(lighthouse-ci): unblock webpack build and surface LCP/TBT/bytes in report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two issues found by running the workflow end-to-end locally: 1. **default-browser webpack build was aborting in all 3 modes.** The dynamic-import refactor in 36c3f7676 caused @sentry/browser to be emitted as a separate code-split chunk (~265 KiB raw). Webpack's default performance.hints='warning' fires AssetsOverSizeLimitWarning for assets > 244 KiB, and build.mjs treats any warning as fatal (process.exit(1)). The original static-import build came in at 138 KiB so the warning never fired before. Disabling performance hints in this app's webpack config is the right scope — bundle size is tracked separately via .size-limit.js at the repo root. 2. **post-comment.mjs reported only categories.performance.score, which tops out at 100 for fast static apps across all 3 modes.** Local measurement of default-browser confirmed: score is 100 for no-sentry, init-only, AND tracing-replay, even though LCP jumps from 916 ms to 1836 ms when the SDK is loaded. The score column alone would have rendered as a sea of '100 | 100 | 0 | 100 | 0' across all 14 apps, making the comment useless for fast frameworks. Now the report has four tables (one per metric): performance score, LCP, TBT, and total bytes downloaded. Each follows the same App × Mode × Δ shape. LCP is the primary SDK-overhead indicator; TBT captures runtime cost of integrations; bytes captures download cost. For slower apps where scores actually differ, the score table still carries signal; for fast static apps the metric tables show what's happening. Verified locally end-to-end: built default-browser in all 3 modes, ran lhci collect (2 runs each) + lhci upload --target=filesystem, staged the resulting manifests as the workflow would, and confirmed post-comment.mjs renders all four tables with correct deltas and units. Webpack's constant-folding successfully eliminates the dynamic @sentry/browser import entirely in no-sentry mode — the entry never references the SDK chunk, so the browser fetches just 3 KB total (clean baseline confirmed). Known limitation tracked in plan: dynamic import doesn't propagate treeshaking, so init-only and tracing-replay download the same 87 KB gzipped chunk. Only TBT differentiates them (runtime cost of integration instantiation). LCP / bytes are mostly indistinguishable between those two modes for now; can be improved later via per-mode entry files. Co-Authored-By: Claude claude-opus-4-5 --- .../default-browser/build.mjs | 7 ++ .../lighthouse-tests/post-comment.mjs | 102 ++++++++++++------ 2 files changed, 74 insertions(+), 35 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/default-browser/build.mjs b/dev-packages/e2e-tests/test-applications/default-browser/build.mjs index 26dbad87140f..6a4254cc8238 100644 --- a/dev-packages/e2e-tests/test-applications/default-browser/build.mjs +++ b/dev-packages/e2e-tests/test-applications/default-browser/build.mjs @@ -17,6 +17,13 @@ webpack( minimize: true, minimizer: [new TerserPlugin()], }, + // The Lighthouse-CI baseline mode (SENTRY_LIGHTHOUSE_MODE=no-sentry) emits the + // @sentry/browser chunk as a separately code-split asset (~265 KiB raw). Webpack's + // default performance.hints='warning' fires AssetsOverSizeLimitWarning for that + // chunk, and build.mjs's `if (stats.hasWarnings()) process.exit(1)` would then + // abort the build. This is a test app, not a size-tracked production bundle, so + // hints are disabled — size is tracked separately via .size-limit.js at the repo root. + performance: { hints: false }, plugins: [ new webpack.EnvironmentPlugin({ E2E_TEST_DSN: '', SENTRY_LIGHTHOUSE_MODE: '' }), new HtmlWebpackPlugin({ diff --git a/dev-packages/lighthouse-tests/post-comment.mjs b/dev-packages/lighthouse-tests/post-comment.mjs index 2661c4f5ef1c..0894cf245a5f 100644 --- a/dev-packages/lighthouse-tests/post-comment.mjs +++ b/dev-packages/lighthouse-tests/post-comment.mjs @@ -9,7 +9,7 @@ const MODES = ['no-sentry', 'init-only', 'tracing-replay']; /** * Apps and their human-readable SDK labels, matching lighthouse-matrix.mjs. - * Order here determines row order in the table. + * Order here determines row order in each table. */ const APPS = [ { app: 'default-browser', sdk: 'browser' }, @@ -28,9 +28,22 @@ const APPS = [ { app: 'nuxt-5', sdk: 'nuxt' }, ]; +/** + * Metrics surfaced in the report. Lighthouse's category score is too coarse on its own + * (fast static apps cap at 100 across all modes), so we also report the underlying + * metric values. LCP is the primary regression indicator for SDK overhead; TBT captures + * runtime cost of instrumentation; total bytes captures download cost. + */ +const SECTIONS = [ + { label: 'Performance score', metric: 'score', unit: '', betterIs: 'higher' }, + { label: 'Largest Contentful Paint (LCP)', metric: 'lcp', unit: ' ms', betterIs: 'lower' }, + { label: 'Total Blocking Time (TBT)', metric: 'tbt', unit: ' ms', betterIs: 'lower' }, + { label: 'Bytes downloaded', metric: 'bytes', unit: ' KB', betterIs: 'lower' }, +]; + /** * Read the median-run LHR from an LHCI artifact directory. - * Returns { score, url } or null if missing/invalid. + * Returns { score, lcp, tbt, bytes } or null if missing/invalid. */ async function readResult(resultsDir, app, mode) { const dir = path.join(resultsDir, `lighthouse-${app}-${mode}`); @@ -57,23 +70,48 @@ async function readResult(resultsDir, app, mode) { const score = lhr.categories?.performance?.score; if (score == null) return null; - // NOTE: LHCI's manifest.json only carries local filesystem paths (htmlPath, jsonPath) - // for the runner, not clickable URLs. The temporaryPublicStorage URLs live in the - // treosh action's `links` output, which isn't currently piped into the artifact. - // For MVP we drop the link and direct readers to workflow artifacts via the footer. - return { score: Math.round(score * 100) }; + const audit = id => lhr.audits?.[id]?.numericValue; + const round = v => (typeof v === 'number' ? Math.round(v) : undefined); + + return { + score: Math.round(score * 100), + lcp: round(audit('largest-contentful-paint')), + tbt: round(audit('total-blocking-time')), + bytes: round(audit('total-byte-weight') / 1024), + }; } -function formatCell(result) { - if (!result) return '⚠️'; - return `${result.score}`; +function formatValue(value, unit) { + if (value == null || Number.isNaN(value)) return '⚠️'; + return `${value}${unit}`; } -function formatDelta(a, b) { - if (!a || !b) return '—'; - const diff = b.score - a.score; +function formatDelta(before, after, unit) { + if (before == null || after == null || Number.isNaN(before) || Number.isNaN(after)) { + return '—'; + } + const diff = after - before; + if (diff === 0) return '0'; const sign = diff > 0 ? '+' : ''; - return `${sign}${diff}`; + return `${sign}${diff}${unit}`; +} + +function buildSectionTable(rows, metric, unit) { + const header = ['App', 'No Sentry', 'Init Only', 'Δ (SDK)', 'Tracing+Replay', 'Δ (Features)']; + const body = rows.map(({ sdk, results }) => { + const n = results['no-sentry']?.[metric]; + const i = results['init-only']?.[metric]; + const t = results['tracing-replay']?.[metric]; + return [ + sdk, + formatValue(n, unit), + formatValue(i, unit), + formatDelta(n, i, unit), + formatValue(t, unit), + formatDelta(i, t, unit), + ]; + }); + return markdownTable([header, ...body]); } async function run() { @@ -81,7 +119,6 @@ async function run() { const isPR = process.env.IS_PR === 'true'; const prNumber = process.env.PR_NUMBER ? Number(process.env.PR_NUMBER) : undefined; - // Collect all results const rows = []; let totalCells = 0; let filledCells = 0; @@ -96,37 +133,32 @@ async function run() { rows.push({ sdk, results }); } - // If <50% of cells have data, warn and skip if (totalCells > 0 && filledCells / totalCells < 0.5) { core.warning(`Only ${filledCells}/${totalCells} Lighthouse cells have results (< 50%). Skipping comment.`); return; } - // Build markdown table - const header = ['App', 'No Sentry', 'Init Only', 'Δ (SDK)', 'Tracing+Replay', 'Δ (Features)']; - const tableRows = rows.map(({ sdk, results }) => [ - sdk, - formatCell(results['no-sentry']), - formatCell(results['init-only']), - formatDelta(results['no-sentry'], results['init-only']), - formatCell(results['tracing-replay']), - formatDelta(results['init-only'], results['tracing-replay']), - ]); - - const table = markdownTable([header, ...tableRows]); + // Build one table per metric so each metric's deltas are clearly readable. + // Performance score is generous (fast static apps top out at 100 across all modes), + // so the LCP / TBT / Bytes tables are typically the real signal. + const tables = SECTIONS.map( + ({ label, metric, unit }) => `### ${label}\n\n${buildSectionTable(rows, metric, unit)}`, + ).join('\n\n'); const footer = - '\n\n_Median of 5 runs, simulated throttling, localhost. ' + + '\n\n_Median of 5 runs · simulated throttling · localhost. ' + + 'Lower is better for LCP, TBT, and bytes. Higher is better for score. ' + 'Full reports are attached as workflow artifacts (`lighthouse--`)._'; + const body = `${HEADING}\n\n${tables}${footer}`; + if (!isPR || !prNumber) { - // Nightly / non-PR: just log to stdout + // Nightly / non-PR: log to stdout (captured in workflow logs) // eslint-disable-next-line no-console - console.log(`${HEADING}\n\n${table}${footer}`); + console.log(body); return; } - // Post or update PR comment const token = process.env.GITHUB_TOKEN; if (!token) { core.warning('GITHUB_TOKEN not set — cannot post PR comment.'); @@ -136,15 +168,13 @@ async function run() { const octokit = getOctokit(token); const repo = context.repo; - // Find existing comment + // Find existing Lighthouse comment to update (mirror size-limit-gh-action pattern) const { data: comments } = await octokit.rest.issues.listComments({ ...repo, issue_number: prNumber, }); const existing = comments.find(c => c.body?.startsWith(HEADING)); - const body = `${HEADING}\n\n${table}${footer}`; - try { if (existing) { await octokit.rest.issues.updateComment({ @@ -163,6 +193,8 @@ async function run() { } } catch (err) { if (err.status === 403) { + // Fork PRs: GITHUB_TOKEN is read-only. Log the table to the workflow log so the + // data is still discoverable, and exit 0 so the job doesn't fail. core.warning( 'Could not post PR comment (403 Forbidden). This is expected for fork PRs where GITHUB_TOKEN is read-only.', ); From 9e28f298631eca67688b7fc9cca8bda448f10985 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 12 May 2026 22:48:49 +0200 Subject: [PATCH 10/29] chore(lighthouse-ci): drop angular, remix, ember, solidstart from matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These four apps were producing Lighthouse failures in CI due to framework-specific build/serve quirks that we don't want to invest in for the MVP. Their cells are removed from both lighthouse-matrix.mjs and post-comment.mjs's APPS list (the two must stay in sync). Matrix shape goes from 14 apps × 3 modes (42 cells) to 10 apps × 3 modes (30 cells). Remaining coverage: default-browser (browser) react-router-7-spa (react-router) react-19 (react) tanstackstart-react (tanstack-start) vue-3 (vue) nextjs-16 (nextjs) svelte-5 (svelte) nuxt-5 (nuxt) sveltekit-2 (sveltekit) astro-5 (astro) These can be added back later as a follow-up once we're confident the core pipeline is stable. Co-Authored-By: Claude claude-opus-4-5 --- .../lighthouse-tests/lighthouse-matrix.mjs | 26 +++---------------- .../lighthouse-tests/post-comment.mjs | 6 ++--- 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/dev-packages/lighthouse-tests/lighthouse-matrix.mjs b/dev-packages/lighthouse-tests/lighthouse-matrix.mjs index 04fa5641b251..abe364494b63 100644 --- a/dev-packages/lighthouse-tests/lighthouse-matrix.mjs +++ b/dev-packages/lighthouse-tests/lighthouse-matrix.mjs @@ -29,26 +29,13 @@ */ /** @type {AppDefinition[]} */ +// NOTE: angular, remix, ember, and solidstart were intentionally excluded from +// this matrix — their build/serve setups need framework-specific tuning we're +// not investing in for the MVP. They can be added back as a follow-up. const APPS = [ // Plain webpack apps — read process.env directly (no bundler prefix). { app: 'default-browser', sdk: 'browser', serve: 'static', staticDir: 'build', envVarName: 'SENTRY_LIGHTHOUSE_MODE' }, { app: 'react-19', sdk: 'react', serve: 'static', staticDir: 'build', envVarName: 'SENTRY_LIGHTHOUSE_MODE' }, - { app: 'ember-classic', sdk: 'ember', serve: 'static', staticDir: 'dist', envVarName: 'SENTRY_LIGHTHOUSE_MODE' }, - { - app: 'create-remix-app-express', - sdk: 'remix', - serve: 'server', - startCmd: 'cross-env NODE_ENV=production node ./server.mjs', - readyPattern: 'localhost', - envVarName: 'SENTRY_LIGHTHOUSE_MODE', - }, - { - app: 'angular-21', - sdk: 'angular', - serve: 'static', - staticDir: 'dist/angular-21', - envVarName: 'SENTRY_LIGHTHOUSE_MODE', - }, // Vite-based apps with `envPrefix: 'PUBLIC_'` (matches Sentry's repo convention for PUBLIC_E2E_TEST_DSN). { app: 'vue-3', sdk: 'vue', serve: 'static', staticDir: 'dist', envVarName: 'PUBLIC_SENTRY_LIGHTHOUSE_MODE' }, @@ -78,13 +65,6 @@ const APPS = [ }, // Vite-based apps using the default `VITE_` prefix (no custom envPrefix set). - { - app: 'solidstart-spa', - sdk: 'solidstart', - serve: 'static', - staticDir: '.output/public', - envVarName: 'VITE_SENTRY_LIGHTHOUSE_MODE', - }, { app: 'tanstackstart-react', sdk: 'tanstack-start', diff --git a/dev-packages/lighthouse-tests/post-comment.mjs b/dev-packages/lighthouse-tests/post-comment.mjs index 0894cf245a5f..7ac869f02389 100644 --- a/dev-packages/lighthouse-tests/post-comment.mjs +++ b/dev-packages/lighthouse-tests/post-comment.mjs @@ -11,18 +11,16 @@ const MODES = ['no-sentry', 'init-only', 'tracing-replay']; * Apps and their human-readable SDK labels, matching lighthouse-matrix.mjs. * Order here determines row order in each table. */ +// Must mirror lighthouse-matrix.mjs APPS. angular, remix, ember, solidstart are +// intentionally excluded — see note in lighthouse-matrix.mjs. const APPS = [ { app: 'default-browser', sdk: 'browser' }, { app: 'react-19', sdk: 'react' }, - { app: 'ember-classic', sdk: 'ember' }, - { app: 'create-remix-app-express', sdk: 'remix' }, - { app: 'angular-21', sdk: 'angular' }, { app: 'vue-3', sdk: 'vue' }, { app: 'svelte-5', sdk: 'svelte' }, { app: 'sveltekit-2', sdk: 'sveltekit' }, { app: 'astro-5', sdk: 'astro' }, { app: 'react-router-7-spa', sdk: 'react-router' }, - { app: 'solidstart-spa', sdk: 'solidstart' }, { app: 'tanstackstart-react', sdk: 'tanstack-start' }, { app: 'nextjs-16', sdk: 'nextjs' }, { app: 'nuxt-5', sdk: 'nuxt' }, From 254f9735cec4500621f1ff3b07d0f3ce2deca00c Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 12 May 2026 22:53:43 +0200 Subject: [PATCH 11/29] fix(lighthouse-ci): rename lighthouserc.js to .cjs and switch to module.exports All 42 Lighthouse cells in the first CI run failed with: ##[error]Config missing top level 'ci' property Root cause: LHCI's config loader uses Node `require()` to load the config file. The parent package (`dev-packages/lighthouse-tests/`) sets `"type": "module"` so `lighthouserc.js` was treated as ESM. On modern Node, `require()` of an ESM module returns `{ default: }` instead of the export itself, so LHCI saw a config without a top-level `ci` property and bailed out before running Lighthouse. Fix: rename to `lighthouserc.cjs` (explicit CommonJS regardless of the parent package's `type` field) and change `export default` to `module.exports`. Update the workflow's `configPath` accordingly. Verified locally: - `require('./lighthouserc.cjs')` returns an object with `ci` at the top level. - Server-mode branching (LIGHTHOUSE_SERVE_MODE=server) still resolves the expected `startServerCommand` / `url` config. The other files in this package (`lighthouse-matrix.mjs`, `post-comment.mjs`) are explicit-extension ESM modules invoked via `node script.mjs`, so they're unaffected by this change. Co-Authored-By: Claude claude-opus-4-5 --- .github/workflows/build.yml | 4 ++-- .../{lighthouserc.js => lighthouserc.cjs} | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) rename dev-packages/lighthouse-tests/{lighthouserc.js => lighthouserc.cjs} (85%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 031c520957c8..7034871239e4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1313,12 +1313,12 @@ jobs: # SHA is the current tip of v12. Bump deliberately when upgrading. uses: treosh/lighthouse-ci-action@3e7e23fb74242897f95c0ba9cabad3d0227b9b18 # v12 with: - configPath: ${{ github.workspace }}/dev-packages/lighthouse-tests/lighthouserc.js + configPath: ${{ github.workspace }}/dev-packages/lighthouse-tests/lighthouserc.cjs uploadArtifacts: true temporaryPublicStorage: true runs: 5 env: - # lighthouserc.js branches on these env vars at config-load time. + # lighthouserc.cjs branches on these env vars at config-load time. LIGHTHOUSE_SERVE_MODE: ${{ matrix.serve }} LIGHTHOUSE_STATIC_DIR: ${{ runner.temp }}/test-application/${{ matrix.static-dir }} LIGHTHOUSE_START_CMD: cd ${{ runner.temp }}/test-application && ${{ matrix.start-cmd }} diff --git a/dev-packages/lighthouse-tests/lighthouserc.js b/dev-packages/lighthouse-tests/lighthouserc.cjs similarity index 85% rename from dev-packages/lighthouse-tests/lighthouserc.js rename to dev-packages/lighthouse-tests/lighthouserc.cjs index 80f6b0eca26b..a7391c4970d4 100644 --- a/dev-packages/lighthouse-tests/lighthouserc.js +++ b/dev-packages/lighthouse-tests/lighthouserc.cjs @@ -2,6 +2,12 @@ // Used by treosh/lighthouse-ci-action@v12 via the `configPath` input. // Docs: https://github.com/GoogleChrome/lighthouse-ci/blob/main/docs/configuration.md // +// IMPORTANT: This file MUST be CommonJS (`.cjs` extension, `module.exports`). +// LHCI's config loader uses `require()` to load the config file. The parent +// package has `"type": "module"`, so a `.js` file here would be treated as +// ESM by Node and `require()` would return `{ default: }` instead of +// `` — LHCI then fails with "Config missing top level 'ci' property". +// // Per-cell environment variables (set by the GitHub Actions workflow): // LIGHTHOUSE_SERVE_MODE - 'static' | 'server' // LIGHTHOUSE_STATIC_DIR - absolute path to static dist dir (when serve mode = 'static') @@ -11,7 +17,7 @@ const isServer = process.env.LIGHTHOUSE_SERVE_MODE === 'server'; -export default { +module.exports = { ci: { collect: { // Median of 5 runs halves variance vs a single run (per Lighthouse variability docs). From b766f80a57152e2f95afcd25832cb2a8edac6ce2 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 12 May 2026 23:12:57 +0200 Subject: [PATCH 12/29] fix(lighthouse-ci): drop redundant uploadArtifacts from treosh action 29 of 30 matrix cells failed with: ##[error]Failed to CreateArtifact: Received non-retryable error: Failed request: (409) Conflict: an artifact with this name already exists on the workflow run Root cause: `treosh/lighthouse-ci-action` with `uploadArtifacts: true` uploads `.lighthouseci/` to a default-named artifact (`lighthouse-reports`). With 30 matrix cells running in the same workflow run, every cell after the first one to finish hit a 409 name-collision and aborted the action \u2014 even though the Lighthouse audit itself had already succeeded. The one cell that did succeed (`svelte-5 init-only`) was simply the first to call upload-artifact; everything else lost the race. We already do our own per-cell artifact upload one step later with a unique name (`lighthouse-${{ matrix.app }}-${{ matrix.mode }}`), so the treosh action's upload is redundant. Drop it. `temporaryPublicStorage: true` is kept \u2014 it uploads the HTML report to Google's bucket and prints clickable URLs into the action log, which is useful per-cell debug context independent of the artifact. Co-Authored-By: Claude claude-opus-4-5 --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7034871239e4..9b7fa1fb7fd9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1314,7 +1314,10 @@ jobs: uses: treosh/lighthouse-ci-action@3e7e23fb74242897f95c0ba9cabad3d0227b9b18 # v12 with: configPath: ${{ github.workspace }}/dev-packages/lighthouse-tests/lighthouserc.cjs - uploadArtifacts: true + # Do NOT enable uploadArtifacts here — it would upload `.lighthouseci/` under + # the default name `lighthouse-reports` for every cell, causing a 409 conflict + # on every matrix cell after the first one. We do our own per-cell upload + # below with a unique name (lighthouse--). temporaryPublicStorage: true runs: 5 env: From db8a4bbb7cc00956b6e18611f0cd96a066ec3af3 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Tue, 12 May 2026 23:38:35 +0200 Subject: [PATCH 13/29] fix(lighthouse-ci): unblock nextjs-16 + astro-5; drop react-router-7-spa MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Last CI run (b766f80a5) was a big step forward — 21 of 30 cells succeeded, proving the pipeline works end-to-end. Three apps still failed: 1. nextjs-16 — build-time TypeScript error: Type error: '"@sentry/nextjs"' has no exported member named 'Integration'. The previous SENTRY_LIGHTHOUSE_MODE refactor introduced const integrations: Sentry.Integration[] = [] but `@sentry/nextjs` doesn't re-export the `Integration` type. Fix by inlining a spread-based array literal in Sentry.init() so no explicit type annotation is needed. TypeScript infers integration types from the individual integration factories. 2. astro-5 — Lighthouse hit chrome-error://chromewebdata/ (CHROME_INTERSTITIAL_ERROR): The @astrojs/node adapter defaults to port 4321, not 3000. The server started fine and printed "Listening on localhost:4321" — matching our readyPattern: 'localhost' — but LHCI then navigated to http://localhost:3000/, where nothing was listening, and Chrome bounced to its error page. Fix: prefix the start command with PORT=3000 so the Astro server binds to the port Lighthouse audits. 3. react-router-7-spa — Lighthouse reported NO_FCP across all three modes: the bundle loads, no obvious error in the log, but the page never paints within the navigation timeout. Could be a 2-minute fix or a 2-hour debug — and we already have 7 frontend SDKs reporting cleanly (browser, react, vue, svelte, sveltekit, astro, nuxt, tanstack-start, nextjs). Drop it from the matrix to match the previous descope decision (angular, remix, ember, solidstart). Can be re-added once we figure out why React Router 7 SPA doesn't paint under headless Chrome with simulated throttling. Matrix shape goes from 10 apps × 3 modes (30 cells) to 9 apps × 3 modes (27 cells). Co-Authored-By: Claude claude-opus-4-5 --- .../nextjs-16/instrumentation-client.ts | 35 ++++++++----------- .../lighthouse-tests/lighthouse-matrix.mjs | 19 +++++----- .../lighthouse-tests/post-comment.mjs | 6 ++-- 3 files changed, 25 insertions(+), 35 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/instrumentation-client.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/instrumentation-client.ts index 7db8f4e2c584..cede9c55c688 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/instrumentation-client.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/instrumentation-client.ts @@ -10,26 +10,6 @@ import type { Log } from '@sentry/nextjs'; const lighthouseMode = process.env.NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE; if (lighthouseMode !== 'no-sentry') { - const integrations: Sentry.Integration[] = []; - - // Existing E2E behavior (unset) keeps third-party-error-filter on. We disable it in - // init-only and tracing-replay so those modes measure SDK overhead without app-specific - // integrations skewing the result. - if (lighthouseMode === undefined || lighthouseMode === '') { - integrations.push( - Sentry.thirdPartyErrorFilterIntegration({ - filterKeys: ['nextjs-16-e2e'], - behaviour: 'apply-tag-if-contains-third-party-frames', - }), - ); - } - - // tracing-replay mode enables both performance + session replay so we can measure their - // combined overhead. init-only skips both. - if (lighthouseMode === 'tracing-replay') { - integrations.push(Sentry.browserTracingIntegration(), Sentry.replayIntegration()); - } - Sentry.init({ environment: 'qa', // dynamic sampling bias to keep transactions dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN, @@ -38,7 +18,20 @@ if (lighthouseMode !== 'no-sentry') { replaysSessionSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : 0, replaysOnErrorSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : 0, sendDefaultPii: true, - integrations, + // Existing E2E behavior (mode unset/'') keeps third-party-error-filter on. + // init-only / tracing-replay drop it so we measure SDK overhead without app-specific noise. + // tracing-replay additionally enables browserTracing + replay. + integrations: [ + ...(lighthouseMode === undefined || lighthouseMode === '' + ? [ + Sentry.thirdPartyErrorFilterIntegration({ + filterKeys: ['nextjs-16-e2e'], + behaviour: 'apply-tag-if-contains-third-party-frames', + }), + ] + : []), + ...(lighthouseMode === 'tracing-replay' ? [Sentry.browserTracingIntegration(), Sentry.replayIntegration()] : []), + ], // Verify Log type is available beforeSendLog(log: Log) { return log; diff --git a/dev-packages/lighthouse-tests/lighthouse-matrix.mjs b/dev-packages/lighthouse-tests/lighthouse-matrix.mjs index abe364494b63..c736c0ec7f35 100644 --- a/dev-packages/lighthouse-tests/lighthouse-matrix.mjs +++ b/dev-packages/lighthouse-tests/lighthouse-matrix.mjs @@ -29,9 +29,11 @@ */ /** @type {AppDefinition[]} */ -// NOTE: angular, remix, ember, and solidstart were intentionally excluded from -// this matrix — their build/serve setups need framework-specific tuning we're -// not investing in for the MVP. They can be added back as a follow-up. +// NOTE: angular, remix, ember, solidstart, and react-router-7-spa were +// intentionally excluded from this matrix — their build/serve setups need +// framework-specific tuning we're not investing in for the MVP. They can be +// added back as a follow-up. (react-router-7-spa fails with NO_FCP in +// Lighthouse — the bundle loads but doesn't paint within the timeout.) const APPS = [ // Plain webpack apps — read process.env directly (no bundler prefix). { app: 'default-browser', sdk: 'browser', serve: 'static', staticDir: 'build', envVarName: 'SENTRY_LIGHTHOUSE_MODE' }, @@ -52,17 +54,12 @@ const APPS = [ app: 'astro-5', sdk: 'astro', serve: 'server', - startCmd: 'node ./dist/server/entry.mjs', + // Astro's @astrojs/node adapter defaults to PORT=4321; force 3000 so Lighthouse can + // reach the server at the URL it audits (http://localhost:3000/). + startCmd: 'PORT=3000 node ./dist/server/entry.mjs', readyPattern: 'localhost', envVarName: 'PUBLIC_SENTRY_LIGHTHOUSE_MODE', }, - { - app: 'react-router-7-spa', - sdk: 'react-router', - serve: 'static', - staticDir: 'dist', - envVarName: 'PUBLIC_SENTRY_LIGHTHOUSE_MODE', - }, // Vite-based apps using the default `VITE_` prefix (no custom envPrefix set). { diff --git a/dev-packages/lighthouse-tests/post-comment.mjs b/dev-packages/lighthouse-tests/post-comment.mjs index 7ac869f02389..83635c4e9c7f 100644 --- a/dev-packages/lighthouse-tests/post-comment.mjs +++ b/dev-packages/lighthouse-tests/post-comment.mjs @@ -11,8 +11,9 @@ const MODES = ['no-sentry', 'init-only', 'tracing-replay']; * Apps and their human-readable SDK labels, matching lighthouse-matrix.mjs. * Order here determines row order in each table. */ -// Must mirror lighthouse-matrix.mjs APPS. angular, remix, ember, solidstart are -// intentionally excluded — see note in lighthouse-matrix.mjs. +// Must mirror lighthouse-matrix.mjs APPS. angular, remix, ember, solidstart, +// and react-router-7-spa are intentionally excluded — see note in +// lighthouse-matrix.mjs. const APPS = [ { app: 'default-browser', sdk: 'browser' }, { app: 'react-19', sdk: 'react' }, @@ -20,7 +21,6 @@ const APPS = [ { app: 'svelte-5', sdk: 'svelte' }, { app: 'sveltekit-2', sdk: 'sveltekit' }, { app: 'astro-5', sdk: 'astro' }, - { app: 'react-router-7-spa', sdk: 'react-router' }, { app: 'tanstackstart-react', sdk: 'tanstack-start' }, { app: 'nextjs-16', sdk: 'nextjs' }, { app: 'nuxt-5', sdk: 'nuxt' }, From b29aadaf1bb2589d077b3c81bdc267a4a02235ea Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 00:11:51 +0200 Subject: [PATCH 14/29] fix(lighthouse-ci): upload artifacts from the treosh action's resultsPath output All 27 Lighthouse cells succeeded in CI run db8a4bbb7 but NO PR comment was posted. The Lighthouse Report job logged: Only 0/27 Lighthouse cells have results (< 50%). Skipping comment. Root cause: the upload-artifact step had `path: .lighthouseci/` (in our repo root), but `lhci collect` writes to `.lighthouseci/` inside the treosh action's own working directory (`/home/runner/work/_actions/treosh/ lighthouse-ci-action/.../node_modules/lighthouse-ci/` or similar) \u2014 not our repo root. So the upload step found 0 files, and because of `if-no-files-found: ignore`, silently succeeded with no artifact uploaded. All 27 cells therefore reported "success" while uploading nothing. The treosh action exposes the actual path via its `resultsPath` output. Fix: - Add `id: lighthouse` to the action step. - Switch `path:` to `${{ steps.lighthouse.outputs.resultsPath }}`. - Gate the upload on `resultsPath` being set so a failed Lighthouse run doesn't try to upload nothing. - Switch `if-no-files-found` from `ignore` to `error` so a missing or empty dir is now a loud bug instead of a silent zero-upload. Co-Authored-By: Claude claude-opus-4-5 --- .github/workflows/build.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9b7fa1fb7fd9..f9ad07275131 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1307,6 +1307,7 @@ jobs: REACT_APP_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} - name: Run Lighthouse CI + id: lighthouse # Pinned to a commit SHA (rather than the floating @v12 tag) to lock down the # supply chain: this third-party action also npm-installs @lhci/cli, so an # upstream tag-repoint could otherwise change what runs on our runners. @@ -1329,13 +1330,20 @@ jobs: LIGHTHOUSE_URL: 'http://localhost:3000/' - name: Upload Lighthouse results + # `resultsPath` points at the .lighthouseci/ directory inside the treosh action's + # own working directory (NOT our repo root — lhci's cwd when the action invokes it + # is the action's checkout). Use it explicitly so upload-artifact finds the real + # files. Without this, `path: .lighthouseci/` silently finds nothing in our repo + # root and `if-no-files-found: ignore` makes the step pass with zero uploaded. uses: actions/upload-artifact@v7 - if: always() + if: always() && steps.lighthouse.outputs.resultsPath != '' with: name: lighthouse-${{ matrix.app }}-${{ matrix.mode }} - path: .lighthouseci/ + path: ${{ steps.lighthouse.outputs.resultsPath }} retention-days: 7 - if-no-files-found: ignore + # Fail loudly if the dir is empty — we already gated on resultsPath being set, + # so an empty dir means lhci collect produced nothing, which is a real bug. + if-no-files-found: error job_lighthouse_report: name: Lighthouse Report From b96bb65ce810890ad974a6ec10e20098463692a8 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 00:25:02 +0200 Subject: [PATCH 15/29] fix(lighthouse-ci): include hidden files when uploading .lighthouseci/ artifact After the previous fix (pointing upload-artifact at the treosh action's `resultsPath` output), Lighthouse cells started failing with: ##[error]No files were found with the provided path: /home/runner/work/sentry-javascript/sentry-javascript/.lighthouseci. No artifacts will be uploaded. even though the treosh action's manifest output showed real files at that path: "htmlPath":"/home/runner/.../.lighthouseci/localhost-index_html-...html" Root cause: `.lighthouseci` is a dot-prefixed directory, and `actions/upload-artifact@v7` excludes hidden files by default. The docs: > include-hidden-files: If true, hidden files will be included in the > artifact. If false, hidden files will be excluded from the artifact. > Default: false This also explains why the previous run (b29aadaf1, with the same path but `if-no-files-found: ignore`) produced zero `lighthouse-*` artifacts on the workflow run \u2014 the dir was being skipped for the same reason, just silently. Fix: set `include-hidden-files: true` so the dot-prefixed parent dir doesn't cause everything inside to be excluded. Co-Authored-By: Claude claude-opus-4-5 --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f9ad07275131..40e5e242781f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1330,16 +1330,16 @@ jobs: LIGHTHOUSE_URL: 'http://localhost:3000/' - name: Upload Lighthouse results - # `resultsPath` points at the .lighthouseci/ directory inside the treosh action's - # own working directory (NOT our repo root — lhci's cwd when the action invokes it - # is the action's checkout). Use it explicitly so upload-artifact finds the real - # files. Without this, `path: .lighthouseci/` silently finds nothing in our repo - # root and `if-no-files-found: ignore` makes the step pass with zero uploaded. + # `resultsPath` is the absolute path to the .lighthouseci/ output dir written by + # `lhci collect`. The dir is dot-prefixed, and actions/upload-artifact@v7 excludes + # hidden files / dot-prefixed dirs by default — so we must set + # `include-hidden-files: true` or every cell uploads zero files. uses: actions/upload-artifact@v7 if: always() && steps.lighthouse.outputs.resultsPath != '' with: name: lighthouse-${{ matrix.app }}-${{ matrix.mode }} path: ${{ steps.lighthouse.outputs.resultsPath }} + include-hidden-files: true retention-days: 7 # Fail loudly if the dir is empty — we already gated on resultsPath being set, # so an empty dir means lhci collect produced nothing, which is a real bug. From 9809ea1dd5ff29b601767a908cfa9179345ce726 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 08:53:45 +0200 Subject: [PATCH 16/29] fix(lighthouse-ci): attach default-browser listeners synchronously to avoid E2E race Commit 36c3f7676 wrapped default-browser/src/index.js in an async IIFE so the `@sentry/browser` import could be guarded behind `SENTRY_LIGHTHOUSE_MODE=no-sentry` and tree-shaken out of that Lighthouse build. As a side effect the click listeners on `#exception-button` and `#navigation-link` were also moved inside that IIFE, so they now attach only after `await import('@sentry/browser')` resolves. That created a race for the existing E2E tests: `page.goto('/')` is followed immediately by `exceptionButton.click()`, and the click can fire before the dynamic import resolves and the listener registers. Flagged as a HIGH-severity review-bot finding on PR #20850. Move the two listeners back to synchronous module-top-level so they're guaranteed attached before any user interaction. Only the Sentry init is left inside the async IIFE \u2014 it's still gated by `if (lighthouseMode !== 'no-sentry')` so the tree-shaking benefit is preserved. The thrown error from the exception button is still captured by Sentry's global `window.error` listener installed by `Sentry.init()` whenever the IIFE finishes running. --- .../default-browser/src/index.js | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/default-browser/src/index.js b/dev-packages/e2e-tests/test-applications/default-browser/src/index.js index a1018c5a0499..1288bfcfab09 100644 --- a/dev-packages/e2e-tests/test-applications/default-browser/src/index.js +++ b/dev-packages/e2e-tests/test-applications/default-browser/src/index.js @@ -1,7 +1,23 @@ const lighthouseMode = process.env.SENTRY_LIGHTHOUSE_MODE; -(async () => { - if (lighthouseMode !== 'no-sentry') { +// Event listeners are attached synchronously at module top-level so E2E tests that do +// `page.goto('/')` followed immediately by `button.click()` cannot race the dynamic +// `import('@sentry/browser')` below. The handlers don't depend on Sentry being +// initialized — Sentry's global error/transaction handlers attach via window-level +// listeners installed by `Sentry.init()` and pick up the thrown error regardless. +document.getElementById('exception-button').addEventListener('click', () => { + throw new Error('I am an error!'); +}); + +document.getElementById('navigation-link').addEventListener('click', () => { + document.getElementById('navigation-target').scrollIntoView({ behavior: 'smooth' }); +}); + +// Sentry is loaded via dynamic `import()` so the `no-sentry` Lighthouse build can +// tree-shake the SDK out completely. Wrapped in an async IIFE because top-level await +// isn't supported by the webpack target used for this app's bundle. +if (lighthouseMode !== 'no-sentry') { + void (async () => { const Sentry = await import('@sentry/browser'); const integrations = []; @@ -29,13 +45,5 @@ const lighthouseMode = process.env.SENTRY_LIGHTHOUSE_MODE; environment: 'qa', tunnel: 'http://localhost:3031', }); - } - - document.getElementById('exception-button').addEventListener('click', () => { - throw new Error('I am an error!'); - }); - - document.getElementById('navigation-link').addEventListener('click', () => { - document.getElementById('navigation-target').scrollIntoView({ behavior: 'smooth' }); - }); -})(); + })(); +} From 1549ea8d884f0cd0ff874c7987d1323957b6ef2d Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 08:53:54 +0200 Subject: [PATCH 17/29] chore(lighthouse-ci): trim matrix to instrumented apps only (default-browser, nextjs-16) The previous matrix listed nine apps but only two (default-browser, nextjs-16) actually branch on `SENTRY_LIGHTHOUSE_MODE` in their Sentry init code. The other seven (react-19, vue-3, svelte-5, sveltekit-2, astro-5, tanstackstart-react, nuxt-5) ran three identical builds for every mode \u2014 same SDK, same integrations \u2014 so the `\u0394 (SDK)` and `\u0394 (Features)` columns in the PR comment were just run-to-run measurement noise rather than real SDK overhead. Flagged as a MEDIUM-severity review-bot finding on PR #20850. Drop the seven uninstrumented apps from the matrix so the report only shows honest deltas. Matrix is now 2 apps \u00d7 3 modes = 6 cells. The follow-up todo TODO-aeab11f0 already exists to wire `SENTRY_LIGHTHOUSE_MODE` into the remaining apps; once an app is actually instrumented it can be added back here. react-router-7-spa is instrumented but stays out of the matrix until its Lighthouse NO_FCP failure is diagnosed. Also update the file-level docstring \u2014 it still claimed "14 apps \u00d7 3 modes = 42 cells" from the original planner output even though earlier commits had dropped the matrix down to nine apps. --- .../lighthouse-tests/lighthouse-matrix.mjs | 62 ++++--------------- 1 file changed, 11 insertions(+), 51 deletions(-) diff --git a/dev-packages/lighthouse-tests/lighthouse-matrix.mjs b/dev-packages/lighthouse-tests/lighthouse-matrix.mjs index c736c0ec7f35..d178c4bcc488 100644 --- a/dev-packages/lighthouse-tests/lighthouse-matrix.mjs +++ b/dev-packages/lighthouse-tests/lighthouse-matrix.mjs @@ -3,7 +3,15 @@ * * Outputs: `matrix=` to stdout (consumed by $GITHUB_OUTPUT in CI). * - * Matrix shape: 14 representative E2E apps × 3 Sentry feature modes = 42 cells. + * Matrix shape: 2 E2E apps × 3 Sentry feature modes = 6 cells (MVP scope). + * + * Only apps whose Sentry init code actually branches on SENTRY_LIGHTHOUSE_MODE are + * included here. Adding an app without that wiring produces three identical builds + * — same SDK, same integrations — so the `Δ (SDK)` and `Δ (Features)` columns in + * the PR comment become noise. The follow-up todo `TODO-aeab11f0` tracks instrumenting + * react-19, vue-3, svelte-5, sveltekit-2, astro-5, tanstackstart-react, and nuxt-5 so + * they can be added back here. react-router-7-spa is also instrumented but currently + * fails Lighthouse with NO_FCP — kept out of the matrix until that's diagnosed. * * Modes: * no-sentry — app built without any Sentry SDK (baseline) @@ -29,49 +37,11 @@ */ /** @type {AppDefinition[]} */ -// NOTE: angular, remix, ember, solidstart, and react-router-7-spa were -// intentionally excluded from this matrix — their build/serve setups need -// framework-specific tuning we're not investing in for the MVP. They can be -// added back as a follow-up. (react-router-7-spa fails with NO_FCP in -// Lighthouse — the bundle loads but doesn't paint within the timeout.) const APPS = [ - // Plain webpack apps — read process.env directly (no bundler prefix). + // Plain webpack app — reads `process.env.SENTRY_LIGHTHOUSE_MODE` directly. { app: 'default-browser', sdk: 'browser', serve: 'static', staticDir: 'build', envVarName: 'SENTRY_LIGHTHOUSE_MODE' }, - { app: 'react-19', sdk: 'react', serve: 'static', staticDir: 'build', envVarName: 'SENTRY_LIGHTHOUSE_MODE' }, - - // Vite-based apps with `envPrefix: 'PUBLIC_'` (matches Sentry's repo convention for PUBLIC_E2E_TEST_DSN). - { app: 'vue-3', sdk: 'vue', serve: 'static', staticDir: 'dist', envVarName: 'PUBLIC_SENTRY_LIGHTHOUSE_MODE' }, - { app: 'svelte-5', sdk: 'svelte', serve: 'static', staticDir: 'dist', envVarName: 'PUBLIC_SENTRY_LIGHTHOUSE_MODE' }, - { - app: 'sveltekit-2', - sdk: 'sveltekit', - serve: 'server', - startCmd: 'node build', - readyPattern: 'localhost', - envVarName: 'PUBLIC_SENTRY_LIGHTHOUSE_MODE', - }, - { - app: 'astro-5', - sdk: 'astro', - serve: 'server', - // Astro's @astrojs/node adapter defaults to PORT=4321; force 3000 so Lighthouse can - // reach the server at the URL it audits (http://localhost:3000/). - startCmd: 'PORT=3000 node ./dist/server/entry.mjs', - readyPattern: 'localhost', - envVarName: 'PUBLIC_SENTRY_LIGHTHOUSE_MODE', - }, - // Vite-based apps using the default `VITE_` prefix (no custom envPrefix set). - { - app: 'tanstackstart-react', - sdk: 'tanstack-start', - serve: 'server', - startCmd: 'node --import ./.output/server/instrument.server.mjs .output/server/index.mjs', - readyPattern: 'localhost', - envVarName: 'VITE_SENTRY_LIGHTHOUSE_MODE', - }, - - // Next.js — only `NEXT_PUBLIC_*` env vars are exposed to client code. + // Next.js — reads `process.env.NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE` (client-exposed env var prefix). { app: 'nextjs-16', sdk: 'nextjs', @@ -80,16 +50,6 @@ const APPS = [ readyPattern: 'Ready in', envVarName: 'NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE', }, - - // Nuxt — only `NUXT_PUBLIC_*` env vars are exposed to client code (Nuxt convention). - { - app: 'nuxt-5', - sdk: 'nuxt', - serve: 'server', - startCmd: 'node .output/server/index.mjs', - readyPattern: 'Listening on', - envVarName: 'NUXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE', - }, ]; const MODES = /** @type {const} */ (['no-sentry', 'init-only', 'tracing-replay']); From 760b1288064005877489f6a52c4e2e5243feb2bd Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 09:11:32 +0200 Subject: [PATCH 18/29] fix(lighthouse-ci): run on every PR + nightly, drop label gate Two changes that simplify the trigger surface and fix the HIGH-severity review-bot finding on PR #20850 about the workflow-level `labeled` trigger: 1. Remove `labeled` from `pull_request.types` in build.yml. With it included, adding ANY label to a PR (not just `ci:lighthouse`) re-triggered the whole workflow, and the workflow-level `concurrency.cancel-in-progress: true` block cancelled the in-progress build/test/e2e jobs and restarted them from scratch. Labels are no longer a CI control surface here. 2. Drop the `contains(labels, 'ci:lighthouse')` check on `job_lighthouse_matrix`. Lighthouse now runs on every PR push the same way the other test jobs do, plus on the nightly schedule. Simpler mental model, no per-PR opt-in. The Lighthouse jobs are still NOT referenced in `job_required_jobs_passed.needs`, so they never block merges \u2014 only surface measurements via the PR comment and (after the next commit) the workflow run Job Summary. --- .github/workflows/build.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 40e5e242781f..65ca268e6687 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,7 +8,7 @@ on: - v8 - release/** pull_request: - types: [opened, synchronize, reopened, labeled] + types: [opened, synchronize, reopened] merge_group: types: [checks_requested] workflow_dispatch: @@ -1203,7 +1203,7 @@ jobs: if-no-files-found: ignore # ----------------------------------------------------------------------- - # Lighthouse CI — nightly + label-gated (ci:lighthouse) + # Lighthouse CI — runs on every PR and on the nightly schedule. # NOT in job_required_jobs_passed — never blocks merges. # ----------------------------------------------------------------------- @@ -1211,9 +1211,7 @@ jobs: name: Lighthouse Matrix needs: [job_get_metadata] runs-on: ubuntu-24.04 - if: | - github.event_name == 'schedule' || - (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:lighthouse')) + if: github.event_name == 'pull_request' || github.event_name == 'schedule' outputs: matrix: ${{ steps.matrix.outputs.matrix }} steps: From 8288dcbc41e12b8acc7a248f719faca4faaa2640 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 09:11:40 +0200 Subject: [PATCH 19/29] feat(lighthouse-ci): render report as Job Summary; sync APPS list to matrix Two changes to make Lighthouse data visible on every run, regardless of trigger: 1. Always write the report markdown to `$GITHUB_STEP_SUMMARY` via `core.summary.addRaw().write()`. The Job Summary renders as a styled panel at the top of the workflow run page \u2014 visible for PR runs, nightly runs, and manual dispatches alike. For PR runs the existing PR comment is still posted/updated; for nightly runs the Job Summary becomes the only output (was previously just `console.log` to the workflow log, which nobody reads). 2. Trim the hardcoded APPS list from 9 entries down to 2 (default-browser, nextjs-16). Only those two apps actually branch on `SENTRY_LIGHTHOUSE_MODE` today \u2014 the other seven were carried over from the original planner output but were never instrumented. With the stale 9-entry list, the matrix only produced 6 of 27 expected cells, tripping the 50%-fill safety check on every run and skipping the report entirely. The MEDIUM-severity bugbot finding called this out. When more apps gain `SENTRY_LIGHTHOUSE_MODE` support (tracked in TODO-aeab11f0), add them to both APPS arrays at the same time. --- .../lighthouse-tests/post-comment.mjs | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/dev-packages/lighthouse-tests/post-comment.mjs b/dev-packages/lighthouse-tests/post-comment.mjs index 83635c4e9c7f..a48f3aca3b6b 100644 --- a/dev-packages/lighthouse-tests/post-comment.mjs +++ b/dev-packages/lighthouse-tests/post-comment.mjs @@ -11,19 +11,13 @@ const MODES = ['no-sentry', 'init-only', 'tracing-replay']; * Apps and their human-readable SDK labels, matching lighthouse-matrix.mjs. * Order here determines row order in each table. */ -// Must mirror lighthouse-matrix.mjs APPS. angular, remix, ember, solidstart, -// and react-router-7-spa are intentionally excluded — see note in -// lighthouse-matrix.mjs. +// Must mirror the APPS array in lighthouse-matrix.mjs. Only apps whose Sentry init +// code actually branches on SENTRY_LIGHTHOUSE_MODE are listed here — listing +// uninstrumented apps would dilute the 50%-fill safety check below and produce +// meaningless Δ columns. See lighthouse-matrix.mjs for the full rationale. const APPS = [ { app: 'default-browser', sdk: 'browser' }, - { app: 'react-19', sdk: 'react' }, - { app: 'vue-3', sdk: 'vue' }, - { app: 'svelte-5', sdk: 'svelte' }, - { app: 'sveltekit-2', sdk: 'sveltekit' }, - { app: 'astro-5', sdk: 'astro' }, - { app: 'tanstackstart-react', sdk: 'tanstack-start' }, { app: 'nextjs-16', sdk: 'nextjs' }, - { app: 'nuxt-5', sdk: 'nuxt' }, ]; /** @@ -150,10 +144,14 @@ async function run() { const body = `${HEADING}\n\n${tables}${footer}`; + // Always render the report as a GitHub Actions Job Summary so it's visible on the + // workflow run page for every trigger (PR, nightly, dispatch). For PR runs we also + // post/update a sticky comment on the PR below. + await core.summary.addRaw(body).write(); + core.info('Wrote Lighthouse report to Job Summary.'); + if (!isPR || !prNumber) { - // Nightly / non-PR: log to stdout (captured in workflow logs) - // eslint-disable-next-line no-console - console.log(body); + // Nightly / non-PR: Job Summary above is the only output. Nothing to post. return; } From 7c24cff05156a3a7bf9af2aa7c24810190d86b8a Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 09:25:43 +0200 Subject: [PATCH 20/29] chore(lighthouse-ci): extract to standalone nightly workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the Lighthouse jobs out of build.yml into a new .github/workflows/lighthouse.yml that runs only on the nightly schedule and on workflow_dispatch. No pull_request trigger — per @mydea's review feedback on PR #20850, running on every PR was too noisy and the per-PR comment was unwanted. Concretely: - Delete job_lighthouse_matrix, job_lighthouse, job_lighthouse_report from build.yml. - Add .github/workflows/lighthouse.yml with three jobs that mirror the deleted set: * job_build does its own `yarn build:ci` + `yarn build:tarball` and uploads them as 'lighthouse-build-tarball-output' — fully decoupled from build.yml. * job_lighthouse matrix unchanged from before (2 apps × 3 modes = 6 cells on ubuntu-24.04-large-js with continue-on-error so a single bad cell doesn't tank the run). * job_lighthouse_report runs report.mjs and writes the result to the workflow Job Summary (added in the next commit). - Trim the build-time env-var prefix block down to the two prefixes we actually consume today (SENTRY_LIGHTHOUSE_MODE + NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE). PUBLIC_*, VITE_*, NUXT_PUBLIC_*, REACT_APP_* are gone — they implied broader coverage than we actually have. Same for the DSN block. - Use a unique concurrency group (lighthouse-${{ run_id }}) with cancel-in-progress: false — nightly runs are independent and shouldn't cancel each other. Adds ~5 min of self-build overhead per nightly run; nothing waits on it. --- .github/workflows/build.yml | 171 --------------- .github/workflows/lighthouse.yml | 202 ++++++++++++++++++ .../{post-comment.mjs => report.mjs} | 0 3 files changed, 202 insertions(+), 171 deletions(-) create mode 100644 .github/workflows/lighthouse.yml rename dev-packages/lighthouse-tests/{post-comment.mjs => report.mjs} (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 65ca268e6687..34e5c0be6d54 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1202,177 +1202,6 @@ jobs: retention-days: 7 if-no-files-found: ignore - # ----------------------------------------------------------------------- - # Lighthouse CI — runs on every PR and on the nightly schedule. - # NOT in job_required_jobs_passed — never blocks merges. - # ----------------------------------------------------------------------- - - job_lighthouse_matrix: - name: Lighthouse Matrix - needs: [job_get_metadata] - runs-on: ubuntu-24.04 - if: github.event_name == 'pull_request' || github.event_name == 'schedule' - outputs: - matrix: ${{ steps.matrix.outputs.matrix }} - steps: - - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) - uses: actions/checkout@v6 - with: - ref: ${{ env.HEAD_COMMIT }} - - name: Set up Node - uses: actions/setup-node@v6 - with: - node-version-file: 'package.json' - - name: Generate matrix - id: matrix - # Script prints `matrix=` to stdout; redirect to $GITHUB_OUTPUT so downstream - # jobs can pick it up via ${{ needs.job_lighthouse_matrix.outputs.matrix }}. - run: node dev-packages/lighthouse-tests/lighthouse-matrix.mjs >> "$GITHUB_OUTPUT" - - job_lighthouse: - name: Lighthouse ${{ matrix.app }} (${{ matrix.mode }}) - needs: [job_get_metadata, job_build, job_build_tarballs, job_lighthouse_matrix] - if: always() && needs.job_build_tarballs.result == 'success' && needs.job_lighthouse_matrix.result == 'success' - runs-on: ubuntu-24.04-large-js - timeout-minutes: 20 - continue-on-error: true - strategy: - fail-fast: false - max-parallel: 15 - matrix: ${{ fromJson(needs.job_lighthouse_matrix.outputs.matrix) }} - env: - # Mirrors job_e2e_tests env block — apps expect framework-specific DSN prefixes. - E2E_TEST_DSN: 'https://username@domain/123' - NEXT_PUBLIC_E2E_TEST_DSN: 'https://username@domain/123' - PUBLIC_E2E_TEST_DSN: 'https://username@domain/123' - REACT_APP_E2E_TEST_DSN: 'https://username@domain/123' - VITE_E2E_TEST_DSN: 'https://username@domain/123' - NUXT_PUBLIC_E2E_TEST_DSN: 'https://username@domain/123' - steps: - - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) - uses: actions/checkout@v6 - with: - ref: ${{ env.HEAD_COMMIT }} - - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 - with: - version: 9.15.9 - - name: Set up Node - uses: actions/setup-node@v6 - with: - node-version-file: 'dev-packages/e2e-tests/test-applications/${{ matrix.app }}/package.json' - - name: Restore caches - uses: ./.github/actions/restore-cache - with: - dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - - - name: Restore tarball artifacts - uses: actions/download-artifact@v7 - with: - name: build-tarball-output - path: ${{ env.TARBALL_ARTIFACT_DOWNLOAD_PATH }} - - - name: Prepare E2E tests - run: yarn test:prepare - working-directory: dev-packages/e2e-tests - - - name: Validate E2E tests setup - run: yarn test:validate - working-directory: dev-packages/e2e-tests - - - name: Copy app to temp - run: yarn ci:copy-to-temp ./test-applications/${{ matrix.app }} ${{ runner.temp }}/test-application - working-directory: dev-packages/e2e-tests - - - name: Add pnpm overrides - run: - yarn ci:pnpm-overrides ${{ runner.temp }}/test-application ${{ github.workspace - }}/dev-packages/e2e-tests/packed - working-directory: dev-packages/e2e-tests - - - name: Build E2E app for Lighthouse - working-directory: ${{ runner.temp }}/test-application - timeout-minutes: 7 - run: pnpm test:build - env: - SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} - # Set under every common bundler env prefix so each app reads whichever its bundler exposes - # (matrix.env-var-name is informational; we set all common prefixes here for simplicity). - SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} - NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} - PUBLIC_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} - VITE_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} - NUXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} - REACT_APP_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} - - - name: Run Lighthouse CI - id: lighthouse - # Pinned to a commit SHA (rather than the floating @v12 tag) to lock down the - # supply chain: this third-party action also npm-installs @lhci/cli, so an - # upstream tag-repoint could otherwise change what runs on our runners. - # SHA is the current tip of v12. Bump deliberately when upgrading. - uses: treosh/lighthouse-ci-action@3e7e23fb74242897f95c0ba9cabad3d0227b9b18 # v12 - with: - configPath: ${{ github.workspace }}/dev-packages/lighthouse-tests/lighthouserc.cjs - # Do NOT enable uploadArtifacts here — it would upload `.lighthouseci/` under - # the default name `lighthouse-reports` for every cell, causing a 409 conflict - # on every matrix cell after the first one. We do our own per-cell upload - # below with a unique name (lighthouse--). - temporaryPublicStorage: true - runs: 5 - env: - # lighthouserc.cjs branches on these env vars at config-load time. - LIGHTHOUSE_SERVE_MODE: ${{ matrix.serve }} - LIGHTHOUSE_STATIC_DIR: ${{ runner.temp }}/test-application/${{ matrix.static-dir }} - LIGHTHOUSE_START_CMD: cd ${{ runner.temp }}/test-application && ${{ matrix.start-cmd }} - LIGHTHOUSE_READY_PATTERN: ${{ matrix.ready-pattern }} - LIGHTHOUSE_URL: 'http://localhost:3000/' - - - name: Upload Lighthouse results - # `resultsPath` is the absolute path to the .lighthouseci/ output dir written by - # `lhci collect`. The dir is dot-prefixed, and actions/upload-artifact@v7 excludes - # hidden files / dot-prefixed dirs by default — so we must set - # `include-hidden-files: true` or every cell uploads zero files. - uses: actions/upload-artifact@v7 - if: always() && steps.lighthouse.outputs.resultsPath != '' - with: - name: lighthouse-${{ matrix.app }}-${{ matrix.mode }} - path: ${{ steps.lighthouse.outputs.resultsPath }} - include-hidden-files: true - retention-days: 7 - # Fail loudly if the dir is empty — we already gated on resultsPath being set, - # so an empty dir means lhci collect produced nothing, which is a real bug. - if-no-files-found: error - - job_lighthouse_report: - name: Lighthouse Report - needs: [job_get_metadata, job_build, job_lighthouse, job_lighthouse_matrix] - if: always() && needs.job_lighthouse_matrix.result == 'success' - runs-on: ubuntu-24.04 - steps: - - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) - uses: actions/checkout@v6 - with: - ref: ${{ env.HEAD_COMMIT }} - - name: Set up Node - uses: actions/setup-node@v6 - with: - node-version-file: 'package.json' - - name: Restore caches - uses: ./.github/actions/restore-cache - with: - dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - - name: Download all Lighthouse artifacts - uses: actions/download-artifact@v7 - with: - pattern: lighthouse-* - path: lighthouse-results/ - - name: Generate report - run: node dev-packages/lighthouse-tests/post-comment.mjs - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - IS_PR: ${{ github.event_name == 'pull_request' }} - PR_NUMBER: ${{ github.event.pull_request.number }} - job_required_jobs_passed: name: All required jobs passed or were skipped needs: diff --git a/.github/workflows/lighthouse.yml b/.github/workflows/lighthouse.yml new file mode 100644 index 000000000000..58b9eabedd04 --- /dev/null +++ b/.github/workflows/lighthouse.yml @@ -0,0 +1,202 @@ +name: 'Nightly: Lighthouse' + +# Standalone workflow that runs Lighthouse audits across instrumented E2E test apps +# on a nightly schedule (and on demand via workflow_dispatch). Intentionally NOT +# wired to pull_request triggers — the per-PR comment was too noisy to keep on +# every push, so the report now surfaces only as a Job Summary on each workflow +# run page. Never blocks merges (not referenced by build.yml's required-jobs gate). + +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +# Independent concurrency group from build.yml so nothing in this workflow can +# cancel or queue against the main CI pipeline. +concurrency: + group: lighthouse-${{ github.run_id }} + cancel-in-progress: false + +env: + CACHED_DEPENDENCY_PATHS: | + ${{ github.workspace }}/node_modules + ${{ github.workspace }}/packages/*/node_modules + ${{ github.workspace }}/dev-packages/*/node_modules + ~/.cache/mongodb-binaries/ + TARBALL_ARTIFACT_GLOB: packages/*/*.tgz + TARBALL_ARTIFACT_DOWNLOAD_PATH: ${{ github.workspace }}/packages + DISABLE_V8_COMPILE_CACHE: '1' + +jobs: + job_build: + name: Build SDK + Generate Matrix + runs-on: ubuntu-24.04 + timeout-minutes: 20 + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + dependency_cache_key: ${{ steps.install_dependencies.outputs.cache_key }} + steps: + - name: Check out current commit + uses: actions/checkout@v6 + + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version-file: 'package.json' + + - name: Install Dependencies + uses: ./.github/actions/install-dependencies + id: install_dependencies + + - name: Build packages + run: yarn build:ci + + - name: Build tarballs + run: yarn build:tarball + + - name: Upload tarball artifacts + uses: actions/upload-artifact@v7 + with: + name: lighthouse-build-tarball-output + path: ${{ env.TARBALL_ARTIFACT_GLOB }} + if-no-files-found: error + retention-days: 4 + compression-level: 6 + overwrite: true + + - name: Generate matrix + id: matrix + # Script prints `matrix=` to stdout; redirect to $GITHUB_OUTPUT so the + # downstream matrix job can pick it up via needs.job_build.outputs.matrix. + run: node dev-packages/lighthouse-tests/lighthouse-matrix.mjs >> "$GITHUB_OUTPUT" + + job_lighthouse: + name: Lighthouse ${{ matrix.app }} (${{ matrix.mode }}) + needs: [job_build] + runs-on: ubuntu-24.04-large-js + timeout-minutes: 20 + # One bad cell shouldn't tank the rest of the matrix — the Report job marks + # missing/empty cells in the Job Summary. + continue-on-error: true + strategy: + fail-fast: false + max-parallel: 15 + matrix: ${{ fromJson(needs.job_build.outputs.matrix) }} + env: + # Only the two prefixes the currently-matrixed apps actually consume: + # - SENTRY_LIGHTHOUSE_MODE → default-browser (plain webpack) + # - NEXT_PUBLIC_* → nextjs-16 (Next.js client-side env exposure) + # If more frameworks are added to the matrix later, add the corresponding + # prefix here. (Tracked in TODO-aeab11f0.) + E2E_TEST_DSN: 'https://username@domain/123' + NEXT_PUBLIC_E2E_TEST_DSN: 'https://username@domain/123' + steps: + - name: Check out current commit + uses: actions/checkout@v6 + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + with: + version: 9.15.9 + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version-file: 'dev-packages/e2e-tests/test-applications/${{ matrix.app }}/package.json' + - name: Restore caches + uses: ./.github/actions/restore-cache + with: + dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} + + - name: Restore tarball artifacts + uses: actions/download-artifact@v7 + with: + name: lighthouse-build-tarball-output + path: ${{ env.TARBALL_ARTIFACT_DOWNLOAD_PATH }} + + - name: Prepare E2E tests + run: yarn test:prepare + working-directory: dev-packages/e2e-tests + + - name: Validate E2E tests setup + run: yarn test:validate + working-directory: dev-packages/e2e-tests + + - name: Copy app to temp + run: yarn ci:copy-to-temp ./test-applications/${{ matrix.app }} ${{ runner.temp }}/test-application + working-directory: dev-packages/e2e-tests + + - name: Add pnpm overrides + run: + yarn ci:pnpm-overrides ${{ runner.temp }}/test-application ${{ github.workspace + }}/dev-packages/e2e-tests/packed + working-directory: dev-packages/e2e-tests + + - name: Build E2E app for Lighthouse + working-directory: ${{ runner.temp }}/test-application + timeout-minutes: 7 + run: pnpm test:build + env: + SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} + SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} + NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} + + - name: Run Lighthouse CI + id: lighthouse + # Pinned to a commit SHA (rather than the floating @v12 tag) to lock down the + # supply chain: this third-party action also npm-installs @lhci/cli, so an + # upstream tag-repoint could otherwise change what runs on our runners. + # SHA is the current tip of v12. Bump deliberately when upgrading. + uses: treosh/lighthouse-ci-action@3e7e23fb74242897f95c0ba9cabad3d0227b9b18 # v12 + with: + configPath: ${{ github.workspace }}/dev-packages/lighthouse-tests/lighthouserc.cjs + # Do NOT enable uploadArtifacts here — it would upload `.lighthouseci/` under + # the default name `lighthouse-reports` for every cell, causing a 409 conflict + # on every matrix cell after the first one. We do our own per-cell upload + # below with a unique name (lighthouse--). + temporaryPublicStorage: true + runs: 5 + env: + # lighthouserc.cjs branches on these env vars at config-load time. + LIGHTHOUSE_SERVE_MODE: ${{ matrix.serve }} + LIGHTHOUSE_STATIC_DIR: ${{ runner.temp }}/test-application/${{ matrix.static-dir }} + LIGHTHOUSE_START_CMD: cd ${{ runner.temp }}/test-application && ${{ matrix.start-cmd }} + LIGHTHOUSE_READY_PATTERN: ${{ matrix.ready-pattern }} + LIGHTHOUSE_URL: 'http://localhost:3000/' + + - name: Upload Lighthouse results + # `resultsPath` is the absolute path to the .lighthouseci/ output dir written by + # `lhci collect`. The dir is dot-prefixed, and actions/upload-artifact@v7 excludes + # hidden files / dot-prefixed dirs by default — so we must set + # `include-hidden-files: true` or every cell uploads zero files. + uses: actions/upload-artifact@v7 + if: always() && steps.lighthouse.outputs.resultsPath != '' + with: + name: lighthouse-${{ matrix.app }}-${{ matrix.mode }} + path: ${{ steps.lighthouse.outputs.resultsPath }} + include-hidden-files: true + retention-days: 7 + # Fail loudly if the dir is empty — we already gated on resultsPath being set, + # so an empty dir means lhci collect produced nothing, which is a real bug. + if-no-files-found: error + + job_lighthouse_report: + name: Lighthouse Report + needs: [job_build, job_lighthouse] + if: always() && needs.job_build.result == 'success' + runs-on: ubuntu-24.04 + steps: + - name: Check out current commit + uses: actions/checkout@v6 + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version-file: 'package.json' + - name: Restore caches + uses: ./.github/actions/restore-cache + with: + dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} + - name: Download all Lighthouse artifacts + uses: actions/download-artifact@v7 + with: + pattern: lighthouse-* + path: lighthouse-results/ + - name: Generate report + run: node dev-packages/lighthouse-tests/report.mjs diff --git a/dev-packages/lighthouse-tests/post-comment.mjs b/dev-packages/lighthouse-tests/report.mjs similarity index 100% rename from dev-packages/lighthouse-tests/post-comment.mjs rename to dev-packages/lighthouse-tests/report.mjs From 06f87a5f73d3c2fd446fd871dc1df36d0470bb54 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 09:25:58 +0200 Subject: [PATCH 21/29] refactor(lighthouse-ci): drop PR-commenting logic from report script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename post-comment.mjs to report.mjs and remove all PR-commenting code paths (octokit, listComments + find-and-update, GITHUB_TOKEN handling, fork-403 fallback, IS_PR / PR_NUMBER env vars). The Lighthouse workflow no longer runs on pull_request triggers (see the previous commit), so this code was dead. The script now does one thing: read .lighthouseci/ artifacts, build the markdown report, and write it to $GITHUB_STEP_SUMMARY via core.summary. That panel renders at the top of the workflow run page and is the workflow's only public output. Also drop the @actions/github dependency from dev-packages/lighthouse-tests/package.json — it was only used by the deleted octokit calls. yarn.lock retains an orphaned "@actions/github@^5.0.0" entry that the existing 9.x usage in other gh-action packages doesn't touch; left for natural cleanup on the next clean install. --- dev-packages/lighthouse-tests/package.json | 1 - dev-packages/lighthouse-tests/report.mjs | 66 ++-------------------- 2 files changed, 4 insertions(+), 63 deletions(-) diff --git a/dev-packages/lighthouse-tests/package.json b/dev-packages/lighthouse-tests/package.json index b37aed9c9af9..e1dc154836f3 100644 --- a/dev-packages/lighthouse-tests/package.json +++ b/dev-packages/lighthouse-tests/package.json @@ -11,7 +11,6 @@ }, "dependencies": { "@actions/core": "1.10.1", - "@actions/github": "^5.0.0", "markdown-table": "3.0.3" } } diff --git a/dev-packages/lighthouse-tests/report.mjs b/dev-packages/lighthouse-tests/report.mjs index a48f3aca3b6b..3e7e20a6afa6 100644 --- a/dev-packages/lighthouse-tests/report.mjs +++ b/dev-packages/lighthouse-tests/report.mjs @@ -1,16 +1,11 @@ import { promises as fs } from 'node:fs'; import path from 'node:path'; import * as core from '@actions/core'; -import { context, getOctokit } from '@actions/github'; import { markdownTable } from 'markdown-table'; const HEADING = '## 🔦 Lighthouse Report'; const MODES = ['no-sentry', 'init-only', 'tracing-replay']; -/** - * Apps and their human-readable SDK labels, matching lighthouse-matrix.mjs. - * Order here determines row order in each table. - */ // Must mirror the APPS array in lighthouse-matrix.mjs. Only apps whose Sentry init // code actually branches on SENTRY_LIGHTHOUSE_MODE are listed here — listing // uninstrumented apps would dilute the 50%-fill safety check below and produce @@ -108,8 +103,6 @@ function buildSectionTable(rows, metric, unit) { async function run() { const resultsDir = process.env.LIGHTHOUSE_RESULTS_DIR || 'lighthouse-results'; - const isPR = process.env.IS_PR === 'true'; - const prNumber = process.env.PR_NUMBER ? Number(process.env.PR_NUMBER) : undefined; const rows = []; let totalCells = 0; @@ -126,7 +119,7 @@ async function run() { } if (totalCells > 0 && filledCells / totalCells < 0.5) { - core.warning(`Only ${filledCells}/${totalCells} Lighthouse cells have results (< 50%). Skipping comment.`); + core.warning(`Only ${filledCells}/${totalCells} Lighthouse cells have results (< 50%). Skipping report.`); return; } @@ -144,62 +137,11 @@ async function run() { const body = `${HEADING}\n\n${tables}${footer}`; - // Always render the report as a GitHub Actions Job Summary so it's visible on the - // workflow run page for every trigger (PR, nightly, dispatch). For PR runs we also - // post/update a sticky comment on the PR below. + // Render the report as a GitHub Actions Job Summary so it's visible on the workflow + // run page. This is the workflow's only public output — there is intentionally no + // PR comment (the workflow doesn't run on PRs; see lighthouse.yml). await core.summary.addRaw(body).write(); core.info('Wrote Lighthouse report to Job Summary.'); - - if (!isPR || !prNumber) { - // Nightly / non-PR: Job Summary above is the only output. Nothing to post. - return; - } - - const token = process.env.GITHUB_TOKEN; - if (!token) { - core.warning('GITHUB_TOKEN not set — cannot post PR comment.'); - return; - } - - const octokit = getOctokit(token); - const repo = context.repo; - - // Find existing Lighthouse comment to update (mirror size-limit-gh-action pattern) - const { data: comments } = await octokit.rest.issues.listComments({ - ...repo, - issue_number: prNumber, - }); - const existing = comments.find(c => c.body?.startsWith(HEADING)); - - try { - if (existing) { - await octokit.rest.issues.updateComment({ - ...repo, - comment_id: existing.id, - body, - }); - core.info('Updated existing Lighthouse comment.'); - } else { - await octokit.rest.issues.createComment({ - ...repo, - issue_number: prNumber, - body, - }); - core.info('Created Lighthouse PR comment.'); - } - } catch (err) { - if (err.status === 403) { - // Fork PRs: GITHUB_TOKEN is read-only. Log the table to the workflow log so the - // data is still discoverable, and exit 0 so the job doesn't fail. - core.warning( - 'Could not post PR comment (403 Forbidden). This is expected for fork PRs where GITHUB_TOKEN is read-only.', - ); - // eslint-disable-next-line no-console - console.log(`\n${body}`); - return; - } - throw err; - } } run().catch(err => { From bb5ee63bddcbf7b81b026d5a3a547afca472abd0 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 11:27:01 +0200 Subject: [PATCH 22/29] feat(react-19-e2e): add SENTRY_LIGHTHOUSE_MODE instrumentation Mirrors the dynamic-import pattern from default-browser/src/index.js so the react-19 E2E test app can serve as a third (CRA) cell in the Lighthouse matrix alongside default-browser (webpack) and nextjs-16 (Next.js SSR). CRA exposes only env vars prefixed REACT_APP_*, so the gate is process.env.REACT_APP_SENTRY_LIGHTHOUSE_MODE. Three branches: - 'no-sentry': sync render with no Sentry import; CRA dead-code-eliminates the @sentry/react chunk entirely (the dynamic import in the else branch becomes unreachable when the env var inlines to 'no-sentry'). - 'tracing-replay': Sentry.init + browserTracingIntegration + replayIntegration. - 'init-only' / unset: Sentry.init with no integrations, error handlers attached to createRoot (preserves the existing E2E behavior when the env var is unset). The createRoot call moves inside the async IIFE in the else branch because the onUncaughtError / onCaughtError options need the dynamically-imported Sentry.reactErrorHandler. React handles its own scheduling so the extra microtask before render is invisible to Playwright tests. --- .../test-applications/react-19/src/index.tsx | 72 +++++++++++++------ 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/react-19/src/index.tsx b/dev-packages/e2e-tests/test-applications/react-19/src/index.tsx index 16209793ae07..365319374905 100644 --- a/dev-packages/e2e-tests/test-applications/react-19/src/index.tsx +++ b/dev-packages/e2e-tests/test-applications/react-19/src/index.tsx @@ -1,26 +1,58 @@ -import * as Sentry from '@sentry/react'; import React from 'react'; import ReactDOM from 'react-dom/client'; import Index from './pages/Index'; -Sentry.init({ - environment: 'qa', // dynamic sampling bias to keep transactions - dsn: process.env.REACT_APP_E2E_TEST_DSN, - release: 'e2e-test', - tunnel: 'http://localhost:3031/', // proxy server -}); +const lighthouseMode = process.env.REACT_APP_SENTRY_LIGHTHOUSE_MODE; -const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement, { - onUncaughtError: Sentry.reactErrorHandler((error, errorInfo) => { - console.warn(error, errorInfo); - }), - onCaughtError: Sentry.reactErrorHandler((error, errorInfo) => { - console.warn(error, errorInfo); - }), -}); +if (lighthouseMode === 'no-sentry') { + // No Sentry at all — sync render so webpack/CRA can fully dead-code-eliminate the + // @sentry/react import below. CRA inlines `process.env.REACT_APP_*` at build time, + // so this branch becomes the only one in the bundle when the env var is set. + const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement); + root.render( +
+ +
, + ); +} else { + // Dynamic-import Sentry so it lives in a separate chunk that webpack/CRA drops + // entirely from the no-sentry build above. Preserves existing E2E behavior when + // the env var is unset (init + error handlers, no tracing/replay). + void (async () => { + const Sentry = await import('@sentry/react'); -root.render( -
- -
, -); + const integrations: unknown[] = []; + if (lighthouseMode === 'tracing-replay') { + integrations.push(Sentry.browserTracingIntegration()); + integrations.push(Sentry.replayIntegration()); + } + + Sentry.init({ + environment: 'qa', // dynamic sampling bias to keep transactions + dsn: process.env.REACT_APP_E2E_TEST_DSN, + release: 'e2e-test', + tunnel: 'http://localhost:3031/', // proxy server + integrations: integrations as Parameters[0]['integrations'], + tracesSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : undefined, + replaysSessionSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : 0, + replaysOnErrorSampleRate: lighthouseMode === 'tracing-replay' ? 1.0 : 0, + }); + + const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement, { + onUncaughtError: Sentry.reactErrorHandler((error, errorInfo) => { + // oxlint-disable-next-line no-console + console.warn(error, errorInfo); + }), + onCaughtError: Sentry.reactErrorHandler((error, errorInfo) => { + // oxlint-disable-next-line no-console + console.warn(error, errorInfo); + }), + }); + + root.render( +
+ +
, + ); + })(); +} From df529c5cade48bf20c9d4f8d27635d794b2194aa Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 11:27:32 +0200 Subject: [PATCH 23/29] feat(lighthouse-ci): bundle test apps and upload to lighthouse.sentry.gg lab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the in-CI Lighthouse runs with a build-bundle-upload pipeline that ships prebuilt test apps to the dedicated Sentry Lighthouse lab service (https://lighthouse.sentry.gg). The lab runs Lighthouse on stable, dedicated hardware — eliminating the measurement noise we hit on shared GitHub-hosted runners — persists results, and ships every metric to Sentry under the lighthouse.* namespace. Companion to the spec at ~/Projects/sentry-lhci/docs/sentry-javascript-handoff.md. Concretely: - Rename dev-packages/lighthouse-tests/ to dev-packages/lighthouse-bundle/. The directory's job is no longer 'tests' — it's bundle preparation. - Delete the in-CI Lighthouse infrastructure: lighthouserc.cjs (lab owns the LHCI config now), report.mjs (Job Summary moves into the new script), and lighthouse-matrix.mjs (matrix is just a hardcoded constant in the upload script — no GitHub Actions matrix expansion needed anymore). - Add bundle-and-upload.mjs: zero-deps (Node 22 builtins only — fetch, FormData, Blob, system tar). For each (app, mode) cell, copies the app to a temp dir, applies the existing ci:pnpm-overrides helper, builds with the right SENTRY_LIGHTHOUSE_MODE / NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE / REACT_APP_SENTRY_LIGHTHOUSE_MODE env vars (every prefix set at once — each app's bundler picks up whichever it knows), and tars the result. Static cells (default-browser, react-19) bundle only their build/ dir. SSR cell (nextjs-16) strips node_modules + pnpm-lock.yaml, copies the packed SDK tgzs into ./packed/ inside the bundle, and rewrites every workspace-absolute file: path in package.json (dependencies, devDependencies, pnpm.overrides) to file:./packed/... so pnpm install can resolve them after the lab extracts the tarball. All 9 bundles are POSTed as one multipart request to /api/builds; the script then polls /api/builds/:id every 15s for up to 25 minutes and appends a markdown summary table to $GITHUB_STEP_SUMMARY with per-cell status, median score, and Lighthouse-report links. - Strip dependencies from dev-packages/lighthouse-bundle/package.json. The script only needs Node builtins. - Rewrite .github/workflows/lighthouse.yml. Two jobs: * job_build — SDK build + tarball generation (unchanged). * job_bundle_and_upload — single non-matrix job; runs the script after restoring tarballs and seeding dev-packages/e2e-tests/packed/ via yarn test:prepare. Per-cell parallelism gained nothing once Lighthouse moved off the runner. Requires two new repo secrets in getsentry/sentry-javascript: - LIGHTHOUSE_LAB_URL (https://lighthouse.sentry.gg) - LIGHTHOUSE_UPLOAD_TOKEN (mirror of the lab's Northflank UPLOAD_TOKEN) Triggers: nightly cron at 00:00 UTC plus workflow_dispatch for ad-hoc runs. --- .github/workflows/lighthouse.yml | 148 ++------ .../lighthouse-bundle/bundle-and-upload.mjs | 328 ++++++++++++++++++ dev-packages/lighthouse-bundle/package.json | 12 + .../lighthouse-tests/lighthouse-matrix.mjs | 76 ---- .../lighthouse-tests/lighthouserc.cjs | 57 --- dev-packages/lighthouse-tests/package.json | 16 - dev-packages/lighthouse-tests/report.mjs | 150 -------- 7 files changed, 374 insertions(+), 413 deletions(-) create mode 100644 dev-packages/lighthouse-bundle/bundle-and-upload.mjs create mode 100644 dev-packages/lighthouse-bundle/package.json delete mode 100644 dev-packages/lighthouse-tests/lighthouse-matrix.mjs delete mode 100644 dev-packages/lighthouse-tests/lighthouserc.cjs delete mode 100644 dev-packages/lighthouse-tests/package.json delete mode 100644 dev-packages/lighthouse-tests/report.mjs diff --git a/.github/workflows/lighthouse.yml b/.github/workflows/lighthouse.yml index 58b9eabedd04..f8f5aa6eedd9 100644 --- a/.github/workflows/lighthouse.yml +++ b/.github/workflows/lighthouse.yml @@ -1,18 +1,23 @@ name: 'Nightly: Lighthouse' -# Standalone workflow that runs Lighthouse audits across instrumented E2E test apps -# on a nightly schedule (and on demand via workflow_dispatch). Intentionally NOT -# wired to pull_request triggers — the per-PR comment was too noisy to keep on -# every push, so the report now surfaces only as a Job Summary on each workflow -# run page. Never blocks merges (not referenced by build.yml's required-jobs gate). +# Builds the instrumented Sentry test apps (default-browser, react-19, nextjs-16) +# in three Sentry feature modes (no-sentry, init-only, tracing-replay), tars each +# of the 9 cells, and uploads them to the Sentry Lighthouse lab at +# https://lighthouse.sentry.gg. +# +# The lab runs Lighthouse on dedicated hardware (no shared-runner flake), saves +# results to its own DB + dashboard, and ships metrics to Sentry. See the wire +# protocol in ~/Projects/sentry-lhci/docs/sentry-javascript-handoff.md. +# +# This workflow only builds + uploads — no Lighthouse runs on GitHub Actions. +# Never blocks merges (not in build.yml's required-jobs gate). on: schedule: - cron: '0 0 * * *' workflow_dispatch: -# Independent concurrency group from build.yml so nothing in this workflow can -# cancel or queue against the main CI pipeline. +# Independent of build.yml so nothing here can cancel or queue against main CI. concurrency: group: lighthouse-${{ github.run_id }} cancel-in-progress: false @@ -29,11 +34,10 @@ env: jobs: job_build: - name: Build SDK + Generate Matrix + name: Build SDK + Tarballs runs-on: ubuntu-24.04 timeout-minutes: 20 outputs: - matrix: ${{ steps.matrix.outputs.matrix }} dependency_cache_key: ${{ steps.install_dependencies.outputs.cache_key }} steps: - name: Check out current commit @@ -64,42 +68,33 @@ jobs: compression-level: 6 overwrite: true - - name: Generate matrix - id: matrix - # Script prints `matrix=` to stdout; redirect to $GITHUB_OUTPUT so the - # downstream matrix job can pick it up via needs.job_build.outputs.matrix. - run: node dev-packages/lighthouse-tests/lighthouse-matrix.mjs >> "$GITHUB_OUTPUT" - - job_lighthouse: - name: Lighthouse ${{ matrix.app }} (${{ matrix.mode }}) + job_bundle_and_upload: + name: Bundle test apps and upload to Lighthouse lab needs: [job_build] - runs-on: ubuntu-24.04-large-js - timeout-minutes: 20 - # One bad cell shouldn't tank the rest of the matrix — the Report job marks - # missing/empty cells in the Job Summary. - continue-on-error: true - strategy: - fail-fast: false - max-parallel: 15 - matrix: ${{ fromJson(needs.job_build.outputs.matrix) }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 env: - # Only the two prefixes the currently-matrixed apps actually consume: - # - SENTRY_LIGHTHOUSE_MODE → default-browser (plain webpack) - # - NEXT_PUBLIC_* → nextjs-16 (Next.js client-side env exposure) - # If more frameworks are added to the matrix later, add the corresponding - # prefix here. (Tracked in TODO-aeab11f0.) + LIGHTHOUSE_LAB_URL: ${{ secrets.LIGHTHOUSE_LAB_URL }} + LIGHTHOUSE_UPLOAD_TOKEN: ${{ secrets.LIGHTHOUSE_UPLOAD_TOKEN }} + # Synthetic DSNs — the apps only need *something* parseable at build time; + # nothing actually phones home (the Lighthouse run is what we care about). E2E_TEST_DSN: 'https://username@domain/123' NEXT_PUBLIC_E2E_TEST_DSN: 'https://username@domain/123' + PUBLIC_E2E_TEST_DSN: 'https://username@domain/123' + REACT_APP_E2E_TEST_DSN: 'https://username@domain/123' steps: - name: Check out current commit uses: actions/checkout@v6 + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: version: 9.15.9 + - name: Set up Node uses: actions/setup-node@v6 with: - node-version-file: 'dev-packages/e2e-tests/test-applications/${{ matrix.app }}/package.json' + node-version-file: 'package.json' + - name: Restore caches uses: ./.github/actions/restore-cache with: @@ -111,7 +106,7 @@ jobs: name: lighthouse-build-tarball-output path: ${{ env.TARBALL_ARTIFACT_DOWNLOAD_PATH }} - - name: Prepare E2E tests + - name: Prepare E2E tests (copy packed tgzs into dev-packages/e2e-tests/packed) run: yarn test:prepare working-directory: dev-packages/e2e-tests @@ -119,84 +114,9 @@ jobs: run: yarn test:validate working-directory: dev-packages/e2e-tests - - name: Copy app to temp - run: yarn ci:copy-to-temp ./test-applications/${{ matrix.app }} ${{ runner.temp }}/test-application - working-directory: dev-packages/e2e-tests - - - name: Add pnpm overrides - run: - yarn ci:pnpm-overrides ${{ runner.temp }}/test-application ${{ github.workspace - }}/dev-packages/e2e-tests/packed - working-directory: dev-packages/e2e-tests - - - name: Build E2E app for Lighthouse - working-directory: ${{ runner.temp }}/test-application - timeout-minutes: 7 - run: pnpm test:build - env: - SENTRY_E2E_WORKSPACE_ROOT: ${{ github.workspace }} - SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} - NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE: ${{ matrix.mode }} - - - name: Run Lighthouse CI - id: lighthouse - # Pinned to a commit SHA (rather than the floating @v12 tag) to lock down the - # supply chain: this third-party action also npm-installs @lhci/cli, so an - # upstream tag-repoint could otherwise change what runs on our runners. - # SHA is the current tip of v12. Bump deliberately when upgrading. - uses: treosh/lighthouse-ci-action@3e7e23fb74242897f95c0ba9cabad3d0227b9b18 # v12 - with: - configPath: ${{ github.workspace }}/dev-packages/lighthouse-tests/lighthouserc.cjs - # Do NOT enable uploadArtifacts here — it would upload `.lighthouseci/` under - # the default name `lighthouse-reports` for every cell, causing a 409 conflict - # on every matrix cell after the first one. We do our own per-cell upload - # below with a unique name (lighthouse--). - temporaryPublicStorage: true - runs: 5 - env: - # lighthouserc.cjs branches on these env vars at config-load time. - LIGHTHOUSE_SERVE_MODE: ${{ matrix.serve }} - LIGHTHOUSE_STATIC_DIR: ${{ runner.temp }}/test-application/${{ matrix.static-dir }} - LIGHTHOUSE_START_CMD: cd ${{ runner.temp }}/test-application && ${{ matrix.start-cmd }} - LIGHTHOUSE_READY_PATTERN: ${{ matrix.ready-pattern }} - LIGHTHOUSE_URL: 'http://localhost:3000/' - - - name: Upload Lighthouse results - # `resultsPath` is the absolute path to the .lighthouseci/ output dir written by - # `lhci collect`. The dir is dot-prefixed, and actions/upload-artifact@v7 excludes - # hidden files / dot-prefixed dirs by default — so we must set - # `include-hidden-files: true` or every cell uploads zero files. - uses: actions/upload-artifact@v7 - if: always() && steps.lighthouse.outputs.resultsPath != '' - with: - name: lighthouse-${{ matrix.app }}-${{ matrix.mode }} - path: ${{ steps.lighthouse.outputs.resultsPath }} - include-hidden-files: true - retention-days: 7 - # Fail loudly if the dir is empty — we already gated on resultsPath being set, - # so an empty dir means lhci collect produced nothing, which is a real bug. - if-no-files-found: error - - job_lighthouse_report: - name: Lighthouse Report - needs: [job_build, job_lighthouse] - if: always() && needs.job_build.result == 'success' - runs-on: ubuntu-24.04 - steps: - - name: Check out current commit - uses: actions/checkout@v6 - - name: Set up Node - uses: actions/setup-node@v6 - with: - node-version-file: 'package.json' - - name: Restore caches - uses: ./.github/actions/restore-cache - with: - dependency_cache_key: ${{ needs.job_build.outputs.dependency_cache_key }} - - name: Download all Lighthouse artifacts - uses: actions/download-artifact@v7 - with: - pattern: lighthouse-* - path: lighthouse-results/ - - name: Generate report - run: node dev-packages/lighthouse-tests/report.mjs + - name: Bundle every cell and upload to the lab + # bundle-and-upload.mjs handles the full 3 × 3 matrix in one process: + # copies each app to a temp dir, applies pnpm overrides, builds with the + # right SENTRY_LIGHTHOUSE_MODE env var, tars, POSTs all 9 bundles to the + # lab, polls until terminal, writes a Job Summary. + run: node dev-packages/lighthouse-bundle/bundle-and-upload.mjs diff --git a/dev-packages/lighthouse-bundle/bundle-and-upload.mjs b/dev-packages/lighthouse-bundle/bundle-and-upload.mjs new file mode 100644 index 000000000000..ffe69938d94c --- /dev/null +++ b/dev-packages/lighthouse-bundle/bundle-and-upload.mjs @@ -0,0 +1,328 @@ +/** + * Bundle the instrumented Sentry test apps for every (app, mode) cell, POST the + * tarballs to the Sentry Lighthouse lab (https://lighthouse.sentry.gg), poll + * until the lab finishes running Lighthouse, then write a Job Summary. + * + * This script runs inside .github/workflows/lighthouse.yml. It is the only + * script that talks to the lab — see ~/Projects/sentry-lhci/docs/sentry-javascript-handoff.md + * for the wire protocol. + * + * Zero runtime dependencies — uses Node 22 builtins (fetch, FormData, Blob) + * and the system `tar` available on every ubuntu runner. + */ + +/* eslint-disable no-console */ + +import { execSync } from 'node:child_process'; +import { copyFile, readdir, readFile, mkdir, rm, writeFile, appendFile } from 'node:fs/promises'; +import path from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; + +const LAB_URL = process.env.LIGHTHOUSE_LAB_URL; +const TOKEN = process.env.LIGHTHOUSE_UPLOAD_TOKEN; +if (!LAB_URL || !TOKEN) { + throw new Error('LIGHTHOUSE_LAB_URL and LIGHTHOUSE_UPLOAD_TOKEN must be set'); +} + +const WORKSPACE = process.env.GITHUB_WORKSPACE ?? process.cwd(); +const RUNNER_TEMP = process.env.RUNNER_TEMP ?? path.join(WORKSPACE, '.tmp'); +const PACKED_DIR = path.join(WORKSPACE, 'dev-packages/e2e-tests/packed'); +const E2E_DIR = path.join(WORKSPACE, 'dev-packages/e2e-tests'); + +/** + * The matrix. Adding an app here requires: + * 1. The test app reads `` and branches its Sentry init. + * 2. The lab's runner understands `serve` / `staticDir` / `startCmd` (it does). + */ +const APPS = [ + { + app: 'default-browser', + serve: 'static', + staticDir: 'build', + envVarName: 'SENTRY_LIGHTHOUSE_MODE', + }, + { + app: 'react-19', + serve: 'static', + staticDir: 'build', + envVarName: 'REACT_APP_SENTRY_LIGHTHOUSE_MODE', + }, + { + app: 'nextjs-16', + serve: 'server', + startCmd: 'pnpm start', + readyPattern: 'Ready in', + // Lab side: pnpm 9.15.9 is on the image via corepack. We strip the lockfile + // from the bundle (CI generates it with workspace-absolute override paths + // that don't survive the move to the lab), so --no-frozen-lockfile lets pnpm + // re-resolve from the rewritten package.json. --prefer-offline uses the + // lab's persistent pnpm store (/data/.pnpm-store). + installCmd: 'pnpm install --no-frozen-lockfile --prefer-offline', + envVarName: 'NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE', + }, +]; + +const MODES = ['no-sentry', 'init-only', 'tracing-replay']; + +async function run() { + // Fail fast if the lab is down before we waste minutes building bundles. + console.log(`Liveness check: ${LAB_URL}/healthz`); + const health = await fetch(`${LAB_URL}/healthz`); + if (!health.ok) throw new Error(`Lab healthcheck failed: ${health.status} ${await health.text()}`); + console.log('Lab is reachable.'); + + await mkdir(path.join(RUNNER_TEMP, 'bundles'), { recursive: true }); + const bundles = []; + + for (const def of APPS) { + for (const mode of MODES) { + const fieldName = `bundle-${bundles.length}`; + console.log(`\n=== Preparing ${def.app} (${mode}) → ${fieldName} ===`); + const bundle = await prepareCell(def, mode, fieldName); + bundles.push(bundle); + } + } + + console.log(`\n=== Uploading ${bundles.length} bundles to ${LAB_URL}/api/builds ===`); + const buildResp = await uploadBundles(bundles); + console.log(`Build queued: ${buildResp.buildId}`); + console.log(`Build URL: ${LAB_URL}${buildResp.buildUrl}`); + console.log(`Dashboard: ${LAB_URL}${buildResp.dashboardUrl}`); + + const final = await pollUntilDone(buildResp.buildId); + await writeJobSummary(final); + + // Surface failure to CI when the lab marks the build failed or any cell failed. + const failedCells = (final.cells ?? []).filter(c => c.status === 'failed'); + if (final.status === 'failed' || failedCells.length > 0) { + console.error(`Build status=${final.status}, ${failedCells.length} cell(s) failed.`); + process.exit(1); + } +} + +/** + * Build a single (app, mode) cell: + * 1. Copy the app to a unique temp dir (so concurrent cells don't collide). + * 2. Apply pnpm overrides (existing helper). + * 3. Run `pnpm test:build` with the right SENTRY_LIGHTHOUSE_MODE env vars. + * 4. For static cells, tar just the build dir. + * For SSR cells, copy packed tgzs into the bundle, rewrite package.json + * to use relative `file:./packed/...` paths, then tar (no node_modules). + * 5. Return cell metadata for the upload. + */ +async function prepareCell(def, mode, fieldName) { + const tempApp = path.join(RUNNER_TEMP, `app-${def.app}-${mode}`); + await rm(tempApp, { recursive: true, force: true }); + + // 1. Copy app to temp (fixes file: deps to workspace-absolute paths) + execSync(`yarn ci:copy-to-temp ./test-applications/${def.app} ${tempApp}`, { + cwd: E2E_DIR, + stdio: 'inherit', + }); + + // 2. Add pnpm overrides (workspace-absolute paths pointing at packed dir) + execSync(`yarn ci:pnpm-overrides ${tempApp} ${PACKED_DIR}`, { + cwd: E2E_DIR, + stdio: 'inherit', + }); + + // 3. Build with the right mode env var. We set all common bundler prefixes so each + // app's bundler picks up whichever variant it knows about — apps that don't read a + // prefix simply ignore extra vars. + execSync('pnpm test:build', { + cwd: tempApp, + stdio: 'inherit', + env: { + ...process.env, + SENTRY_E2E_WORKSPACE_ROOT: WORKSPACE, + SENTRY_LIGHTHOUSE_MODE: mode, + NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE: mode, + REACT_APP_SENTRY_LIGHTHOUSE_MODE: mode, + }, + }); + + const tarPath = path.join(RUNNER_TEMP, 'bundles', `${def.app}-${mode}.tar.gz`); + + if (def.serve === 'static') { + // Static cell — tar the build dir only. Lab serves it with a static HTTP server. + execSync(`tar -czf ${tarPath} -C ${tempApp} ${def.staticDir}`, { stdio: 'inherit' }); + const bytes = Number(execSync(`wc -c < ${tarPath}`, { encoding: 'utf8' }).trim()); + console.log(`Static bundle: ${tarPath} (${formatBytes(bytes)})`); + return { + fieldName, + tarPath, + cell: { + app: def.app, + mode, + bundleField: fieldName, + serve: 'static', + staticDir: def.staticDir, + }, + }; + } + + // SSR cell — prep for `pnpm install && pnpm start` on the lab. + await prepareSsrBundle(tempApp); + execSync( + `tar -czf ${tarPath} --exclude=node_modules --exclude=.git --exclude=pnpm-lock.yaml ` + + `-C ${path.dirname(tempApp)} ${path.basename(tempApp)}`, + { stdio: 'inherit' }, + ); + const bytes = Number(execSync(`wc -c < ${tarPath}`, { encoding: 'utf8' }).trim()); + console.log(`SSR bundle: ${tarPath} (${formatBytes(bytes)})`); + return { + fieldName, + tarPath, + cell: { + app: def.app, + mode, + bundleField: fieldName, + serve: 'server', + startCmd: def.startCmd, + readyPattern: def.readyPattern, + installCmd: def.installCmd, + }, + }; +} + +/** + * For SSR cells: copy the packed Sentry tarballs into the bundle and rewrite + * package.json deps + pnpm.overrides to relative `file:./packed/...` paths so + * the lab's `pnpm install` can resolve them from inside the extracted bundle. + * + * `node_modules` and `pnpm-lock.yaml` are stripped by the tar exclude list — + * the lab regenerates both. + */ +async function prepareSsrBundle(tempApp) { + // Copy packed tgz files into /packed/ + const inBundlePacked = path.join(tempApp, 'packed'); + await mkdir(inBundlePacked, { recursive: true }); + const entries = await readdir(PACKED_DIR); + for (const name of entries) { + if (!name.endsWith('.tgz')) continue; + await copyFile(path.join(PACKED_DIR, name), path.join(inBundlePacked, name)); + } + + // Rewrite all workspace-absolute `file:.../sentry-*-packed.tgz` references in + // package.json (in dependencies, devDependencies, pnpm.overrides) to point at + // `./packed/`. + const pkgPath = path.join(tempApp, 'package.json'); + const pkg = JSON.parse(await readFile(pkgPath, 'utf8')); + const rewrite = obj => { + if (!obj) return; + for (const [name, val] of Object.entries(obj)) { + const m = typeof val === 'string' ? val.match(/sentry-[a-z0-9-]+-packed\.tgz$/) : null; + if (m && (val.startsWith('file:') || val.startsWith('link:'))) { + obj[name] = `file:./packed/${m[0]}`; + } + } + }; + rewrite(pkg.dependencies); + rewrite(pkg.devDependencies); + rewrite(pkg.pnpm?.overrides); + await writeFile(pkgPath, JSON.stringify(pkg, null, 2)); +} + +/** + * POST the multipart form. Returns the parsed 202 response body. + */ +async function uploadBundles(bundles) { + const metadata = { + commit: process.env.GITHUB_SHA ?? 'unknown', + branch: process.env.GITHUB_REF_NAME ?? 'unknown', + triggeredBy: 'github-actions', + workflowRunUrl: + process.env.GITHUB_SERVER_URL && process.env.GITHUB_REPOSITORY && process.env.GITHUB_RUN_ID + ? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` + : undefined, + cells: bundles.map(b => b.cell), + }; + + const form = new FormData(); + form.append('metadata', JSON.stringify(metadata)); + for (const b of bundles) { + const buf = await readFile(b.tarPath); + form.append(b.fieldName, new Blob([buf], { type: 'application/gzip' }), path.basename(b.tarPath)); + } + + const res = await fetch(`${LAB_URL}/api/builds`, { + method: 'POST', + headers: { Authorization: `Bearer ${TOKEN}` }, + body: form, + }); + if (!res.ok) { + throw new Error(`Upload failed: ${res.status} ${await res.text()}`); + } + return res.json(); +} + +/** + * Poll GET /api/builds/:id every 15 seconds until status is terminal, or the + * 25-minute ceiling is reached. + */ +async function pollUntilDone(buildId) { + const deadline = Date.now() + 25 * 60 * 1000; + while (Date.now() < deadline) { + const r = await fetch(`${LAB_URL}/api/builds/${buildId}`); + if (!r.ok) { + console.warn(`Poll failed: ${r.status} ${await r.text()} — retrying`); + await sleep(15000); + continue; + } + const build = await r.json(); + const cells = build.cells ?? []; + const done = cells.filter(c => c.status === 'completed').length; + const failed = cells.filter(c => c.status === 'failed').length; + console.log(`status=${build.status} cells=${done + failed}/${cells.length} (${failed} failed)`); + if (build.status === 'completed' || build.status === 'failed') return build; + await sleep(15000); + } + throw new Error(`Build ${buildId} did not finish within 25 minutes`); +} + +/** + * Append a markdown table of the results to $GITHUB_STEP_SUMMARY so it renders + * on the workflow run page. Skipped when running locally. + */ +async function writeJobSummary(build) { + const out = process.env.GITHUB_STEP_SUMMARY; + if (!out) return; + const cells = build.cells ?? []; + const lines = [ + `## 🔦 Lighthouse — ${build.status}`, + '', + `- Build ID: \`${build.buildId}\``, + `- Commit: \`${build.commit ?? 'unknown'}\``, + `- Branch: \`${build.branch ?? 'unknown'}\``, + `- Dashboard: ${LAB_URL}${build.buildUrl ?? `/api/builds/${build.buildId}`}`, + '', + '| App | Mode | Status | Median score | Runs | Report |', + '| --- | --- | --- | --- | --- | --- |', + ]; + for (const c of cells) { + const runs = c.runs ?? []; + const scores = runs.map(r => r.performanceScore).filter(s => s != null); + const median = scores.length === 0 ? '—' : computeMedian(scores).toFixed(2); + const reportUrl = runs[0]?.reportUrl ? `[view](${LAB_URL}${runs[0].reportUrl})` : '—'; + lines.push(`| ${c.app} | ${c.mode} | ${c.status} | ${median} | ${runs.length} | ${reportUrl} |`); + } + lines.push(''); + await appendFile(out, `${lines.join('\n')}\n`); +} + +function computeMedian(xs) { + const sorted = [...xs].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; +} + +function formatBytes(n) { + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`; + return `${(n / 1024 / 1024).toFixed(1)} MB`; +} + +run().catch(err => { + console.error(err.stack || err.message); + process.exit(1); +}); diff --git a/dev-packages/lighthouse-bundle/package.json b/dev-packages/lighthouse-bundle/package.json new file mode 100644 index 000000000000..14480a2ebb34 --- /dev/null +++ b/dev-packages/lighthouse-bundle/package.json @@ -0,0 +1,12 @@ +{ + "name": "@sentry-internal/lighthouse-bundle", + "version": "0.0.0", + "private": true, + "type": "module", + "volta": { + "extends": "../../package.json" + }, + "scripts": { + "bundle-and-upload": "node bundle-and-upload.mjs" + } +} diff --git a/dev-packages/lighthouse-tests/lighthouse-matrix.mjs b/dev-packages/lighthouse-tests/lighthouse-matrix.mjs deleted file mode 100644 index d178c4bcc488..000000000000 --- a/dev-packages/lighthouse-tests/lighthouse-matrix.mjs +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Generates the GitHub Actions matrix for Lighthouse CI runs. - * - * Outputs: `matrix=` to stdout (consumed by $GITHUB_OUTPUT in CI). - * - * Matrix shape: 2 E2E apps × 3 Sentry feature modes = 6 cells (MVP scope). - * - * Only apps whose Sentry init code actually branches on SENTRY_LIGHTHOUSE_MODE are - * included here. Adding an app without that wiring produces three identical builds - * — same SDK, same integrations — so the `Δ (SDK)` and `Δ (Features)` columns in - * the PR comment become noise. The follow-up todo `TODO-aeab11f0` tracks instrumenting - * react-19, vue-3, svelte-5, sveltekit-2, astro-5, tanstackstart-react, and nuxt-5 so - * they can be added back here. react-router-7-spa is also instrumented but currently - * fails Lighthouse with NO_FCP — kept out of the matrix until that's diagnosed. - * - * Modes: - * no-sentry — app built without any Sentry SDK (baseline) - * init-only — Sentry.init() only, no additional integrations - * tracing-replay — Sentry.init() with browserTracingIntegration + replayIntegration - * - * Each E2E app reads SENTRY_LIGHTHOUSE_MODE at build time. Because bundlers expose - * env vars under different prefixes (NEXT_PUBLIC_*, PUBLIC_*, VITE_*, etc.), the - * workflow sets the var under every common prefix; each app reads whichever its - * bundler exposes. The `envVarName` field below is informational (used in the - * report) and documents which prefix the app code actually consumes. - */ - -/** - * @typedef {Object} AppDefinition - * @property {string} app - Directory name under dev-packages/e2e-tests/test-applications/ - * @property {string} sdk - Human-readable SDK label for reports - * @property {'static'|'server'} serve - How the built app is served by Lighthouse - * @property {string} [staticDir] - Relative path to built static assets (serve === 'static') - * @property {string} [startCmd] - Command to start the SSR server (serve === 'server') - * @property {string} [readyPattern] - Log pattern that signals server is ready (server only) - * @property {string} envVarName - Env var the app's Sentry init code reads at build time - */ - -/** @type {AppDefinition[]} */ -const APPS = [ - // Plain webpack app — reads `process.env.SENTRY_LIGHTHOUSE_MODE` directly. - { app: 'default-browser', sdk: 'browser', serve: 'static', staticDir: 'build', envVarName: 'SENTRY_LIGHTHOUSE_MODE' }, - - // Next.js — reads `process.env.NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE` (client-exposed env var prefix). - { - app: 'nextjs-16', - sdk: 'nextjs', - serve: 'server', - startCmd: 'pnpm start', - readyPattern: 'Ready in', - envVarName: 'NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE', - }, -]; - -const MODES = /** @type {const} */ (['no-sentry', 'init-only', 'tracing-replay']); - -const include = []; - -for (const appDef of APPS) { - for (const mode of MODES) { - include.push({ - app: appDef.app, - sdk: appDef.sdk, - 'app-dir': `dev-packages/e2e-tests/test-applications/${appDef.app}`, - mode, - serve: appDef.serve, - 'static-dir': appDef.staticDir ?? '', - 'start-cmd': appDef.startCmd ?? '', - 'ready-pattern': appDef.readyPattern ?? 'localhost', - 'env-var-name': appDef.envVarName, - }); - } -} - -// eslint-disable-next-line no-console -console.log(`matrix=${JSON.stringify({ include })}`); diff --git a/dev-packages/lighthouse-tests/lighthouserc.cjs b/dev-packages/lighthouse-tests/lighthouserc.cjs deleted file mode 100644 index a7391c4970d4..000000000000 --- a/dev-packages/lighthouse-tests/lighthouserc.cjs +++ /dev/null @@ -1,57 +0,0 @@ -// Lighthouse CI configuration for Sentry JavaScript SDK performance testing. -// Used by treosh/lighthouse-ci-action@v12 via the `configPath` input. -// Docs: https://github.com/GoogleChrome/lighthouse-ci/blob/main/docs/configuration.md -// -// IMPORTANT: This file MUST be CommonJS (`.cjs` extension, `module.exports`). -// LHCI's config loader uses `require()` to load the config file. The parent -// package has `"type": "module"`, so a `.js` file here would be treated as -// ESM by Node and `require()` would return `{ default: }` instead of -// `` — LHCI then fails with "Config missing top level 'ci' property". -// -// Per-cell environment variables (set by the GitHub Actions workflow): -// LIGHTHOUSE_SERVE_MODE - 'static' | 'server' -// LIGHTHOUSE_STATIC_DIR - absolute path to static dist dir (when serve mode = 'static') -// LIGHTHOUSE_START_CMD - shell command to start the server (when serve mode = 'server') -// LIGHTHOUSE_READY_PATTERN - server-ready log pattern (default: 'localhost') -// LIGHTHOUSE_URL - URL to audit (default: http://localhost:3000/) - -const isServer = process.env.LIGHTHOUSE_SERVE_MODE === 'server'; - -module.exports = { - ci: { - collect: { - // Median of 5 runs halves variance vs a single run (per Lighthouse variability docs). - numberOfRuns: 5, - ...(isServer - ? { - startServerCommand: process.env.LIGHTHOUSE_START_CMD, - startServerReadyPattern: process.env.LIGHTHOUSE_READY_PATTERN || 'localhost', - startServerReadyTimeout: 30000, - url: [process.env.LIGHTHOUSE_URL || 'http://localhost:3000/'], - } - : { - staticDistDir: process.env.LIGHTHOUSE_STATIC_DIR, - }), - settings: { - // Simulated throttling (LHCI default) — more deterministic on shared CI runners - // than DevTools throttling. Do NOT switch to 'devtools' without dedicated hardware. - chromeFlags: ['--no-sandbox', '--headless=new'], - // Only measure performance — skip accessibility/SEO/PWA/best-practices for now. - // These can be added later once the performance signal is stable. - onlyCategories: ['performance'], - }, - }, - assert: { - // Warn-only — Lighthouse never blocks PR merges (ISC-A-1). - // Floor is set very low (0.5) so we only catch catastrophic regressions while we - // measure baseline variance. Tighten after 30+ days of nightly data. - assertions: { - 'categories:performance': ['warn', { minScore: 0.5, aggregationMethod: 'median-run' }], - }, - }, - upload: { - // 7-day retention; zero infra required. Links appear in action output and the PR comment. - target: 'temporary-public-storage', - }, - }, -}; diff --git a/dev-packages/lighthouse-tests/package.json b/dev-packages/lighthouse-tests/package.json deleted file mode 100644 index e1dc154836f3..000000000000 --- a/dev-packages/lighthouse-tests/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "@sentry-internal/lighthouse-tests", - "version": "0.0.0", - "private": true, - "type": "module", - "volta": { - "extends": "../../package.json" - }, - "scripts": { - "generate-matrix": "node lighthouse-matrix.mjs" - }, - "dependencies": { - "@actions/core": "1.10.1", - "markdown-table": "3.0.3" - } -} diff --git a/dev-packages/lighthouse-tests/report.mjs b/dev-packages/lighthouse-tests/report.mjs deleted file mode 100644 index 3e7e20a6afa6..000000000000 --- a/dev-packages/lighthouse-tests/report.mjs +++ /dev/null @@ -1,150 +0,0 @@ -import { promises as fs } from 'node:fs'; -import path from 'node:path'; -import * as core from '@actions/core'; -import { markdownTable } from 'markdown-table'; - -const HEADING = '## 🔦 Lighthouse Report'; -const MODES = ['no-sentry', 'init-only', 'tracing-replay']; - -// Must mirror the APPS array in lighthouse-matrix.mjs. Only apps whose Sentry init -// code actually branches on SENTRY_LIGHTHOUSE_MODE are listed here — listing -// uninstrumented apps would dilute the 50%-fill safety check below and produce -// meaningless Δ columns. See lighthouse-matrix.mjs for the full rationale. -const APPS = [ - { app: 'default-browser', sdk: 'browser' }, - { app: 'nextjs-16', sdk: 'nextjs' }, -]; - -/** - * Metrics surfaced in the report. Lighthouse's category score is too coarse on its own - * (fast static apps cap at 100 across all modes), so we also report the underlying - * metric values. LCP is the primary regression indicator for SDK overhead; TBT captures - * runtime cost of instrumentation; total bytes captures download cost. - */ -const SECTIONS = [ - { label: 'Performance score', metric: 'score', unit: '', betterIs: 'higher' }, - { label: 'Largest Contentful Paint (LCP)', metric: 'lcp', unit: ' ms', betterIs: 'lower' }, - { label: 'Total Blocking Time (TBT)', metric: 'tbt', unit: ' ms', betterIs: 'lower' }, - { label: 'Bytes downloaded', metric: 'bytes', unit: ' KB', betterIs: 'lower' }, -]; - -/** - * Read the median-run LHR from an LHCI artifact directory. - * Returns { score, lcp, tbt, bytes } or null if missing/invalid. - */ -async function readResult(resultsDir, app, mode) { - const dir = path.join(resultsDir, `lighthouse-${app}-${mode}`); - let manifest; - try { - manifest = JSON.parse(await fs.readFile(path.join(dir, 'manifest.json'), 'utf8')); - } catch { - return null; - } - - // LHCI manifest is an array of entries. Pick the representative one - // (aggregationMethod: median-run) or fall back to the first entry. - const entry = manifest.find(e => e.isRepresentativeRun) || manifest[0]; - if (!entry) return null; - - const lhrPath = path.join(dir, path.basename(entry.jsonPath)); - let lhr; - try { - lhr = JSON.parse(await fs.readFile(lhrPath, 'utf8')); - } catch { - return null; - } - - const score = lhr.categories?.performance?.score; - if (score == null) return null; - - const audit = id => lhr.audits?.[id]?.numericValue; - const round = v => (typeof v === 'number' ? Math.round(v) : undefined); - - return { - score: Math.round(score * 100), - lcp: round(audit('largest-contentful-paint')), - tbt: round(audit('total-blocking-time')), - bytes: round(audit('total-byte-weight') / 1024), - }; -} - -function formatValue(value, unit) { - if (value == null || Number.isNaN(value)) return '⚠️'; - return `${value}${unit}`; -} - -function formatDelta(before, after, unit) { - if (before == null || after == null || Number.isNaN(before) || Number.isNaN(after)) { - return '—'; - } - const diff = after - before; - if (diff === 0) return '0'; - const sign = diff > 0 ? '+' : ''; - return `${sign}${diff}${unit}`; -} - -function buildSectionTable(rows, metric, unit) { - const header = ['App', 'No Sentry', 'Init Only', 'Δ (SDK)', 'Tracing+Replay', 'Δ (Features)']; - const body = rows.map(({ sdk, results }) => { - const n = results['no-sentry']?.[metric]; - const i = results['init-only']?.[metric]; - const t = results['tracing-replay']?.[metric]; - return [ - sdk, - formatValue(n, unit), - formatValue(i, unit), - formatDelta(n, i, unit), - formatValue(t, unit), - formatDelta(i, t, unit), - ]; - }); - return markdownTable([header, ...body]); -} - -async function run() { - const resultsDir = process.env.LIGHTHOUSE_RESULTS_DIR || 'lighthouse-results'; - - const rows = []; - let totalCells = 0; - let filledCells = 0; - - for (const { app, sdk } of APPS) { - const results = {}; - for (const mode of MODES) { - totalCells++; - results[mode] = await readResult(resultsDir, app, mode); - if (results[mode]) filledCells++; - } - rows.push({ sdk, results }); - } - - if (totalCells > 0 && filledCells / totalCells < 0.5) { - core.warning(`Only ${filledCells}/${totalCells} Lighthouse cells have results (< 50%). Skipping report.`); - return; - } - - // Build one table per metric so each metric's deltas are clearly readable. - // Performance score is generous (fast static apps top out at 100 across all modes), - // so the LCP / TBT / Bytes tables are typically the real signal. - const tables = SECTIONS.map( - ({ label, metric, unit }) => `### ${label}\n\n${buildSectionTable(rows, metric, unit)}`, - ).join('\n\n'); - - const footer = - '\n\n_Median of 5 runs · simulated throttling · localhost. ' + - 'Lower is better for LCP, TBT, and bytes. Higher is better for score. ' + - 'Full reports are attached as workflow artifacts (`lighthouse--`)._'; - - const body = `${HEADING}\n\n${tables}${footer}`; - - // Render the report as a GitHub Actions Job Summary so it's visible on the workflow - // run page. This is the workflow's only public output — there is intentionally no - // PR comment (the workflow doesn't run on PRs; see lighthouse.yml). - await core.summary.addRaw(body).write(); - core.info('Wrote Lighthouse report to Job Summary.'); -} - -run().catch(err => { - core.setFailed(err.message); - process.exit(1); -}); From d876721688d966ab32f47570d7fc4bd20bcd2a3e Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 11:36:23 +0200 Subject: [PATCH 24/29] =?UTF-8?q?ci(lighthouse):=20TEMP=20=E2=80=94=20fire?= =?UTF-8?q?=20workflow=20on=20push=20to=20feat/lighthouse-ci?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub only registers the workflow_dispatch button (and accepts API dispatches) once the workflow file lives on the default branch. This PR's workflow file is only on feat/lighthouse-ci, so neither the Actions UI nor `gh workflow run` can launch it. Add a temporary push trigger for this branch only so we can verify the end-to-end flow against the live lab before merging to develop. Once we have one green run, drop this trigger — the long-term contract is schedule + workflow_dispatch only. Tracking: remove this commit (or just this block) before opening the PR for review on develop. --- .github/workflows/lighthouse.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/lighthouse.yml b/.github/workflows/lighthouse.yml index f8f5aa6eedd9..02ca122d7cf5 100644 --- a/.github/workflows/lighthouse.yml +++ b/.github/workflows/lighthouse.yml @@ -13,6 +13,14 @@ name: 'Nightly: Lighthouse' # Never blocks merges (not in build.yml's required-jobs gate). on: + # TEMP: fires on every push to feat/lighthouse-ci so we can verify the workflow + # end-to-end before it lands on develop. GitHub's workflow_dispatch button only + # registers once a workflow file is on the default branch, and we don't want to + # merge this until it's proven green. REMOVE before merging — the schedule + + # workflow_dispatch triggers below are the long-term contract. + push: + branches: + - feat/lighthouse-ci schedule: - cron: '0 0 * * *' workflow_dispatch: From a271b873d07770adf47183bf183dba6607070459 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 11:38:23 +0200 Subject: [PATCH 25/29] fix(lighthouse-ci): update root workspaces to lighthouse-bundle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit renamed dev-packages/lighthouse-tests/ to dev-packages/lighthouse-bundle/ but missed the corresponding entry in the root package.json workspaces array. scripts/dependency-hash-key.js iterates `workspaces` and require()s each package.json — the stale entry pointing at the missing lighthouse-tests/package.json crashed the install-dependencies composite action with MODULE_NOT_FOUND on the first push of the new workflow. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 19979e65bded..ddd85333d3f3 100644 --- a/package.json +++ b/package.json @@ -110,7 +110,7 @@ "dev-packages/external-contributor-gh-action", "dev-packages/rollup-utils", "dev-packages/bundler-tests", - "dev-packages/lighthouse-tests" + "dev-packages/lighthouse-bundle" ], "devDependencies": { "@rollup/plugin-commonjs": "^25.0.7", From 1800bf0af927a0e7af42c9c59c1677ae6430365b Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 11:49:31 +0200 Subject: [PATCH 26/29] fix(lighthouse-ci): upload build-output artifact for restore-cache The downstream `job_bundle_and_upload` job uses `.github/actions/restore-cache`, which unconditionally downloads a `build-output` artifact (the transpiled package outputs from Nx). My new `job_build` was only uploading the SDK tarballs, so the restore step failed with 'Artifact not found for name: build-output'. Mirror build.yml's job_build: compute the Nx build artifact paths via `yarn ci:print-build-artifact-paths` and upload them as `build-output` in addition to the existing `lighthouse-build-tarball-output`. --- .github/workflows/lighthouse.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/lighthouse.yml b/.github/workflows/lighthouse.yml index 02ca122d7cf5..dc03a6e48e6c 100644 --- a/.github/workflows/lighthouse.yml +++ b/.github/workflows/lighthouse.yml @@ -63,6 +63,27 @@ jobs: - name: Build packages run: yarn build:ci + - name: Compute build artifact paths from Nx + id: nx_build_paths + run: | + { + echo 'paths<> "$GITHUB_OUTPUT" + + - name: Upload build artifacts + # Required by .github/actions/restore-cache in the downstream job — mirrors + # the `build-output` artifact that build.yml's job_build produces. Without + # it, the bundle job fails on restore-cache with "Artifact not found". + uses: actions/upload-artifact@v7 + with: + name: build-output + path: ${{ steps.nx_build_paths.outputs.paths }} + retention-days: 4 + compression-level: 6 + overwrite: true + - name: Build tarballs run: yarn build:tarball From 48c2db9dfb499739a05f38a5058c646abe46f040 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 12:14:53 +0200 Subject: [PATCH 27/29] refactor(lighthouse-ci): fire-and-forget upload, fix SSR install, harden exec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five tightly related cleanups to the upload script — all motivated by the first end-to-end CI run (which uploaded successfully but had all three nextjs-16 cells fail at the lab's `pnpm install` step) and by the review-bot findings on the same commit: 1. Fire-and-forget upload (per maintainer feedback). The script no longer polls the lab's `GET /api/builds/:id` endpoint or appends a Job Summary table. As soon as the POST succeeds we log the dashboard URL and exit 0. Cell results live in the Sentry dashboard, not the GH Actions UI. 2. Fix nextjs-16 install on the lab. The previous SSR bundle preserved `@sentry-internal/test-utils: link:/abs/path/...` in devDependencies — ciCopyToTemp rewrites workspace-relative `link:` deps to absolute paths anchored at the runner's workspace, which obviously doesn't exist after the bundle moves to the lab. `pnpm install` then exited with code 1. Fix: `prepareSsrBundle` now deletes `devDependencies` wholesale before writing package.json. The lab only needs to run `pnpm start` (no test tools, no type-checker, no linter), and dropping devDeps eliminates the workspace-link footgun entirely. 3. Replace every `execSync('cmd ' + var)` with `execFileSync('cmd', [argv])`. CodeQL flagged four sites where shell commands were built from env-derived strings (RUNNER_TEMP-rooted paths, app names). Even though inputs are controlled, the argv-array form is the correct shape — no shell layer, no interpolation, no escape-quoting concerns. As a bonus, paths with spaces would now work. 4. Drop dead `envVarName` field from APPS entries. Three matrix definitions carried this property but nothing reads it — the build step hardcodes all three prefix variants. Cursor flagged this as a maintenance trap. 5. Drop unused imports (`setTimeout as sleep`, `appendFile`) now that the polling and Job Summary code is gone. --- .../lighthouse-bundle/bundle-and-upload.mjs | 219 ++++++------------ 1 file changed, 77 insertions(+), 142 deletions(-) diff --git a/dev-packages/lighthouse-bundle/bundle-and-upload.mjs b/dev-packages/lighthouse-bundle/bundle-and-upload.mjs index ffe69938d94c..f33aa8f77eb6 100644 --- a/dev-packages/lighthouse-bundle/bundle-and-upload.mjs +++ b/dev-packages/lighthouse-bundle/bundle-and-upload.mjs @@ -1,22 +1,24 @@ /** - * Bundle the instrumented Sentry test apps for every (app, mode) cell, POST the - * tarballs to the Sentry Lighthouse lab (https://lighthouse.sentry.gg), poll - * until the lab finishes running Lighthouse, then write a Job Summary. + * Bundle the instrumented Sentry test apps for every (app, mode) cell and POST + * the tarballs to the Sentry Lighthouse lab (https://lighthouse.sentry.gg). The + * lab runs Lighthouse asynchronously and ships results to Sentry on its own + * schedule — this script exits as soon as the upload succeeds, it does NOT wait + * for the lab to finish. * - * This script runs inside .github/workflows/lighthouse.yml. It is the only - * script that talks to the lab — see ~/Projects/sentry-lhci/docs/sentry-javascript-handoff.md - * for the wire protocol. + * Wire protocol: ~/Projects/sentry-lhci/docs/sentry-javascript-handoff.md * - * Zero runtime dependencies — uses Node 22 builtins (fetch, FormData, Blob) - * and the system `tar` available on every ubuntu runner. + * Zero runtime dependencies — uses Node 22 builtins (fetch, FormData, Blob) and + * the system `tar`. Every external command is invoked via `execFileSync` with + * an argv array so no shell interpolation happens — needed both for safety + * (CodeQL flags any env-derived string concatenated into a shell command, even + * when the inputs are controlled) and to keep paths with spaces working. */ /* eslint-disable no-console */ -import { execSync } from 'node:child_process'; -import { copyFile, readdir, readFile, mkdir, rm, writeFile, appendFile } from 'node:fs/promises'; +import { execFileSync } from 'node:child_process'; +import { copyFile, mkdir, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; import path from 'node:path'; -import { setTimeout as sleep } from 'node:timers/promises'; const LAB_URL = process.env.LIGHTHOUSE_LAB_URL; const TOKEN = process.env.LIGHTHOUSE_UPLOAD_TOKEN; @@ -31,44 +33,37 @@ const E2E_DIR = path.join(WORKSPACE, 'dev-packages/e2e-tests'); /** * The matrix. Adding an app here requires: - * 1. The test app reads `` and branches its Sentry init. - * 2. The lab's runner understands `serve` / `staticDir` / `startCmd` (it does). + * 1. The test app reads SENTRY_LIGHTHOUSE_MODE (or its bundler-specific + * prefix variant) and branches its Sentry init. + * 2. For SSR apps, the lab's runner must accept a startCmd + readyPattern. */ const APPS = [ - { - app: 'default-browser', - serve: 'static', - staticDir: 'build', - envVarName: 'SENTRY_LIGHTHOUSE_MODE', - }, - { - app: 'react-19', - serve: 'static', - staticDir: 'build', - envVarName: 'REACT_APP_SENTRY_LIGHTHOUSE_MODE', - }, + { app: 'default-browser', serve: 'static', staticDir: 'build' }, + { app: 'react-19', serve: 'static', staticDir: 'build' }, { app: 'nextjs-16', serve: 'server', startCmd: 'pnpm start', readyPattern: 'Ready in', // Lab side: pnpm 9.15.9 is on the image via corepack. We strip the lockfile - // from the bundle (CI generates it with workspace-absolute override paths - // that don't survive the move to the lab), so --no-frozen-lockfile lets pnpm - // re-resolve from the rewritten package.json. --prefer-offline uses the - // lab's persistent pnpm store (/data/.pnpm-store). + // and devDependencies from the SSR bundle (CI generates the lockfile with + // workspace-absolute paths that don't survive the move; devDeps include + // workspace links like @sentry-internal/test-utils that would also fail). + // --no-frozen-lockfile lets pnpm regenerate from the rewritten package.json, + // --prefer-offline uses the lab's persistent pnpm store (/data/.pnpm-store). installCmd: 'pnpm install --no-frozen-lockfile --prefer-offline', - envVarName: 'NEXT_PUBLIC_SENTRY_LIGHTHOUSE_MODE', }, ]; const MODES = ['no-sentry', 'init-only', 'tracing-replay']; async function run() { - // Fail fast if the lab is down before we waste minutes building bundles. + // Fail fast if the lab is down so we don't waste minutes building bundles. console.log(`Liveness check: ${LAB_URL}/healthz`); const health = await fetch(`${LAB_URL}/healthz`); - if (!health.ok) throw new Error(`Lab healthcheck failed: ${health.status} ${await health.text()}`); + if (!health.ok) { + throw new Error(`Lab healthcheck failed: ${health.status} ${await health.text()}`); + } console.log('Lab is reachable.'); await mkdir(path.join(RUNNER_TEMP, 'bundles'), { recursive: true }); @@ -86,50 +81,44 @@ async function run() { console.log(`\n=== Uploading ${bundles.length} bundles to ${LAB_URL}/api/builds ===`); const buildResp = await uploadBundles(bundles); console.log(`Build queued: ${buildResp.buildId}`); - console.log(`Build URL: ${LAB_URL}${buildResp.buildUrl}`); - console.log(`Dashboard: ${LAB_URL}${buildResp.dashboardUrl}`); - - const final = await pollUntilDone(buildResp.buildId); - await writeJobSummary(final); - - // Surface failure to CI when the lab marks the build failed or any cell failed. - const failedCells = (final.cells ?? []).filter(c => c.status === 'failed'); - if (final.status === 'failed' || failedCells.length > 0) { - console.error(`Build status=${final.status}, ${failedCells.length} cell(s) failed.`); - process.exit(1); - } + console.log(`Dashboard: ${LAB_URL}${buildResp.dashboardUrl}`); + console.log(`API: ${LAB_URL}${buildResp.buildUrl}`); + console.log('\nUpload succeeded. The lab runs Lighthouse asynchronously — track results in the Sentry dashboard.'); } /** * Build a single (app, mode) cell: - * 1. Copy the app to a unique temp dir (so concurrent cells don't collide). + * 1. Copy the app to a unique temp dir (concurrent cells can't collide). * 2. Apply pnpm overrides (existing helper). - * 3. Run `pnpm test:build` with the right SENTRY_LIGHTHOUSE_MODE env vars. - * 4. For static cells, tar just the build dir. - * For SSR cells, copy packed tgzs into the bundle, rewrite package.json - * to use relative `file:./packed/...` paths, then tar (no node_modules). + * 3. Run `pnpm test:build` with SENTRY_LIGHTHOUSE_MODE + the framework + * prefix variants (NEXT_PUBLIC_*, REACT_APP_*) — each app's bundler picks + * up whichever it knows about. + * 4. Static cells: tar just the build dir. + * SSR cells: copy packed tgzs into the bundle, rewrite package.json to + * relative `file:./packed/...` paths, drop devDependencies, tar without + * node_modules and the lockfile. * 5. Return cell metadata for the upload. */ async function prepareCell(def, mode, fieldName) { const tempApp = path.join(RUNNER_TEMP, `app-${def.app}-${mode}`); await rm(tempApp, { recursive: true, force: true }); - // 1. Copy app to temp (fixes file: deps to workspace-absolute paths) - execSync(`yarn ci:copy-to-temp ./test-applications/${def.app} ${tempApp}`, { + // Copy app to temp (fixes file:/link: deps to workspace-absolute paths) + execFileSync('yarn', ['ci:copy-to-temp', `./test-applications/${def.app}`, tempApp], { cwd: E2E_DIR, stdio: 'inherit', }); - // 2. Add pnpm overrides (workspace-absolute paths pointing at packed dir) - execSync(`yarn ci:pnpm-overrides ${tempApp} ${PACKED_DIR}`, { + // Add pnpm overrides (workspace-absolute paths pointing at packed dir) + execFileSync('yarn', ['ci:pnpm-overrides', tempApp, PACKED_DIR], { cwd: E2E_DIR, stdio: 'inherit', }); - // 3. Build with the right mode env var. We set all common bundler prefixes so each - // app's bundler picks up whichever variant it knows about — apps that don't read a - // prefix simply ignore extra vars. - execSync('pnpm test:build', { + // Build with the right mode env var. We set all common bundler prefixes so each + // app's bundler picks up whichever variant it knows about — apps that don't read + // a prefix simply ignore extra vars. + execFileSync('pnpm', ['test:build'], { cwd: tempApp, stdio: 'inherit', env: { @@ -144,10 +133,9 @@ async function prepareCell(def, mode, fieldName) { const tarPath = path.join(RUNNER_TEMP, 'bundles', `${def.app}-${mode}.tar.gz`); if (def.serve === 'static') { - // Static cell — tar the build dir only. Lab serves it with a static HTTP server. - execSync(`tar -czf ${tarPath} -C ${tempApp} ${def.staticDir}`, { stdio: 'inherit' }); - const bytes = Number(execSync(`wc -c < ${tarPath}`, { encoding: 'utf8' }).trim()); - console.log(`Static bundle: ${tarPath} (${formatBytes(bytes)})`); + // Static cell — tar just the build dir. Lab serves it with a static HTTP server. + execFileSync('tar', ['-czf', tarPath, '-C', tempApp, def.staticDir], { stdio: 'inherit' }); + console.log(`Static bundle: ${tarPath} (${await formatSize(tarPath)})`); return { fieldName, tarPath, @@ -163,13 +151,21 @@ async function prepareCell(def, mode, fieldName) { // SSR cell — prep for `pnpm install && pnpm start` on the lab. await prepareSsrBundle(tempApp); - execSync( - `tar -czf ${tarPath} --exclude=node_modules --exclude=.git --exclude=pnpm-lock.yaml ` + - `-C ${path.dirname(tempApp)} ${path.basename(tempApp)}`, + execFileSync( + 'tar', + [ + '-czf', + tarPath, + '--exclude=node_modules', + '--exclude=.git', + '--exclude=pnpm-lock.yaml', + '-C', + path.dirname(tempApp), + path.basename(tempApp), + ], { stdio: 'inherit' }, ); - const bytes = Number(execSync(`wc -c < ${tarPath}`, { encoding: 'utf8' }).trim()); - console.log(`SSR bundle: ${tarPath} (${formatBytes(bytes)})`); + console.log(`SSR bundle: ${tarPath} (${await formatSize(tarPath)})`); return { fieldName, tarPath, @@ -186,26 +182,24 @@ async function prepareCell(def, mode, fieldName) { } /** - * For SSR cells: copy the packed Sentry tarballs into the bundle and rewrite + * For SSR cells: copy the packed Sentry tarballs into the bundle, rewrite * package.json deps + pnpm.overrides to relative `file:./packed/...` paths so - * the lab's `pnpm install` can resolve them from inside the extracted bundle. - * - * `node_modules` and `pnpm-lock.yaml` are stripped by the tar exclude list — - * the lab regenerates both. + * the lab's `pnpm install` can resolve them from inside the extracted bundle, + * and drop devDependencies entirely (not needed at runtime; some are workspace + * links like @sentry-internal/test-utils that don't survive the move). */ async function prepareSsrBundle(tempApp) { // Copy packed tgz files into /packed/ const inBundlePacked = path.join(tempApp, 'packed'); await mkdir(inBundlePacked, { recursive: true }); - const entries = await readdir(PACKED_DIR); - for (const name of entries) { - if (!name.endsWith('.tgz')) continue; - await copyFile(path.join(PACKED_DIR, name), path.join(inBundlePacked, name)); + for (const name of await readdir(PACKED_DIR)) { + if (name.endsWith('.tgz')) { + await copyFile(path.join(PACKED_DIR, name), path.join(inBundlePacked, name)); + } } // Rewrite all workspace-absolute `file:.../sentry-*-packed.tgz` references in - // package.json (in dependencies, devDependencies, pnpm.overrides) to point at - // `./packed/`. + // package.json to `./packed/`, and drop devDependencies wholesale. const pkgPath = path.join(tempApp, 'package.json'); const pkg = JSON.parse(await readFile(pkgPath, 'utf8')); const rewrite = obj => { @@ -218,8 +212,8 @@ async function prepareSsrBundle(tempApp) { } }; rewrite(pkg.dependencies); - rewrite(pkg.devDependencies); rewrite(pkg.pnpm?.overrides); + delete pkg.devDependencies; await writeFile(pkgPath, JSON.stringify(pkg, null, 2)); } @@ -256,70 +250,11 @@ async function uploadBundles(bundles) { return res.json(); } -/** - * Poll GET /api/builds/:id every 15 seconds until status is terminal, or the - * 25-minute ceiling is reached. - */ -async function pollUntilDone(buildId) { - const deadline = Date.now() + 25 * 60 * 1000; - while (Date.now() < deadline) { - const r = await fetch(`${LAB_URL}/api/builds/${buildId}`); - if (!r.ok) { - console.warn(`Poll failed: ${r.status} ${await r.text()} — retrying`); - await sleep(15000); - continue; - } - const build = await r.json(); - const cells = build.cells ?? []; - const done = cells.filter(c => c.status === 'completed').length; - const failed = cells.filter(c => c.status === 'failed').length; - console.log(`status=${build.status} cells=${done + failed}/${cells.length} (${failed} failed)`); - if (build.status === 'completed' || build.status === 'failed') return build; - await sleep(15000); - } - throw new Error(`Build ${buildId} did not finish within 25 minutes`); -} - -/** - * Append a markdown table of the results to $GITHUB_STEP_SUMMARY so it renders - * on the workflow run page. Skipped when running locally. - */ -async function writeJobSummary(build) { - const out = process.env.GITHUB_STEP_SUMMARY; - if (!out) return; - const cells = build.cells ?? []; - const lines = [ - `## 🔦 Lighthouse — ${build.status}`, - '', - `- Build ID: \`${build.buildId}\``, - `- Commit: \`${build.commit ?? 'unknown'}\``, - `- Branch: \`${build.branch ?? 'unknown'}\``, - `- Dashboard: ${LAB_URL}${build.buildUrl ?? `/api/builds/${build.buildId}`}`, - '', - '| App | Mode | Status | Median score | Runs | Report |', - '| --- | --- | --- | --- | --- | --- |', - ]; - for (const c of cells) { - const runs = c.runs ?? []; - const scores = runs.map(r => r.performanceScore).filter(s => s != null); - const median = scores.length === 0 ? '—' : computeMedian(scores).toFixed(2); - const reportUrl = runs[0]?.reportUrl ? `[view](${LAB_URL}${runs[0].reportUrl})` : '—'; - lines.push(`| ${c.app} | ${c.mode} | ${c.status} | ${median} | ${runs.length} | ${reportUrl} |`); - } - lines.push(''); - await appendFile(out, `${lines.join('\n')}\n`); -} - -function computeMedian(xs) { - const sorted = [...xs].sort((a, b) => a - b); - const mid = Math.floor(sorted.length / 2); - return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; -} - -function formatBytes(n) { - if (n < 1024) return `${n} B`; - if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`; - return `${(n / 1024 / 1024).toFixed(1)} MB`; +async function formatSize(filePath) { + const { size } = await stat(filePath); + if (size < 1024) return `${size} B`; + if (size < 1024 * 1024) return `${(size / 1024).toFixed(1)} KB`; + return `${(size / 1024 / 1024).toFixed(1)} MB`; } run().catch(err => { From cad23e06be1de093c26d4b3e111524028fc2ae64 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 13:21:12 +0200 Subject: [PATCH 28/29] revert(react-router-7-spa-e2e): restore original entry, drop dead lighthouse gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit react-router-7-spa was instrumented for SENTRY_LIGHTHOUSE_MODE early in this PR but never made it into the upload matrix (the app's static build serves an empty FCP under Lighthouse — NO_FCP). The dynamic-import-gated initialisation has therefore been dead code that: - adds an async IIFE wrapper around a sync render path, - keeps the unused PUBLIC_SENTRY_LIGHTHOUSE_MODE branch in source, - and risks subtly changing existing E2E test behaviour for an app whose Lighthouse coverage is on hold anyway. Restore the file to its develop state. The lighthouse-bundle matrix is now the only source of truth for which apps are touched, and it lists exactly three: default-browser, react-19, nextjs-16. --- .../react-router-7-spa/src/main.tsx | 91 +++++++++---------- 1 file changed, 41 insertions(+), 50 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/react-router-7-spa/src/main.tsx b/dev-packages/e2e-tests/test-applications/react-router-7-spa/src/main.tsx index ca48ea98bebe..baf12f7ff574 100644 --- a/dev-packages/e2e-tests/test-applications/react-router-7-spa/src/main.tsx +++ b/dev-packages/e2e-tests/test-applications/react-router-7-spa/src/main.tsx @@ -1,3 +1,4 @@ +import * as Sentry from '@sentry/react'; import React from 'react'; import ReactDOM from 'react-dom/client'; import { @@ -13,53 +14,43 @@ import Index from './pages/Index'; import SSE from './pages/SSE'; import User from './pages/User'; -const lighthouseMode = import.meta.env.PUBLIC_SENTRY_LIGHTHOUSE_MODE; - -let SentryRoutes = Routes; - -(async () => { - if (lighthouseMode !== 'no-sentry') { - const Sentry = await import('@sentry/react'); - - const integrations: Sentry.Integration[] = []; - - if (lighthouseMode !== 'init-only') { - integrations.push( - Sentry.reactRouterV7BrowserTracingIntegration({ - useEffect: React.useEffect, - useLocation, - useNavigationType, - createRoutesFromChildren, - matchRoutes, - trackFetchStreamPerformance: true, - }), - Sentry.replayIntegration(), - ); - } - - Sentry.init({ - environment: 'qa', // dynamic sampling bias to keep transactions - dsn: import.meta.env.PUBLIC_E2E_TEST_DSN, - integrations, - tracesSampleRate: 1.0, - release: 'e2e-test', - replaysSessionSampleRate: lighthouseMode !== 'init-only' ? 1.0 : 0.0, - replaysOnErrorSampleRate: 0.0, - tunnel: 'http://localhost:3031', - sendDefaultPii: true, - }); - - SentryRoutes = Sentry.withSentryReactRouterV7Routing(Routes); - } - - const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement); - root.render( - - - } /> - } /> - } /> - - , - ); -})(); +const replay = Sentry.replayIntegration(); + +Sentry.init({ + environment: 'qa', // dynamic sampling bias to keep transactions + dsn: import.meta.env.PUBLIC_E2E_TEST_DSN, + integrations: [ + Sentry.reactRouterV7BrowserTracingIntegration({ + useEffect: React.useEffect, + useLocation, + useNavigationType, + createRoutesFromChildren, + matchRoutes, + trackFetchStreamPerformance: true, + }), + replay, + ], + // We recommend adjusting this value in production, or using tracesSampler + // for finer control + tracesSampleRate: 1.0, + release: 'e2e-test', + + // Always capture replays, so we can test this properly + replaysSessionSampleRate: 1.0, + replaysOnErrorSampleRate: 0.0, + tunnel: 'http://localhost:3031', + sendDefaultPii: true, +}); + +const SentryRoutes = Sentry.withSentryReactRouterV7Routing(Routes); + +const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement); +root.render( + + + } /> + } /> + } /> + + , +); From 9ba92f4fede4d3e4b8e33d430c3c51c7904c1df8 Mon Sep 17 00:00:00 2001 From: Daniel Griesser Date: Wed, 13 May 2026 13:21:12 +0200 Subject: [PATCH 29/29] ci(lighthouse): drop temp push trigger on feat/lighthouse-ci MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verification trigger was only needed because GitHub doesn't register the workflow_dispatch button (or accept REST dispatches) until the workflow file lives on the default branch. Now that the workflow has been exercised end-to-end (bundle build, upload, lab acceptance, fire-and-forget exit) the long-term contract — schedule cron + workflow_dispatch — is sufficient. After merge to develop, the workflow will fire nightly at 00:00 UTC and be manually triggerable from the Actions UI on any branch. --- .github/workflows/lighthouse.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/lighthouse.yml b/.github/workflows/lighthouse.yml index dc03a6e48e6c..81a25c967c58 100644 --- a/.github/workflows/lighthouse.yml +++ b/.github/workflows/lighthouse.yml @@ -13,14 +13,6 @@ name: 'Nightly: Lighthouse' # Never blocks merges (not in build.yml's required-jobs gate). on: - # TEMP: fires on every push to feat/lighthouse-ci so we can verify the workflow - # end-to-end before it lands on develop. GitHub's workflow_dispatch button only - # registers once a workflow file is on the default branch, and we don't want to - # merge this until it's proven green. REMOVE before merging — the schedule + - # workflow_dispatch triggers below are the long-term contract. - push: - branches: - - feat/lighthouse-ci schedule: - cron: '0 0 * * *' workflow_dispatch: