From 5c48a10b04623f0fe1138e10169c139a29406bdf Mon Sep 17 00:00:00 2001 From: Tak Hoffman <781889+Takhoffman@users.noreply.github.com> Date: Tue, 19 May 2026 21:15:14 -0500 Subject: [PATCH] fix(review): inspect video proof with ffmpeg --- src/clawsweeper.ts | 271 ++++++++++++++++++++++++++++++++++++++- test/clawsweeper.test.ts | 95 ++++++++++++++ 2 files changed, 362 insertions(+), 4 deletions(-) diff --git a/src/clawsweeper.ts b/src/clawsweeper.ts index d4caebe71a..9de151a3b0 100644 --- a/src/clawsweeper.ts +++ b/src/clawsweeper.ts @@ -477,6 +477,21 @@ interface ReviewPromptBuild { telemetry: ReviewPromptTelemetry; } +interface PreparedMediaProofArtifact { + url: string; + downloadedPath: string | null; + metadataPath: string | null; + contactSheetPath: string | null; + status: "prepared" | "failed"; + detail: string; +} + +interface PreparedMediaProof { + manifestPath: string | null; + summaryPath: string | null; + artifacts: PreparedMediaProofArtifact[]; +} + interface ReviewContextLedgerEntry { section: string; label: string; @@ -489,6 +504,8 @@ interface ReviewContextLedgerEntry { interface ReviewPromptRuntimeHints { proofScratchDir?: string; + mediaProofManifestPath?: string; + mediaProofSummary?: string; } interface DashboardItem { @@ -4431,6 +4448,223 @@ function contextJsonForPrompt(context: ItemContext): string { return JSON.stringify(context, null, 2); } +type MediaProofCommandRunner = ( + command: string, + args: readonly string[], +) => { + status: number | null; + stdout?: string | Buffer; + stderr?: string | Buffer; + error?: Error; +}; + +const VIDEO_PROOF_EXTENSIONS = new Set([".mov", ".mp4", ".m4v", ".webm", ".avi", ".mkv"]); +const MEDIA_PROOF_MANIFEST_FILE = "media-proof-manifest.json"; +const MEDIA_PROOF_SUMMARY_FILE = "media-proof-summary.md"; +const MAX_MEDIA_PROOF_URLS = 4; + +function mediaProofCommandRunner(command: string, args: readonly string[]) { + return spawnSync(command, [...args], { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }); +} + +function trimTrailingUrlPunctuation(raw: string): string { + let end = raw.length; + while (end > 0) { + const char = raw.charCodeAt(end - 1); + if (char !== 44 && char !== 46 && char !== 58 && char !== 59) break; + end -= 1; + } + return raw.slice(0, end); +} + +function proofVideoUrlsFromContext(context: ItemContext): string[] { + const text = JSON.stringify(context); + const matches = text.match(/https?:\/\/[^\s<>"'\\)]+/g) ?? []; + const urls: string[] = []; + const seen = new Set(); + for (const raw of matches) { + const cleaned = trimTrailingUrlPunctuation(raw); + let parsed: URL; + try { + parsed = new URL(cleaned); + } catch { + continue; + } + const pathname = parsed.pathname.toLowerCase(); + const isVideo = [...VIDEO_PROOF_EXTENSIONS].some((extension) => pathname.endsWith(extension)); + if (!isVideo || seen.has(parsed.href)) continue; + seen.add(parsed.href); + urls.push(parsed.href); + if (urls.length >= MAX_MEDIA_PROOF_URLS) break; + } + return urls; +} + +function mediaProofFileExtension(url: string): string { + try { + const pathname = new URL(url).pathname.toLowerCase(); + const extension = [...VIDEO_PROOF_EXTENSIONS].find((candidate) => pathname.endsWith(candidate)); + return extension ?? ".video"; + } catch { + return ".video"; + } +} + +function mediaProofSpawnDetail(result: ReturnType): string { + if (result.status === 0) return "ok"; + const stderr = String(result.stderr ?? "").trim(); + const stdout = String(result.stdout ?? "").trim(); + const error = result.error?.message ?? ""; + const detail = stderr || stdout || error || "command failed without output"; + return trimMiddle(detail, 1000); +} + +function prepareMediaProofArtifacts( + context: ItemContext, + proofScratchDir: string, + runner: MediaProofCommandRunner = mediaProofCommandRunner, +): PreparedMediaProof { + const urls = proofVideoUrlsFromContext(context); + if (urls.length === 0) return { manifestPath: null, summaryPath: null, artifacts: [] }; + ensureDir(proofScratchDir); + const artifacts: PreparedMediaProofArtifact[] = []; + for (const [index, url] of urls.entries()) { + const ordinal = index + 1; + const downloadedPath = join( + proofScratchDir, + `proof-video-${ordinal}${mediaProofFileExtension(url)}`, + ); + const metadataPath = join(proofScratchDir, `proof-video-${ordinal}.ffprobe.json`); + const contactSheetPath = join(proofScratchDir, `proof-video-${ordinal}.contact-sheet.jpg`); + const download = runner("curl", [ + "-L", + "--fail", + "--silent", + "--show-error", + "--max-time", + "90", + "--output", + downloadedPath, + url, + ]); + if (download.status !== 0) { + artifacts.push({ + url, + downloadedPath: null, + metadataPath: null, + contactSheetPath: null, + status: "failed", + detail: `download failed: ${mediaProofSpawnDetail(download)}`, + }); + continue; + } + const metadata = runner("ffprobe", [ + "-v", + "error", + "-print_format", + "json", + "-show_format", + "-show_streams", + downloadedPath, + ]); + if (metadata.status !== 0) { + artifacts.push({ + url, + downloadedPath, + metadataPath: null, + contactSheetPath: null, + status: "failed", + detail: `ffprobe failed: ${mediaProofSpawnDetail(metadata)}`, + }); + continue; + } + writeFileSync(metadataPath, String(metadata.stdout ?? "{}"), "utf8"); + const contactSheet = runner("ffmpeg", [ + "-hide_banner", + "-y", + "-i", + downloadedPath, + "-vf", + "fps=1/5,scale=640:-1,tile=5x4", + "-frames:v", + "1", + contactSheetPath, + ]); + if (contactSheet.status !== 0) { + artifacts.push({ + url, + downloadedPath, + metadataPath, + contactSheetPath: null, + status: "failed", + detail: `ffmpeg contact sheet failed: ${mediaProofSpawnDetail(contactSheet)}`, + }); + continue; + } + artifacts.push({ + url, + downloadedPath, + metadataPath, + contactSheetPath, + status: "prepared", + detail: "downloaded, probed, and converted to a contact sheet with ffmpeg", + }); + } + const manifestPath = join(proofScratchDir, MEDIA_PROOF_MANIFEST_FILE); + const summaryPath = join(proofScratchDir, MEDIA_PROOF_SUMMARY_FILE); + const prepared: PreparedMediaProof = { manifestPath, summaryPath, artifacts }; + writeFileSync(manifestPath, JSON.stringify(prepared, null, 2), "utf8"); + writeFileSync(summaryPath, mediaProofSummaryMarkdown(prepared), "utf8"); + return prepared; +} + +function mediaProofSummaryMarkdown(prepared: PreparedMediaProof): string { + const lines = ["# Prepared Media Proof", ""]; + for (const artifact of prepared.artifacts) { + lines.push(`- ${artifact.status}: ${artifact.url}`); + if (artifact.downloadedPath) lines.push(` - downloaded: ${artifact.downloadedPath}`); + if (artifact.metadataPath) lines.push(` - ffprobe metadata: ${artifact.metadataPath}`); + if (artifact.contactSheetPath) lines.push(` - contact sheet: ${artifact.contactSheetPath}`); + lines.push(` - detail: ${artifact.detail}`); + } + return `${lines.join("\n")}\n`; +} + +function mediaProofRuntimePrompt(summary: string | undefined, manifestPath: string | undefined) { + const trimmed = summary?.trim(); + if (!trimmed || !manifestPath) return ""; + return ` +- ClawSweeper preprocessed linked video proof with ffprobe/ffmpeg before this review. Read \`${manifestPath}\` and inspect any generated contact-sheet image paths before trying browser playback. +- If browser playback fails but ffprobe metadata and ffmpeg contact sheets are readable, assess the proof from those generated artifacts instead of treating the video as uninspectable. +- Only fall back to browser playback after checking the prepared ffmpeg artifacts. If both ffmpeg extraction and browser playback fail, report the exact failure from the manifest. +`; +} + +function mediaProofRuntimeHints( + proofScratchDir: string, + preparedMediaProof: PreparedMediaProof, +): ReviewPromptRuntimeHints { + const hints: ReviewPromptRuntimeHints = { proofScratchDir }; + if (preparedMediaProof.manifestPath) + hints.mediaProofManifestPath = preparedMediaProof.manifestPath; + if (preparedMediaProof.summaryPath && preparedMediaProof.artifacts.length) { + hints.mediaProofSummary = mediaProofSummaryMarkdown(preparedMediaProof); + } + return hints; +} + +export function proofVideoUrlsFromContextForTest(context: ItemContext): string[] { + return proofVideoUrlsFromContext(context); +} + +export function prepareMediaProofArtifactsForTest( + context: ItemContext, + proofScratchDir: string, + runner: MediaProofCommandRunner, +): PreparedMediaProof { + return prepareMediaProofArtifacts(context, proofScratchDir, runner); +} + function buildReviewPrompt( item: Item, context: ItemContext, @@ -4442,6 +4676,10 @@ function buildReviewPrompt( const contextJson = contextJsonForPrompt(context); const schema = reviewDecisionSchemaText(); const proofScratchDir = runtimeHints.proofScratchDir?.trim(); + const mediaProofPrompt = mediaProofRuntimePrompt( + runtimeHints.mediaProofSummary, + runtimeHints.mediaProofManifestPath, + ); const extra = additionalPrompt.trim() ? ` @@ -4472,6 +4710,7 @@ ${additionalPrompt.trim()} - You may use the available network and read-only GitHub token to inspect PR body links, comments, screenshots, videos, logs, terminal output, and target-repo artifacts. - Download proof artifacts into ${proofScratchDir ? `\`${proofScratchDir}\`` : "a temporary scratch directory"} before inspecting them. - The target checkout is read-only for review. Do not modify repository files; use the scratch directory or /tmp for downloaded evidence and generated video stills/contact sheets. +${mediaProofPrompt} ## GitHub Context @@ -4510,6 +4749,16 @@ export function reviewPromptTelemetryForTest( return reviewPromptTelemetry(item, context, git, additionalPrompt); } +export function reviewPromptForTest( + item: Item, + context: ItemContext, + git: GitInfo, + additionalPrompt = "", + runtimeHints: ReviewPromptRuntimeHints = {}, +): string { + return buildReviewPrompt(item, context, git, additionalPrompt, runtimeHints).text; +} + function codexFailureReason(detail: string): string { if (detail.includes("Codex dirtied the OpenClaw checkout")) return "dirty checkout"; if (detail.includes("did not produce output")) return "missing structured output"; @@ -4678,13 +4927,20 @@ function runCodex(options: { const proofScratchDir = options.proofScratchDir ?? join(options.workDir, "proof-scratch", String(options.item.number)); ensureDir(proofScratchDir); + const preparedMediaProof = options.prompt + ? { manifestPath: null, summaryPath: null, artifacts: [] } + : prepareMediaProofArtifacts(options.context, proofScratchDir); const promptPath = join(options.workDir, `${options.item.number}.prompt.md`); const outputPath = join(options.workDir, `${options.item.number}.json`); const prompt = options.prompt ?? - buildReviewPrompt(options.item, options.context, options.git, options.additionalPrompt, { - proofScratchDir, - }).text; + buildReviewPrompt( + options.item, + options.context, + options.git, + options.additionalPrompt, + mediaProofRuntimeHints(proofScratchDir, preparedMediaProof), + ).text; writeFileSync(promptPath, prompt, "utf8"); const dirtyBefore = openclawDirtyStatus(options.openclawDir); if (dirtyBefore) { @@ -10502,7 +10758,14 @@ function reviewCommand(args: Args): void { const contextElapsedMs = Date.now() - contextStartedAt; const codexWorkDir = join(artifactDir, "codex"); const proofScratchDir = join(codexWorkDir, "proof-scratch", String(item.number)); - const prompt = buildReviewPrompt(item, context, git, additionalPrompt, { proofScratchDir }); + const preparedMediaProof = prepareMediaProofArtifacts(context, proofScratchDir); + const prompt = buildReviewPrompt( + item, + context, + git, + additionalPrompt, + mediaProofRuntimeHints(proofScratchDir, preparedMediaProof), + ); const snapshotHash = itemSnapshotHash(item, context); try { const startComment = postReviewStartStatusComment({ diff --git a/test/clawsweeper.test.ts b/test/clawsweeper.test.ts index d8966a1558..acb6747aca 100644 --- a/test/clawsweeper.test.ts +++ b/test/clawsweeper.test.ts @@ -58,9 +58,11 @@ import { mergeRiskLabelSchemeForTest, prRatingLabelsForTest, prRatingLabelSchemeForTest, + prepareMediaProofArtifactsForTest, prEggCreatureForTest, prEggImagePromptForTest, prEggSpriteMetricsForTest, + proofVideoUrlsFromContextForTest, renderPrEggCommentForTest, prStatusLabelsForTest, prStatusLabelSchemeForTest, @@ -75,6 +77,7 @@ import { reviewAutomationMarkersFromReport, reviewActionForDecision, reviewPriority, + reviewPromptForTest, renderReviewCommentFromReport, renderReviewContextBudgetForTest, renderWorkPlanFromReport, @@ -7045,6 +7048,98 @@ test("review prompt requires real behavior proof for PR reviews", () => { assert.match(prompt, /do not request ClawSweeper repair markers/); }); +test("media proof preparation extracts browser-unplayable ffmpeg-decodeable video proof", () => { + const dir = mkdtempSync(join(tmpdir(), "clawsweeper-media-proof-")); + try { + const context = { + issue: {}, + comments: [ + { + body: [ + "Chromium media error code 4 on this upload, but ffmpeg can decode it:", + "https://github.com/user/repo/releases/download/proof/Screen.Recording.mov", + ].join("\n"), + }, + ], + timeline: [], + }; + const calls: string[] = []; + const prepared = prepareMediaProofArtifactsForTest(context, dir, (command, args) => { + calls.push(`${command} ${args.join(" ")}`); + if (command === "curl") { + const outputIndex = args.indexOf("--output"); + assert.notEqual(outputIndex, -1); + writeFileSync(String(args[outputIndex + 1]), "fake mov bytes"); + return { status: 0, stdout: "", stderr: "" }; + } + if (command === "ffprobe") { + return { + status: 0, + stdout: JSON.stringify({ + format: { duration: "46.49" }, + streams: [{ codec_name: "h264", width: 734, height: 1038 }], + }), + stderr: "", + }; + } + if (command === "ffmpeg") { + const output = String(args.at(-1)); + writeFileSync(output, "fake contact sheet"); + return { status: 0, stdout: "", stderr: "" }; + } + return { status: 1, stdout: "", stderr: `unexpected command: ${command}` }; + }); + + assert.equal(prepared.artifacts.length, 1); + assert.equal(prepared.artifacts[0]?.status, "prepared"); + assert.ok(prepared.manifestPath); + assert.ok(prepared.summaryPath); + assert.ok(prepared.artifacts[0]?.metadataPath); + assert.ok(prepared.artifacts[0]?.contactSheetPath); + assert.equal(existsSync(prepared.manifestPath), true); + assert.equal(existsSync(prepared.artifacts[0].metadataPath), true); + assert.equal(existsSync(prepared.artifacts[0].contactSheetPath), true); + assert.match(calls.join("\n"), /^curl /m); + assert.match(calls.join("\n"), /^ffprobe /m); + assert.match(calls.join("\n"), /^ffmpeg /m); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("runtime prompt tells Codex to inspect ffmpeg video artifacts before browser fallback", () => { + const context = { + issue: {}, + comments: [{ body: "Proof: https://github.com/user/repo/releases/download/proof/demo.mov" }], + timeline: [], + }; + const prompt = reviewPromptForTest( + item({ kind: "pull_request" }), + context, + { mainSha: "abc123", latestRelease: null }, + "", + { + proofScratchDir: "/tmp/proof", + mediaProofManifestPath: "/tmp/proof/media-proof-manifest.json", + mediaProofSummary: "prepared: https://github.com/user/repo/releases/download/proof/demo.mov", + }, + ); + + assert.deepEqual(proofVideoUrlsFromContextForTest(context), [ + "https://github.com/user/repo/releases/download/proof/demo.mov", + ]); + assert.match(prompt, /preprocessed linked video proof with ffprobe\/ffmpeg/); + assert.match(prompt, /generated contact-sheet image paths before trying browser playback/); + assert.match( + prompt, + /Only fall back to browser playback after checking the prepared ffmpeg artifacts/, + ); + assert.match( + prompt, + /If browser playback fails but ffprobe metadata and ffmpeg contact sheets are readable/, + ); +}); + test("review prompt keeps draft and protected workflow state out of PR rank", () => { const prompt = readFileSync("prompts/review-item.md", "utf8");