From f645399ef3e53be288fd3f215c2edd687e21b89b Mon Sep 17 00:00:00 2001 From: CTO Agent Date: Tue, 14 Apr 2026 21:00:59 +0000 Subject: [PATCH] feat: add static image rendering to editorial content pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Editorial content now renders both a video and a static image (editorial base with playlist cover overlays). The static image is passed through the callback chain as `staticImageUrl` alongside the video. - New `buildStaticImageArgs` — ffmpeg args for single-frame image render - New `renderStaticImage` — downloads base + overlays, runs ffmpeg, uploads PNG - `createContentTask` renders static image when template uses overlay - `pollContentRuns` extracts `imageUrl` from task output - 12 new tests (all green) Co-Authored-By: Paperclip --- .../__tests__/buildStaticImageArgs.test.ts | 77 +++++++++++++ src/content/__tests__/pollContentRuns.test.ts | 10 +- .../__tests__/renderStaticImage.test.ts | 104 ++++++++++++++++++ src/content/buildStaticImageArgs.ts | 63 +++++++++++ src/content/pollContentRuns.ts | 3 + src/content/renderStaticImage.ts | 64 +++++++++++ src/tasks/__tests__/createContentTask.test.ts | 7 ++ .../__tests__/createContentTaskApi.test.ts | 7 ++ src/tasks/createContentTask.ts | 16 ++- 9 files changed, 345 insertions(+), 6 deletions(-) create mode 100644 src/content/__tests__/buildStaticImageArgs.test.ts create mode 100644 src/content/__tests__/renderStaticImage.test.ts create mode 100644 src/content/buildStaticImageArgs.ts create mode 100644 src/content/renderStaticImage.ts diff --git a/src/content/__tests__/buildStaticImageArgs.test.ts b/src/content/__tests__/buildStaticImageArgs.test.ts new file mode 100644 index 0000000..4c5053c --- /dev/null +++ b/src/content/__tests__/buildStaticImageArgs.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect } from "vitest"; + +import { buildStaticImageArgs } from "../buildStaticImageArgs"; + +describe("buildStaticImageArgs", () => { + it("builds ffmpeg args with image input and overlay images", () => { + const args = buildStaticImageArgs({ + imagePath: "/tmp/base.png", + overlayImagePaths: ["/tmp/ovr-0.png", "/tmp/ovr-1.png"], + outputPath: "/tmp/output.png", + }); + + expect(args).toContain("-y"); + expect(args).toContain("/tmp/base.png"); + expect(args).toContain("/tmp/ovr-0.png"); + expect(args).toContain("/tmp/ovr-1.png"); + expect(args).toContain("/tmp/output.png"); + }); + + it("includes filter_complex with crop, scale, and overlays", () => { + const args = buildStaticImageArgs({ + imagePath: "/tmp/base.png", + overlayImagePaths: ["/tmp/ovr-0.png"], + outputPath: "/tmp/output.png", + }); + + const filterIdx = args.indexOf("-filter_complex"); + expect(filterIdx).toBeGreaterThan(-1); + + const filter = args[filterIdx + 1]; + expect(filter).toContain("crop=ih*9/16:ih"); + expect(filter).toContain("scale=720:1280"); + expect(filter).toContain("scale=150:150"); + expect(filter).toContain("overlay=30:30"); + }); + + it("stacks multiple overlays vertically", () => { + const args = buildStaticImageArgs({ + imagePath: "/tmp/base.png", + overlayImagePaths: ["/tmp/a.png", "/tmp/b.png"], + outputPath: "/tmp/output.png", + }); + + const filter = args[args.indexOf("-filter_complex") + 1]; + expect(filter).toContain("overlay=30:30"); + expect(filter).toContain("overlay=30:200"); + }); + + it("handles no overlay images — just crops and scales", () => { + const args = buildStaticImageArgs({ + imagePath: "/tmp/base.png", + overlayImagePaths: [], + outputPath: "/tmp/output.png", + }); + + expect(args).toContain("-vf"); + expect(args).not.toContain("-filter_complex"); + + const vfIdx = args.indexOf("-vf"); + const vf = args[vfIdx + 1]; + expect(vf).toContain("crop=ih*9/16:ih"); + expect(vf).toContain("scale=720:1280"); + }); + + it("maps [out] and outputs single frame", () => { + const args = buildStaticImageArgs({ + imagePath: "/tmp/base.png", + overlayImagePaths: ["/tmp/ovr.png"], + outputPath: "/tmp/output.png", + }); + + expect(args).toContain("-map"); + expect(args).toContain("[out]"); + expect(args).toContain("-frames:v"); + expect(args).toContain("1"); + }); +}); diff --git a/src/content/__tests__/pollContentRuns.test.ts b/src/content/__tests__/pollContentRuns.test.ts index 9caf66e..767ffe6 100644 --- a/src/content/__tests__/pollContentRuns.test.ts +++ b/src/content/__tests__/pollContentRuns.test.ts @@ -23,14 +23,14 @@ describe("pollContentRuns", () => { it("returns completed results when all runs finish", async () => { mockPoll.mockResolvedValue({ status: "COMPLETED", - output: { videoSourceUrl: "https://v.mp4", captionText: "caption" }, + output: { videoSourceUrl: "https://v.mp4", captionText: "caption", staticImageUrl: "https://img.png" }, } as any); const results = await pollContentRuns(["run-1", "run-2"]); expect(results).toEqual([ - { runId: "run-1", status: "completed", videoUrl: "https://v.mp4", captionText: "caption" }, - { runId: "run-2", status: "completed", videoUrl: "https://v.mp4", captionText: "caption" }, + { runId: "run-1", status: "completed", videoUrl: "https://v.mp4", captionText: "caption", imageUrl: "https://img.png" }, + { runId: "run-2", status: "completed", videoUrl: "https://v.mp4", captionText: "caption", imageUrl: "https://img.png" }, ]); expect(mockPoll).toHaveBeenCalledTimes(2); expect(mockPoll).toHaveBeenCalledWith("run-1", { pollIntervalMs: 30_000 }); @@ -76,7 +76,7 @@ describe("pollContentRuns", () => { const results = await pollContentRuns(["run-1"]); expect(results).toEqual([ - { runId: "run-1", status: "completed", videoUrl: undefined, captionText: undefined }, + { runId: "run-1", status: "completed", videoUrl: undefined, captionText: undefined, imageUrl: undefined }, ]); }); @@ -105,7 +105,7 @@ describe("pollContentRuns", () => { const results = await pollContentRuns(["run-1", "run-2", "run-3"]); expect(results).toEqual([ - { runId: "run-1", status: "completed", videoUrl: "https://v.mp4", captionText: undefined }, + { runId: "run-1", status: "completed", videoUrl: "https://v.mp4", captionText: undefined, imageUrl: undefined }, { runId: "run-2", status: "failed", error: "Run failed" }, { runId: "run-3", status: "failed", error: "Network error" }, ]); diff --git a/src/content/__tests__/renderStaticImage.test.ts b/src/content/__tests__/renderStaticImage.test.ts new file mode 100644 index 0000000..7401a8c --- /dev/null +++ b/src/content/__tests__/renderStaticImage.test.ts @@ -0,0 +1,104 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +vi.mock("node:fs/promises", () => ({ + writeFile: vi.fn().mockResolvedValue(undefined), + unlink: vi.fn().mockResolvedValue(undefined), + mkdir: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("../downloadMediaToFile", () => ({ + downloadMediaToFile: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("../downloadOverlayImages", () => ({ + downloadOverlayImages: vi.fn().mockResolvedValue(["/tmp/ovr-0.png"]), +})); + +vi.mock("../runFfmpeg", () => ({ + runFfmpeg: vi.fn().mockResolvedValue(undefined), +})); + +vi.mock("../uploadToFalStorage", () => ({ + uploadToFalStorage: vi.fn().mockResolvedValue({ + url: "https://fal.ai/static-image.png", + mimeType: "image/png", + sizeBytes: 12345, + }), +})); + +vi.mock("../../sandboxes/logStep", () => ({ + logStep: vi.fn(), +})); + +import { renderStaticImage } from "../renderStaticImage"; +import { runFfmpeg } from "../runFfmpeg"; +import { uploadToFalStorage } from "../uploadToFalStorage"; +import { downloadOverlayImages } from "../downloadOverlayImages"; +import { downloadMediaToFile } from "../downloadMediaToFile"; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("renderStaticImage", () => { + it("returns the uploaded image URL and size", async () => { + const result = await renderStaticImage({ + imageUrl: "https://example.com/base.png", + overlayImageUrls: ["https://example.com/cover.png"], + }); + + expect(result).toEqual({ + imageUrl: "https://fal.ai/static-image.png", + mimeType: "image/png", + sizeBytes: 12345, + }); + }); + + it("downloads the base image to a temp file", async () => { + await renderStaticImage({ + imageUrl: "https://example.com/base.png", + overlayImageUrls: [], + }); + + expect(downloadMediaToFile).toHaveBeenCalledWith( + "https://example.com/base.png", + expect.stringContaining("base-image.png"), + ); + }); + + it("downloads overlay images", async () => { + await renderStaticImage({ + imageUrl: "https://example.com/base.png", + overlayImageUrls: ["https://example.com/a.png", "https://example.com/b.png"], + }); + + expect(downloadOverlayImages).toHaveBeenCalledWith( + ["https://example.com/a.png", "https://example.com/b.png"], + expect.any(String), + ); + }); + + it("calls runFfmpeg with args", async () => { + await renderStaticImage({ + imageUrl: "https://example.com/base.png", + overlayImageUrls: ["https://example.com/cover.png"], + }); + + expect(runFfmpeg).toHaveBeenCalledTimes(1); + const args = vi.mocked(runFfmpeg).mock.calls[0][0]; + expect(args).toContain("-y"); + }); + + it("uploads the output as image/png", async () => { + await renderStaticImage({ + imageUrl: "https://example.com/base.png", + overlayImageUrls: [], + }); + + expect(uploadToFalStorage).toHaveBeenCalledWith( + expect.stringContaining("static-image.png"), + "static-image.png", + "image/png", + ); + }); +}); diff --git a/src/content/buildStaticImageArgs.ts b/src/content/buildStaticImageArgs.ts new file mode 100644 index 0000000..83bacde --- /dev/null +++ b/src/content/buildStaticImageArgs.ts @@ -0,0 +1,63 @@ +/** Overlay image size */ +const OVERLAY_SIZE = 150; +const EDGE_PADDING = 30; +const OVERLAY_GAP = 20; + +/** + * Builds ffmpeg arguments to render a static image with overlay images. + * + * Pipeline: crop 16:9 → 9:16, scale to 720×1280, overlay images top-left stacked vertically. + * Outputs a single PNG frame. + */ +export function buildStaticImageArgs({ + imagePath, + overlayImagePaths, + outputPath, +}: { + imagePath: string; + overlayImagePaths: string[]; + outputPath: string; +}): string[] { + const hasOverlays = overlayImagePaths.length > 0; + const args = ["-y"]; + + if (hasOverlays) { + args.push("-i", imagePath); + for (const p of overlayImagePaths) { + args.push("-i", p); + } + + const parts: string[] = []; + + // Crop + scale base image + parts.push("[0:v]crop=ih*9/16:ih,scale=720:1280[img_base]"); + + // Scale each overlay + for (let i = 0; i < overlayImagePaths.length; i++) { + parts.push(`[${1 + i}:v]scale=${OVERLAY_SIZE}:${OVERLAY_SIZE}[ovr_${i}]`); + } + + // Chain overlays — stacked vertically from top-left + let prevLabel = "img_base"; + for (let i = 0; i < overlayImagePaths.length; i++) { + const x = EDGE_PADDING; + const y = EDGE_PADDING + i * (OVERLAY_SIZE + OVERLAY_GAP); + const outLabel = i < overlayImagePaths.length - 1 ? `ovr_out_${i}` : "out"; + parts.push(`[${prevLabel}][ovr_${i}]overlay=${x}:${y}[${outLabel}]`); + prevLabel = outLabel; + } + + args.push("-filter_complex", parts.join(";")); + args.push("-map", "[out]"); + args.push("-frames:v", "1", outputPath); + } else { + args.push( + "-i", imagePath, + "-vf", "crop=ih*9/16:ih,scale=720:1280", + "-frames:v", "1", + outputPath, + ); + } + + return args; +} diff --git a/src/content/pollContentRuns.ts b/src/content/pollContentRuns.ts index 2cd6cbc..fb20d53 100644 --- a/src/content/pollContentRuns.ts +++ b/src/content/pollContentRuns.ts @@ -7,6 +7,7 @@ export type ContentRunResult = { runId: string; status: "completed" | "failed" | "timeout"; videoUrl?: string; + imageUrl?: string; captionText?: string; error?: string; }; @@ -41,6 +42,7 @@ export async function pollContentRuns( if (run.status === "COMPLETED") { const output = run.output as { videoSourceUrl?: string; + staticImageUrl?: string; captionText?: string; } | null; @@ -49,6 +51,7 @@ export async function pollContentRuns( runId, status: "completed" as const, videoUrl: output?.videoSourceUrl, + imageUrl: output?.staticImageUrl, captionText: output?.captionText, }; } diff --git a/src/content/renderStaticImage.ts b/src/content/renderStaticImage.ts new file mode 100644 index 0000000..1dcba4b --- /dev/null +++ b/src/content/renderStaticImage.ts @@ -0,0 +1,64 @@ +import { randomUUID } from "node:crypto"; +import { unlink, mkdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { logStep } from "../sandboxes/logStep"; +import { buildStaticImageArgs } from "./buildStaticImageArgs"; +import { downloadOverlayImages } from "./downloadOverlayImages"; +import { downloadMediaToFile } from "./downloadMediaToFile"; +import { runFfmpeg } from "./runFfmpeg"; +import { uploadToFalStorage } from "./uploadToFalStorage"; + +export interface RenderStaticImageInput { + imageUrl: string; + overlayImageUrls: string[]; +} + +export interface RenderStaticImageOutput { + imageUrl: string; + mimeType: string; + sizeBytes: number; +} + +/** + * Renders a static image by cropping to 9:16, scaling to 720×1280, + * and overlaying playlist cover images. Uploads the result to fal.ai storage. + */ +export async function renderStaticImage( + input: RenderStaticImageInput, +): Promise { + const tempDir = join(tmpdir(), `content-static-${randomUUID()}`); + await mkdir(tempDir, { recursive: true }); + + const imagePath = join(tempDir, "base-image.png"); + const outputPath = join(tempDir, "static-image.png"); + let overlayPaths: string[] = []; + + try { + logStep("Downloading base image for static render"); + await downloadMediaToFile(input.imageUrl, imagePath); + + overlayPaths = await downloadOverlayImages(input.overlayImageUrls, tempDir); + + const ffmpegArgs = buildStaticImageArgs({ + imagePath, + overlayImagePaths: overlayPaths, + outputPath, + }); + + logStep("Running ffmpeg static image render", true, { + overlayCount: overlayPaths.length, + }); + + await runFfmpeg(ffmpegArgs); + + logStep("Static image rendered, uploading to fal.ai storage"); + const result = await uploadToFalStorage(outputPath, "static-image.png", "image/png"); + logStep("Static image uploaded", false, { imageUrl: result.url, sizeBytes: result.sizeBytes }); + + return { imageUrl: result.url, mimeType: result.mimeType, sizeBytes: result.sizeBytes }; + } finally { + const cleanupPaths = [imagePath, outputPath, ...overlayPaths]; + await Promise.all(cleanupPaths.map(p => unlink(p).catch(() => undefined))); + } +} diff --git a/src/tasks/__tests__/createContentTask.test.ts b/src/tasks/__tests__/createContentTask.test.ts index cf33939..7a46188 100644 --- a/src/tasks/__tests__/createContentTask.test.ts +++ b/src/tasks/__tests__/createContentTask.test.ts @@ -80,6 +80,13 @@ vi.mock("../../content/renderFinalVideo", () => ({ sizeBytes: 5000, }), })); +vi.mock("../../content/renderStaticImage", () => ({ + renderStaticImage: vi.fn().mockResolvedValue({ + imageUrl: "https://fal.ai/storage/static-image.png", + mimeType: "image/png", + sizeBytes: 12345, + }), +})); vi.mock("../../content/loadTemplate", () => ({ loadTemplate: vi.fn().mockResolvedValue({ diff --git a/src/tasks/__tests__/createContentTaskApi.test.ts b/src/tasks/__tests__/createContentTaskApi.test.ts index 5d0198d..43d5337 100644 --- a/src/tasks/__tests__/createContentTaskApi.test.ts +++ b/src/tasks/__tests__/createContentTaskApi.test.ts @@ -63,6 +63,13 @@ vi.mock("../../content/renderFinalVideo", () => ({ sizeBytes: 5000, }), })); +vi.mock("../../content/renderStaticImage", () => ({ + renderStaticImage: vi.fn().mockResolvedValue({ + imageUrl: "https://fal.ai/storage/static-image.png", + mimeType: "image/png", + sizeBytes: 12345, + }), +})); vi.mock("../../content/loadTemplate", () => ({ loadTemplate: vi.fn().mockResolvedValue({ name: "artist-caption-bedroom", diff --git a/src/tasks/createContentTask.ts b/src/tasks/createContentTask.ts index cad1e64..fbaec1f 100644 --- a/src/tasks/createContentTask.ts +++ b/src/tasks/createContentTask.ts @@ -15,6 +15,7 @@ import { generateVideo, } from "../recoup/contentApi"; import { resolveCaptionText } from "../content/resolveCaptionText"; +import { renderStaticImage } from "../content/renderStaticImage"; /** * Content-creation task — full pipeline that generates a social-ready video. @@ -133,6 +134,7 @@ export const createContentTask = schemaTask({ // --- Step 10: Final render (ffmpeg) --- logStep("Rendering final video (ffmpeg)"); + const overlayUrls = template.usesImageOverlay ? additionalImageUrls : undefined; const finalVideo = await renderFinalVideo({ videoUrl, songBuffer: audioClip.songBuffer, @@ -140,9 +142,20 @@ export const createContentTask = schemaTask({ audioDurationSeconds: audioClip.durationSeconds, captionText, hasAudio: payload.lipsync, - overlayImageUrls: template.usesImageOverlay ? additionalImageUrls : undefined, + overlayImageUrls: overlayUrls, }); + // --- Step 11: Render static image with overlays (when template uses overlays) --- + let staticImageUrl: string | undefined; + if (template.usesImageOverlay && overlayUrls && overlayUrls.length > 0) { + logStep("Rendering static image with overlays"); + const staticImage = await renderStaticImage({ + imageUrl, + overlayImageUrls: overlayUrls, + }); + staticImageUrl = staticImage.imageUrl; + } + // --- Return result --- const result = { status: "completed", @@ -152,6 +165,7 @@ export const createContentTask = schemaTask({ lipsync: payload.lipsync, videoSourceUrl: finalVideo.videoUrl, renderedVideoBytes: finalVideo.sizeBytes, + staticImageUrl, imageUrl, video: null, audio: {