From 4a84fee3d016b23b0160ced705da17587e774d2c Mon Sep 17 00:00:00 2001
From: Abu <abuahmed0821@gmail.com>
Date: Wed, 18 Mar 2026 16:16:49 -0400
Subject: [PATCH 1/3] abu/multimodal

---
 apps/web/src/components/attachment-chip.tsx   |  55 ++++++++
 apps/web/src/hooks/use-file-attachments.ts    | 128 ++++++++++++++++++
 apps/web/src/routes/chat/index.tsx            | 112 ++++++++++++++-
 .../convex-backend/convex/_generated/api.d.ts |   2 +
 packages/convex-backend/convex/files.ts       |  29 ++++
 5 files changed, 325 insertions(+), 1 deletion(-)
 create mode 100644 apps/web/src/components/attachment-chip.tsx
 create mode 100644 apps/web/src/hooks/use-file-attachments.ts
 create mode 100644 packages/convex-backend/convex/files.ts

diff --git a/apps/web/src/components/attachment-chip.tsx b/apps/web/src/components/attachment-chip.tsx
new file mode 100644
index 0000000..0ba9762
--- /dev/null
+++ b/apps/web/src/components/attachment-chip.tsx
@@ -0,0 +1,55 @@
+import { FileText, Loader2, X } from "lucide-react";
+import type { PendingAttachment } from "../hooks/use-file-attachments";
+
+export function AttachmentChip({
+	attachment,
+	onRemove,
+}: {
+	attachment: PendingAttachment;
+	onRemove: () => void;
+}) {
+	const isImage = attachment.mimeType.startsWith("image/");
+	const isPdf = attachment.mimeType === "application/pdf";
+
+	return (
+		<div className="group relative flex h-16 w-16 shrink-0 items-center justify-center overflow-hidden border border-border bg-muted">
+			{isImage && attachment.previewUrl ? (
+				<img
+					src={attachment.previewUrl}
+					alt={attachment.fileName}
+					className="h-full w-full object-cover"
+				/>
+			) : isPdf ? (
+				<div className="flex flex-col items-center gap-1 px-1">
+					<FileText size={20} className="text-muted-foreground" />
+					<span className="w-full truncate text-center text-[9px] leading-tight text-muted-foreground">
+						{attachment.fileName}
+					</span>
+				</div>
+			) : null}
+
+			{/* Uploading spinner overlay */}
+			{attachment.status === "uploading" && (
+				<div className="absolute inset-0 flex items-center justify-center bg-background/60">
+					<Loader2 size={14} className="animate-spin text-foreground" />
+				</div>
+			)}
+
+			{/* Error overlay */}
+			{attachment.status === "error" && (
+				<div className="absolute inset-0 flex items-center justify-center bg-destructive/20">
+					<span className="text-[9px] font-medium text-destructive">Error</span>
+				</div>
+			)}
+
+			{/* Remove button — appears on hover */}
+			<button
+				type="button"
+				onClick={onRemove}
+				className="absolute right-0.5 top-0.5 flex h-4 w-4 items-center justify-center bg-background/80 text-foreground opacity-0 transition-opacity group-hover:opacity-100"
+			>
+				<X size={10} />
+			</button>
+		</div>
+	);
+}
diff --git a/apps/web/src/hooks/use-file-attachments.ts b/apps/web/src/hooks/use-file-attachments.ts
new file mode 100644
index 0000000..eec8e5d
--- /dev/null
+++ b/apps/web/src/hooks/use-file-attachments.ts
@@ -0,0 +1,128 @@
+import { api } from "@harness/convex-backend/convex/_generated/api";
+import { useConvex } from "convex/react";
+import { useCallback, useRef, useState } from "react";
+import toast from "react-hot-toast";
+
+const ACCEPTED_TYPES = new Set([
+	"image/png",
+	"image/jpeg",
+	"image/gif",
+	"image/webp",
+	"application/pdf",
+]);
+
+const MAX_IMAGE_BYTES = 10 * 1024 * 1024; // 10 MB
+const MAX_PDF_BYTES = 20 * 1024 * 1024; // 20 MB
+
+export interface PendingAttachment {
+	localId: string;
+	previewUrl: string | null; // object URL for images, null for PDFs
+	mimeType: string;
+	status: "uploading" | "ready" | "error";
+	storageId?: string;
+	fileName: string;
+	fileSize: number;
+}
+
+export function useFileAttachments() {
+	const [attachments, setAttachments] = useState<PendingAttachment[]>([]);
+	const convex = useConvex();
+	const localIdCounter = useRef(0);
+
+	const uploadOne = useCallback(
+		async (file: File, localId: string) => {
+			try {
+				const uploadUrl = await convex.mutation(
+					api.files.generateUploadUrl,
+					{},
+				);
+				const res = await fetch(uploadUrl, {
+					method: "POST",
+					headers: { "Content-Type": file.type },
+					body: file,
+				});
+				if (!res.ok) throw new Error("Upload failed");
+				const { storageId } = await res.json();
+				setAttachments((prev) =>
+					prev.map((a) =>
+						a.localId === localId ? { ...a, status: "ready", storageId } : a,
+					),
+				);
+			} catch {
+				setAttachments((prev) =>
+					prev.map((a) =>
+						a.localId === localId ? { ...a, status: "error" } : a,
+					),
+				);
+				toast.error(`Failed to upload ${file.name}`);
+			}
+		},
+		[convex],
+	);
+
+	const addFiles = useCallback(
+		(files: File[]) => {
+			for (const file of files) {
+				if (!ACCEPTED_TYPES.has(file.type)) {
+					toast.error(`${file.name}: only images (PNG, JPG, GIF, WebP) and PDFs are supported`);
+					continue;
+				}
+				const maxBytes =
+					file.type === "application/pdf" ? MAX_PDF_BYTES : MAX_IMAGE_BYTES;
+				if (file.size > maxBytes) {
+					const limit = file.type === "application/pdf" ? "20 MB" : "10 MB";
+					toast.error(`${file.name}: exceeds ${limit} limit`);
+					continue;
+				}
+
+				const localId = String(++localIdCounter.current);
+				const previewUrl = file.type.startsWith("image/")
+					? URL.createObjectURL(file)
+					: null;
+
+				setAttachments((prev) => [
+					...prev,
+					{
+						localId,
+						previewUrl,
+						mimeType: file.type,
+						status: "uploading",
+						fileName: file.name,
+						fileSize: file.size,
+					},
+				]);
+
+				// Fire and forget — state updates happen inside uploadOne
+				uploadOne(file, localId);
+			}
+		},
+		[uploadOne],
+	);
+
+	const removeAttachment = useCallback((localId: string) => {
+		setAttachments((prev) => {
+			const target = prev.find((a) => a.localId === localId);
+			if (target?.previewUrl) URL.revokeObjectURL(target.previewUrl);
+			return prev.filter((a) => a.localId !== localId);
+		});
+	}, []);
+
+	const clearAttachments = useCallback(() => {
+		setAttachments((prev) => {
+			for (const a of prev) {
+				if (a.previewUrl) URL.revokeObjectURL(a.previewUrl);
+			}
+			return [];
+		});
+	}, []);
+
+	const hasUploading = attachments.some((a) => a.status === "uploading");
+
+	return {
+		attachments,
+		addFiles,
+		removeAttachment,
+		clearAttachments,
+		hasUploading,
+	};
+}
diff --git a/apps/web/src/routes/chat/index.tsx b/apps/web/src/routes/chat/index.tsx
index 45d42a4..8af31e9 100644
--- a/apps/web/src/routes/chat/index.tsx
+++ b/apps/web/src/routes/chat/index.tsx
@@ -23,6 +23,7 @@ import {
 	MessageSquare,
 	PanelLeftClose,
 	PanelLeftOpen,
+	Paperclip,
 	Plus,
 	Search, // Icon for search
 	Settings,
@@ -102,6 +103,11 @@ import {
 	useChatStream,
 } from "../../lib/use-chat-stream";
 import { cn } from "../../lib/utils";
+import { AttachmentChip } from "../../components/attachment-chip";
+import {
+	type PendingAttachment,
+	useFileAttachments,
+} from "../../hooks/use-file-attachments";
 
 export const Route = createFileRoute("/chat/")({
 	validateSearch: (search: Record<string, unknown>) => ({
@@ -2595,6 +2601,11 @@ function ChatInput({
 }) {
 	const [text, setText] = useState("");
 	const textareaRef = useRef<HTMLTextAreaElement>(null);
+	const fileInputRef = useRef<HTMLInputElement>(null);
+	const [isDragOver, setIsDragOver] = useState(false);
+
+	const { attachments, addFiles, removeAttachment, clearAttachments, hasUploading } =
+		useFileAttachments();
 
 	// Fill input from suggested prompt click
 	useEffect(() => {
@@ -2648,6 +2659,7 @@ function ChatInput({
 		setText("");
 		setHistoryIndex(-1);
 		setDraft("");
+		clearAttachments();
 
 		// If streaming, just enqueue — don't interrupt
 		if (isStreaming && conversationId) {
@@ -2742,10 +2754,72 @@ function ChatInput({
 		}
 	};
 
+	const handlePaste = (e: React.ClipboardEvent<HTMLTextAreaElement>) => {
+		const files = Array.from(e.clipboardData.files).filter(
+			(f) => f.type.startsWith("image/") || f.type === "application/pdf",
+		);
+		if (files.length > 0) {
+			e.preventDefault();
+			addFiles(files);
+		}
+	};
+
+	const handleDragOver = (e: React.DragEvent<HTMLDivElement>) => {
+		e.preventDefault();
+		if (e.dataTransfer.types.includes("Files")) {
+			setIsDragOver(true);
+		}
+	};
+
+	const handleDragLeave = (e: React.DragEvent<HTMLDivElement>) => {
+		// Only clear if leaving the container entirely (not moving to a child)
+		if (!e.currentTarget.contains(e.relatedTarget as Node)) {
+			setIsDragOver(false);
+		}
+	};
+
+	const handleDrop = (e: React.DragEvent<HTMLDivElement>) => {
+		e.preventDefault();
+		setIsDragOver(false);
+		const files = Array.from(e.dataTransfer.files);
+		if (files.length > 0) addFiles(files);
+	};
+
 	const showStopButton = isStreaming && !text.trim();
 
 	return (
-		<div className="border-t border-border px-4 py-3">
+		<div
+			className={cn(
+				"relative border-t border-border px-4 py-3 transition-colors",
+				isDragOver && "bg-primary/5",
+			)}
+			onDragOver={handleDragOver}
+			onDragLeave={handleDragLeave}
+			onDrop={handleDrop}
+		>
+			{/* Drop overlay */}
+			{isDragOver && (
+				<div className="pointer-events-none absolute inset-0 z-10 flex items-center justify-center border-2 border-dashed border-primary/50 bg-primary/5">
+					<div className="flex items-center gap-2 text-sm font-medium text-primary">
+						<Paperclip size={16} />
+						Drop files to attach
+					</div>
+				</div>
+			)}
+
+			{/* Hidden file input */}
+			<input
+				ref={fileInputRef}
+				type="file"
+				accept="image/png,image/jpeg,image/gif,image/webp,application/pdf"
+				multiple
+				className="hidden"
+				onChange={(e) => {
+					if (e.target.files) addFiles(Array.from(e.target.files));
+					e.target.value = "";
+				}}
+			/>
+
 			<div className="mx-auto max-w-3xl">
 				{/* Queued messages as chips above the input */}
 				<AnimatePresence>
@@ -2799,7 +2873,41 @@ function ChatInput({
 					)}
 				</AnimatePresence>
 
+				{/* Attachment preview strip */}
+				<AnimatePresence>
+					{attachments.length > 0 && (
+						<motion.div
+							initial={{ opacity: 0, height: 0 }}
+							animate={{ opacity: 1, height: "auto" }}
+							exit={{ opacity: 0, height: 0 }}
+							className="mb-2 flex flex-wrap gap-2 overflow-hidden"
+						>
+							{attachments.map((attachment) => (
+								<AttachmentChip
+									key={attachment.localId}
+									attachment={attachment}
+									onRemove={() => removeAttachment(attachment.localId)}
+								/>
+							))}
+						</motion.div>
+					)}
+				</AnimatePresence>
+
 				<div className="flex items-end gap-2 border border-border bg-background px-3 py-2 focus-within:border-foreground/30">
+					{/* Attach button */}
+					<Tooltip>
+						<TooltipTrigger asChild>
+							<button
+								type="button"
+								onClick={() => fileInputRef.current?.click()}
+								className="mb-0.5 shrink-0 text-muted-foreground transition-colors hover:text-foreground"
+							>
+								<Paperclip size={15} />
+							</button>
+						</TooltipTrigger>
+						<TooltipContent>Attach image or PDF</TooltipContent>
+					</Tooltip>
+
 					<textarea
 						ref={textareaRef}
 						value={text}
@@ -2811,6 +2919,7 @@ function ChatInput({
 							}
 						}}
 						onKeyDown={handleKeyDown}
+						onPaste={handlePaste}
 						placeholder="Send a message..."
 						rows={1}
 						className="max-h-[200px] min-h-[24px] flex-1 resize-none bg-transparent text-sm text-foreground outline-none placeholder:text-muted-foreground"
@@ -2829,6 +2938,7 @@ function ChatInput({
 								disabled={
 									!showStopButton &&
 									(!text.trim() ||
+										hasUploading ||
 										sendMessage.isPending ||
 										createConvo.isPending)
 								}
diff --git a/packages/convex-backend/convex/_generated/api.d.ts b/packages/convex-backend/convex/_generated/api.d.ts
index 7d74e6e..fb2193f 100644
--- a/packages/convex-backend/convex/_generated/api.d.ts
+++ b/packages/convex-backend/convex/_generated/api.d.ts
@@ -9,6 +9,7 @@
  */
 
 import type * as conversations from "../conversations.js";
+import type * as files from "../files.js";
 import type * as harnesses from "../harnesses.js";
 import type * as mcpOAuthTokens from "../mcpOAuthTokens.js";
 import type * as messages from "../messages.js";
@@ -23,6 +24,7 @@ import type {
 
 declare const fullApi: ApiFromModules<{
   conversations: typeof conversations;
+  files: typeof files;
   harnesses: typeof harnesses;
   mcpOAuthTokens: typeof mcpOAuthTokens;
   messages: typeof messages;
diff --git a/packages/convex-backend/convex/files.ts b/packages/convex-backend/convex/files.ts
new file mode 100644
index 0000000..30c40b6
--- /dev/null
+++ b/packages/convex-backend/convex/files.ts
@@ -0,0 +1,29 @@
+import { v } from "convex/values";
+import { mutation, query } from "./_generated/server";
+
+/**
+ * Generates a short-lived presigned upload URL for Convex file storage.
+ * The client POSTs the file bytes directly to this URL, then receives
+ * a storageId in the response JSON.
+ */
+export const generateUploadUrl = mutation({
+	handler: async (ctx) => {
+		const identity = await ctx.auth.getUserIdentity();
+		if (!identity) throw new Error("Unauthenticated");
+		return await ctx.storage.generateUploadUrl();
+	},
+});
+
+/**
+ * Returns the serving URL for a stored file, or null if not found.
+ * Used to render attachment thumbnails in message history and to
+ * resolve storageIds to URLs before sending to FastAPI.
+ */
+export const getFileUrl = query({
+	args: { storageId: v.id("_storage") },
+	handler: async (ctx, args) => {
+		const identity = await ctx.auth.getUserIdentity();
+		if (!identity) return null;
+		return await ctx.storage.getUrl(args.storageId);
+	},
+});

From 7064bc2e9782080735aa1e44108ef928699b940d Mon Sep 17 00:00:00 2001
From: Abu <abuahmed0821@gmail.com>
Date: Wed, 18 Mar 2026 17:05:54 -0400
Subject: [PATCH 2/3] multimodal progress

---
 apps/web/src/components/attachment-chip.tsx   | 21 ++--
 .../src/components/message-attachments.tsx    | 96 +++++++++++++++++++
 apps/web/src/hooks/use-file-attachments.ts    | 23 +++++
 apps/web/src/lib/use-chat-stream.ts           |  1 +
 apps/web/src/routes/chat/index.tsx            | 38 +++++++-
 packages/convex-backend/convex/messages.ts    | 13 +++
 packages/convex-backend/convex/schema.ts      | 10 ++
 packages/fastapi/app/models.py                |  7 ++
 packages/fastapi/app/routes/chat.py           | 22 +++++
 9 files changed, 215 insertions(+), 16 deletions(-)
 create mode 100644 apps/web/src/components/message-attachments.tsx

diff --git a/apps/web/src/components/attachment-chip.tsx b/apps/web/src/components/attachment-chip.tsx
index 0ba9762..8bcb3a6 100644
--- a/apps/web/src/components/attachment-chip.tsx
+++ b/apps/web/src/components/attachment-chip.tsx
@@ -12,43 +12,40 @@ export function AttachmentChip({
 	const isPdf = attachment.mimeType === "application/pdf";
 
 	return (
-		<div className="group relative flex h-16 w-16 shrink-0 items-center justify-center overflow-hidden border border-border bg-muted">
+		<div className="group relative flex h-8 shrink-0 items-center overflow-hidden border border-border bg-muted">
 			{isImage && attachment.previewUrl ? (
 				<img
 					src={attachment.previewUrl}
 					alt={attachment.fileName}
-					className="h-full w-full object-cover"
+					className="h-full w-auto object-cover"
 				/>
 			) : isPdf ? (
-				<div className="flex flex-col items-center gap-1 px-1">
-					<FileText size={20} className="text-muted-foreground" />
-					<span className="w-full truncate text-center text-[9px] leading-tight text-muted-foreground">
+				<div className="flex items-center gap-1.5 px-2">
+					<FileText size={12} className="shrink-0 text-muted-foreground" />
+					<span className="max-w-[120px] truncate text-[11px] text-muted-foreground">
 						{attachment.fileName}
 					</span>
 				</div>
 			) : null}
 
-			{/* Uploading spinner overlay */}
 			{attachment.status === "uploading" && (
-				<div className="absolute inset-0 flex items-center justify-center bg-background/60">
-					<Loader2 size={14} className="animate-spin text-foreground" />
+				<div className="absolute inset-0 flex items-center justify-center bg-background/70">
+					<Loader2 size={10} className="animate-spin text-foreground" />
 				</div>
 			)}
 
-			{/* Error overlay */}
 			{attachment.status === "error" && (
 				<div className="absolute inset-0 flex items-center justify-center bg-destructive/20">
 					<span className="text-[9px] font-medium text-destructive">Error</span>
 				</div>
 			)}
 
-			{/* Remove button — appears on hover */}
 			<button
 				type="button"
 				onClick={onRemove}
-				className="absolute right-0.5 top-0.5 flex h-4 w-4 items-center justify-center bg-background/80 text-foreground opacity-0 transition-opacity group-hover:opacity-100"
+				className="absolute right-0.5 top-0.5 flex h-3.5 w-3.5 items-center justify-center bg-background/80 text-foreground opacity-0 transition-opacity group-hover:opacity-100"
 			>
-				<X size={10} />
+				<X size={8} />
 			</button>
 		</div>
 	);
diff --git a/apps/web/src/components/message-attachments.tsx b/apps/web/src/components/message-attachments.tsx
new file mode 100644
index 0000000..039dde0
--- /dev/null
+++ b/apps/web/src/components/message-attachments.tsx
@@ -0,0 +1,96 @@
+import { convexQuery } from "@convex-dev/react-query";
+import { api } from "@harness/convex-backend/convex/_generated/api";
+import type { Id } from "@harness/convex-backend/convex/_generated/dataModel";
+import { useQuery } from "@tanstack/react-query";
+import { FileText } from "lucide-react";
+import {
+	Dialog,
+	DialogContent,
+	DialogTrigger,
+} from "./ui/dialog";
+
+interface Attachment {
+	storageId: Id<"_storage">;
+	mimeType: string;
+	fileName: string;
+	fileSize: number;
+}
+
+function AttachmentItem({ attachment }: { attachment: Attachment }) {
+	const { data: url } = useQuery(
+		convexQuery(api.files.getFileUrl, { storageId: attachment.storageId }),
+	);
+
+	if (!url) return null;
+
+	const square = "h-16 w-16 shrink-0 overflow-hidden rounded-lg border border-border";
+
+	if (attachment.mimeType.startsWith("image/")) {
+		return (
+			<Dialog>
+				<DialogTrigger asChild>
+					<button type="button" className={`${square} cursor-zoom-in bg-muted`}>
+						<img
+							src={url}
+							alt={attachment.fileName}
+							className="h-full w-full object-cover"
+						/>
+					</button>
+				</DialogTrigger>
+				<DialogContent
+					className="max-w-[90vw] border-0 bg-transparent p-0 shadow-none"
+					showCloseButton={false}
+				>
+					<img
+						src={url}
+						alt={attachment.fileName}
+						className="max-h-[90vh] max-w-full rounded-lg object-contain"
+					/>
+				</DialogContent>
+			</Dialog>
+		);
+	}
+
+	if (attachment.mimeType === "application/pdf") {
+		return (
+			<Dialog>
+				<DialogTrigger asChild>
+					<button
+						type="button"
+						className={`${square} flex flex-col items-center justify-center gap-1 bg-muted px-1 transition-colors hover:bg-muted/80`}
+					>
+						<FileText size={20} className="shrink-0 text-muted-foreground" />
+						<span className="w-full truncate text-center text-[9px] leading-tight text-muted-foreground">
+							{attachment.fileName}
+						</span>
+					</button>
+				</DialogTrigger>
+				<DialogContent className="max-w-4xl border-0 bg-transparent p-0 shadow-none" showCloseButton={false}>
+					<iframe
+						src={url}
+						title={attachment.fileName}
+						className="h-[90vh] w-full rounded-lg"
+					/>
+				</DialogContent>
+			</Dialog>
+		);
+	}
+
+	return null;
+}
+
+export function MessageAttachments({
+	attachments,
+}: {
+	attachments: Attachment[];
+}) {
+	if (attachments.length === 0) return null;
+
+	return (
+		<div className="mb-1.5 flex flex-wrap items-end gap-1.5">
+			{attachments.map((a) => (
+				<AttachmentItem key={a.storageId} attachment={a} />
+			))}
+		</div>
+	);
+}
diff --git a/apps/web/src/hooks/use-file-attachments.ts b/apps/web/src/hooks/use-file-attachments.ts
index eec8e5d..14cfa42 100644
--- a/apps/web/src/hooks/use-file-attachments.ts
+++ b/apps/web/src/hooks/use-file-attachments.ts
@@ -1,4 +1,5 @@
 import { api } from "@harness/convex-backend/convex/_generated/api";
+import type { Id } from "@harness/convex-backend/convex/_generated/dataModel";
 import { useConvex } from "convex/react";
 import { useCallback, useRef, useState } from "react";
 import toast from "react-hot-toast";
@@ -118,11 +119,33 @@ export function useFileAttachments() {
 
 	const hasUploading = attachments.some((a) => a.status === "uploading");
 
+	const resolveSignedUrls = useCallback(
+		async (
+			readyAttachments: Array<{
+				storageId: string;
+				mimeType: string;
+				fileName: string;
+			}>,
+		): Promise<Array<{ url: string; mime_type: string; file_name: string }>> => {
+			const results = await Promise.all(
+				readyAttachments.map(async (a) => {
+					const url = await convex.query(api.files.getFileUrl, {
+						storageId: a.storageId as Id<"_storage">,
+					});
+					return url ? { url, mime_type: a.mimeType, file_name: a.fileName } : null;
+				}),
+			);
+			return results.filter((r): r is NonNullable<typeof r> => r !== null);
+		},
+		[convex],
+	);
+
 	return {
 		attachments,
 		addFiles,
 		removeAttachment,
 		clearAttachments,
 		hasUploading,
+		resolveSignedUrls,
 	};
 }
diff --git a/apps/web/src/lib/use-chat-stream.ts b/apps/web/src/lib/use-chat-stream.ts
index f44d8e5..ff0ca95 100644
--- a/apps/web/src/lib/use-chat-stream.ts
+++ b/apps/web/src/lib/use-chat-stream.ts
@@ -72,6 +72,7 @@ export interface ChatStreamRequest {
 		name: string;
 	};
 	conversation_id: string;
+	attachments?: Array<{ url: string; mime_type: string; file_name: string }>;
 }
 
 export function useChatStream(callbacks: UseChatStreamCallbacks) {
diff --git a/apps/web/src/routes/chat/index.tsx b/apps/web/src/routes/chat/index.tsx
index 8af31e9..f0034f3 100644
--- a/apps/web/src/routes/chat/index.tsx
+++ b/apps/web/src/routes/chat/index.tsx
@@ -104,6 +104,7 @@ import {
 } from "../../lib/use-chat-stream";
 import { cn } from "../../lib/utils";
 import { AttachmentChip } from "../../components/attachment-chip";
+import { MessageAttachments } from "../../components/message-attachments";
 import {
 	type PendingAttachment,
 	useFileAttachments,
@@ -1760,6 +1761,12 @@ function ChatMessages({
 		};
 		model?: string;
 		interrupted?: boolean;
+		attachments?: Array<{
+			storageId: Id<"_storage">;
+			mimeType: string;
+			fileName: string;
+			fileSize: number;
+		}>;
 	}>;
 	streamingContent: string | null;
 	streamingReasoning: string | null;
@@ -2018,7 +2025,13 @@ function ChatMessages({
 										</AvatarFallback>
 									</Avatar>
 								)}
-								<div className="max-w-[80%]">
+								<div className={cn(
+									"max-w-[80%]",
+									msg.role === "user" && "flex flex-col items-end",
+								)}>
+									{msg.role === "user" && msg.attachments && msg.attachments.length > 0 && (
+										<MessageAttachments attachments={msg.attachments} />
+									)}
 									<div
 										className={cn(
 											"text-sm leading-relaxed",
@@ -2604,7 +2617,7 @@ function ChatInput({
 	const fileInputRef = useRef<HTMLInputElement>(null);
 	const [isDragOver, setIsDragOver] = useState(false);
 
-	const { attachments, addFiles, removeAttachment, clearAttachments, hasUploading } =
+	const { attachments, addFiles, removeAttachment, clearAttachments, hasUploading, resolveSignedUrls } =
 		useFileAttachments();
 
 	// Fill input from suggested prompt click
@@ -2689,12 +2702,23 @@ function ChatInput({
 			onConvoCreated(newId);
 		}
 
+		// Capture ready attachments before clearing
+		const readyAttachments = attachments
+			.filter((a) => a.status === "ready" && a.storageId)
+			.map((a) => ({
+				storageId: a.storageId as Id<"_storage">,
+				mimeType: a.mimeType,
+				fileName: a.fileName,
+				fileSize: a.fileSize,
+			}));
+
 		// Save user message to Convex
 		await sendMessage.mutateAsync({
 			conversationId: convoId,
 			role: "user",
 			content,
 			harnessId: activeHarness._id,
+			...(readyAttachments.length > 0 ? { attachments: readyAttachments } : {}),
 		});
 
 		// Build message history for the LLM
@@ -2705,11 +2729,17 @@ function ChatInput({
 			})) ?? [];
 		history.push({ role: "user", content });
 
+		// Resolve signed URLs for any attachments
+		const signedAttachments = readyAttachments.length > 0
+			? await resolveSignedUrls(readyAttachments)
+			: undefined;
+
 		// Start streaming from FastAPI
 		onStream({
 			messages: history,
 			harness: harnessConfig,
 			conversation_id: convoId,
+			...(signedAttachments && signedAttachments.length > 0 ? { attachments: signedAttachments } : {}),
 		});
 	};
 
@@ -2790,7 +2820,7 @@ function ChatInput({
 	return (
 		<div
 			className={cn(
-				"relative border-t border-border px-4 py-3 transition-colors",
+				"relative border-t border-border px-4 py-2 transition-colors",
 				isDragOver && "bg-primary/5",
 			)}
 			onDragOver={handleDragOver}
@@ -2820,7 +2850,7 @@ function ChatInput({
 				}}
 			/>
 
-			<div className="mx-auto max-w-3xl">
+			<div className="mx-auto max-w-xl">
 				{/* Queued messages as chips above the input */}
 				<AnimatePresence>
 					{messageQueue.length > 0 && (
diff --git a/packages/convex-backend/convex/messages.ts b/packages/convex-backend/convex/messages.ts
index b12ec3a..29b672e 100644
--- a/packages/convex-backend/convex/messages.ts
+++ b/packages/convex-backend/convex/messages.ts
@@ -23,6 +23,16 @@ export const send = mutation({
 		role: v.union(v.literal("user"), v.literal("assistant")),
 		content: v.string(),
 		harnessId: v.optional(v.id("harnesses")),
+		attachments: v.optional(
+			v.array(
+				v.object({
+					storageId: v.id("_storage"),
+					mimeType: v.string(),
+					fileName: v.string(),
+					fileSize: v.number(),
+				}),
+			),
+		),
 	},
 	handler: async (ctx, args) => {
 		const identity = await ctx.auth.getUserIdentity();
@@ -36,6 +46,9 @@ export const send = mutation({
 			userId: identity.subject,
 			role: args.role,
 			content: args.content,
+			...(args.attachments && args.attachments.length > 0
+				? { attachments: args.attachments }
+				: {}),
 		});
 
 		const patch: { lastMessageAt: number; lastHarnessId?: typeof args.harnessId } = {
diff --git a/packages/convex-backend/convex/schema.ts b/packages/convex-backend/convex/schema.ts
index 6cb8fa0..3666de2 100644
--- a/packages/convex-backend/convex/schema.ts
+++ b/packages/convex-backend/convex/schema.ts
@@ -83,6 +83,16 @@ export default defineSchema({
 		),
 		model: v.optional(v.string()),
 		interrupted: v.optional(v.boolean()),
+		attachments: v.optional(
+			v.array(
+				v.object({
+					storageId: v.id("_storage"),
+					mimeType: v.string(),
+					fileName: v.string(),
+					fileSize: v.number(),
+				}),
+			),
+		),
 	})
 		.index("by_conversation", ["conversationId"])
 		.searchIndex("search_content", {
diff --git a/packages/fastapi/app/models.py b/packages/fastapi/app/models.py
index 22bab27..50617d0 100644
--- a/packages/fastapi/app/models.py
+++ b/packages/fastapi/app/models.py
@@ -16,6 +16,12 @@ class HarnessConfig(BaseModel):
     name: str
 
 
+class AttachmentPayload(BaseModel):
+    url: str
+    mime_type: str
+    file_name: str
+
+
 class MessagePayload(BaseModel):
     role: str
     content: str
@@ -25,3 +31,4 @@ class ChatRequest(BaseModel):
     messages: list[MessagePayload]
     harness: HarnessConfig
     conversation_id: str
+    attachments: list[AttachmentPayload] | None = None
diff --git a/packages/fastapi/app/routes/chat.py b/packages/fastapi/app/routes/chat.py
index fdd00b7..888a480 100644
--- a/packages/fastapi/app/routes/chat.py
+++ b/packages/fastapi/app/routes/chat.py
@@ -57,6 +57,28 @@ async def event_generator():
 
         messages = [m.model_dump() for m in body.messages]
 
+        # If the last user message has attachments, convert it to multimodal content
+        if body.attachments and messages and messages[-1]["role"] == "user":
+            text_content = messages[-1]["content"]
+            content_parts: list[dict] = []
+            for att in body.attachments:
+                if att.mime_type.startswith("image/"):
+                    content_parts.append({
+                        "type": "image_url",
+                        "image_url": {"url": att.url},
+                    })
+                elif att.mime_type == "application/pdf":
+                    content_parts.append({
+                        "type": "file",
+                        "file": {
+                            "url": att.url,
+                            "mime_type": "application/pdf",
+                        },
+                    })
+            if text_content:
+                content_parts.append({"type": "text", "text": text_content})
+            messages[-1] = {"role": "user", "content": content_parts}
+
         # Accumulate across all iterations so reasoning/tool history isn't lost
         all_reasoning = ""
         all_tool_calls_history: list[dict] = []  # [{tool, arguments, call_id, result}]

From f6de8d79636892456b9e164cedd253a9ca0e9b45 Mon Sep 17 00:00:00 2001
From: Abu <abuahmed0821@gmail.com>
Date: Sun, 22 Mar 2026 20:58:36 -0400
Subject: [PATCH 3/3] completed multimodal, added audio support

---
 apps/web/src/components/attachment-chip.tsx   |   9 +-
 .../src/components/message-attachments.tsx    |  50 +++---
 apps/web/src/hooks/use-file-attachments.ts    |  48 ++++--
 apps/web/src/lib/models.ts                    | 102 +++++++++---
 apps/web/src/lib/multimodal.ts                |  64 ++++++++
 apps/web/src/lib/use-chat-stream.ts           |   5 +-
 apps/web/src/routes/chat/index.tsx            | 154 +++++++++++++-----
 packages/fastapi/app/models.py                |  11 +-
 packages/fastapi/app/routes/chat.py           |  22 ---
 9 files changed, 334 insertions(+), 131 deletions(-)
 create mode 100644 apps/web/src/lib/multimodal.ts

diff --git a/apps/web/src/components/attachment-chip.tsx b/apps/web/src/components/attachment-chip.tsx
index 8bcb3a6..d68d83b 100644
--- a/apps/web/src/components/attachment-chip.tsx
+++ b/apps/web/src/components/attachment-chip.tsx
@@ -1,4 +1,4 @@
-import { FileText, Loader2, X } from "lucide-react";
+import { FileText, Loader2, Music, X } from "lucide-react";
 import type { PendingAttachment } from "../hooks/use-file-attachments";
 
 export function AttachmentChip({
@@ -10,6 +10,9 @@ export function AttachmentChip({
 }) {
 	const isImage = attachment.mimeType.startsWith("image/");
 	const isPdf = attachment.mimeType === "application/pdf";
+	const isAudio = attachment.mimeType.startsWith("audio/");
+
+	const Icon = isAudio ? Music : FileText;
 
 	return (
 		<div className="group relative flex h-8 shrink-0 items-center overflow-hidden border border-border bg-muted">
@@ -19,9 +22,9 @@ export function AttachmentChip({
 					alt={attachment.fileName}
 					className="h-full w-auto object-cover"
 				/>
-			) : isPdf ? (
+			) : (isPdf || isAudio) ? (
 				<div className="flex items-center gap-1.5 px-2">
-					<FileText size={12} className="shrink-0 text-muted-foreground" />
+					<Icon size={12} className="shrink-0 text-muted-foreground" />
 					<span className="max-w-[120px] truncate text-[11px] text-muted-foreground">
 						{attachment.fileName}
 					</span>
diff --git a/apps/web/src/components/message-attachments.tsx b/apps/web/src/components/message-attachments.tsx
index 039dde0..d01222c 100644
--- a/apps/web/src/components/message-attachments.tsx
+++ b/apps/web/src/components/message-attachments.tsx
@@ -2,7 +2,7 @@ import { convexQuery } from "@convex-dev/react-query";
 import { api } from "@harness/convex-backend/convex/_generated/api";
 import type { Id } from "@harness/convex-backend/convex/_generated/dataModel";
 import { useQuery } from "@tanstack/react-query";
-import { FileText } from "lucide-react";
+import { FileText, Music } from "lucide-react";
 import {
 	Dialog,
 	DialogContent,
@@ -53,26 +53,32 @@ function AttachmentItem({ attachment }: { attachment: Attachment }) {
 
 	if (attachment.mimeType === "application/pdf") {
 		return (
-			<Dialog>
-				<DialogTrigger asChild>
-					<button
-						type="button"
-						className={`${square} flex flex-col items-center justify-center gap-1 bg-muted px-1 transition-colors hover:bg-muted/80`}
-					>
-						<FileText size={20} className="shrink-0 text-muted-foreground" />
-						<span className="w-full truncate text-center text-[9px] leading-tight text-muted-foreground">
-							{attachment.fileName}
-						</span>
-					</button>
-				</DialogTrigger>
-				<DialogContent className="max-w-4xl border-0 bg-transparent p-0 shadow-none" showCloseButton={false}>
-					<iframe
-						src={url}
-						title={attachment.fileName}
-						className="h-[90vh] w-full rounded-lg"
-					/>
-				</DialogContent>
-			</Dialog>
+			<a
+				href={url}
+				target="_blank"
+				rel="noopener noreferrer"
+				className={`${square} flex flex-col items-center justify-center gap-1 bg-muted px-1 transition-colors hover:bg-muted/80`}
+			>
+				<FileText size={20} className="shrink-0 text-muted-foreground" />
+				<span className="w-full truncate text-center text-[9px] leading-tight text-muted-foreground">
+					{attachment.fileName}
+				</span>
+			</a>
+		);
+	}
+
+	if (attachment.mimeType.startsWith("audio/")) {
+		return (
+			<div className="flex w-56 shrink-0 flex-col gap-1 rounded-lg border border-border bg-muted p-2">
+				<div className="flex items-center gap-1.5">
+					<Music size={14} className="shrink-0 text-muted-foreground" />
+					<span className="truncate text-[11px] text-muted-foreground">
+						{attachment.fileName}
+					</span>
+				</div>
+				{/* eslint-disable-next-line jsx-a11y/media-has-caption */}
+				<audio controls src={url} className="h-8 w-full" preload="metadata" />
+			</div>
 		);
 	}
 
@@ -87,7 +93,7 @@ export function MessageAttachments({
 	if (attachments.length === 0) return null;
 
 	return (
-		<div className="mb-1.5 flex flex-wrap items-end gap-1.5">
+		<div className="mb-1.5 flex flex-wrap items-end justify-end gap-1.5">
 			{attachments.map((a) => (
 				<AttachmentItem key={a.storageId} attachment={a} />
 			))}
diff --git a/apps/web/src/hooks/use-file-attachments.ts b/apps/web/src/hooks/use-file-attachments.ts
index 14cfa42..65c1015 100644
--- a/apps/web/src/hooks/use-file-attachments.ts
+++ b/apps/web/src/hooks/use-file-attachments.ts
@@ -4,20 +4,26 @@ import { useConvex } from "convex/react";
 import { useCallback, useRef, useState } from "react";
 import toast from "react-hot-toast";
 
-const ACCEPTED_TYPES = new Set([
-	"image/png",
-	"image/jpeg",
-	"image/gif",
-	"image/webp",
-	"application/pdf",
-]);
-
 const MAX_IMAGE_BYTES = 10 * 1024 * 1024; // 10 MB
 const MAX_PDF_BYTES = 20 * 1024 * 1024; // 20 MB
+const MAX_AUDIO_BYTES = 25 * 1024 * 1024; // 25 MB
+const MAX_ATTACHMENTS = 5;
+
+function getMaxBytes(mimeType: string): number {
+	if (mimeType === "application/pdf") return MAX_PDF_BYTES;
+	if (mimeType.startsWith("audio/")) return MAX_AUDIO_BYTES;
+	return MAX_IMAGE_BYTES;
+}
+
+function getSizeLabel(mimeType: string): string {
+	if (mimeType === "application/pdf") return "20 MB";
+	if (mimeType.startsWith("audio/")) return "25 MB";
+	return "10 MB";
+}
 
 export interface PendingAttachment {
 	localId: string;
-	previewUrl: string | null; // object URL for images, null for PDFs
+	previewUrl: string | null; // object URL for images, null for PDFs/audio
 	mimeType: string;
 	status: "uploading" | "ready" | "error";
 	storageId?: string;
@@ -25,7 +31,11 @@ export interface PendingAttachment {
 	fileSize: number;
 }
 
-export function useFileAttachments() {
+/**
+ * @param allowedMimes – the set of MIME types the current model accepts.
+ *   Passed in from the caller so validation stays in sync with model capabilities.
+ */
+export function useFileAttachments(allowedMimes: Set<string>) {
 	const [attachments, setAttachments] = useState<PendingAttachment[]>([]);
 	const convex = useConvex();
 	const localIdCounter = useRef(0);
@@ -63,16 +73,19 @@ export function useFileAttachments() {
 
 	const addFiles = useCallback(
 		(files: File[]) => {
+			let current = attachments.length;
 			for (const file of files) {
-				if (!ACCEPTED_TYPES.has(file.type)) {
-					toast.error(`${file.name}: only images (PNG, JPG, GIF, WebP) and PDFs are supported`);
+				if (current >= MAX_ATTACHMENTS) {
+					toast.error(`Maximum ${MAX_ATTACHMENTS} attachments per message`);
+					break;
+				}
+				if (!allowedMimes.has(file.type)) {
+					toast.error(`${file.name}: not supported by this model`);
 					continue;
 				}
-				const maxBytes =
-					file.type === "application/pdf" ? MAX_PDF_BYTES : MAX_IMAGE_BYTES;
+				const maxBytes = getMaxBytes(file.type);
 				if (file.size > maxBytes) {
-					const limit = file.type === "application/pdf" ? "20 MB" : "10 MB";
-					toast.error(`${file.name}: exceeds ${limit} limit`);
+					toast.error(`${file.name}: exceeds ${getSizeLabel(file.type)} limit`);
 					continue;
 				}
 
@@ -95,9 +108,10 @@ export function useFileAttachments() {
 
 				// Fire and forget — state updates happen inside uploadOne
 				uploadOne(file, localId);
+				current++;
 			}
 		},
-		[uploadOne],
+		[attachments.length, uploadOne, allowedMimes],
 	);
 
 	const removeAttachment = useCallback((localId: string) => {
diff --git a/apps/web/src/lib/models.ts b/apps/web/src/lib/models.ts
index 72fcc48..188c958 100644
--- a/apps/web/src/lib/models.ts
+++ b/apps/web/src/lib/models.ts
@@ -1,20 +1,84 @@
-export const MODELS = [
-	{ value: "openai/gpt-5.4", label: "GPT-5.4" },
-	{ value: "gpt-4o", label: "GPT-4o" },
-	{ value: "gpt-4.1", label: "GPT-4.1" },
-	{ value: "gpt-4.1-mini", label: "GPT-4.1 Mini" },
-	{ value: "claude-sonnet-4", label: "Claude Sonnet 4" },
-	{ value: "claude-sonnet-4-thinking", label: "Claude Sonnet 4 (Thinking)" },
-	{ value: "claude-opus-4", label: "Claude Opus 4" },
-	{ value: "claude-opus-4-thinking", label: "Claude Opus 4 (Thinking)" },
-	{
-		value: "google/gemini-3.1-flash-lite-preview",
-		label: "Gemini 3.1 Flash Lite Preview",
-	},
-	{ value: "gemini-2.5-pro", label: "Gemini 2.5 Pro" },
-	{ value: "gemini-2.5-flash", label: "Gemini 2.5 Flash" },
-	{ value: "deepseek-r1", label: "DeepSeek R1" },
-	{ value: "deepseek-v3", label: "DeepSeek V3" },
-	{ value: "grok-3", label: "Grok 3" },
-	{ value: "grok-3-mini", label: "Grok 3 Mini" },
+// Per-model modality support on OpenRouter
+// "image" = image inputs, "pdf" = PDF inputs, "audio" = audio inputs
+type Modality = "image" | "pdf" | "audio";
+
+export const MODELS: Array<{
+	value: string;
+	label: string;
+	modalities: Modality[];
+}> = [
+	// Audio input: only Gemini models are confirmed on OpenRouter
+	{ value: "openai/gpt-5.4", label: "GPT-5.4", modalities: ["image", "pdf"] },
+	{ value: "gpt-4o", label: "GPT-4o", modalities: ["image", "pdf"] },
+	{ value: "gpt-4.1", label: "GPT-4.1", modalities: ["image", "pdf"] },
+	{ value: "gpt-4.1-mini", label: "GPT-4.1 Mini", modalities: ["image", "pdf"] },
+	{ value: "claude-sonnet-4", label: "Claude Sonnet 4", modalities: ["image", "pdf"] },
+	{ value: "claude-sonnet-4-thinking", label: "Claude Sonnet 4 (Thinking)", modalities: ["image", "pdf"] },
+	{ value: "claude-opus-4", label: "Claude Opus 4", modalities: ["image", "pdf"] },
+	{ value: "claude-opus-4-thinking", label: "Claude Opus 4 (Thinking)", modalities: ["image", "pdf"] },
+	{ value: "google/gemini-3.1-flash-lite-preview", label: "Gemini 3.1 Flash Lite Preview", modalities: ["image", "pdf", "audio"] },
+	{ value: "gemini-2.5-pro", label: "Gemini 2.5 Pro", modalities: ["image", "pdf", "audio"] },
+	{ value: "gemini-2.5-flash", label: "Gemini 2.5 Flash", modalities: ["image", "pdf", "audio"] },
+	{ value: "deepseek-r1", label: "DeepSeek R1", modalities: [] },
+	{ value: "deepseek-v3", label: "DeepSeek V3", modalities: [] },
+	{ value: "grok-3", label: "Grok 3", modalities: ["image"] },
+	{ value: "grok-3-mini", label: "Grok 3 Mini", modalities: [] },
 ];
+
+// Lookup index built once from the MODELS array
+const modalityIndex = new Map(MODELS.map((m) => [m.value, new Set(m.modalities)]));
+
+function modelHas(model: string | undefined, modality: Modality): boolean {
+	if (!model) return false;
+	return modalityIndex.get(model)?.has(modality) ?? false;
+}
+
+export function modelSupportsMedia(model: string | undefined): boolean {
+	return modelHas(model, "image");
+}
+
+export function modelSupportsAudio(model: string | undefined): boolean {
+	return modelHas(model, "audio");
+}
+
+// ── MIME mappings ────────────────────────────────────────────────────
+
+const IMAGE_MIMES = ["image/png", "image/jpeg", "image/gif", "image/webp"];
+const PDF_MIMES = ["application/pdf"];
+const AUDIO_MIMES = [
+	"audio/wav", "audio/mpeg", "audio/mp3", "audio/aiff",
+	"audio/aac", "audio/ogg", "audio/flac", "audio/mp4",
+	"audio/x-m4a", "audio/webm",
+];
+
+/** Returns the set of MIME types the model can accept, or empty set if none. */
+export function allowedMimeTypes(model: string | undefined): Set<string> {
+	const mimes: string[] = [];
+	if (modelHas(model, "image")) mimes.push(...IMAGE_MIMES);
+	if (modelHas(model, "pdf")) mimes.push(...PDF_MIMES);
+	if (modelHas(model, "audio")) mimes.push(...AUDIO_MIMES);
+	return new Set(mimes);
+}
+
+/** Returns an `accept` string for <input type="file"> based on model capabilities. */
+export function acceptString(model: string | undefined): string {
+	return [...allowedMimeTypes(model)].join(",");
+}
+
+/** Map MIME type to OpenRouter audio format identifier */
+const AUDIO_FORMAT_MAP: Record<string, string> = {
+	"audio/wav": "wav",
+	"audio/mpeg": "mp3",
+	"audio/mp3": "mp3",
+	"audio/aiff": "aiff",
+	"audio/aac": "aac",
+	"audio/ogg": "ogg",
+	"audio/flac": "flac",
+	"audio/mp4": "m4a",
+	"audio/x-m4a": "m4a",
+	"audio/webm": "webm",
+};
+
+export function mimeToAudioFormat(mime: string): string {
+	return AUDIO_FORMAT_MAP[mime] ?? "wav";
+}
diff --git a/apps/web/src/lib/multimodal.ts b/apps/web/src/lib/multimodal.ts
new file mode 100644
index 0000000..9d7c864
--- /dev/null
+++ b/apps/web/src/lib/multimodal.ts
@@ -0,0 +1,64 @@
+import { mimeToAudioFormat } from "./models";
+
+type ContentPart = Record<string, unknown>;
+export type MessageContent = string | ContentPart[];
+
+/** Read a Blob into a raw base64 string (no data-URL prefix). */
+function blobToBase64(blob: Blob): Promise<string> {
+	return new Promise((resolve, reject) => {
+		const reader = new FileReader();
+		reader.onloadend = () => {
+			const result = reader.result as string;
+			resolve(result.split(",")[1]);
+		};
+		reader.onerror = () => reject(new Error("Failed to read audio data"));
+		reader.readAsDataURL(blob);
+	});
+}
+
+type ResolveSignedUrls = (
+	atts: Array<{ storageId: string; mimeType: string; fileName: string }>,
+) => Promise<Array<{ url: string; mime_type: string; file_name: string }>>;
+
+/**
+ * Build an OpenRouter multimodal content array from text + attachment metadata.
+ * Returns a plain string when there are no attachments.
+ */
+export async function buildMultimodalContent(
+	text: string,
+	atts: Array<{ storageId: string; mimeType: string; fileName: string }>,
+	resolveSignedUrls: ResolveSignedUrls,
+): Promise<MessageContent> {
+	const signed = await resolveSignedUrls(atts);
+	if (signed.length === 0) return text;
+
+	const parts: ContentPart[] = [];
+	if (text) parts.push({ type: "text", text });
+
+	for (const a of signed) {
+		if (a.mime_type.startsWith("image/")) {
+			parts.push({ type: "image_url", image_url: { url: a.url } });
+		} else if (a.mime_type === "application/pdf") {
+			parts.push({ type: "file", file: { filename: a.file_name, file_data: a.url } });
+		} else if (a.mime_type.startsWith("audio/")) {
+			try {
+				const res = await fetch(a.url);
+				if (!res.ok) throw new Error(`HTTP ${res.status}`);
+				const blob = await res.blob();
+				const base64 = await blobToBase64(blob);
+				parts.push({
+					type: "input_audio",
+					input_audio: {
+						data: base64,
+						format: mimeToAudioFormat(a.mime_type),
+					},
+				});
+			} catch (err) {
+				const msg = err instanceof Error ? err.message : "unknown error";
+				throw new Error(`Failed to encode audio "${a.file_name}": ${msg}`);
+			}
+		}
+	}
+
+	return parts;
+}
diff --git a/apps/web/src/lib/use-chat-stream.ts b/apps/web/src/lib/use-chat-stream.ts
index ff0ca95..fb9c918 100644
--- a/apps/web/src/lib/use-chat-stream.ts
+++ b/apps/web/src/lib/use-chat-stream.ts
@@ -59,8 +59,10 @@ interface UseChatStreamCallbacks {
 	onAbort?: (conversationId: string) => void;
 }
 
+export type MessageContent = string | Array<Record<string, unknown>>;
+
 export interface ChatStreamRequest {
-	messages: Array<{ role: string; content: string }>;
+	messages: Array<{ role: string; content: MessageContent }>;
 	harness: {
 		model: string;
 		mcp_servers: Array<{
@@ -72,7 +74,6 @@ export interface ChatStreamRequest {
 		name: string;
 	};
 	conversation_id: string;
-	attachments?: Array<{ url: string; mime_type: string; file_name: string }>;
 }
 
 export function useChatStream(callbacks: UseChatStreamCallbacks) {
diff --git a/apps/web/src/routes/chat/index.tsx b/apps/web/src/routes/chat/index.tsx
index f0034f3..72df8c0 100644
--- a/apps/web/src/routes/chat/index.tsx
+++ b/apps/web/src/routes/chat/index.tsx
@@ -21,6 +21,7 @@ import {
 	Loader2,
 	LogOut,
 	MessageSquare,
+	Mic,
 	PanelLeftClose,
 	PanelLeftOpen,
 	Paperclip,
@@ -95,6 +96,9 @@ import {
 	TooltipTrigger,
 } from "../../components/ui/tooltip";
 import { env } from "../../env";
+import { acceptString, allowedMimeTypes, modelSupportsAudio, modelSupportsMedia } from "../../lib/models";
+import { buildMultimodalContent } from "../../lib/multimodal";
+import { useFileAttachments } from "../../hooks/use-file-attachments";
 import {
 	type ConvoStreamState,
 	type StreamPart,
@@ -105,10 +109,6 @@ import {
 import { cn } from "../../lib/utils";
 import { AttachmentChip } from "../../components/attachment-chip";
 import { MessageAttachments } from "../../components/message-attachments";
-import {
-	type PendingAttachment,
-	useFileAttachments,
-} from "../../hooks/use-file-attachments";
 
 export const Route = createFileRoute("/chat/")({
 	validateSearch: (search: Record<string, unknown>) => ({
@@ -2590,7 +2590,7 @@ function ChatInput({
 	onConvoCreated: (id: Id<"conversations">) => void;
 	isStreaming: boolean;
 	onStream: (body: {
-		messages: Array<{ role: string; content: string }>;
+		messages: Array<{ role: string; content: string | Array<Record<string, unknown>> }>;
 		harness: {
 			model: string;
 			mcp_servers: Array<{
@@ -2605,7 +2605,16 @@ function ChatInput({
 	}) => Promise<void>;
 	onInterrupt: (convoId: string) => void;
 	onEnqueue: (content: string) => void;
-	messages?: Array<{ role: string; content: string }>;
+	messages?: Array<{
+		role: string;
+		content: string;
+		attachments?: Array<{
+			storageId: Id<"_storage">;
+			mimeType: string;
+			fileName: string;
+			fileSize: number;
+		}>;
+	}>;
 	messageQueue: { id: number; content: string }[];
 	onDequeue: (index: number) => void;
 	onSendNow: (index: number) => void;
@@ -2617,8 +2626,52 @@ function ChatInput({
 	const fileInputRef = useRef<HTMLInputElement>(null);
 	const [isDragOver, setIsDragOver] = useState(false);
 
+	const supportsMedia = modelSupportsMedia(activeHarness?.model);
+	const supportsAudio = modelSupportsAudio(activeHarness?.model);
+	const supportsAnyAttachment = supportsMedia || supportsAudio;
+	const modelAccept = acceptString(activeHarness?.model);
+	const modelAllowedMimes = useMemo(() => allowedMimeTypes(activeHarness?.model), [activeHarness?.model]);
+
 	const { attachments, addFiles, removeAttachment, clearAttachments, hasUploading, resolveSignedUrls } =
-		useFileAttachments();
+		useFileAttachments(modelAllowedMimes);
+
+	// Clear attachments if the active model switches to one that doesn't support media
+	useEffect(() => {
+		if (!supportsAnyAttachment) clearAttachments();
+	}, [supportsAnyAttachment, clearAttachments]);
+
+	// ── Voice recording ──────────────────────────────────────────────
+	const [isRecording, setIsRecording] = useState(false);
+	const mediaRecorderRef = useRef<MediaRecorder | null>(null);
+	const chunksRef = useRef<Blob[]>([]);
+
+	const startRecording = useCallback(async () => {
+		try {
+			const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+			const recorder = new MediaRecorder(stream, { mimeType: "audio/webm" });
+			chunksRef.current = [];
+			recorder.ondataavailable = (e) => {
+				if (e.data.size > 0) chunksRef.current.push(e.data);
+			};
+			recorder.onstop = () => {
+				const blob = new Blob(chunksRef.current, { type: "audio/webm" });
+				const file = new File([blob], `recording-${Date.now()}.webm`, { type: "audio/webm" });
+				addFiles([file]);
+				stream.getTracks().forEach((t) => t.stop());
+			};
+			mediaRecorderRef.current = recorder;
+			recorder.start();
+			setIsRecording(true);
+		} catch {
+			toast.error("Microphone access denied");
+		}
+	}, [addFiles]);
+
+	const stopRecording = useCallback(() => {
+		mediaRecorderRef.current?.stop();
+		mediaRecorderRef.current = null;
+		setIsRecording(false);
+	}, []);
 
 	// Fill input from suggested prompt click
 	useEffect(() => {
@@ -2702,7 +2755,7 @@ function ChatInput({
 			onConvoCreated(newId);
 		}
 
-		// Capture ready attachments before clearing
+		// Snapshot ready attachments from the current render's state (clearAttachments above is async)
 		const readyAttachments = attachments
 			.filter((a) => a.status === "ready" && a.storageId)
 			.map((a) => ({
@@ -2721,25 +2774,28 @@ function ChatInput({
 			...(readyAttachments.length > 0 ? { attachments: readyAttachments } : {}),
 		});
 
-		// Build message history for the LLM
-		const history: Array<{ role: string; content: string }> =
-			existingMessages?.map((m) => ({
-				role: m.role,
-				content: m.content,
-			})) ?? [];
-		history.push({ role: "user", content });
+		// Build message history for the LLM (with multimodal content where applicable)
+		const history: Array<{ role: string; content: string | Array<Record<string, unknown>> }> = [];
+		for (const m of existingMessages ?? []) {
+			if (m.role === "user" && m.attachments && m.attachments.length > 0) {
+				history.push({ role: m.role, content: await buildMultimodalContent(m.content, m.attachments, resolveSignedUrls) });
+			} else {
+				history.push({ role: m.role, content: m.content });
+			}
+		}
 
-		// Resolve signed URLs for any attachments
-		const signedAttachments = readyAttachments.length > 0
-			? await resolveSignedUrls(readyAttachments)
-			: undefined;
+		// Add the new user message (with any current attachments)
+		if (readyAttachments.length > 0) {
+			history.push({ role: "user", content: await buildMultimodalContent(content, readyAttachments, resolveSignedUrls) });
+		} else {
+			history.push({ role: "user", content });
+		}
 
 		// Start streaming from FastAPI
 		onStream({
 			messages: history,
 			harness: harnessConfig,
 			conversation_id: convoId,
-			...(signedAttachments && signedAttachments.length > 0 ? { attachments: signedAttachments } : {}),
 		});
 	};
 
@@ -2785,8 +2841,9 @@ function ChatInput({
 	};
 
 	const handlePaste = (e: React.ClipboardEvent<HTMLTextAreaElement>) => {
+		if (!supportsAnyAttachment) return;
 		const files = Array.from(e.clipboardData.files).filter(
-			(f) => f.type.startsWith("image/") || f.type === "application/pdf",
+			(f) => f.type.startsWith("image/") || f.type === "application/pdf" || f.type.startsWith("audio/"),
 		);
 		if (files.length > 0) {
 			e.preventDefault();
@@ -2796,7 +2853,7 @@ function ChatInput({
 
 	const handleDragOver = (e: React.DragEvent<HTMLDivElement>) => {
 		e.preventDefault();
-		if (e.dataTransfer.types.includes("Files")) {
+		if (supportsAnyAttachment && e.dataTransfer.types.includes("Files")) {
 			setIsDragOver(true);
 		}
 	};
@@ -2811,6 +2868,7 @@ function ChatInput({
 	const handleDrop = (e: React.DragEvent<HTMLDivElement>) => {
 		e.preventDefault();
 		setIsDragOver(false);
+		if (!supportsAnyAttachment) return;
 		const files = Array.from(e.dataTransfer.files);
 		if (files.length > 0) addFiles(files);
 	};
@@ -2841,7 +2899,7 @@ function ChatInput({
 			<input
 				ref={fileInputRef}
 				type="file"
-				accept="image/png,image/jpeg,image/gif,image/webp,application/pdf"
+				accept={modelAccept}
 				multiple
 				className="hidden"
 				onChange={(e) => {
@@ -2923,20 +2981,42 @@ function ChatInput({
 					)}
 				</AnimatePresence>
 
-				<div className="flex items-end gap-2 border border-border bg-background px-3 py-2 focus-within:border-foreground/30">
-					{/* Attach button */}
-					<Tooltip>
-						<TooltipTrigger asChild>
-							<button
-								type="button"
-								onClick={() => fileInputRef.current?.click()}
-								className="mb-0.5 shrink-0 text-muted-foreground transition-colors hover:text-foreground"
-							>
-								<Paperclip size={15} />
-							</button>
-						</TooltipTrigger>
-						<TooltipContent>Attach image or PDF</TooltipContent>
-					</Tooltip>
+				<div className="flex items-center gap-2 border border-border bg-background px-3 py-2 focus-within:border-foreground/30">
+					{/* Attach button — hidden for models that don't support media */}
+					{supportsAnyAttachment && (
+						<Tooltip>
+							<TooltipTrigger asChild>
+								<button
+									type="button"
+									onClick={() => fileInputRef.current?.click()}
+									className="shrink-0 text-muted-foreground transition-colors hover:text-foreground"
+								>
+									<Paperclip size={15} />
+								</button>
+							</TooltipTrigger>
+							<TooltipContent>Attach files</TooltipContent>
+						</Tooltip>
+					)}
+
+					{supportsAudio && (
+						<Tooltip>
+							<TooltipTrigger asChild>
+								<button
+									type="button"
+									onClick={isRecording ? stopRecording : startRecording}
+									className={cn(
+										"shrink-0 transition-colors",
+										isRecording
+											? "animate-pulse text-destructive"
+											: "text-muted-foreground hover:text-foreground",
+									)}
+								>
+									{isRecording ? <Square size={15} /> : <Mic size={15} />}
+								</button>
+							</TooltipTrigger>
+							<TooltipContent>{isRecording ? "Stop recording" : "Record audio"}</TooltipContent>
+						</Tooltip>
+					)}
 
 					<textarea
 						ref={textareaRef}
diff --git a/packages/fastapi/app/models.py b/packages/fastapi/app/models.py
index 50617d0..416c7b8 100644
--- a/packages/fastapi/app/models.py
+++ b/packages/fastapi/app/models.py
@@ -1,4 +1,4 @@
-from typing import Literal
+from typing import Any, Literal
 
 from pydantic import BaseModel
 
@@ -16,19 +16,12 @@ class HarnessConfig(BaseModel):
     name: str
 
 
-class AttachmentPayload(BaseModel):
-    url: str
-    mime_type: str
-    file_name: str
-
-
 class MessagePayload(BaseModel):
     role: str
-    content: str
+    content: Any  # str for text-only, list[dict] for multimodal
 
 
 class ChatRequest(BaseModel):
     messages: list[MessagePayload]
     harness: HarnessConfig
     conversation_id: str
-    attachments: list[AttachmentPayload] | None = None
diff --git a/packages/fastapi/app/routes/chat.py b/packages/fastapi/app/routes/chat.py
index 888a480..fdd00b7 100644
--- a/packages/fastapi/app/routes/chat.py
+++ b/packages/fastapi/app/routes/chat.py
@@ -57,28 +57,6 @@ async def event_generator():
 
         messages = [m.model_dump() for m in body.messages]
 
-        # If the last user message has attachments, convert it to multimodal content
-        if body.attachments and messages and messages[-1]["role"] == "user":
-            text_content = messages[-1]["content"]
-            content_parts: list[dict] = []
-            for att in body.attachments:
-                if att.mime_type.startswith("image/"):
-                    content_parts.append({
-                        "type": "image_url",
-                        "image_url": {"url": att.url},
-                    })
-                elif att.mime_type == "application/pdf":
-                    content_parts.append({
-                        "type": "file",
-                        "file": {
-                            "url": att.url,
-                            "mime_type": "application/pdf",
-                        },
-                    })
-            if text_content:
-                content_parts.append({"type": "text", "text": text_content})
-            messages[-1] = {"role": "user", "content": content_parts}
-
         # Accumulate across all iterations so reasoning/tool history isn't lost
         all_reasoning = ""
         all_tool_calls_history: list[dict] = []  # [{tool, arguments, call_id, result}]