diff --git a/TYPE_FIX_SUMMARY.md b/TYPE_FIX_SUMMARY.md new file mode 100644 index 00000000..f11e3ad2 --- /dev/null +++ b/TYPE_FIX_SUMMARY.md @@ -0,0 +1,63 @@ +# AI SDK v6 Type Error Fix Summary + +## Problem +The build fails with TypeScript errors after upgrading to AI SDK v6. The main issues are: +1. `ToolCallPart` type now requires `input` field (not optional), but stored data may only have deprecated `args` field +2. Tool-result output types missing newer types like `execution-denied` and extended content types +3. Generated component types out of sync with updated validators + +## Changes Made + +### 1. Fixed `tool-call` Part Handling (src/mapping.ts) +- Updated `toModelMessageContent()` to ensure `input` is always present by falling back to `args` or `{}` +- Updated `serializeContent()` and `fromModelMessageContent()` to handle both `input` and legacy `args` fields +- This fixes the core issue where AI SDK v6's `ToolCallPart` expects non-nullable `input` + +### 2. Fixed Tool Approval Response Handling (src/client/search.ts) +- Updated `filterOutOrphanedToolMessages()` to handle tool-approval-response parts that don't have `toolCallId` +- Tool-approval-response only has `approvalId`, not `toolCallId` + +### 3. Updated Generated Component Types (src/component/_generated/component.ts) +Made manual updates to sync with validators (normally done via `convex codegen`): +- Added `input: any` field to all tool-call type definitions +- Made `args` optional (`args?: any`) in tool-call types +- Added `execution-denied` output type to tool-result +- Added extended content types: `file-data`, `file-url`, `file-id`, `image-data`, `image-url`, `image-file-id`, `custom` +- Added `providerOptions` to text types in content values + +## Remaining Issues (5 TypeScript errors) + +The remaining errors are due to a structural mismatch in the generated component types: +- Generated types have BOTH `experimental_content` (deprecated) and `output` (new) fields on tool-result +- Our validators only define `output`, not `experimental_content` +- TypeScript is comparing our new output types against the old experimental_content types +- This cannot be fixed manually - requires proper component regeneration + +### To Complete the Fix: +1. Run `convex codegen --component-dir ./src/component` with a valid Convex deployment +2. This will regenerate `src/component/_generated/component.ts` from the validators +3. The regenerated types will: + - Remove the deprecated `experimental_content` field + - Use only the `output` field with correct types + - Properly match the validator definitions + +### Error Locations: +- `src/client/index.ts:1052` - addMessages call +- `src/client/index.ts:1103` - addMessages call +- `src/client/index.ts:1169` - updateMessage call +- `src/client/messages.ts:141` - addMessages call +- `src/client/start.ts:265` - addMessages call + +All errors have the same root cause: content value types in tool-result output don't match experimental_content expectations. + +## Testing Plan +Once component types are regenerated: +1. Run `npm run build` - should complete without errors +2. Run `npm test` - ensure no regressions +3. Test with actual AI SDK v6 workflow - verify tool-call handling works with both new `input` and legacy `args` fields + +## Notes +- The mapping functions in `src/mapping.ts` correctly handle both old and new formats +- Data with only `args` will be converted to have `input` (with `args` as fallback) +- Data with `input` will work directly +- This provides backward compatibility while supporting AI SDK v6's requirements diff --git a/src/UIMessages.ts b/src/UIMessages.ts index d65de779..45011f77 100644 --- a/src/UIMessages.ts +++ b/src/UIMessages.ts @@ -114,9 +114,10 @@ export async function fromUIMessages( ); if (partWithProviderOptions?.providerOptions) { // convertToModelMessages changes providerMetadata to providerOptions - const providerOptions = partWithProviderOptions.providerOptions as - | Record> - | undefined; + const providerOptions = + partWithProviderOptions.providerOptions as + | Record> + | undefined; if (providerOptions) { doc.providerMetadata = providerOptions; doc.providerOptions ??= providerOptions; @@ -477,8 +478,39 @@ function createAssistantUIMessage< } case "tool-result": { const typedPart = contentPart as unknown as ToolResultPart & { - output: { type: string; value: unknown }; + output: { type: string; value?: unknown; reason?: string }; }; + + // Check if this is an execution-denied result + if (typedPart.output?.type === "execution-denied") { + const call = allParts.find( + (part) => + part.type === `tool-${contentPart.toolName}` && + "toolCallId" in part && + part.toolCallId === contentPart.toolCallId, + ) as ToolUIPart | undefined; + + if (call) { + call.state = "output-denied"; + if (!("approval" in call) || !call.approval) { + (call as ToolUIPart & { approval?: object }).approval = { + id: "", + approved: false, + reason: typedPart.output.reason, + }; + } else { + const approval = ( + call as ToolUIPart & { + approval: { approved?: boolean; reason?: string }; + } + ).approval; + approval.approved = false; + approval.reason = typedPart.output.reason; + } + } + break; + } + const output = typeof typedPart.output?.type === "string" ? typedPart.output.value @@ -531,6 +563,68 @@ function createAssistantUIMessage< } break; } + case "tool-approval-request": { + // Find the matching tool call + const typedPart = contentPart as { + toolCallId: string; + approvalId: string; + }; + const toolCallPart = allParts.find( + (part) => + "toolCallId" in part && part.toolCallId === typedPart.toolCallId, + ) as ToolUIPart | undefined; + + if (toolCallPart) { + toolCallPart.state = "approval-requested"; + (toolCallPart as ToolUIPart & { approval?: object }).approval = { + id: typedPart.approvalId, + }; + } else { + console.warn( + "Tool approval request without preceding tool call", + contentPart, + ); + } + break; + } + case "tool-approval-response": { + // Find the tool call that has this approval by matching approval.id + const typedPart = contentPart as { + approvalId: string; + approved: boolean; + reason?: string; + }; + const toolCallPart = allParts.find( + (part) => + "approval" in part && + (part as ToolUIPart & { approval?: { id: string } }).approval + ?.id === typedPart.approvalId, + ) as ToolUIPart | undefined; + + if (toolCallPart) { + if (typedPart.approved) { + toolCallPart.state = "approval-responded"; + (toolCallPart as ToolUIPart & { approval?: object }).approval = { + id: typedPart.approvalId, + approved: true, + reason: typedPart.reason, + }; + } else { + toolCallPart.state = "output-denied"; + (toolCallPart as ToolUIPart & { approval?: object }).approval = { + id: typedPart.approvalId, + approved: false, + reason: typedPart.reason, + }; + } + } else { + console.warn( + "Tool approval response without matching approval request", + contentPart, + ); + } + break; + } default: { const maybeSource = contentPart as unknown as SourcePart; if (maybeSource.type === "source") { diff --git a/src/client/createTool.ts b/src/client/createTool.ts index 83ca5288..05767e83 100644 --- a/src/client/createTool.ts +++ b/src/client/createTool.ts @@ -1,9 +1,15 @@ -import type { FlexibleSchema } from "@ai-sdk/provider-utils"; -import type { Tool, ToolExecutionOptions, ToolSet } from "ai"; +import type { ToolResultOutput } from "@ai-sdk/provider-utils"; +import type { + FlexibleSchema, + ModelMessage, + Tool, + ToolExecutionOptions, + ToolSet, +} from "ai"; import { tool } from "ai"; -import type { Agent } from "./index.js"; import type { GenericActionCtx, GenericDataModel } from "convex/server"; import type { ProviderOptions } from "../validators.js"; +import type { Agent } from "./index.js"; export type ToolCtx = GenericActionCtx & { @@ -13,79 +19,241 @@ export type ToolCtx = messageId?: string; }; +/** + * Function that is called to determine if the tool needs approval before it can be executed. + */ +export type ToolNeedsApprovalFunctionCtx< + INPUT, + Ctx extends ToolCtx = ToolCtx, +> = ( + ctx: Ctx, + input: INPUT, + options: { + /** + * The ID of the tool call. You can use it e.g. when sending tool-call related information with stream data. + */ + toolCallId: string; + /** + * Messages that were sent to the language model to initiate the response that contained the tool call. + * The messages **do not** include the system prompt nor the assistant response that contained the tool call. + */ + messages: ModelMessage[]; + /** + * Additional context. + * + * Experimental (can break in patch releases). + */ + experimental_context?: unknown; + }, +) => boolean | PromiseLike; + +export type ToolExecuteFunctionCtx< + INPUT, + OUTPUT, + Ctx extends ToolCtx = ToolCtx, +> = ( + ctx: Ctx, + input: INPUT, + options: ToolExecutionOptions, +) => AsyncIterable | PromiseLike; + +type NeverOptional = 0 extends 1 & N + ? Partial + : [N] extends [never] + ? Partial> + : T; + +export type ToolOutputPropertiesCtx< + INPUT, + OUTPUT, + Ctx extends ToolCtx = ToolCtx, +> = NeverOptional< + OUTPUT, + | { + /** + * An async function that is called with the arguments from the tool call and produces a result. + * If `execute` (or `handler`) is not provided, the tool will not be executed automatically. + * + * @param input - The input of the tool call. + * @param options.abortSignal - A signal that can be used to abort the tool call. + */ + execute: ToolExecuteFunctionCtx; + outputSchema?: FlexibleSchema; + handler?: never; + } + | { + /** @deprecated Use execute instead. */ + handler: ToolExecuteFunctionCtx; + outputSchema?: FlexibleSchema; + execute?: never; + } + | { + outputSchema: FlexibleSchema; + execute?: never; + handler?: never; + } +>; + +export type ToolInputProperties = + | { + /** + * The schema of the input that the tool expects. + * The language model will use this to generate the input. + * It is also used to validate the output of the language model. + * + * You can use descriptions on the schema properties to make the input understandable for the language model. + */ + inputSchema: FlexibleSchema; + args?: never; + } + | { + /** + * The schema of the input that the tool expects. The language model will use this to generate the input. + * It is also used to validate the output of the language model. + * Use descriptions to make the input understandable for the language model. + * + * @deprecated Use inputSchema instead. + */ + args: FlexibleSchema; + inputSchema?: never; + }; + /** * This is a wrapper around the ai.tool function that adds extra context to the * tool call, including the action context, userId, threadId, and messageId. * @param tool The tool. See https://sdk.vercel.ai/docs/ai-sdk-core/tools-and-tool-calling - * but swap parameters for args and handler for execute. + * Currently contains deprecated parameters `args` and `handler` to maintain backwards compatibility + * but these will be removed in the future. Use `inputSchema` and `execute` instead, respectively. + * * @returns A tool to be used with the AI SDK. */ -export function createTool(def: { - /** - An optional description of what the tool does. - Will be used by the language model to decide whether to use the tool. - Not used for provider-defined tools. +export function createTool( + def: { + /** + * An optional description of what the tool does. + * Will be used by the language model to decide whether to use the tool. + * Not used for provider-defined tools. */ - description?: string; - /** - The schema of the input that the tool expects. The language model will use this to generate the input. - It is also used to validate the output of the language model. - Use descriptions to make the input understandable for the language model. + description?: string; + /** + * An optional title of the tool. */ - args: FlexibleSchema; - /** - An async function that is called with the arguments from the tool call and produces a result. - If not provided, the tool will not be executed automatically. - - @args is the input of the tool call. - @options.abortSignal is a signal that can be used to abort the tool call. + title?: string; + /** + * Additional provider-specific metadata. They are passed through + * to the provider from the AI SDK and enable provider-specific + * functionality that can be fully encapsulated in the provider. */ - handler: ( - ctx: Ctx, - args: INPUT, - options: ToolExecutionOptions, - ) => PromiseLike | AsyncIterable; - /** - * Provide the context to use, e.g. when defining the tool at runtime. - */ - ctx?: Ctx; - /** - * Optional function that is called when the argument streaming starts. - * Only called when the tool is used in a streaming context. - */ - onInputStart?: ( - ctx: Ctx, - options: ToolExecutionOptions, - ) => void | PromiseLike; - /** - * Optional function that is called when an argument streaming delta is available. - * Only called when the tool is used in a streaming context. - */ - onInputDelta?: ( - ctx: Ctx, - options: { inputTextDelta: string } & ToolExecutionOptions, - ) => void | PromiseLike; - /** - * Optional function that is called when a tool call can be started, - * even if the execute function is not provided. - */ - onInputAvailable?: ( - ctx: Ctx, - options: { - input: [INPUT] extends [never] ? unknown : INPUT; - } & ToolExecutionOptions, - ) => void | PromiseLike; + providerOptions?: ProviderOptions; + } & ToolInputProperties & { + /** + * An optional list of input examples that show the language + * model what the input should look like. + */ + inputExamples?: Array<{ + input: NoInfer; + }>; + /** + * Whether the tool needs approval before it can be executed. + */ + needsApproval?: + | boolean + | ToolNeedsApprovalFunctionCtx< + [INPUT] extends [never] ? unknown : INPUT, + Ctx + >; + /** + * Strict mode setting for the tool. + * + * Providers that support strict mode will use this setting to determine + * how the input should be generated. Strict mode will always produce + * valid inputs, but it might limit what input schemas are supported. + */ + strict?: boolean; + /** + * Provide the context to use, e.g. when defining the tool at runtime. + */ + ctx?: Ctx; + /** + * Optional function that is called when the argument streaming starts. + * Only called when the tool is used in a streaming context. + */ + onInputStart?: ( + ctx: Ctx, + options: ToolExecutionOptions, + ) => void | PromiseLike; + /** + * Optional function that is called when an argument streaming delta is available. + * Only called when the tool is used in a streaming context. + */ + onInputDelta?: ( + ctx: Ctx, + options: { inputTextDelta: string } & ToolExecutionOptions, + ) => void | PromiseLike; + /** + * Optional function that is called when a tool call can be started, + * even if the execute function is not provided. + */ + onInputAvailable?: ( + ctx: Ctx, + options: { + input: [INPUT] extends [never] ? unknown : INPUT; + } & ToolExecutionOptions, + ) => void | PromiseLike; + } & ToolOutputPropertiesCtx & { + /** + * Optional conversion function that maps the tool result to an output that can be used by the language model. + * + * If not provided, the tool result will be sent as a JSON object. + */ + toModelOutput?: ( + ctx: Ctx, + options: { + /** + * The ID of the tool call. You can use it e.g. when sending tool-call related information with stream data. + */ + toolCallId: string; + /** + * The input of the tool call. + */ + input: [INPUT] extends [never] ? unknown : INPUT; + /** + * The output of the tool call. + */ + output: 0 extends 1 & OUTPUT + ? any + : [OUTPUT] extends [never] + ? any + : NoInfer; + }, + ) => ToolResultOutput | PromiseLike; + }, +): Tool { + const inputSchema = def.inputSchema ?? def.args; + if (!inputSchema) + throw new Error("To use a Convex tool, you must provide an `inputSchema` (or `args`)"); - // Extra AI SDK pass-through options. - providerOptions?: ProviderOptions; -}): Tool { - const t = tool({ + const executeHandler = def.execute ?? def.handler; + if (!executeHandler && !def.outputSchema) + throw new Error( + "To use a Convex tool, you must either provide an execute" + + " handler function, define an outputSchema, or both", + ); + + const t = tool({ type: "function", __acceptsCtx: true, ctx: def.ctx, description: def.description, - inputSchema: def.args, - execute(args: INPUT, options: ToolExecutionOptions) { + title: def.title, + providerOptions: def.providerOptions, + inputSchema, + inputExamples: def.inputExamples, + needsApproval(this: Tool, input, options) { + const needsApproval = def.needsApproval; + if (!needsApproval || typeof needsApproval === "boolean") + return Boolean(needsApproval); + if (!getCtx(this)) { throw new Error( "To use a Convex tool, you must either provide the ctx" + @@ -93,9 +261,28 @@ export function createTool(def: { " call it (which injects the ctx, userId and threadId)", ); } - return def.handler(getCtx(this), args, options); + return needsApproval(getCtx(this), input, options); }, - providerOptions: def.providerOptions, + strict: def.strict, + ...(executeHandler + ? { + execute( + this: Tool, + input: INPUT, + options: ToolExecutionOptions, + ) { + if (!getCtx(this)) { + throw new Error( + "To use a Convex tool, you must either provide the ctx" + + " at definition time (dynamically in an action), or use the Agent to" + + " call it (which injects the ctx, userId and threadId)", + ); + } + return executeHandler(getCtx(this), input, options); + }, + } + : {}), + outputSchema: def.outputSchema, }); if (def.onInputStart) { t.onInputStart = def.onInputStart.bind(t, getCtx(t)); @@ -106,6 +293,9 @@ export function createTool(def: { if (def.onInputAvailable) { t.onInputAvailable = def.onInputAvailable.bind(t, getCtx(t)); } + if (def.toModelOutput) { + t.toModelOutput = def.toModelOutput.bind(t, getCtx(t)); + } return t; } diff --git a/src/client/files.ts b/src/client/files.ts index 6ee78506..ae56581e 100644 --- a/src/client/files.ts +++ b/src/client/files.ts @@ -92,7 +92,7 @@ export async function storeFile( storageId: newStorageId, hash, filename, - mimeType: blob.type, + mediaType: blob.type, }); const url = (await ctx.storage.getUrl(storageId as Id<"_storage">))!; if (storageId !== newStorageId) { @@ -142,8 +142,10 @@ export async function getFile( if (!url) { throw new Error(`File not found in storage: ${file.storageId}`); } + // Support both mediaType (preferred) and mimeType (deprecated) + const mediaType = file.mediaType ?? file.mimeType ?? ""; return { - ...getParts(url, file.mimeType, file.filename), + ...getParts(url, mediaType, file.filename), file: { fileId, url, diff --git a/src/client/index.test.ts b/src/client/index.test.ts index f6e97269..9a2c4e18 100644 --- a/src/client/index.test.ts +++ b/src/client/index.test.ts @@ -193,6 +193,7 @@ describe("filterOutOrphanedToolMessages", () => { type: "tool-call", toolCallId: "1", toolName: "tool1", + input: { test: "test" }, args: { test: "test" }, }, ], diff --git a/src/client/search.test.ts b/src/client/search.test.ts index c2749c92..a3c1c034 100644 --- a/src/client/search.test.ts +++ b/src/client/search.test.ts @@ -159,6 +159,7 @@ describe("search.ts", () => { type: "tool-call", toolCallId: "call_123", toolName: "test", + input: {}, args: {}, }, ], @@ -202,6 +203,7 @@ describe("search.ts", () => { type: "tool-call", toolCallId: "call_orphaned", toolName: "test", + input: {}, args: {}, }, ], @@ -234,6 +236,170 @@ describe("search.ts", () => { expect(result[0]._id).toBe("0"); expect(result[1]._id).toBe("3"); }); + + it("should keep tool calls with approval responses (but no tool-result yet)", () => { + const messages: MessageDoc[] = [ + { + _id: "1", + message: { + role: "assistant", + content: [ + { type: "text", text: "I'll run the dangerous tool" }, + { + type: "tool-call", + toolCallId: "call_123", + toolName: "dangerousTool", + input: { action: "delete" }, + args: { action: "delete" }, + }, + { + type: "tool-approval-request", + toolCallId: "call_123", + approvalId: "approval_456", + }, + ], + }, + order: 1, + } as MessageDoc, + { + _id: "2", + message: { + role: "tool", + content: [ + { + type: "tool-approval-response", + approvalId: "approval_456", + approved: true, + }, + ], + }, + order: 2, + } as MessageDoc, + ]; + + const result = filterOutOrphanedToolMessages(messages); + expect(result).toHaveLength(2); + // The assistant message should still contain the tool-call + expect(result[0]._id).toBe("1"); + const assistantContent = result[0].message?.content; + expect(Array.isArray(assistantContent)).toBe(true); + if (Array.isArray(assistantContent)) { + const toolCall = assistantContent.find((p) => p.type === "tool-call"); + expect(toolCall).toBeDefined(); + expect(toolCall?.toolCallId).toBe("call_123"); + } + // The tool message with approval response should be kept + expect(result[1]._id).toBe("2"); + }); + + it("should filter out tool calls with approval request but NO approval response", () => { + const messages: MessageDoc[] = [ + { + _id: "1", + message: { + role: "assistant", + content: [ + { type: "text", text: "I'll run the dangerous tool" }, + { + type: "tool-call", + toolCallId: "call_123", + toolName: "dangerousTool", + input: { action: "delete" }, + args: { action: "delete" }, + }, + { + type: "tool-approval-request", + toolCallId: "call_123", + approvalId: "approval_456", + }, + ], + }, + order: 1, + } as MessageDoc, + // No approval response provided + ]; + + const result = filterOutOrphanedToolMessages(messages); + expect(result).toHaveLength(1); + // The assistant message should have the tool-call filtered out + const assistantContent = result[0].message?.content; + expect(Array.isArray(assistantContent)).toBe(true); + if (Array.isArray(assistantContent)) { + // Text and approval-request should remain, but tool-call should be filtered + expect(assistantContent).toHaveLength(2); + expect(assistantContent.find((p) => p.type === "text")).toBeDefined(); + expect( + assistantContent.find((p) => p.type === "tool-approval-request"), + ).toBeDefined(); + expect( + assistantContent.find((p) => p.type === "tool-call"), + ).toBeUndefined(); + } + }); + + it("should handle mix of tool calls with results and with approvals", () => { + const messages: MessageDoc[] = [ + { + _id: "1", + message: { + role: "assistant", + content: [ + { + type: "tool-call", + toolCallId: "call_with_result", + toolName: "safeTool", + input: {}, + args: {}, + }, + { + type: "tool-call", + toolCallId: "call_with_approval", + toolName: "dangerousTool", + input: {}, + args: {}, + }, + { + type: "tool-approval-request", + toolCallId: "call_with_approval", + approvalId: "approval_789", + }, + ], + }, + order: 1, + } as MessageDoc, + { + _id: "2", + message: { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call_with_result", + result: "success", + }, + { + type: "tool-approval-response", + approvalId: "approval_789", + approved: true, + }, + ], + }, + order: 2, + } as MessageDoc, + ]; + + const result = filterOutOrphanedToolMessages(messages); + expect(result).toHaveLength(2); + // Both tool calls should be kept + const assistantContent = result[0].message?.content; + expect(Array.isArray(assistantContent)).toBe(true); + if (Array.isArray(assistantContent)) { + const toolCalls = assistantContent.filter( + (p) => p.type === "tool-call", + ); + expect(toolCalls).toHaveLength(2); + } + }); }); describe("fetchContextMessages", () => { diff --git a/src/client/search.ts b/src/client/search.ts index 05cf42d5..e3a73dc4 100644 --- a/src/client/search.ts +++ b/src/client/search.ts @@ -36,14 +36,20 @@ const DEFAULT_VECTOR_SCORE_THRESHOLD = 0.0; // the 8k token limit for some models. const MAX_EMBEDDING_TEXT_LENGTH = 10_000; -export type GetEmbedding = (text: string) => Promise<{ - embedding: number[]; - /** - * @deprecated Use embeddingModel instead. - */ - textEmbeddingModel?: string | EmbeddingModel; - embeddingModel: string | EmbeddingModel; -}>; +export type GetEmbedding = (text: string) => Promise< + | { + embedding: number[]; + /** @deprecated Use embeddingModel instead. */ + textEmbeddingModel: string | EmbeddingModel; + embeddingModel?: string | EmbeddingModel; + } + | { + embedding: number[]; + /** @deprecated Use embeddingModel instead. */ + textEmbeddingModel?: string | EmbeddingModel; + embeddingModel: string | EmbeddingModel; + } +>; /** * Fetch the context messages for a thread. @@ -231,12 +237,19 @@ export async function fetchRecentAndSearchMessages( /** * Filter out tool messages that don't have both a tool call and response. + * For the approval workflow, tool calls with approval responses (but no tool-results yet) + * should also be kept. * @param docs The messages to filter. * @returns The filtered messages. */ export function filterOutOrphanedToolMessages(docs: MessageDoc[]) { const toolCallIds = new Set(); const toolResultIds = new Set(); + // Track approval workflow: toolCallId → approvalId + const approvalRequestsByToolCallId = new Map(); + // Track which approvalIds have responses + const approvalResponseIds = new Set(); + const result: MessageDoc[] = []; for (const doc of docs) { if (doc.message && Array.isArray(doc.message.content)) { @@ -245,17 +258,43 @@ export function filterOutOrphanedToolMessages(docs: MessageDoc[]) { toolCallIds.add(content.toolCallId); } else if (content.type === "tool-result") { toolResultIds.add(content.toolCallId); + } else if (content.type === "tool-approval-request") { + const approvalRequest = content as { + type: "tool-approval-request"; + toolCallId: string; + approvalId: string; + }; + approvalRequestsByToolCallId.set( + approvalRequest.toolCallId, + approvalRequest.approvalId, + ); + } else if (content.type === "tool-approval-response") { + const approvalResponse = content as { + type: "tool-approval-response"; + approvalId: string; + }; + approvalResponseIds.add(approvalResponse.approvalId); } } } } + + // Helper: check if tool call has a corresponding approval response + const hasApprovalResponse = (toolCallId: string) => { + const approvalId = approvalRequestsByToolCallId.get(toolCallId); + return approvalId !== undefined && approvalResponseIds.has(approvalId); + }; + for (const doc of docs) { if ( doc.message?.role === "assistant" && Array.isArray(doc.message.content) ) { const content = doc.message.content.filter( - (p) => p.type !== "tool-call" || toolResultIds.has(p.toolCallId), + (p) => + p.type !== "tool-call" || + toolResultIds.has(p.toolCallId) || + hasApprovalResponse(p.toolCallId), ); if (content.length) { result.push({ @@ -267,9 +306,14 @@ export function filterOutOrphanedToolMessages(docs: MessageDoc[]) { }); } } else if (doc.message?.role === "tool") { - const content = doc.message.content.filter((c) => - toolCallIds.has(c.toolCallId), - ); + const content = doc.message.content.filter((c) => { + // tool-result parts have toolCallId + if (c.type === "tool-result") { + return toolCallIds.has(c.toolCallId); + } + // tool-approval-response parts don't have toolCallId, so include them + return true; + }); if (content.length) { result.push({ ...doc, @@ -300,7 +344,10 @@ export async function embedMessages( userId: string | undefined; threadId: string | undefined; agentName?: string; - } & Pick, + } & Pick< + Config, + "usageHandler" | "textEmbeddingModel" | "embeddingModel" | "callSettings" + >, messages: (ModelMessage | Message)[], ): Promise< | { @@ -310,7 +357,8 @@ export async function embedMessages( } | undefined > { - const textEmbeddingModel = options.embeddingModel ?? options.textEmbeddingModel; + const textEmbeddingModel = + options.embeddingModel ?? options.textEmbeddingModel; if (!textEmbeddingModel) { return undefined; } @@ -369,7 +417,10 @@ export async function embedMany( abortSignal?: AbortSignal; headers?: Record; agentName?: string; - } & Pick, + } & Pick< + Config, + "usageHandler" | "textEmbeddingModel" | "embeddingModel" | "callSettings" + >, ): Promise<{ embeddings: number[][] }> { const { userId, @@ -437,7 +488,8 @@ export async function generateAndSaveEmbeddings( } & Pick, messages: MessageDoc[], ) { - const effectiveEmbeddingModel = args.embeddingModel ?? args.textEmbeddingModel; + const effectiveEmbeddingModel = + args.embeddingModel ?? args.textEmbeddingModel; if (!effectiveEmbeddingModel) { throw new Error( "an embeddingModel (or textEmbeddingModel) is required to generate and save embeddings", diff --git a/src/component/_generated/component.ts b/src/component/_generated/component.ts index 228108ff..f6708edc 100644 --- a/src/component/_generated/component.ts +++ b/src/component/_generated/component.ts @@ -56,7 +56,8 @@ export type ComponentApi = { filename?: string; hash: string; - mimeType: string; + mediaType?: string; + mimeType?: string; storageId: string; }, { fileId: string; storageId: string }, @@ -86,7 +87,8 @@ export type ComponentApi = filename?: string; hash: string; lastTouchedAt: number; - mimeType: string; + mediaType?: string; + mimeType?: string; refcount: number; storageId: string; }, @@ -114,7 +116,8 @@ export type ComponentApi = filename?: string; hash: string; lastTouchedAt: number; - mimeType: string; + mediaType?: string; + mimeType?: string; refcount: number; storageId: string; }>; @@ -182,6 +185,7 @@ export type ComponentApi = } | { image: string | ArrayBuffer; + mediaType?: string; mimeType?: string; providerOptions?: Record< string, @@ -192,7 +196,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -226,7 +231,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -262,8 +268,25 @@ export type ComponentApi = >; type: "redacted-reasoning"; } + | { + args?: any; + input: any; + providerExecuted?: boolean; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record< + string, + Record + >; + toolCallId: string; + toolName: string; + type: "tool-call"; + } | { args: any; + input?: any; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -289,19 +312,120 @@ export type ComponentApi = >; isError?: boolean; output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } | { type: "content"; value: Array< - | { text: string; type: "text" } + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } | { data: string; mediaType: string; type: "media"; } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } >; }; providerExecuted?: boolean; @@ -349,38 +473,167 @@ export type ComponentApi = title: string; type: "source"; } + | { + approvalId: string; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record< + string, + Record + >; + toolCallId: string; + type: "tool-approval-request"; + } >; providerOptions?: Record>; role: "assistant"; } | { - content: Array<{ - args?: any; - experimental_content?: Array< - | { text: string; type: "text" } - | { data: string; mimeType?: string; type: "image" } - >; - isError?: boolean; - output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } - | { - type: "content"; - value: Array< - | { text: string; type: "text" } - | { data: string; mediaType: string; type: "media" } - >; - }; - providerExecuted?: boolean; - providerMetadata?: Record>; - providerOptions?: Record>; - result?: any; - toolCallId: string; - toolName: string; - type: "tool-result"; - }>; + content: Array< + | { + args?: any; + experimental_content?: Array< + | { text: string; type: "text" } + | { data: string; mimeType?: string; type: "image" } + >; + isError?: boolean; + output?: + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } + | { + type: "content"; + value: Array< + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } + | { + data: string; + mediaType: string; + type: "media"; + } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } + >; + }; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + result?: any; + toolCallId: string; + toolName: string; + type: "tool-result"; + } + | { + approvalId: string; + approved: boolean; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + reason?: string; + type: "tool-approval-response"; + } + >; providerOptions?: Record>; role: "tool"; } @@ -485,6 +738,7 @@ export type ComponentApi = } | { image: string | ArrayBuffer; + mediaType?: string; mimeType?: string; providerOptions?: Record< string, @@ -495,7 +749,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -529,7 +784,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -565,8 +821,25 @@ export type ComponentApi = >; type: "redacted-reasoning"; } + | { + args?: any; + input: any; + providerExecuted?: boolean; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record< + string, + Record + >; + toolCallId: string; + toolName: string; + type: "tool-call"; + } | { args: any; + input?: any; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -592,19 +865,120 @@ export type ComponentApi = >; isError?: boolean; output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } | { type: "content"; value: Array< - | { text: string; type: "text" } + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } | { data: string; mediaType: string; type: "media"; } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } >; }; providerExecuted?: boolean; @@ -652,38 +1026,167 @@ export type ComponentApi = title: string; type: "source"; } + | { + approvalId: string; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record< + string, + Record + >; + toolCallId: string; + type: "tool-approval-request"; + } >; providerOptions?: Record>; role: "assistant"; } | { - content: Array<{ - args?: any; - experimental_content?: Array< - | { text: string; type: "text" } - | { data: string; mimeType?: string; type: "image" } - >; - isError?: boolean; - output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } - | { - type: "content"; - value: Array< - | { text: string; type: "text" } - | { data: string; mediaType: string; type: "media" } - >; - }; - providerExecuted?: boolean; - providerMetadata?: Record>; - providerOptions?: Record>; - result?: any; - toolCallId: string; - toolName: string; - type: "tool-result"; - }>; + content: Array< + | { + args?: any; + experimental_content?: Array< + | { text: string; type: "text" } + | { data: string; mimeType?: string; type: "image" } + >; + isError?: boolean; + output?: + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } + | { + type: "content"; + value: Array< + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } + | { + data: string; + mediaType: string; + type: "media"; + } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } + >; + }; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + result?: any; + toolCallId: string; + toolName: string; + type: "tool-result"; + } + | { + approvalId: string; + approved: boolean; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + reason?: string; + type: "tool-approval-response"; + } + >; providerOptions?: Record>; role: "tool"; } @@ -839,6 +1342,7 @@ export type ComponentApi = } | { image: string | ArrayBuffer; + mediaType?: string; mimeType?: string; providerOptions?: Record>; type: "image"; @@ -846,7 +1350,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -874,7 +1379,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -902,7 +1408,8 @@ export type ComponentApi = type: "redacted-reasoning"; } | { - args: any; + args?: any; + input: any; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -913,6 +1420,22 @@ export type ComponentApi = toolName: string; type: "tool-call"; } + | { + args: any; + input?: any; + providerExecuted?: boolean; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record< + string, + Record + >; + toolCallId: string; + toolName: string; + type: "tool-call"; + } | { args?: any; experimental_content?: Array< @@ -921,19 +1444,120 @@ export type ComponentApi = >; isError?: boolean; output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } | { type: "content"; value: Array< - | { text: string; type: "text" } + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } | { data: string; mediaType: string; type: "media"; } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } >; }; providerExecuted?: boolean; @@ -972,88 +1596,214 @@ export type ComponentApi = title: string; type: "source"; } + | { + approvalId: string; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record>; + toolCallId: string; + type: "tool-approval-request"; + } >; providerOptions?: Record>; role: "assistant"; } | { - content: Array<{ - args?: any; - experimental_content?: Array< - | { text: string; type: "text" } - | { data: string; mimeType?: string; type: "image" } - >; - isError?: boolean; - output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } - | { - type: "content"; - value: Array< - | { text: string; type: "text" } - | { data: string; mediaType: string; type: "media" } - >; - }; - providerExecuted?: boolean; - providerMetadata?: Record>; - providerOptions?: Record>; - result?: any; - toolCallId: string; - toolName: string; - type: "tool-result"; - }>; - providerOptions?: Record>; - role: "tool"; - } - | { - content: string; - providerOptions?: Record>; - role: "system"; - }; - model?: string; - order: number; - provider?: string; - providerMetadata?: Record>; - providerOptions?: Record>; - reasoning?: string; - reasoningDetails?: Array< - | { - providerMetadata?: Record>; - providerOptions?: Record>; - signature?: string; - text: string; - type: "reasoning"; - } - | { signature?: string; text: string; type: "text" } - | { data: string; type: "redacted" } - >; - sources?: Array< - | { - id: string; - providerMetadata?: Record>; - providerOptions?: Record>; - sourceType: "url"; - title?: string; - type?: "source"; - url: string; - } - | { - filename?: string; - id: string; - mediaType: string; - providerMetadata?: Record>; - providerOptions?: Record>; - sourceType: "document"; - title: string; - type: "source"; - } - >; - status: "pending" | "success" | "failed"; - stepOrder: number; - text?: string; - threadId: string; + content: Array< + | { + args?: any; + experimental_content?: Array< + | { text: string; type: "text" } + | { data: string; mimeType?: string; type: "image" } + >; + isError?: boolean; + output?: + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } + | { + type: "content"; + value: Array< + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } + | { + data: string; + mediaType: string; + type: "media"; + } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } + >; + }; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + result?: any; + toolCallId: string; + toolName: string; + type: "tool-result"; + } + | { + approvalId: string; + approved: boolean; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + reason?: string; + type: "tool-approval-response"; + } + >; + providerOptions?: Record>; + role: "tool"; + } + | { + content: string; + providerOptions?: Record>; + role: "system"; + }; + model?: string; + order: number; + provider?: string; + providerMetadata?: Record>; + providerOptions?: Record>; + reasoning?: string; + reasoningDetails?: Array< + | { + providerMetadata?: Record>; + providerOptions?: Record>; + signature?: string; + text: string; + type: "reasoning"; + } + | { signature?: string; text: string; type: "text" } + | { data: string; type: "redacted" } + >; + sources?: Array< + | { + id: string; + providerMetadata?: Record>; + providerOptions?: Record>; + sourceType: "url"; + title?: string; + type?: "source"; + url: string; + } + | { + filename?: string; + id: string; + mediaType: string; + providerMetadata?: Record>; + providerOptions?: Record>; + sourceType: "document"; + title: string; + type: "source"; + } + >; + status: "pending" | "success" | "failed"; + stepOrder: number; + text?: string; + threadId: string; tool: boolean; usage?: { cachedInputTokens?: number; @@ -1134,6 +1884,7 @@ export type ComponentApi = } | { image: string | ArrayBuffer; + mediaType?: string; mimeType?: string; providerOptions?: Record< string, @@ -1144,7 +1895,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -1178,7 +1930,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -1214,8 +1967,25 @@ export type ComponentApi = >; type: "redacted-reasoning"; } + | { + args?: any; + input: any; + providerExecuted?: boolean; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record< + string, + Record + >; + toolCallId: string; + toolName: string; + type: "tool-call"; + } | { args: any; + input?: any; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -1241,19 +2011,120 @@ export type ComponentApi = >; isError?: boolean; output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } | { type: "content"; value: Array< - | { text: string; type: "text" } + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } | { data: string; mediaType: string; type: "media"; } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } >; }; providerExecuted?: boolean; @@ -1301,38 +2172,167 @@ export type ComponentApi = title: string; type: "source"; } + | { + approvalId: string; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record< + string, + Record + >; + toolCallId: string; + type: "tool-approval-request"; + } >; providerOptions?: Record>; role: "assistant"; } | { - content: Array<{ - args?: any; - experimental_content?: Array< - | { text: string; type: "text" } - | { data: string; mimeType?: string; type: "image" } - >; - isError?: boolean; - output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } - | { - type: "content"; - value: Array< - | { text: string; type: "text" } - | { data: string; mediaType: string; type: "media" } - >; - }; - providerExecuted?: boolean; - providerMetadata?: Record>; - providerOptions?: Record>; - result?: any; - toolCallId: string; - toolName: string; - type: "tool-result"; - }>; + content: Array< + | { + args?: any; + experimental_content?: Array< + | { text: string; type: "text" } + | { data: string; mimeType?: string; type: "image" } + >; + isError?: boolean; + output?: + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } + | { + type: "content"; + value: Array< + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } + | { + data: string; + mediaType: string; + type: "media"; + } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } + >; + }; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + result?: any; + toolCallId: string; + toolName: string; + type: "tool-result"; + } + | { + approvalId: string; + approved: boolean; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + reason?: string; + type: "tool-approval-response"; + } + >; providerOptions?: Record>; role: "tool"; } @@ -1455,6 +2455,7 @@ export type ComponentApi = } | { image: string | ArrayBuffer; + mediaType?: string; mimeType?: string; providerOptions?: Record>; type: "image"; @@ -1462,7 +2463,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -1490,7 +2492,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -1518,7 +2521,8 @@ export type ComponentApi = type: "redacted-reasoning"; } | { - args: any; + args?: any; + input: any; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -1529,6 +2533,22 @@ export type ComponentApi = toolName: string; type: "tool-call"; } + | { + args: any; + input?: any; + providerExecuted?: boolean; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record< + string, + Record + >; + toolCallId: string; + toolName: string; + type: "tool-call"; + } | { args?: any; experimental_content?: Array< @@ -1537,19 +2557,120 @@ export type ComponentApi = >; isError?: boolean; output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } | { type: "content"; value: Array< - | { text: string; type: "text" } + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } | { data: string; mediaType: string; type: "media"; } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } >; }; providerExecuted?: boolean; @@ -1588,47 +2709,173 @@ export type ComponentApi = title: string; type: "source"; } + | { + approvalId: string; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record>; + toolCallId: string; + type: "tool-approval-request"; + } >; providerOptions?: Record>; role: "assistant"; } | { - content: Array<{ - args?: any; - experimental_content?: Array< - | { text: string; type: "text" } - | { data: string; mimeType?: string; type: "image" } - >; - isError?: boolean; - output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } - | { - type: "content"; - value: Array< - | { text: string; type: "text" } - | { data: string; mediaType: string; type: "media" } - >; - }; - providerExecuted?: boolean; - providerMetadata?: Record>; - providerOptions?: Record>; - result?: any; - toolCallId: string; - toolName: string; - type: "tool-result"; - }>; - providerOptions?: Record>; - role: "tool"; - } - | { - content: string; - providerOptions?: Record>; - role: "system"; - }; - model?: string; + content: Array< + | { + args?: any; + experimental_content?: Array< + | { text: string; type: "text" } + | { data: string; mimeType?: string; type: "image" } + >; + isError?: boolean; + output?: + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } + | { + type: "content"; + value: Array< + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } + | { + data: string; + mediaType: string; + type: "media"; + } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } + >; + }; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + result?: any; + toolCallId: string; + toolName: string; + type: "tool-result"; + } + | { + approvalId: string; + approved: boolean; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + reason?: string; + type: "tool-approval-response"; + } + >; + providerOptions?: Record>; + role: "tool"; + } + | { + content: string; + providerOptions?: Record>; + role: "system"; + }; + model?: string; order: number; provider?: string; providerMetadata?: Record>; @@ -1729,6 +2976,7 @@ export type ComponentApi = } | { image: string | ArrayBuffer; + mediaType?: string; mimeType?: string; providerOptions?: Record>; type: "image"; @@ -1736,7 +2984,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -1764,7 +3013,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -1792,7 +3042,8 @@ export type ComponentApi = type: "redacted-reasoning"; } | { - args: any; + args?: any; + input: any; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -1803,6 +3054,22 @@ export type ComponentApi = toolName: string; type: "tool-call"; } + | { + args: any; + input?: any; + providerExecuted?: boolean; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record< + string, + Record + >; + toolCallId: string; + toolName: string; + type: "tool-call"; + } | { args?: any; experimental_content?: Array< @@ -1811,19 +3078,120 @@ export type ComponentApi = >; isError?: boolean; output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } | { type: "content"; value: Array< - | { text: string; type: "text" } + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } | { data: string; mediaType: string; type: "media"; } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } >; }; providerExecuted?: boolean; @@ -1862,38 +3230,164 @@ export type ComponentApi = title: string; type: "source"; } + | { + approvalId: string; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record>; + toolCallId: string; + type: "tool-approval-request"; + } >; providerOptions?: Record>; role: "assistant"; } | { - content: Array<{ - args?: any; - experimental_content?: Array< - | { text: string; type: "text" } - | { data: string; mimeType?: string; type: "image" } - >; - isError?: boolean; - output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } - | { - type: "content"; - value: Array< - | { text: string; type: "text" } - | { data: string; mediaType: string; type: "media" } - >; - }; - providerExecuted?: boolean; - providerMetadata?: Record>; - providerOptions?: Record>; - result?: any; - toolCallId: string; - toolName: string; - type: "tool-result"; - }>; + content: Array< + | { + args?: any; + experimental_content?: Array< + | { text: string; type: "text" } + | { data: string; mimeType?: string; type: "image" } + >; + isError?: boolean; + output?: + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } + | { + type: "content"; + value: Array< + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } + | { + data: string; + mediaType: string; + type: "media"; + } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } + >; + }; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + result?: any; + toolCallId: string; + toolName: string; + type: "tool-result"; + } + | { + approvalId: string; + approved: boolean; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + reason?: string; + type: "tool-approval-response"; + } + >; providerOptions?: Record>; role: "tool"; } @@ -1996,6 +3490,7 @@ export type ComponentApi = } | { image: string | ArrayBuffer; + mediaType?: string; mimeType?: string; providerOptions?: Record< string, @@ -2006,7 +3501,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -2040,7 +3536,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -2076,8 +3573,25 @@ export type ComponentApi = >; type: "redacted-reasoning"; } + | { + args?: any; + input: any; + providerExecuted?: boolean; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record< + string, + Record + >; + toolCallId: string; + toolName: string; + type: "tool-call"; + } | { args: any; + input?: any; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -2103,19 +3617,120 @@ export type ComponentApi = >; isError?: boolean; output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } | { type: "content"; value: Array< - | { text: string; type: "text" } + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } | { data: string; mediaType: string; type: "media"; } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } >; }; providerExecuted?: boolean; @@ -2163,38 +3778,167 @@ export type ComponentApi = title: string; type: "source"; } + | { + approvalId: string; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record< + string, + Record + >; + toolCallId: string; + type: "tool-approval-request"; + } >; providerOptions?: Record>; role: "assistant"; } | { - content: Array<{ - args?: any; - experimental_content?: Array< - | { text: string; type: "text" } - | { data: string; mimeType?: string; type: "image" } - >; - isError?: boolean; - output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } - | { - type: "content"; - value: Array< - | { text: string; type: "text" } - | { data: string; mediaType: string; type: "media" } - >; - }; - providerExecuted?: boolean; - providerMetadata?: Record>; - providerOptions?: Record>; - result?: any; - toolCallId: string; - toolName: string; - type: "tool-result"; - }>; + content: Array< + | { + args?: any; + experimental_content?: Array< + | { text: string; type: "text" } + | { data: string; mimeType?: string; type: "image" } + >; + isError?: boolean; + output?: + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } + | { + type: "content"; + value: Array< + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } + | { + data: string; + mediaType: string; + type: "media"; + } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } + >; + }; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + result?: any; + toolCallId: string; + toolName: string; + type: "tool-result"; + } + | { + approvalId: string; + approved: boolean; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + reason?: string; + type: "tool-approval-response"; + } + >; providerOptions?: Record>; role: "tool"; } @@ -2241,6 +3985,7 @@ export type ComponentApi = } | { image: string | ArrayBuffer; + mediaType?: string; mimeType?: string; providerOptions?: Record>; type: "image"; @@ -2248,7 +3993,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -2276,7 +4022,8 @@ export type ComponentApi = | { data: string | ArrayBuffer; filename?: string; - mimeType: string; + mediaType?: string; + mimeType?: string; providerMetadata?: Record< string, Record @@ -2304,7 +4051,8 @@ export type ComponentApi = type: "redacted-reasoning"; } | { - args: any; + args?: any; + input: any; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -2315,6 +4063,22 @@ export type ComponentApi = toolName: string; type: "tool-call"; } + | { + args: any; + input?: any; + providerExecuted?: boolean; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record< + string, + Record + >; + toolCallId: string; + toolName: string; + type: "tool-call"; + } | { args?: any; experimental_content?: Array< @@ -2323,19 +4087,120 @@ export type ComponentApi = >; isError?: boolean; output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } | { type: "content"; value: Array< - | { text: string; type: "text" } + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } | { data: string; mediaType: string; type: "media"; } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } >; }; providerExecuted?: boolean; @@ -2374,38 +4239,164 @@ export type ComponentApi = title: string; type: "source"; } + | { + approvalId: string; + providerMetadata?: Record< + string, + Record + >; + providerOptions?: Record>; + toolCallId: string; + type: "tool-approval-request"; + } >; providerOptions?: Record>; role: "assistant"; } | { - content: Array<{ - args?: any; - experimental_content?: Array< - | { text: string; type: "text" } - | { data: string; mimeType?: string; type: "image" } - >; - isError?: boolean; - output?: - | { type: "text"; value: string } - | { type: "json"; value: any } - | { type: "error-text"; value: string } - | { type: "error-json"; value: any } - | { - type: "content"; - value: Array< - | { text: string; type: "text" } - | { data: string; mediaType: string; type: "media" } - >; - }; - providerExecuted?: boolean; - providerMetadata?: Record>; - providerOptions?: Record>; - result?: any; - toolCallId: string; - toolName: string; - type: "tool-result"; - }>; + content: Array< + | { + args?: any; + experimental_content?: Array< + | { text: string; type: "text" } + | { data: string; mimeType?: string; type: "image" } + >; + isError?: boolean; + output?: + | { + providerOptions?: Record< + string, + Record + >; + type: "text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-text"; + value: string; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "error-json"; + value: any; + } + | { + providerOptions?: Record< + string, + Record + >; + reason?: string; + type: "execution-denied"; + } + | { + type: "content"; + value: Array< + | { + providerOptions?: Record< + string, + Record + >; + text: string; + type: "text"; + } + | { + data: string; + mediaType: string; + type: "media"; + } + | { + data: string; + filename?: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "file-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "file-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "file-id"; + } + | { + data: string; + mediaType: string; + providerOptions?: Record< + string, + Record + >; + type: "image-data"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "image-url"; + url: string; + } + | { + fileId: string | Record; + providerOptions?: Record< + string, + Record + >; + type: "image-file-id"; + } + | { + providerOptions?: Record< + string, + Record + >; + type: "custom"; + } + >; + }; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + result?: any; + toolCallId: string; + toolName: string; + type: "tool-result"; + } + | { + approvalId: string; + approved: boolean; + providerExecuted?: boolean; + providerMetadata?: Record>; + providerOptions?: Record>; + reason?: string; + type: "tool-approval-response"; + } + >; providerOptions?: Record>; role: "tool"; } diff --git a/src/component/files.ts b/src/component/files.ts index c6ce165b..89506f4c 100644 --- a/src/component/files.ts +++ b/src/component/files.ts @@ -9,7 +9,9 @@ const addFileArgs = v.object({ storageId: v.string(), hash: v.string(), filename: v.optional(v.string()), - mimeType: v.string(), + mediaType: v.optional(v.string()), + /** @deprecated Use `mediaType` instead. */ + mimeType: v.optional(v.string()), }); export const addFile = mutation({ @@ -25,6 +27,9 @@ export async function addFileHandler( ctx: MutationCtx, args: Infer, ) { + // Support both mediaType (preferred) and mimeType (deprecated) + const mediaType = args.mediaType ?? args.mimeType; + const existingFile = await ctx.db .query("files") .withIndex("hash", (q) => q.eq("hash", args.hash)) @@ -42,7 +47,11 @@ export async function addFileHandler( }; } const fileId = await ctx.db.insert("files", { - ...args, + storageId: args.storageId, + hash: args.hash, + filename: args.filename, + mediaType, + mimeType: args.mimeType, // Keep for backwards compatibility // We start out with it unused - when it's saved in a message we increment. refcount: 0, lastTouchedAt: Date.now(), diff --git a/src/component/messages.test.ts b/src/component/messages.test.ts index 9581b8f6..0c273307 100644 --- a/src/component/messages.test.ts +++ b/src/component/messages.test.ts @@ -68,7 +68,7 @@ describe("agent", () => { content: [ { type: "tool-call", - args: { a: 1 }, + input: { a: 1 }, toolCallId: "1", toolName: "tool", }, @@ -258,7 +258,7 @@ describe("agent", () => { content: [ { type: "tool-call", - args: { a: 1 }, + input: { a: 1 }, toolCallId: "1", toolName: "tool", }, @@ -389,7 +389,7 @@ describe("agent", () => { content: [ { type: "tool-call", - args: { a: 1 }, + input: { a: 1 }, toolCallId: "1", toolName: "tool", }, @@ -408,7 +408,7 @@ describe("agent", () => { content: [ { type: "tool-call", - args: { a: 2, b: 3 }, + input: { a: 2, b: 3 }, toolCallId: "1", toolName: "tool", }, @@ -422,7 +422,7 @@ describe("agent", () => { content: [ { type: "tool-call", - args: { a: 2, b: 3 }, + input: { a: 2, b: 3 }, toolCallId: "1", toolName: "tool", }, diff --git a/src/component/schema.ts b/src/component/schema.ts index da319828..644cc62e 100644 --- a/src/component/schema.ts +++ b/src/component/schema.ts @@ -149,7 +149,9 @@ export const schema = defineSchema({ files: defineTable({ storageId: v.string(), - mimeType: v.string(), + mediaType: v.optional(v.string()), + /** @deprecated Use `mediaType` instead. */ + mimeType: v.optional(v.string()), filename: v.optional(v.string()), hash: v.string(), refcount: v.number(), diff --git a/src/deltas.test.ts b/src/deltas.test.ts index fab52758..6be75868 100644 --- a/src/deltas.test.ts +++ b/src/deltas.test.ts @@ -156,6 +156,49 @@ describe("UIMessageChunks", () => { }); }); +describe("UIMessageChunks - continuation stream", () => { + it("gracefully handles tool-result without tool-call in continuation stream after approval", async () => { + // This simulates what happens after tool approval: + // Stream A: tool-call, tool-approval-request -> finishes + // User approves + // Stream B: tool-result (referencing tool-call from Stream A) -> this test + // + // The AI SDK's readUIMessageStream expects tool-call before tool-result, + // but they're in different streams. The onError handler should gracefully + // ignore this error since stored messages provide the fallback. + const uiMessage = blankUIMessage( + { + streamId: "continuation-stream", + status: "streaming", + order: 1, + stepOrder: 0, + format: "UIMessageChunk", + agentName: "agent1", + }, + "thread1", + ); + + // Send a tool-result without the corresponding tool-call in this stream + // This would normally throw "No tool invocation found" error + const updatedMessage = await updateFromUIMessageChunks(uiMessage, [ + { type: "start" }, + { type: "start-step" }, + { + type: "tool-output-available", + toolCallId: "call_from_previous_stream", + output: "Tool execution result", + }, + { type: "finish-step" }, + { type: "finish" }, + ]); + + // The message should NOT be marked as failed - the error should be suppressed + expect(updatedMessage.status).not.toBe("failed"); + // The stream still processes (even if tool-output isn't reflected without tool-input) + expect(updatedMessage).toBeDefined(); + }); +}); + describe("mergeDeltas", () => { it("merges a single text-delta into a message", () => { const streamId = "s1"; @@ -533,4 +576,51 @@ describe("mergeDeltas", () => { }, ]); }); + + it("handles streaming tool-approval-request and updates tool state", () => { + const streamId = "s10"; + const deltas = [ + { + streamId, + start: 0, + end: 1, + parts: [ + { + type: "tool-call", + toolCallId: "call1", + toolName: "dangerousTool", + input: { action: "delete" }, + }, + ], + } satisfies StreamDelta, + { + streamId, + start: 1, + end: 2, + parts: [ + { + type: "tool-approval-request", + toolCallId: "call1", + approvalId: "approval1", + }, + ], + } satisfies StreamDelta, + ]; + const [[message], _, changed] = deriveUIMessagesFromTextStreamParts( + "thread1", + [{ streamId, order: 10, stepOrder: 0, status: "streaming" }], + [], + deltas, + ); + expect(message).toBeDefined(); + expect(message.role).toBe("assistant"); + expect(changed).toBe(true); + + const toolPart = message.parts.find( + (p) => p.type === "tool-dangerousTool", + ) as any; + expect(toolPart).toBeDefined(); + expect(toolPart.state).toBe("approval-requested"); + expect(toolPart.approval).toEqual({ id: "approval1" }); + }); }); diff --git a/src/deltas.ts b/src/deltas.ts index e5764fd5..b62f2338 100644 --- a/src/deltas.ts +++ b/src/deltas.ts @@ -66,22 +66,40 @@ export async function updateFromUIMessageChunks( }, }); let failed = false; + let suppressError = false; const messageStream = readUIMessageStream({ message: uiMessage, stream: partsStream, onError: (e) => { + const errorMessage = e instanceof Error ? e.message : String(e); + // Tool invocation errors can be safely ignored when streaming continuation + // after tool approval - the stored messages have the complete tool context + if (errorMessage.toLowerCase().includes("no tool invocation found")) { + // Silently suppress - this is expected after tool approval when the + // continuation stream has tool-result without the original tool-call + suppressError = true; + return; + } failed = true; console.error("Error in stream", e); }, terminateOnError: true, }); let message = uiMessage; - for await (const messagePart of messageStream) { - assert( - messagePart.id === message.id, - `Expecting to only make one UIMessage in a stream`, - ); - message = messagePart; + try { + for await (const messagePart of messageStream) { + assert( + messagePart.id === message.id, + `Expecting to only make one UIMessage in a stream`, + ); + message = messagePart; + } + } catch (e) { + // If we've already handled this error in onError and marked it as suppressed, + // don't rethrow - the stored messages provide the fallback + if (!suppressError) { + throw e; + } } if (failed) { message.status = "failed"; @@ -472,6 +490,25 @@ export function updateFromTextStreamParts( } break; } + case "tool-approval-request": { + const typedPart = part as unknown as { + type: "tool-approval-request"; + toolCallId: string; + approvalId: string; + }; + const toolPart = toolPartsById.get(typedPart.toolCallId); + if (toolPart) { + toolPart.state = "approval-requested"; + (toolPart as ToolUIPart & { approval?: object }).approval = { + id: typedPart.approvalId, + }; + } else { + console.warn( + `Expected tool call part ${typedPart.toolCallId} for approval request`, + ); + } + break; + } case "file": case "text-end": case "finish-step": diff --git a/src/fromUIMessages.test.ts b/src/fromUIMessages.test.ts index 4d295d43..4413c3c1 100644 --- a/src/fromUIMessages.test.ts +++ b/src/fromUIMessages.test.ts @@ -166,6 +166,7 @@ describe("fromUIMessages round-trip tests", () => { type: "tool-call", toolName: "calculator", toolCallId: "call1", + input: { operation: "add", a: 2, b: 3 }, args: { operation: "add", a: 2, b: 3 }, }, ], @@ -277,7 +278,7 @@ describe("fromUIMessages round-trip tests", () => { expect(fileContent).toBeDefined(); expect(fileContent).toMatchObject({ type: "file", - mimeType: "image/png", + mediaType: "image/png", }); } }); @@ -442,7 +443,9 @@ describe("fromUIMessages functionality tests", () => { ], }; - const result = await fromUIMessages([toolUIMessage], { threadId: "thread1" }); + const result = await fromUIMessages([toolUIMessage], { + threadId: "thread1", + }); expect(result.length).toBeGreaterThan(0); // Should have tool messages @@ -471,7 +474,9 @@ describe("fromUIMessages functionality tests", () => { ], }; - const result = await fromUIMessages([toolUIMessage], { threadId: "thread1" }); + const result = await fromUIMessages([toolUIMessage], { + threadId: "thread1", + }); expect(result.length).toBeGreaterThan(0); // Should have tool messages diff --git a/src/mapping.test.ts b/src/mapping.test.ts index f730a775..c802b468 100644 --- a/src/mapping.test.ts +++ b/src/mapping.test.ts @@ -209,4 +209,52 @@ describe("mapping", () => { const { fileIds } = await serializeContent(ctx, component, content); expect(fileIds).toBeUndefined(); }); + + test("tool-approval-request is preserved after serialization", async () => { + const approvalRequest = { + type: "tool-approval-request" as const, + approvalId: "approval-123", + toolCallId: "tool-call-456", + }; + const { content } = await serializeContent( + {} as ActionCtx, + {} as AgentComponent, + [approvalRequest], + ); + expect(content).toHaveLength(1); + expect((content as unknown[])[0]).toMatchObject(approvalRequest); + }); + + test("tool-approval-response with approved: true is preserved", async () => { + const approvalResponse = { + type: "tool-approval-response" as const, + approvalId: "approval-123", + approved: true, + reason: "User approved", + }; + const { content } = await serializeContent( + {} as ActionCtx, + {} as AgentComponent, + [approvalResponse], + ); + expect(content).toHaveLength(1); + expect((content as unknown[])[0]).toMatchObject(approvalResponse); + }); + + test("tool-approval-response with approved: false is preserved", async () => { + const approvalResponse = { + type: "tool-approval-response" as const, + approvalId: "approval-123", + approved: false, + reason: "User denied", + providerExecuted: false, + }; + const { content } = await serializeContent( + {} as ActionCtx, + {} as AgentComponent, + [approvalResponse], + ); + expect(content).toHaveLength(1); + expect((content as unknown[])[0]).toMatchObject(approvalResponse); + }); }); diff --git a/src/mapping.ts b/src/mapping.ts index 2d69c0ac..cdfd6806 100644 --- a/src/mapping.ts +++ b/src/mapping.ts @@ -36,6 +36,8 @@ import { type SourcePart, vToolResultOutput, type MessageDoc, + vToolApprovalRequest, + vToolApprovalResponse, } from "./validators.js"; import type { ActionCtx, AgentComponent } from "./client/types.js"; import type { MutationCtx } from "./client/types.js"; @@ -45,6 +47,8 @@ import { convertUint8ArrayToBase64, type ProviderOptions, type ReasoningPart, + type ToolApprovalRequest, + type ToolApprovalResponse, } from "@ai-sdk/provider-utils"; import { parse, validate } from "convex-helpers/validators"; import { @@ -324,7 +328,7 @@ export async function serializeContent( } return { type: part.type, - mimeType: getMimeOrMediaType(part), + mediaType: getMimeOrMediaType(part), ...metadata, image, } satisfies Infer; @@ -344,15 +348,18 @@ export async function serializeContent( type: part.type, data, filename: part.filename, - mimeType: getMimeOrMediaType(part)!, + mediaType: getMimeOrMediaType(part)!, ...metadata, } satisfies Infer; } case "tool-call": { - const args = "input" in part ? part.input : part.args; + // Handle legacy data where only args field exists + const input = part.input ?? (part as any)?.args ?? {}; return { type: part.type, - args: args ?? null, + input, + /** @deprecated Use `input` instead. */ + args: input, toolCallId: part.toolCallId, toolName: part.toolName, providerExecuted: part.providerExecuted, @@ -380,6 +387,24 @@ export async function serializeContent( case "source": { return part satisfies Infer; } + case "tool-approval-request": { + return { + type: part.type, + approvalId: part.approvalId, + toolCallId: part.toolCallId, + ...metadata, + } satisfies Infer; + } + case "tool-approval-response": { + return { + type: part.type, + approvalId: part.approvalId, + approved: part.approved, + reason: part.reason, + providerExecuted: part.providerExecuted, + ...metadata, + } satisfies Infer; + } default: return null; } @@ -413,7 +438,7 @@ export function fromModelMessageContent(content: Content): Message["content"] { case "image": return { type: part.type, - mimeType: getMimeOrMediaType(part), + mediaType: getMimeOrMediaType(part), ...metadata, image: serializeDataOrUrl(part.image), } satisfies Infer; @@ -422,13 +447,16 @@ export function fromModelMessageContent(content: Content): Message["content"] { type: part.type, data: serializeDataOrUrl(part.data), filename: part.filename, - mimeType: getMimeOrMediaType(part)!, + mediaType: getMimeOrMediaType(part)!, ...metadata, } satisfies Infer; case "tool-call": + // Handle legacy data where only args field exists return { type: part.type, - args: part.input ?? null, + input: part.input ?? (part as any)?.args ?? {}, + /** @deprecated Use `input` instead. */ + args: part.input ?? (part as any)?.args ?? {}, toolCallId: part.toolCallId, toolName: part.toolName, providerExecuted: part.providerExecuted, @@ -442,6 +470,22 @@ export function fromModelMessageContent(content: Content): Message["content"] { text: part.text, ...metadata, } satisfies Infer; + case "tool-approval-request": + return { + type: part.type, + approvalId: part.approvalId, + toolCallId: part.toolCallId, + ...metadata, + } satisfies Infer; + case "tool-approval-response": + return { + type: part.type, + approvalId: part.approvalId, + approved: part.approved, + reason: part.reason, + providerExecuted: part.providerExecuted, + ...metadata, + } satisfies Infer; // Not in current generation output, but could be in historical messages default: return null; @@ -491,10 +535,11 @@ export function toModelMessageContent( ...metadata, } satisfies FilePart; case "tool-call": { - const input = "input" in part ? part.input : part.args; + // Handle legacy data where only args field exists + const input = part.input ?? (part as any)?.args ?? {}; return { type: part.type, - input: input ?? null, + input, toolCallId: part.toolCallId, toolName: part.toolName, providerExecuted: part.providerExecuted, @@ -531,6 +576,22 @@ export function toModelMessageContent( } satisfies ReasoningPart; case "source": return part satisfies SourcePart; + case "tool-approval-request": + return { + type: part.type, + approvalId: part.approvalId, + toolCallId: part.toolCallId, + ...metadata, + } satisfies ToolApprovalRequest; + case "tool-approval-response": + return { + type: part.type, + approvalId: part.approvalId, + approved: part.approved, + reason: part.reason, + providerExecuted: part.providerExecuted, + ...metadata, + } satisfies ToolApprovalResponse; default: return null; } diff --git a/src/toUIMessages.test.ts b/src/toUIMessages.test.ts index f871f8d3..73707cbe 100644 --- a/src/toUIMessages.test.ts +++ b/src/toUIMessages.test.ts @@ -90,6 +90,7 @@ describe("toUIMessages", () => { type: "tool-call", toolName: "myTool", toolCallId: "call1", + input: "an arg", args: "an arg", }, ], @@ -219,6 +220,7 @@ describe("toUIMessages", () => { }, { type: "tool-call", + input: "What's the meaning of life?", args: "What's the meaning of life?", toolCallId: "call1", toolName: "myTool", @@ -310,6 +312,7 @@ describe("toUIMessages", () => { type: "tool-call", toolName: "myTool", toolCallId: "call1", + input: { query: "test" }, args: { query: "test" }, }, ], @@ -356,6 +359,7 @@ describe("toUIMessages", () => { type: "tool-call", toolName: "myTool", toolCallId: "call1", + input: "hi", args: "hi", }, ], @@ -387,6 +391,7 @@ describe("toUIMessages", () => { type: "tool-call", toolName: "myTool", toolCallId: "call1", + input: "", args: "", }, ], @@ -448,6 +453,7 @@ describe("toUIMessages", () => { type: "tool-call", toolName: "calculator", toolCallId: "call1", + input: { operation: "add", a: 2, b: 3 }, args: { operation: "add", a: 2, b: 3 }, }, { @@ -490,6 +496,7 @@ describe("toUIMessages", () => { type: "tool-call", toolName: "calculator", toolCallId: "call1", + input: { operation: "add", a: 1, b: 2 }, args: { operation: "add", a: 1, b: 2 }, }, ], @@ -561,6 +568,9 @@ describe("toUIMessages", () => { text: "**Finding the Time**\n\nI've pinpointed the core task: obtaining the current time in Paris. It involves using the `dateTime` tool. I've identified \"Europe/Paris\" as the necessary timezone identifier to provide to the tool. My next step is to test the tool.\n\n\n", }, { + input: { + timezone: "Europe/Paris", + }, args: { timezone: "Europe/Paris", }, @@ -690,6 +700,7 @@ describe("toUIMessages", () => { type: "tool-call", toolName: "calculator", toolCallId: "call1", + input: { operation: "add", a: 40, b: 2 }, args: { operation: "add", a: 40, b: 2 }, }, ], @@ -744,6 +755,7 @@ describe("toUIMessages", () => { type: "tool-call", toolName: "generateImage", toolCallId: "call1", + input: { id: "invalid-id" }, args: { id: "invalid-id" }, }, ], @@ -898,6 +910,7 @@ describe("toUIMessages", () => { type: "tool-call", toolName: "myTool", toolCallId: "call1", + input: {}, args: {}, }, ], @@ -957,4 +970,300 @@ describe("toUIMessages", () => { expect(uiMessages[0].userId).toBeUndefined(); }); }); + + describe("tool approval workflow", () => { + it("sets state to approval-requested when tool-approval-request is present", () => { + const messages = [ + baseMessageDoc({ + _id: "msg1", + order: 1, + stepOrder: 1, + tool: true, + message: { + role: "assistant", + content: [ + { + type: "tool-call", + toolName: "dangerousTool", + toolCallId: "call1", + input: { action: "delete" }, + args: { action: "delete" }, + }, + { + type: "tool-approval-request", + approvalId: "approval1", + toolCallId: "call1", + }, + ], + }, + }), + ]; + + const uiMessages = toUIMessages(messages); + + expect(uiMessages).toHaveLength(1); + const toolPart = uiMessages[0].parts.find( + (p) => p.type === "tool-dangerousTool", + ) as any; + expect(toolPart).toBeDefined(); + expect(toolPart.state).toBe("approval-requested"); + expect(toolPart.approval).toEqual({ id: "approval1" }); + }); + + it("sets state to approval-responded when tool-approval-response with approved: true", () => { + const messages = [ + baseMessageDoc({ + _id: "msg1", + order: 1, + stepOrder: 1, + tool: true, + message: { + role: "assistant", + content: [ + { + type: "tool-call", + toolName: "dangerousTool", + toolCallId: "call1", + input: { action: "delete" }, + args: { action: "delete" }, + }, + { + type: "tool-approval-request", + approvalId: "approval1", + toolCallId: "call1", + }, + ], + }, + }), + baseMessageDoc({ + _id: "msg2", + order: 1, + stepOrder: 2, + tool: true, + message: { + role: "tool", + content: [ + { + type: "tool-approval-response", + approvalId: "approval1", + approved: true, + reason: "User confirmed", + }, + ], + }, + }), + ]; + + const uiMessages = toUIMessages(messages); + + expect(uiMessages).toHaveLength(1); + const toolPart = uiMessages[0].parts.find( + (p) => p.type === "tool-dangerousTool", + ) as any; + expect(toolPart).toBeDefined(); + expect(toolPart.state).toBe("approval-responded"); + expect(toolPart.approval).toEqual({ + id: "approval1", + approved: true, + reason: "User confirmed", + }); + }); + + it("sets state to output-denied when tool-approval-response with approved: false", () => { + const messages = [ + baseMessageDoc({ + _id: "msg1", + order: 1, + stepOrder: 1, + tool: true, + message: { + role: "assistant", + content: [ + { + type: "tool-call", + toolName: "dangerousTool", + toolCallId: "call1", + input: { action: "delete" }, + args: { action: "delete" }, + }, + { + type: "tool-approval-request", + approvalId: "approval1", + toolCallId: "call1", + }, + ], + }, + }), + baseMessageDoc({ + _id: "msg2", + order: 1, + stepOrder: 2, + tool: true, + message: { + role: "tool", + content: [ + { + type: "tool-approval-response", + approvalId: "approval1", + approved: false, + reason: "User declined the operation", + }, + ], + }, + }), + ]; + + const uiMessages = toUIMessages(messages); + + expect(uiMessages).toHaveLength(1); + const toolPart = uiMessages[0].parts.find( + (p) => p.type === "tool-dangerousTool", + ) as any; + expect(toolPart).toBeDefined(); + expect(toolPart.state).toBe("output-denied"); + expect(toolPart.approval).toEqual({ + id: "approval1", + approved: false, + reason: "User declined the operation", + }); + }); + + it("sets state to output-denied when tool-result has execution-denied output", () => { + const messages = [ + baseMessageDoc({ + _id: "msg1", + order: 1, + stepOrder: 1, + tool: true, + message: { + role: "assistant", + content: [ + { + type: "tool-call", + toolName: "dangerousTool", + toolCallId: "call1", + input: { action: "delete" }, + args: { action: "delete" }, + }, + ], + }, + }), + baseMessageDoc({ + _id: "msg2", + order: 1, + stepOrder: 2, + tool: true, + message: { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call1", + toolName: "dangerousTool", + output: { + type: "execution-denied", + reason: "Tool execution was denied by the user", + }, + }, + ], + }, + }), + ]; + + const uiMessages = toUIMessages(messages); + + expect(uiMessages).toHaveLength(1); + const toolPart = uiMessages[0].parts.find( + (p) => p.type === "tool-dangerousTool", + ) as any; + expect(toolPart).toBeDefined(); + expect(toolPart.state).toBe("output-denied"); + expect(toolPart.approval).toEqual({ + id: "", + approved: false, + reason: "Tool execution was denied by the user", + }); + }); + + it("handles full approval flow: request → approved → executed → output-available", () => { + const messages = [ + baseMessageDoc({ + _id: "msg1", + order: 1, + stepOrder: 1, + tool: true, + message: { + role: "assistant", + content: [ + { + type: "tool-call", + toolName: "dangerousTool", + toolCallId: "call1", + input: { action: "delete" }, + args: { action: "delete" }, + }, + { + type: "tool-approval-request", + approvalId: "approval1", + toolCallId: "call1", + }, + ], + }, + }), + baseMessageDoc({ + _id: "msg2", + order: 1, + stepOrder: 2, + tool: true, + message: { + role: "tool", + content: [ + { + type: "tool-approval-response", + approvalId: "approval1", + approved: true, + }, + ], + }, + }), + baseMessageDoc({ + _id: "msg3", + order: 1, + stepOrder: 3, + tool: true, + message: { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call1", + toolName: "dangerousTool", + output: { + type: "json", + value: { deleted: true }, + }, + }, + ], + }, + }), + ]; + + const uiMessages = toUIMessages(messages); + + expect(uiMessages).toHaveLength(1); + const toolPart = uiMessages[0].parts.find( + (p) => p.type === "tool-dangerousTool", + ) as any; + expect(toolPart).toBeDefined(); + // After tool-result, state should be output-available + expect(toolPart.state).toBe("output-available"); + expect(toolPart.output).toEqual({ deleted: true }); + // approval should still be preserved from earlier + expect(toolPart.approval).toEqual({ + id: "approval1", + approved: true, + reason: undefined, + }); + }); + }); }); diff --git a/src/validators.ts b/src/validators.ts index 8d664ea4..f8e03132 100644 --- a/src/validators.ts +++ b/src/validators.ts @@ -1,4 +1,5 @@ -import { v, type Infer, type Validator, type Value } from "convex/values"; +import type { Infer, Validator, Value } from "convex/values"; +import { v } from "convex/values"; import { vVectorDimension } from "./component/vector/tables.js"; // const deprecated = v.optional(v.any()) as unknown as VNull; @@ -42,6 +43,8 @@ export const vTextPart = v.object({ export const vImagePart = v.object({ type: v.literal("image"), image: v.union(v.string(), v.bytes()), + mediaType: v.optional(v.string()), + /** @deprecated Use `mediaType` instead. */ mimeType: v.optional(v.string()), providerOptions, }); @@ -50,7 +53,9 @@ export const vFilePart = v.object({ type: v.literal("file"), data: v.union(v.string(), v.bytes()), filename: v.optional(v.string()), - mimeType: v.string(), + mediaType: v.optional(v.string()), + /** @deprecated Use `mediaType` instead. */ + mimeType: v.optional(v.string()), providerOptions, providerMetadata, }); @@ -110,15 +115,34 @@ export const vSourcePart = v.union( ); export type SourcePart = Infer; -export const vToolCallPart = v.object({ - type: v.literal("tool-call"), - toolCallId: v.string(), - toolName: v.string(), - args: v.any(), - providerExecuted: v.optional(v.boolean()), - providerOptions, - providerMetadata, -}); +// Union type to support both old (args) and new (input) formats +// Both include input for type hint support +export const vToolCallPart = v.union( + // New format: input is primary, args is optional for backwards compat + v.object({ + type: v.literal("tool-call"), + toolCallId: v.string(), + toolName: v.string(), + input: v.any(), + /** @deprecated Use `input` instead. */ + args: v.optional(v.any()), + providerExecuted: v.optional(v.boolean()), + providerOptions, + providerMetadata, + }), + // Legacy format: args is present, input is optional + v.object({ + type: v.literal("tool-call"), + toolCallId: v.string(), + toolName: v.string(), + /** @deprecated Use `input` instead. */ + args: v.any(), + input: v.optional(v.any()), + providerExecuted: v.optional(v.boolean()), + providerOptions, + providerMetadata, + }), +); const vToolResultContent = v.array( v.union( @@ -132,25 +156,155 @@ const vToolResultContent = v.array( ); export const vToolResultOutput = v.union( - v.object({ type: v.literal("text"), value: v.string() }), - v.object({ type: v.literal("json"), value: v.any() }), - v.object({ type: v.literal("error-text"), value: v.string() }), - v.object({ type: v.literal("error-json"), value: v.any() }), + v.object({ type: v.literal("text"), value: v.string(), providerOptions }), + v.object({ type: v.literal("json"), value: v.any(), providerOptions }), + v.object({ + type: v.literal("error-text"), + value: v.string(), + providerOptions, + }), + v.object({ type: v.literal("error-json"), value: v.any(), providerOptions }), + v.object({ + type: v.literal("execution-denied"), + reason: v.optional(v.string()), + providerOptions, + }), v.object({ type: v.literal("content"), value: v.array( v.union( - v.object({ type: v.literal("text"), text: v.string() }), + v.object({ + type: v.literal("text"), + text: v.string(), + providerOptions, + }), + /** @deprecated Use `image-data` or `file-data` instead. */ v.object({ type: v.literal("media"), data: v.string(), mediaType: v.string(), }), + v.object({ + type: v.literal("file-data"), + /** Base-64 encoded */ + data: v.string(), + /** + * IANA media type. + * @see https://www.iana.org/assignments/media-types/media-types.xhtml + */ + mediaType: v.string(), + filename: v.optional(v.string()), + providerOptions, + }), + v.object({ + type: v.literal("file-url"), + url: v.string(), + providerOptions, + }), + v.object({ + type: v.literal("file-id"), + /** + * ID of the file. + * + * If you use multiple providers, you need to + * specify the provider specific ids using + * the Record option. The key is the provider + * name, e.g. 'openai' or 'anthropic'. + */ + fileId: v.union(v.string(), v.record(v.string(), v.string())), + providerOptions, + }), + v.object({ + type: v.literal("image-data"), + data: v.string(), + /** + * IANA media type. + * @see https://www.iana.org/assignments/media-types/media-types.xhtml + */ + mediaType: v.string(), + providerOptions, + }), + v.object({ + type: v.literal("image-url"), + url: v.string(), + providerOptions, + }), + v.object({ + /** + * Images that are referenced using a provider file id. + */ + type: v.literal("image-file-id"), + /** + * Image that is referenced using a provider file id. + * + * If you use multiple providers, you need to + * specify the provider specific ids using + * the Record option. The key is the provider + * name, e.g. 'openai' or 'anthropic'. + */ + fileId: v.union(v.string(), v.record(v.string(), v.string())), + providerOptions, + }), + v.object({ + /** + * Custom content part. This can be used to implement + * provider-specific content parts. + */ + type: v.literal("custom"), + providerOptions, + }), ), ), }), ); +/** + * Tool approval request prompt part. + */ +export const vToolApprovalRequest = v.object({ + type: v.literal("tool-approval-request"), + /** + * ID of the tool approval. + */ + approvalId: v.string(), + /** + * ID of the tool call that the approval request is for. + */ + toolCallId: v.string(), + /** @todo Should we continue to include? */ + providerMetadata, + /** @todo Should we continue to include? */ + providerOptions, +}); + +/** + * Tool approval response prompt part. + */ +export const vToolApprovalResponse = v.object({ + type: v.literal("tool-approval-response"), + /** + * ID of the tool approval. + */ + approvalId: v.string(), + /** + * Flag indicating whether the approval was granted or denied. + */ + approved: v.boolean(), + /** + * Optional reason for the approval or denial. + */ + reason: v.optional(v.string()), + /** + * Flag indicating whether the tool call is provider-executed. + * Only provider-executed tool approval responses should be sent to the model. + */ + providerExecuted: v.optional(v.boolean()), + /** @todo Should we continue to include? */ + providerMetadata, + /** @todo Should we continue to include? */ + providerOptions, +}); + export const vToolResultPart = v.object({ type: v.literal("tool-result"), toolCallId: v.string(), @@ -169,7 +323,9 @@ export const vToolResultPart = v.object({ args: v.optional(v.any()), experimental_content: v.optional(vToolResultContent), }); -export const vToolContent = v.array(vToolResultPart); +export const vToolContent = v.array( + v.union(vToolResultPart, vToolApprovalResponse), +); export const vAssistantContent = v.union( v.string(), @@ -182,6 +338,7 @@ export const vAssistantContent = v.union( vToolCallPart, vToolResultPart, vSourcePart, + vToolApprovalRequest, ), ), ); @@ -462,7 +619,7 @@ export const vStreamMessage = v.object({ agentName: v.optional(v.string()), model: v.optional(v.string()), provider: v.optional(v.string()), - providerOptions: v.optional(vProviderOptions), // Sent to model + providerOptions, // Sent to model }); export type StreamMessage = Infer; @@ -490,7 +647,7 @@ export const vMessageDoc = v.object({ agentName: v.optional(v.string()), model: v.optional(v.string()), provider: v.optional(v.string()), - providerOptions: v.optional(vProviderOptions), // Sent to model + providerOptions, // Sent to model // The result message: v.optional(vMessage),