diff --git a/js/src/eval-parameters.ts b/js/src/eval-parameters.ts index e33d5c911..f06e56a2d 100644 --- a/js/src/eval-parameters.ts +++ b/js/src/eval-parameters.ts @@ -151,5 +151,51 @@ function validateParametersWithJsonSchema>( } // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return parameters as T; + return hydrateRemoteParameters(parameters, schema) as T; +} + +function hydrateRemoteParameters( + parameters: Record, + schema: Record, +): Record { + const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null; + const properties = + "properties" in schema && isRecord(schema.properties) + ? schema.properties + : undefined; + if (!properties) { + return parameters; + } + + return Object.fromEntries( + Object.entries(parameters).map(([name, value]) => { + const propertySchema = properties[name]; + if (!propertySchema || typeof propertySchema !== "object") { + return [name, value]; + } + + if ( + "x-bt-type" in propertySchema && + propertySchema["x-bt-type"] === "prompt" + ) { + return [ + name, + Prompt.fromPromptData(name, promptDataSchema.parse(value)), + ]; + } + + if ( + "x-bt-type" in propertySchema && + propertySchema["x-bt-type"] === "model" && + typeof value !== "string" + ) { + throw Error( + `Invalid parameter '${name}': model values must be strings`, + ); + } + + return [name, value]; + }), + ); } diff --git a/js/src/parameters.test.ts b/js/src/parameters.test.ts index dbba8ea49..4d5764b78 100644 --- a/js/src/parameters.test.ts +++ b/js/src/parameters.test.ts @@ -3,6 +3,8 @@ import { runEvaluator } from "./framework"; import { z } from "zod/v3"; import { type ProgressReporter } from "./reporters/types"; import { configureNode } from "./node/config"; +import { RemoteEvalParameters } from "./logger"; +import { validateParameters } from "./eval-parameters"; beforeAll(() => { configureNode(); @@ -238,3 +240,44 @@ test("model parameter is required when default is missing", async () => { ), ).rejects.toThrow("Parameter 'model' is required"); }); + +test("remote prompt parameters are hydrated to Prompt objects", async () => { + const parameters = new RemoteEvalParameters({ + id: "params-123", + project_id: "project-123", + name: "Saved parameters", + slug: "saved-parameters", + _xact_id: "v1", + function_type: "parameters", + function_data: { + type: "parameters", + data: { + main: { + prompt: { + type: "chat", + messages: [{ role: "user", content: "{{input}}" }], + }, + options: { + model: "gpt-5-mini", + }, + }, + }, + __schema: { + type: "object", + properties: { + main: { + type: "object", + "x-bt-type": "prompt", + }, + }, + additionalProperties: true, + }, + }, + created: "2023-10-01T00:00:00Z", + }); + + const result = await validateParameters({}, parameters); + + expect(result.main).toBeDefined(); + expect(typeof result.main.build).toBe("function"); +});