Hydrate remote prompt parameters in evals

Author: joshuawootonn

js/src/eval-parameters.ts

@@ -151,5 +151,51 @@ function validateParametersWithJsonSchema<T extends Record<string, unknown>>(
   }
 
   // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-  return parameters as T;
+  return hydrateRemoteParameters(parameters, schema) as T;
+}
+
+function hydrateRemoteParameters(
+  parameters: Record<string, unknown>,
+  schema: Record<string, unknown>,
+): Record<string, unknown> {
+  const properties =
+    "properties" in schema &&
+    schema.properties &&
+    typeof schema.properties === "object"
+      ? (schema.properties as Record<string, unknown>)
+      : undefined;
+  if (!properties) {
+    return parameters;
+  }
+
+  return Object.fromEntries(
+    Object.entries(parameters).map(([name, value]) => {
+      const propertySchema = properties[name];
+      if (!propertySchema || typeof propertySchema !== "object") {
+        return [name, value];
+      }
+
+      if (
+        "x-bt-type" in propertySchema &&
+        propertySchema["x-bt-type"] === "prompt"
+      ) {
+        return [
+          name,
+          Prompt.fromPromptData(name, promptDataSchema.parse(value)),
+        ];
+      }
+
+      if (
+        "x-bt-type" in propertySchema &&
+        propertySchema["x-bt-type"] === "model" &&
+        typeof value !== "string"
+      ) {
+        throw Error(
+          `Invalid parameter '${name}': model values must be strings`,
+        );
+      }
+
+      return [name, value];
+    }),
+  );
 }

js/src/parameters.test.ts

@@ -3,6 +3,8 @@ import { runEvaluator } from "./framework";
 import { z } from "zod/v3";
 import { type ProgressReporter } from "./reporters/types";
 import { configureNode } from "./node/config";
+import { RemoteEvalParameters } from "./logger";
+import { validateParameters } from "./eval-parameters";
 
 beforeAll(() => {
   configureNode();
@@ -238,3 +240,44 @@ test("model parameter is required when default is missing", async () => {
     ),
   ).rejects.toThrow("Parameter 'model' is required");
 });
+
+test("remote prompt parameters are hydrated to Prompt objects", async () => {
+  const parameters = new RemoteEvalParameters({
+    id: "params-123",
+    project_id: "project-123",
+    name: "Saved parameters",
+    slug: "saved-parameters",
+    _xact_id: "v1",
+    function_type: "parameters",
+    function_data: {
+      type: "parameters",
+      data: {
+        main: {
+          prompt: {
+            type: "chat",
+            messages: [{ role: "user", content: "{{input}}" }],
+          },
+          options: {
+            model: "gpt-5-mini",
+          },
+        },
+      },
+      __schema: {
+        type: "object",
+        properties: {
+          main: {
+            type: "object",
+            "x-bt-type": "prompt",
+          },
+        },
+        additionalProperties: true,
+      },
+    },
+    created: "2023-10-01T00:00:00Z",
+  });
+
+  const result = await validateParameters({}, parameters);
+
+  expect(result.main).toBeDefined();
+  expect(typeof result.main.build).toBe("function");
+});