Fix inline remote eval parameter handling

Author: joshuawootonn

py/src/braintrust/devserver/server.py

@@ -28,7 +28,7 @@
 from ..framework import EvalAsync, EvalScorer, Evaluator, ExperimentSummary, SSEProgressEvent
 from ..generated_types import FunctionId
 from ..logger import BraintrustState, bt_iscoroutinefunction
-from ..parameters import serialize_remote_eval_parameters_container, validate_parameters
+from ..parameters import RemoteEvalParameters, serialize_remote_eval_parameters_container, validate_parameters
 from ..span_identifier_v4 import parse_parent
 from .auth import AuthorizationMiddleware
 from .cache import cached_login
@@ -228,6 +228,8 @@ def stream_fn(event: SSEProgressEvent):
     eval_kwargs = {
         k: v for (k, v) in evaluator.__dict__.items() if k not in ["eval_name", "project_name", "parameter_values"]
     }
+    if validated_parameters is not None and not RemoteEvalParameters.is_parameters(evaluator.parameters):
+        eval_kwargs["parameters"] = validated_parameters
 
     try:
         eval_task = asyncio.create_task(

py/src/braintrust/devserver/test_server_integration.py

@@ -8,6 +8,9 @@
 from braintrust.test_helpers import has_devserver_installed
 
 
+HAS_PYDANTIC = __import__("importlib.util").util.find_spec("pydantic") is not None
+
+
 @pytest.fixture
 def client():
     """Create test client using the real simple_eval.py example."""
@@ -205,3 +208,67 @@ def test_eval_error_handling(client, api_key, org_name):
     error = response.json()
     assert "error" in error
     assert "not found" in error["error"].lower()
+
+
+@pytest.mark.skipif(not HAS_PYDANTIC, reason="pydantic not installed")
+def test_eval_uses_inline_request_parameters(api_key, org_name, monkeypatch):
+    from braintrust import Evaluator
+    from braintrust.devserver import server as devserver_module
+    from braintrust.devserver.server import create_app
+    from braintrust.logger import BraintrustState
+    from pydantic import BaseModel
+    from starlette.testclient import TestClient
+
+    class RequiredInt(BaseModel):
+        value: int
+
+    def task(input: str, hooks) -> dict[str, Any]:
+        return {"input": input, "num_samples": hooks.parameters["num_samples_without_default"]}
+
+    evaluator = Evaluator(
+        project_name="test-math-eval",
+        eval_name="inline-parameter-eval",
+        data=lambda: [{"input": "What is 2+2?", "expected": "4"}],
+        task=task,
+        scores=[],
+        experiment_name=None,
+        metadata=None,
+        parameters={"num_samples_without_default": RequiredInt},
+    )
+
+    async def fake_cached_login(**_kwargs):
+        return BraintrustState()
+
+    class FakeSummary:
+        def as_dict(self):
+            return {"experiment_name": "inline-parameter-eval", "project_name": "test-math-eval", "scores": {}}
+
+    class FakeResult:
+        summary = FakeSummary()
+
+    async def fake_eval_async(*, task, data, parameters, **_kwargs):
+        assert parameters == {"num_samples_without_default": 1}
+        datum = data[0]
+        hooks = type("Hooks", (), {"parameters": parameters, "report_progress": lambda self, _progress: None})()
+        await task(datum["input"], hooks)
+        return FakeResult()
+
+    monkeypatch.setattr(devserver_module, "cached_login", fake_cached_login)
+    monkeypatch.setattr(devserver_module, "EvalAsync", fake_eval_async)
+
+    response = TestClient(create_app([evaluator])).post(
+        "/eval",
+        headers={
+            "x-bt-auth-token": api_key,
+            "x-bt-org-name": org_name,
+            "Content-Type": "application/json",
+        },
+        json={
+            "name": "inline-parameter-eval",
+            "stream": False,
+            "parameters": {"num_samples_without_default": 1},
+            "data": [{"input": "What is 2+2?", "expected": "4"}],
+        },
+    )
+
+    assert response.status_code == 200

py/src/braintrust/parameters.py

@@ -321,27 +321,38 @@ def serialize_eval_parameters(parameters: EvalParameters) -> dict[str, Any]:
 
     for name, schema in parameters.items():
         if _is_prompt_parameter(schema):
-            result[name] = {
+            parameter_data = {
                 "type": "prompt",
-                "default": _prompt_data_to_dict(schema.get("default")),
                 "description": schema.get("description"),
             }
+            default = schema.get("default")
+            if default is not None:
+                parameter_data["default"] = _prompt_data_to_dict(default)
+            result[name] = parameter_data
         elif _is_model_parameter(schema):
-            result[name] = {
+            parameter_data = {
                 "type": "model",
-                "default": schema.get("default"),
                 "description": schema.get("description"),
             }
+            default = schema.get("default")
+            if default is not None:
+                parameter_data["default"] = default
+            result[name] = parameter_data
         elif schema is None:
             result[name] = {
                 "type": "data",
                 "schema": {},
             }
         else:
-            result[name] = {
+            schema_json = _serialize_pydantic_parameter_schema(schema)
+            parameter_data = {
                 "type": "data",
-                "schema": _serialize_pydantic_parameter_schema(schema),
+                "schema": schema_json,
+                "description": schema_json.get("description"),
             }
+            if "default" in schema_json:
+                parameter_data["default"] = schema_json["default"]
+            result[name] = parameter_data
 
     return result