feat: Port load_parameters to the Python SDK (#90)

* Add saved parameters support to Python SDK

* Fix inline remote eval parameter handling

* Address PR comments for python parameters impl

* Prefer version over environment in Python loaders

* Update json schema creation to dereference pydantic complex data types

Author: joshuawootonn

py/setup.py

@@ -18,6 +18,7 @@
     "chevron",
     "tqdm",
     "exceptiongroup>=1.2.0",
+    "jsonschema",
     "python-dotenv",
     "sseclient-py",
     "python-slugify",

py/src/braintrust/cli/eval.py

@@ -22,6 +22,7 @@
     set_thread_pool_max_workers,
 )
 from ..logger import Dataset
+from ..parameters import RemoteEvalParameters
 from ..util import eprint
 
 
@@ -131,6 +132,12 @@ async def run_evaluator_task(evaluator, position, opts: EvaluatorOpts):
         if isinstance(evaluator.data, Dataset):
             dataset = evaluator.data
 
+        parameters = None
+        if isinstance(evaluator.parameters, RemoteEvalParameters) and evaluator.parameters.id is not None:
+            parameters = {"id": evaluator.parameters.id}
+            if evaluator.parameters.version is not None:
+                parameters["version"] = evaluator.parameters.version
+
         # NOTE: This code is duplicated with _EvalCommon in py/src/braintrust/framework.py.
         # Make sure to update those arguments if you change this.
         experiment = init_experiment(
@@ -147,6 +154,7 @@ async def run_evaluator_task(evaluator, position, opts: EvaluatorOpts):
             git_metadata_settings=evaluator.git_metadata_settings,
             repo_info=evaluator.repo_info,
             dataset=dataset,
+            parameters=parameters,
         )
 
     try:

py/src/braintrust/cli/push.py

@@ -322,6 +322,13 @@ def _collect_evaluator_defs(
         )
 
 
+def _collect_parameters_function_defs(
+    project_ids: ProjectIdCache, functions: list[dict[str, Any]], if_exists: IfExists
+) -> None:
+    for p in global_.parameters:
+        functions.append(p.to_function_definition(if_exists, project_ids))
+
+
 def run(args):
     """Runs the braintrust push subcommand."""
     login(
@@ -379,6 +386,8 @@ def run(args):
 
     if len(global_.prompts) > 0:
         _collect_prompt_function_defs(project_ids, functions, args.if_exists)
+    if len(global_.parameters) > 0:
+        _collect_parameters_function_defs(project_ids, functions, args.if_exists)
 
     if len(functions) > 0:
         api_conn().post_json("insert-functions", {"functions": functions})

py/src/braintrust/devserver/eval_hooks.py

@@ -10,14 +10,16 @@
 from collections.abc import Callable
 from typing import Any
 
+from ..parameters import ValidatedParameters
+
 
 class EvalHooks:
     """Hooks provided to eval tasks for progress reporting."""
 
     def __init__(
         self,
         report_progress: Callable[[dict[str, Any]], None] | None = None,
-        parameters: dict[str, Any] | None = None,
+        parameters: ValidatedParameters | None = None,
     ):
         self._report_progress = report_progress
         self.parameters = parameters or {}

py/src/braintrust/devserver/server.py

@@ -25,10 +25,22 @@
         )
     )
 
-from ..framework import EvalAsync, EvalScorer, Evaluator, ExperimentSummary, SSEProgressEvent
+from ..framework import (
+    EvalAsync,
+    EvalHooks,
+    EvalScorer,
+    Evaluator,
+    ExperimentSummary,
+    SSEProgressEvent,
+)
 from ..generated_types import FunctionId
 from ..logger import BraintrustState, bt_iscoroutinefunction
-from ..parameters import parameters_to_json_schema, validate_parameters
+from ..parameters import (
+    RemoteEvalParameters,
+    ValidatedParameters,
+    serialize_remote_eval_parameters_container,
+    validate_parameters,
+)
 from ..span_identifier_v4 import parse_parent
 from .auth import AuthorizationMiddleware
 from .cache import cached_login
@@ -41,6 +53,19 @@
 _all_evaluators: dict[str, Evaluator[Any, Any]] = {}
 
 
+class _ParameterOverrideHooks:
+    def __init__(self, hooks: EvalHooks[Any], parameters: ValidatedParameters):
+        self._hooks = hooks
+        self._parameters = parameters
+
+    @property
+    def parameters(self) -> ValidatedParameters:
+        return self._parameters
+
+    def __getattr__(self, name: str):
+        return getattr(self._hooks, name)
+
+
 class CheckAuthorizedMiddleware(BaseHTTPMiddleware):
     def __init__(self, app, allowed_org_name: str | None = None):
         super().__init__(app)
@@ -95,7 +120,9 @@ async def list_evaluators(request: Request) -> JSONResponse:
     evaluator_list = {}
     for name, evaluator in _all_evaluators.items():
         evaluator_list[name] = {
-            "parameters": parameters_to_json_schema(evaluator.parameters) if evaluator.parameters else {},
+            "parameters": (
+                serialize_remote_eval_parameters_container(evaluator.parameters) if evaluator.parameters else None
+            ),
             "scores": [{"name": getattr(score, "name", f"score_{i}")} for i, score in enumerate(evaluator.scores)],
         }
 
@@ -154,12 +181,13 @@ async def run_eval(request: Request) -> JSONResponse | StreamingResponse:
     # Set up SSE headers for streaming
     sse_queue = SSEQueue()
 
-    async def task(input, hooks):
+    async def task(input: Any, hooks: EvalHooks[Any]):
+        task_hooks = hooks if validated_parameters is None else _ParameterOverrideHooks(hooks, validated_parameters)
         if bt_iscoroutinefunction(evaluator.task):
-            result = await evaluator.task(input, hooks)
+            result = await evaluator.task(input, task_hooks)
         else:
-            result = evaluator.task(input, hooks)
-        hooks.report_progress(
+            result = evaluator.task(input, task_hooks)
+        task_hooks.report_progress(
             {
                 "format": "code",
                 "output_type": "completion",
@@ -186,9 +214,10 @@ def stream_fn(event: SSEProgressEvent):
     if parent:
         parent = parse_parent(parent)
 
-    # Override evaluator parameters with validated ones if provided
-    eval_kwargs = {k: v for (k, v) in evaluator.__dict__.items() if k not in ["eval_name", "project_name"]}
-    if validated_parameters is not None:
+    eval_kwargs = {
+        k: v for (k, v) in evaluator.__dict__.items() if k not in ["eval_name", "project_name", "parameter_values"]
+    }
+    if validated_parameters is not None and not isinstance(evaluator.parameters, RemoteEvalParameters):
         eval_kwargs["parameters"] = validated_parameters
 
     try:
@@ -289,7 +318,10 @@ def create_app(evaluators: list[Evaluator[Any, Any]], org_name: str | None = Non
 
 
 def run_dev_server(
-    evaluators: list[Evaluator[Any, Any]], host: str = "localhost", port: int = 8300, org_name: str | None = None
+    evaluators: list[Evaluator[Any, Any]],
+    host: str = "localhost",
+    port: int = 8300,
+    org_name: str | None = None,
 ):
     """Start the dev server.
 

py/src/braintrust/devserver/test_server_integration.py

@@ -8,6 +8,9 @@
 from braintrust.test_helpers import has_devserver_installed
 
 
+HAS_PYDANTIC = __import__("importlib.util").util.find_spec("pydantic") is not None
+
+
 @pytest.fixture
 def client():
     """Create test client using the real simple_eval.py example."""
@@ -205,3 +208,67 @@ def test_eval_error_handling(client, api_key, org_name):
     error = response.json()
     assert "error" in error
     assert "not found" in error["error"].lower()
+
+
+@pytest.mark.skipif(not HAS_PYDANTIC, reason="pydantic not installed")
+def test_eval_uses_inline_request_parameters(api_key, org_name, monkeypatch):
+    from braintrust import Evaluator
+    from braintrust.devserver import server as devserver_module
+    from braintrust.devserver.server import create_app
+    from braintrust.logger import BraintrustState
+    from pydantic import BaseModel
+    from starlette.testclient import TestClient
+
+    class RequiredInt(BaseModel):
+        value: int
+
+    def task(input: str, hooks) -> dict[str, Any]:
+        return {"input": input, "num_samples": hooks.parameters["num_samples_without_default"]}
+
+    evaluator = Evaluator(
+        project_name="test-math-eval",
+        eval_name="inline-parameter-eval",
+        data=lambda: [{"input": "What is 2+2?", "expected": "4"}],
+        task=task,
+        scores=[],
+        experiment_name=None,
+        metadata=None,
+        parameters={"num_samples_without_default": RequiredInt},
+    )
+
+    async def fake_cached_login(**_kwargs):
+        return BraintrustState()
+
+    class FakeSummary:
+        def as_dict(self):
+            return {"experiment_name": "inline-parameter-eval", "project_name": "test-math-eval", "scores": {}}
+
+    class FakeResult:
+        summary = FakeSummary()
+
+    async def fake_eval_async(*, task, data, parameters, **_kwargs):
+        assert parameters == {"num_samples_without_default": 1}
+        datum = data[0]
+        hooks = type("Hooks", (), {"parameters": parameters, "report_progress": lambda self, _progress: None})()
+        await task(datum["input"], hooks)
+        return FakeResult()
+
+    monkeypatch.setattr(devserver_module, "cached_login", fake_cached_login)
+    monkeypatch.setattr(devserver_module, "EvalAsync", fake_eval_async)
+
+    response = TestClient(create_app([evaluator])).post(
+        "/eval",
+        headers={
+            "x-bt-auth-token": api_key,
+            "x-bt-org-name": org_name,
+            "Content-Type": "application/json",
+        },
+        json={
+            "name": "inline-parameter-eval",
+            "stream": False,
+            "parameters": {"num_samples_without_default": 1},
+            "data": [{"input": "What is 2+2?", "expected": "4"}],
+        },
+    )
+
+    assert response.status_code == 200

py/src/braintrust/framework.py

@@ -42,7 +42,13 @@
     stringify_exception,
 )
 from .logger import init as _init_experiment
-from .parameters import EvalParameters
+from .parameters import (
+    EvalParameters,
+    RemoteEvalParameters,
+    ValidatedParameters,
+    is_eval_parameter_schema,
+    validate_parameters,
+)
 from .resource_manager import ResourceManager
 from .score import Score, is_score, is_scorer
 from .serializable_data_class import SerializableDataClass
@@ -215,7 +221,7 @@ def meta(self, **info: Any) -> None:
 
     @property
     @abc.abstractmethod
-    def parameters(self) -> dict[str, Any] | None:
+    def parameters(self) -> ValidatedParameters | None:
         """
         The parameters for the current evaluation. These are the validated parameter values
         that were passed to the evaluator.
@@ -439,12 +445,14 @@ class Evaluator(Generic[Input, Output]):
     Whether to summarize the scores of the experiment after it has run.
     """
 
-    parameters: EvalParameters | None = None
+    parameters: EvalParameters | RemoteEvalParameters | None = None
     """
     A set of parameters that will be passed to the evaluator.
     Can be used to define prompts or other configurable values.
     """
 
+    parameter_values: dict[str, Any] | None = None
+
 
 @dataclasses.dataclass
 class EvalResultWithSummary(SerializableDataClass, Generic[Input, Output]):
@@ -675,7 +683,7 @@ def _EvalCommon(
     summarize_scores: bool,
     no_send_logs: bool,
     error_score_handler: ErrorScoreHandler | None = None,
-    parameters: EvalParameters | None = None,
+    parameters: EvalParameters | RemoteEvalParameters | None = None,
     on_start: Callable[[ExperimentSummary], None] | None = None,
     stream: Callable[[SSEProgressEvent], None] | None = None,
     parent: str | None = None,
@@ -741,6 +749,12 @@ async def make_empty_summary():
         if isinstance(evaluator.data, Dataset):
             dataset = evaluator.data
 
+        experiment_parameters = None
+        if isinstance(evaluator.parameters, RemoteEvalParameters) and evaluator.parameters.id is not None:
+            experiment_parameters = {"id": evaluator.parameters.id}
+            if evaluator.parameters.version is not None:
+                experiment_parameters["version"] = evaluator.parameters.version
+
         # NOTE: This code is duplicated with run_evaluator_task in py/src/braintrust/cli/eval.py.
         # Make sure to update those arguments if you change this.
         experiment = None
@@ -759,6 +773,7 @@ async def make_empty_summary():
                 git_metadata_settings=evaluator.git_metadata_settings,
                 repo_info=evaluator.repo_info,
                 dataset=dataset,
+                parameters=experiment_parameters,
                 state=state,
             )
 
@@ -804,7 +819,7 @@ async def EvalAsync(
     description: str | None = None,
     summarize_scores: bool = True,
     no_send_logs: bool = False,
-    parameters: EvalParameters | None = None,
+    parameters: EvalParameters | RemoteEvalParameters | None = None,
     on_start: Callable[[ExperimentSummary], None] | None = None,
     stream: Callable[[SSEProgressEvent], None] | None = None,
     parent: str | None = None,
@@ -931,7 +946,7 @@ def Eval(
     description: str | None = None,
     summarize_scores: bool = True,
     no_send_logs: bool = False,
-    parameters: EvalParameters | None = None,
+    parameters: EvalParameters | RemoteEvalParameters | None = None,
     on_start: Callable[[ExperimentSummary], None] | None = None,
     stream: Callable[[SSEProgressEvent], None] | None = None,
     parent: str | None = None,
@@ -1153,7 +1168,7 @@ def __init__(
         trial_index: int = 0,
         tags: Sequence[str] | None = None,
         report_progress: Callable[[TaskProgressEvent], None] = None,
-        parameters: dict[str, Any] | None = None,
+        parameters: ValidatedParameters | None = None,
     ):
         if metadata is not None:
             self.update({"metadata": metadata})
@@ -1211,7 +1226,7 @@ def report_progress(self, event: TaskProgressEvent):
             return self._report_progress(event)
 
     @property
-    def parameters(self) -> dict[str, Any] | None:
+    def parameters(self) -> ValidatedParameters | None:
         return self._parameters
 
 
@@ -1392,6 +1407,15 @@ def get_other_fields(s):
     scorer_names = [_scorer_name(scorer, i) for i, scorer in enumerate(scorers)]
     unhandled_scores = scorer_names
 
+    if evaluator.parameter_values is not None:
+        resolved_evaluator_parameters = evaluator.parameter_values
+    elif isinstance(evaluator.parameters, RemoteEvalParameters):
+        resolved_evaluator_parameters = validate_parameters({}, evaluator.parameters)
+    elif is_eval_parameter_schema(evaluator.parameters):
+        resolved_evaluator_parameters = validate_parameters({}, evaluator.parameters)
+    else:
+        resolved_evaluator_parameters = evaluator.parameters
+
     async def run_evaluator_task(datum, trial_index=0):
         if isinstance(datum, dict):
             datum = EvalCase.from_dict(datum)
@@ -1451,7 +1475,7 @@ def report_progress(event: TaskProgressEvent):
                     trial_index=trial_index,
                     tags=tags,
                     report_progress=report_progress,
-                    parameters=evaluator.parameters,
+                    parameters=resolved_evaluator_parameters,
                 )
 
                 # Check if the task takes a hooks argument