From 24b1ac5ba1fc2841a57d545144a9192d1452b6aa Mon Sep 17 00:00:00 2001 From: Cristian Pufu Date: Mon, 2 Mar 2026 18:03:07 +0200 Subject: [PATCH] fix: use runtime factory for eval entrypoint discovery and show usage help Replace hardcoded uipath.json-only entrypoint discovery with runtime_factory.discover_entrypoints() which supports all runtimes (langgraph, llama_index, etc). When auto-discovery fails (multiple entrypoints or eval sets), show a clean usage message listing available options instead of a ValueError traceback. Co-Authored-By: Claude Opus 4.6 --- pyproject.toml | 2 +- src/uipath/_cli/cli_eval.py | 193 +++++++++----- src/uipath/eval/_helpers/__init__.py | 4 +- src/uipath/eval/_helpers/helpers.py | 52 +--- src/uipath/eval/helpers.py | 44 ---- tests/cli/eval/test_eval_discovery.py | 355 ++++++++++++++++++++++++++ uv.lock | 2 +- 7 files changed, 495 insertions(+), 157 deletions(-) create mode 100644 tests/cli/eval/test_eval_discovery.py diff --git a/pyproject.toml b/pyproject.toml index b33c94e99..d5d5b4745 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath" -version = "2.10.3" +version = "2.10.4" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/src/uipath/_cli/cli_eval.py b/src/uipath/_cli/cli_eval.py index ed1154516..083cd5cc2 100644 --- a/src/uipath/_cli/cli_eval.py +++ b/src/uipath/_cli/cli_eval.py @@ -3,6 +3,7 @@ import logging import os import uuid +from pathlib import Path from typing import Any import click @@ -15,8 +16,7 @@ from uipath._cli.middlewares import Middlewares from uipath.core.events import EventBus from uipath.core.tracing import UiPathTraceManager -from uipath.eval._helpers import auto_discover_entrypoint -from uipath.eval.helpers import EvalHelpers +from uipath.eval.helpers import EVAL_SETS_DIRECTORY_NAME, EvalHelpers from uipath.eval.models.evaluation_set import EvaluationSet from uipath.eval.runtime import UiPathEvalContext, evaluate from uipath.platform.chat import set_llm_concurrency @@ -135,6 +135,55 @@ def _resolve_model_settings_override( return override if override else None +class _EvalDiscoveryError(Exception): + """Raised when auto-discovery of entrypoint or eval set fails.""" + + def __init__(self, entrypoints: list[str], eval_sets: list[Path]): + self.entrypoints = entrypoints + self.eval_sets = eval_sets + + +def _discover_eval_sets() -> list[Path]: + """Discover available eval set files.""" + eval_sets_dir = Path(EVAL_SETS_DIRECTORY_NAME) + if eval_sets_dir.exists(): + return sorted(eval_sets_dir.glob("*.json")) + return [] + + +def _show_eval_usage_help(entrypoints: list[str], eval_set_files: list[Path]) -> None: + """Show available entrypoints and eval sets with usage examples.""" + lines: list[str] = [] + + if entrypoints: + lines.append("Available entrypoints:") + for name in entrypoints: + lines.append(f" - {name}") + else: + lines.append( + "No entrypoints found. " + "Add a 'functions' or 'agents' section to your config file " + "(e.g. uipath.json, langgraph.json)." + ) + + if eval_set_files: + lines.append("\nAvailable eval sets:") + for f in eval_set_files: + lines.append(f" - {f}") + else: + lines.append( + f"\nNo eval sets found in '{EVAL_SETS_DIRECTORY_NAME}/' directory." + ) + + lines.append("\nUsage: uipath eval ") + if entrypoints and eval_set_files: + ep_name = entrypoints[0] + es_path = eval_set_files[0] + lines.append(f"Example: uipath eval {ep_name} {es_path}") + + click.echo("\n".join(lines)) + + @click.command() @click.argument("entrypoint", required=False) @click.argument("eval_set", required=False) @@ -266,16 +315,9 @@ def eval( if result.should_continue: eval_context = UiPathEvalContext() - - eval_context.entrypoint = entrypoint or auto_discover_entrypoint() eval_context.workers = workers eval_context.eval_set_run_id = eval_set_run_id eval_context.enable_mocker_cache = enable_mocker_cache - - # Load eval set to resolve the path - eval_set_path = eval_set or EvalHelpers.auto_discover_eval_set() - _, resolved_eval_set_path = EvalHelpers.load_eval_set(eval_set_path, eval_ids) - eval_context.report_coverage = report_coverage eval_context.input_overrides = input_overrides eval_context.resume = resume @@ -309,69 +351,103 @@ async def execute_eval(): eval_context.job_id = ctx.job_id runtime_factory = UiPathRuntimeFactoryRegistry.get(context=ctx) - factory_settings = await runtime_factory.get_settings() - trace_settings = ( - factory_settings.trace_settings if factory_settings else None - ) - - if ( - ctx.job_id or should_register_progress_reporter - ) and UiPathConfig.is_tracing_enabled: - # Live tracking for Orchestrator or Studio Web - # Uses UIPATH_TRACE_ID from environment for trace correlation - trace_manager.add_span_processor( - LiveTrackingSpanProcessor( - LlmOpsHttpExporter(), - settings=trace_settings, - ) + + try: + # Auto-discover entrypoint and eval set using the runtime factory + resolved_entrypoint = entrypoint + eval_set_path = eval_set + + available_entrypoints = runtime_factory.discover_entrypoints() + available_eval_sets = _discover_eval_sets() + + if not resolved_entrypoint: + if len(available_entrypoints) == 1: + resolved_entrypoint = available_entrypoints[0] + else: + raise _EvalDiscoveryError( + available_entrypoints, available_eval_sets + ) + + if not eval_set_path: + if len(available_eval_sets) == 1: + eval_set_path = str(available_eval_sets[0]) + else: + raise _EvalDiscoveryError( + available_entrypoints, available_eval_sets + ) + + eval_context.entrypoint = resolved_entrypoint + + # Load eval set to resolve the path + _, resolved_eval_set_path = EvalHelpers.load_eval_set( + eval_set_path, eval_ids ) - if trace_file: + factory_settings = await runtime_factory.get_settings() trace_settings = ( factory_settings.trace_settings if factory_settings else None ) - trace_manager.add_span_exporter( - JsonLinesFileExporter(trace_file), settings=trace_settings - ) - project_id = UiPathConfig.project_id + if ( + ctx.job_id or should_register_progress_reporter + ) and UiPathConfig.is_tracing_enabled: + # Live tracking for Orchestrator or Studio Web + # Uses UIPATH_TRACE_ID from environment for trace correlation + trace_manager.add_span_processor( + LiveTrackingSpanProcessor( + LlmOpsHttpExporter(), + settings=trace_settings, + ) + ) - eval_context.execution_id = ( - eval_context.job_id - or eval_context.eval_set_run_id - or str(uuid.uuid4()) - ) + if trace_file: + trace_settings = ( + factory_settings.trace_settings + if factory_settings + else None + ) + trace_manager.add_span_exporter( + JsonLinesFileExporter(trace_file), + settings=trace_settings, + ) - # Load eval set (path is already resolved in cli_eval.py) - eval_context.evaluation_set, _ = EvalHelpers.load_eval_set( - resolved_eval_set_path, eval_ids - ) + project_id = UiPathConfig.project_id - # Resolve model settings override from eval set - settings_override = _resolve_model_settings_override( - model_settings_id, eval_context.evaluation_set - ) + eval_context.execution_id = ( + eval_context.job_id + or eval_context.eval_set_run_id + or str(uuid.uuid4()) + ) - runtime = await runtime_factory.new_runtime( - entrypoint=eval_context.entrypoint or "", - runtime_id=eval_context.execution_id, - settings=settings_override, - ) + # Load eval set (path is already resolved in cli_eval.py) + eval_context.evaluation_set, _ = EvalHelpers.load_eval_set( + resolved_eval_set_path, eval_ids + ) - eval_context.runtime_schema = await runtime.get_schema() + # Resolve model settings override from eval set + settings_override = _resolve_model_settings_override( + model_settings_id, eval_context.evaluation_set + ) - eval_context.evaluators = await EvalHelpers.load_evaluators( - resolved_eval_set_path, - eval_context.evaluation_set, - _get_agent_model(eval_context.runtime_schema), - ) + runtime = await runtime_factory.new_runtime( + entrypoint=eval_context.entrypoint or "", + runtime_id=eval_context.execution_id, + settings=settings_override, + ) - # Runtime is not required anymore. - await runtime.dispose() + eval_context.runtime_schema = await runtime.get_schema() + + eval_context.evaluators = await EvalHelpers.load_evaluators( + resolved_eval_set_path, + eval_context.evaluation_set, + _get_agent_model(eval_context.runtime_schema), + ) + + # Runtime is not required anymore. + await runtime.dispose() - try: if project_id: studio_client = StudioClient(project_id) @@ -395,11 +471,12 @@ async def execute_eval(): event_bus, ) finally: - if runtime_factory: - await runtime_factory.dispose() + await runtime_factory.dispose() asyncio.run(execute_eval()) + except _EvalDiscoveryError as e: + _show_eval_usage_help(e.entrypoints, e.eval_sets) except Exception as e: console.error( f"Error occurred: {e or 'Execution failed'}", include_traceback=True diff --git a/src/uipath/eval/_helpers/__init__.py b/src/uipath/eval/_helpers/__init__.py index e09b3b051..7c034c3e1 100644 --- a/src/uipath/eval/_helpers/__init__.py +++ b/src/uipath/eval/_helpers/__init__.py @@ -1,3 +1 @@ -from .helpers import auto_discover_entrypoint - -__all__ = ["auto_discover_entrypoint"] +"""Helper functions for evaluation process.""" diff --git a/src/uipath/eval/_helpers/helpers.py b/src/uipath/eval/_helpers/helpers.py index f00f5f5e5..e69976955 100644 --- a/src/uipath/eval/_helpers/helpers.py +++ b/src/uipath/eval/_helpers/helpers.py @@ -1,12 +1,10 @@ +"""Helper functions for evaluation process.""" + import functools -import json -import os import time from collections.abc import Callable from typing import Any -import click - from ..models import ErrorEvaluationResult, EvaluationResult @@ -37,52 +35,6 @@ def is_empty_value(value: Any) -> bool: return False -def auto_discover_entrypoint() -> str: - """Auto-discover entrypoint from config file. - - Returns: - Entrypoint name (key from the functions dict) - - Raises: - ValueError: If no entrypoint found or multiple entrypoints exist - """ - from uipath._cli._utils._console import ConsoleLogger - from uipath._utils.constants import UIPATH_CONFIG_FILE - - console = ConsoleLogger() - - if not os.path.isfile(UIPATH_CONFIG_FILE): - raise ValueError( - f"File '{UIPATH_CONFIG_FILE}' not found. Please run 'uipath init'." - ) - - with open(UIPATH_CONFIG_FILE, "r", encoding="utf-8") as f: - uipath_config = json.loads(f.read()) - - entrypoints: dict[str, str] = uipath_config.get("functions", {}) - - if not entrypoints: - raise ValueError( - f"No entrypoints found in {UIPATH_CONFIG_FILE}. " - "Add a 'functions' section to uipath.json" - ) - - if len(entrypoints) > 1: - entrypoint_list = list(entrypoints.keys()) - raise ValueError( - f"Multiple entrypoints found: {entrypoint_list}. " - "Please specify which entrypoint to use." - ) - - entrypoint_name = next(iter(entrypoints.keys())) - entrypoint_path = entrypoints[entrypoint_name] - console.info( - f"Auto-discovered entrypoint: {click.style(entrypoint_name, fg='cyan')} " - f"({entrypoint_path})" - ) - return entrypoint_name - - def track_evaluation_metrics(func: Callable[..., Any]) -> Callable[..., Any]: """Decorator to track evaluation metrics and handle errors gracefully.""" diff --git a/src/uipath/eval/helpers.py b/src/uipath/eval/helpers.py index 8134f65c5..1c51bd1ae 100644 --- a/src/uipath/eval/helpers.py +++ b/src/uipath/eval/helpers.py @@ -5,7 +5,6 @@ from pathlib import Path from typing import Any -import click from pydantic import ValidationError from uipath._cli._evals._conversational_utils import UiPathLegacyEvalChatMessagesMapper @@ -46,49 +45,6 @@ def discriminate_eval_set(data: dict[str, Any]) -> EvaluationSet | LegacyEvaluat class EvalHelpers: """Helper functions for evaluation commands, including loading and parsing evaluation sets and evaluators.""" - @staticmethod - def auto_discover_eval_set() -> str: - """Auto-discover evaluation set from {EVAL_SETS_DIRECTORY_NAME} directory. - - Returns: - Path to the evaluation set file - - Raises: - ValueError: If no eval set found or multiple eval sets exist - """ - eval_sets_dir = Path(EVAL_SETS_DIRECTORY_NAME) - - if not eval_sets_dir.exists(): - raise ValueError( - f"No '{EVAL_SETS_DIRECTORY_NAME}' directory found. " - "Please set 'UIPATH_PROJECT_ID' env var and run 'uipath pull'." - ) - - eval_set_files = list(eval_sets_dir.glob("*.json")) - - if not eval_set_files: - raise ValueError( - f"No evaluation set files found in '{EVAL_SETS_DIRECTORY_NAME}' directory. " - ) - - if len(eval_set_files) > 1: - file_names = [f.name for f in eval_set_files] - raise ValueError( - f"Multiple evaluation sets found: {file_names}. " - f"Please specify which evaluation set to use: 'uipath eval [entrypoint] '" - ) - - eval_set_path = str(eval_set_files[0]) - logger.info( - f"Auto-discovered evaluation set: {click.style(eval_set_path, fg='cyan')}" - ) - - eval_set_path_obj = Path(eval_set_path) - if not eval_set_path_obj.is_file() or eval_set_path_obj.suffix != ".json": - raise ValueError("Evaluation set must be a JSON file") - - return eval_set_path - @staticmethod def load_eval_set( eval_set_path: str, eval_ids: list[str] | None = None diff --git a/tests/cli/eval/test_eval_discovery.py b/tests/cli/eval/test_eval_discovery.py new file mode 100644 index 000000000..bce44f86c --- /dev/null +++ b/tests/cli/eval/test_eval_discovery.py @@ -0,0 +1,355 @@ +"""Tests for eval CLI auto-discovery of entrypoints and eval sets.""" + +import json +import os +from unittest.mock import AsyncMock, Mock, patch + +from click.testing import CliRunner + +from uipath._cli import cli +from uipath._cli.middlewares import MiddlewareResult + + +def _middleware_continue(): + return MiddlewareResult( + should_continue=True, + error_message=None, + should_include_stacktrace=False, + ) + + +def _make_mock_factory(entrypoints: list[str]): + """Create a mock runtime factory with given entrypoints.""" + mock_factory = Mock() + mock_factory.discover_entrypoints.return_value = entrypoints + mock_factory.get_settings = AsyncMock(return_value=None) + mock_factory.dispose = AsyncMock() + + mock_runtime = Mock() + mock_runtime.get_schema = AsyncMock( + return_value=Mock(metadata=None, input_schema=None, output_schema=None) + ) + mock_runtime.dispose = AsyncMock() + mock_factory.new_runtime = AsyncMock(return_value=mock_runtime) + + return mock_factory + + +class TestEvalDiscoveryMultipleEntrypoints: + """Tests for when multiple entrypoints exist and none is specified.""" + + def test_multiple_entrypoints_shows_usage_help( + self, runner: CliRunner, temp_dir: str + ): + """When multiple entrypoints exist, show available entrypoints and eval sets.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + # Create uipath.json with multiple functions + uipath_config = { + "functions": { + "agent_a": "src/agent_a.py:main", + "agent_b": "src/agent_b.py:main", + } + } + with open("uipath.json", "w") as f: + json.dump(uipath_config, f) + + # Create eval sets directory with one eval set + os.makedirs("evaluations/eval-sets", exist_ok=True) + eval_set = { + "version": 1.0, + "id": "test-set", + "name": "Test Set", + "evaluatorRefs": [], + "evaluations": [], + } + with open("evaluations/eval-sets/test-set.json", "w") as f: + json.dump(eval_set, f) + + mock_factory = _make_mock_factory(["agent_a", "agent_b"]) + + with ( + patch( + "uipath._cli.cli_eval.Middlewares.next", + return_value=_middleware_continue(), + ), + patch( + "uipath._cli.cli_eval.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + patch( + "uipath._cli.cli_eval.setup_reporting_prereq", + return_value=False, + ), + ): + result = runner.invoke(cli, ["eval"]) + + assert result.exit_code == 0 + assert "Available entrypoints:" in result.output + assert "agent_a" in result.output + assert "agent_b" in result.output + assert "Available eval sets:" in result.output + assert "evaluations" in result.output and "test-set.json" in result.output + assert "Usage: uipath eval " in result.output + assert "Example:" in result.output + + def test_multiple_entrypoints_no_eval_sets(self, runner: CliRunner, temp_dir: str): + """When multiple entrypoints exist and no eval sets, show both sections.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump({"functions": {"a": "a.py:main", "b": "b.py:main"}}, f) + + mock_factory = _make_mock_factory(["a", "b"]) + + with ( + patch( + "uipath._cli.cli_eval.Middlewares.next", + return_value=_middleware_continue(), + ), + patch( + "uipath._cli.cli_eval.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + patch( + "uipath._cli.cli_eval.setup_reporting_prereq", + return_value=False, + ), + ): + result = runner.invoke(cli, ["eval"]) + + assert result.exit_code == 0 + assert "Available entrypoints:" in result.output + assert "a" in result.output + assert "b" in result.output + assert "No eval sets found" in result.output + + +class TestEvalDiscoveryMultipleEvalSets: + """Tests for when multiple eval sets exist and none is specified.""" + + def test_multiple_eval_sets_shows_usage_help( + self, runner: CliRunner, temp_dir: str + ): + """When one entrypoint but multiple eval sets, show available eval sets.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump({"functions": {"my_agent": "src/main.py:main"}}, f) + + # Create two eval sets + os.makedirs("evaluations/eval-sets", exist_ok=True) + for name in ["set-a.json", "set-b.json"]: + eval_set = { + "version": 1.0, + "id": name, + "name": name, + "evaluatorRefs": [], + "evaluations": [], + } + with open(f"evaluations/eval-sets/{name}", "w") as f: + json.dump(eval_set, f) + + mock_factory = _make_mock_factory(["my_agent"]) + + with ( + patch( + "uipath._cli.cli_eval.Middlewares.next", + return_value=_middleware_continue(), + ), + patch( + "uipath._cli.cli_eval.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + patch( + "uipath._cli.cli_eval.setup_reporting_prereq", + return_value=False, + ), + ): + result = runner.invoke(cli, ["eval"]) + + assert result.exit_code == 0 + assert "Available entrypoints:" in result.output + assert "my_agent" in result.output + assert "Available eval sets:" in result.output + assert "set-a.json" in result.output + assert "set-b.json" in result.output + assert "Usage: uipath eval " in result.output + assert "Example: uipath eval my_agent" in result.output + + def test_eval_sets_show_full_relative_path(self, runner: CliRunner, temp_dir: str): + """Eval sets in the help message should show the full relative path.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump({"functions": {"agent": "main.py:main"}}, f) + + os.makedirs("evaluations/eval-sets", exist_ok=True) + for name in ["alpha.json", "beta.json"]: + with open(f"evaluations/eval-sets/{name}", "w") as f: + json.dump( + { + "version": 1.0, + "id": name, + "name": name, + "evaluatorRefs": [], + "evaluations": [], + }, + f, + ) + + mock_factory = _make_mock_factory(["agent"]) + + with ( + patch( + "uipath._cli.cli_eval.Middlewares.next", + return_value=_middleware_continue(), + ), + patch( + "uipath._cli.cli_eval.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + patch( + "uipath._cli.cli_eval.setup_reporting_prereq", + return_value=False, + ), + ): + result = runner.invoke(cli, ["eval"]) + + # Should contain full relative paths, not just filenames + output = result.output.replace("\\", "/") + assert "evaluations/eval-sets/alpha.json" in output + assert "evaluations/eval-sets/beta.json" in output + + +class TestEvalDiscoverySingleEntrypointAndEvalSet: + """Tests for auto-discovery when exactly one entrypoint and one eval set exist.""" + + def test_single_entrypoint_and_eval_set_does_not_show_help( + self, runner: CliRunner, temp_dir: str + ): + """When exactly one entrypoint and one eval set, discovery succeeds (no help shown).""" + with runner.isolated_filesystem(temp_dir=temp_dir): + with open("uipath.json", "w") as f: + json.dump({"functions": {"my_agent": "main.py:main"}}, f) + + os.makedirs("evaluations/eval-sets", exist_ok=True) + eval_set = { + "version": 1.0, + "id": "test-set", + "name": "Test Set", + "evaluatorRefs": [], + "evaluations": [], + } + with open("evaluations/eval-sets/test-set.json", "w") as f: + json.dump(eval_set, f) + + mock_factory = _make_mock_factory(["my_agent"]) + + with ( + patch( + "uipath._cli.cli_eval.Middlewares.next", + return_value=_middleware_continue(), + ), + patch( + "uipath._cli.cli_eval.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + patch( + "uipath._cli.cli_eval.setup_reporting_prereq", + return_value=False, + ), + ): + result = runner.invoke(cli, ["eval"]) + + # Should NOT show usage help (discovery succeeded) + assert "Available entrypoints:" not in result.output + assert "Usage: uipath eval" not in result.output + + +class TestEvalDiscoveryNoEntrypoints: + """Tests for when no entrypoints are found.""" + + def test_no_entrypoints_shows_helpful_message( + self, runner: CliRunner, temp_dir: str + ): + """When no entrypoints found, show a helpful message.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + os.makedirs("evaluations/eval-sets", exist_ok=True) + with open("evaluations/eval-sets/test.json", "w") as f: + json.dump( + { + "version": 1.0, + "id": "t", + "name": "t", + "evaluatorRefs": [], + "evaluations": [], + }, + f, + ) + + mock_factory = _make_mock_factory([]) + + with ( + patch( + "uipath._cli.cli_eval.Middlewares.next", + return_value=_middleware_continue(), + ), + patch( + "uipath._cli.cli_eval.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + patch( + "uipath._cli.cli_eval.setup_reporting_prereq", + return_value=False, + ), + ): + result = runner.invoke(cli, ["eval"]) + + assert result.exit_code == 0 + assert "No entrypoints found" in result.output + assert "Usage: uipath eval " in result.output + + +class TestEvalDiscoveryExplicitArgs: + """Tests for when entrypoint and/or eval set are explicitly provided.""" + + def test_explicit_entrypoint_skips_entrypoint_discovery( + self, runner: CliRunner, temp_dir: str + ): + """When entrypoint is provided, skip entrypoint discovery but still discover eval sets.""" + with runner.isolated_filesystem(temp_dir=temp_dir): + os.makedirs("evaluations/eval-sets", exist_ok=True) + for name in ["set-a.json", "set-b.json"]: + with open(f"evaluations/eval-sets/{name}", "w") as f: + json.dump( + { + "version": 1.0, + "id": name, + "name": name, + "evaluatorRefs": [], + "evaluations": [], + }, + f, + ) + + mock_factory = _make_mock_factory(["agent_a", "agent_b"]) + + with ( + patch( + "uipath._cli.cli_eval.Middlewares.next", + return_value=_middleware_continue(), + ), + patch( + "uipath._cli.cli_eval.UiPathRuntimeFactoryRegistry.get", + return_value=mock_factory, + ), + patch( + "uipath._cli.cli_eval.setup_reporting_prereq", + return_value=False, + ), + ): + # Pass entrypoint explicitly, but no eval set + result = runner.invoke(cli, ["eval", "agent_a"]) + + # Should still show usage help because multiple eval sets + assert result.exit_code == 0 + assert "Available eval sets:" in result.output + assert "set-a.json" in result.output + assert "set-b.json" in result.output diff --git a/uv.lock b/uv.lock index 78a95a0ab..b61079688 100644 --- a/uv.lock +++ b/uv.lock @@ -2531,7 +2531,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.10.3" +version = "2.10.4" source = { editable = "." } dependencies = [ { name = "applicationinsights" },