Skip to content

fix: use runtime factory for eval entrypoint discovery and show usage help #1398

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cristipufu wants to merge 1 commit into
base: main
Choose a base branch
from
+495 −157
Open

fix: use runtime factory for eval entrypoint discovery and show usage help #1398

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "uipath"
version = "2.10.3"
version = "2.10.4"
description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
readme = { file = "README.md", content-type = "text/markdown" }
requires-python = ">=3.11"
Expand Down
193 changes: 135 additions & 58 deletions src/uipath/_cli/cli_eval.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import logging
import os
import uuid
from pathlib import Path
from typing import Any

import click
Expand All @@ -15,8 +16,7 @@
from uipath._cli.middlewares import Middlewares
from uipath.core.events import EventBus
from uipath.core.tracing import UiPathTraceManager
from uipath.eval._helpers import auto_discover_entrypoint
from uipath.eval.helpers import EvalHelpers
from uipath.eval.helpers import EVAL_SETS_DIRECTORY_NAME, EvalHelpers
from uipath.eval.models.evaluation_set import EvaluationSet
from uipath.eval.runtime import UiPathEvalContext, evaluate
from uipath.platform.chat import set_llm_concurrency
Expand Down Expand Up @@ -135,6 +135,55 @@ def _resolve_model_settings_override(
return override if override else None


class _EvalDiscoveryError(Exception):
"""Raised when auto-discovery of entrypoint or eval set fails."""

def __init__(self, entrypoints: list[str], eval_sets: list[Path]):
self.entrypoints = entrypoints
self.eval_sets = eval_sets


def _discover_eval_sets() -> list[Path]:
"""Discover available eval set files."""
eval_sets_dir = Path(EVAL_SETS_DIRECTORY_NAME)
if eval_sets_dir.exists():
return sorted(eval_sets_dir.glob("*.json"))
return []


def _show_eval_usage_help(entrypoints: list[str], eval_set_files: list[Path]) -> None:
"""Show available entrypoints and eval sets with usage examples."""
lines: list[str] = []

if entrypoints:
lines.append("Available entrypoints:")
for name in entrypoints:
lines.append(f" - {name}")
else:
lines.append(
"No entrypoints found. "
"Add a 'functions' or 'agents' section to your config file "
"(e.g. uipath.json, langgraph.json)."
)

if eval_set_files:
lines.append("\nAvailable eval sets:")
for f in eval_set_files:
lines.append(f" - {f}")
else:
lines.append(
f"\nNo eval sets found in '{EVAL_SETS_DIRECTORY_NAME}/' directory."
)

lines.append("\nUsage: uipath eval <entrypoint> <eval_set>")
if entrypoints and eval_set_files:
ep_name = entrypoints[0]
es_path = eval_set_files[0]
lines.append(f"Example: uipath eval {ep_name} {es_path}")

click.echo("\n".join(lines))


@click.command()
@click.argument("entrypoint", required=False)
@click.argument("eval_set", required=False)
Expand Down Expand Up @@ -266,16 +315,9 @@ def eval(

if result.should_continue:
eval_context = UiPathEvalContext()

eval_context.entrypoint = entrypoint or auto_discover_entrypoint()
eval_context.workers = workers
eval_context.eval_set_run_id = eval_set_run_id
eval_context.enable_mocker_cache = enable_mocker_cache

# Load eval set to resolve the path
eval_set_path = eval_set or EvalHelpers.auto_discover_eval_set()
_, resolved_eval_set_path = EvalHelpers.load_eval_set(eval_set_path, eval_ids)

eval_context.report_coverage = report_coverage
eval_context.input_overrides = input_overrides
eval_context.resume = resume
Expand Down Expand Up @@ -309,69 +351,103 @@ async def execute_eval():
eval_context.job_id = ctx.job_id

runtime_factory = UiPathRuntimeFactoryRegistry.get(context=ctx)
factory_settings = await runtime_factory.get_settings()
trace_settings = (
factory_settings.trace_settings if factory_settings else None
)

if (
ctx.job_id or should_register_progress_reporter
) and UiPathConfig.is_tracing_enabled:
# Live tracking for Orchestrator or Studio Web
# Uses UIPATH_TRACE_ID from environment for trace correlation
trace_manager.add_span_processor(
LiveTrackingSpanProcessor(
LlmOpsHttpExporter(),
settings=trace_settings,
)

try:
# Auto-discover entrypoint and eval set using the runtime factory
resolved_entrypoint = entrypoint
eval_set_path = eval_set

available_entrypoints = runtime_factory.discover_entrypoints()
available_eval_sets = _discover_eval_sets()

if not resolved_entrypoint:
if len(available_entrypoints) == 1:
resolved_entrypoint = available_entrypoints[0]
else:
raise _EvalDiscoveryError(
available_entrypoints, available_eval_sets
)

if not eval_set_path:
if len(available_eval_sets) == 1:
eval_set_path = str(available_eval_sets[0])
else:
raise _EvalDiscoveryError(
available_entrypoints, available_eval_sets
)

eval_context.entrypoint = resolved_entrypoint

# Load eval set to resolve the path
_, resolved_eval_set_path = EvalHelpers.load_eval_set(
eval_set_path, eval_ids
)

if trace_file:
factory_settings = await runtime_factory.get_settings()
trace_settings = (
factory_settings.trace_settings
if factory_settings
else None
)
trace_manager.add_span_exporter(
JsonLinesFileExporter(trace_file), settings=trace_settings
)

project_id = UiPathConfig.project_id
if (
ctx.job_id or should_register_progress_reporter
) and UiPathConfig.is_tracing_enabled:
# Live tracking for Orchestrator or Studio Web
# Uses UIPATH_TRACE_ID from environment for trace correlation
trace_manager.add_span_processor(
LiveTrackingSpanProcessor(
LlmOpsHttpExporter(),
settings=trace_settings,
)
)

eval_context.execution_id = (
eval_context.job_id
or eval_context.eval_set_run_id
or str(uuid.uuid4())
)
if trace_file:
trace_settings = (
factory_settings.trace_settings
if factory_settings
else None
)
trace_manager.add_span_exporter(
JsonLinesFileExporter(trace_file),
settings=trace_settings,
)

# Load eval set (path is already resolved in cli_eval.py)
eval_context.evaluation_set, _ = EvalHelpers.load_eval_set(
resolved_eval_set_path, eval_ids
)
project_id = UiPathConfig.project_id

# Resolve model settings override from eval set
settings_override = _resolve_model_settings_override(
model_settings_id, eval_context.evaluation_set
)
eval_context.execution_id = (
eval_context.job_id
or eval_context.eval_set_run_id
or str(uuid.uuid4())
)

runtime = await runtime_factory.new_runtime(
entrypoint=eval_context.entrypoint or "",
runtime_id=eval_context.execution_id,
settings=settings_override,
)
# Load eval set (path is already resolved in cli_eval.py)
eval_context.evaluation_set, _ = EvalHelpers.load_eval_set(
resolved_eval_set_path, eval_ids
)

eval_context.runtime_schema = await runtime.get_schema()
# Resolve model settings override from eval set
settings_override = _resolve_model_settings_override(
model_settings_id, eval_context.evaluation_set
)

eval_context.evaluators = await EvalHelpers.load_evaluators(
resolved_eval_set_path,
eval_context.evaluation_set,
_get_agent_model(eval_context.runtime_schema),
)
runtime = await runtime_factory.new_runtime(
entrypoint=eval_context.entrypoint or "",
runtime_id=eval_context.execution_id,
settings=settings_override,
)

# Runtime is not required anymore.
await runtime.dispose()
eval_context.runtime_schema = await runtime.get_schema()

eval_context.evaluators = await EvalHelpers.load_evaluators(
resolved_eval_set_path,
eval_context.evaluation_set,
_get_agent_model(eval_context.runtime_schema),
)

# Runtime is not required anymore.
await runtime.dispose()

try:
if project_id:
studio_client = StudioClient(project_id)

Expand All @@ -395,11 +471,12 @@ async def execute_eval():
event_bus,
)
finally:
if runtime_factory:
await runtime_factory.dispose()
await runtime_factory.dispose()

asyncio.run(execute_eval())

except _EvalDiscoveryError as e:
_show_eval_usage_help(e.entrypoints, e.eval_sets)
except Exception as e:
console.error(
f"Error occurred: {e or 'Execution failed'}", include_traceback=True
Expand Down
4 changes: 1 addition & 3 deletions src/uipath/eval/_helpers/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1 @@
from .helpers import auto_discover_entrypoint

__all__ = ["auto_discover_entrypoint"]
"""Helper functions for evaluation process."""
52 changes: 2 additions & 50 deletions src/uipath/eval/_helpers/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
"""Helper functions for evaluation process."""

import functools
import json
import os
import time
from collections.abc import Callable
from typing import Any

import click

from ..models import ErrorEvaluationResult, EvaluationResult


Expand Down Expand Up @@ -37,52 +35,6 @@ def is_empty_value(value: Any) -> bool:
return False


def auto_discover_entrypoint() -> str:
"""Auto-discover entrypoint from config file.

Returns:
Entrypoint name (key from the functions dict)

Raises:
ValueError: If no entrypoint found or multiple entrypoints exist
"""
from uipath._cli._utils._console import ConsoleLogger
from uipath._utils.constants import UIPATH_CONFIG_FILE

console = ConsoleLogger()

if not os.path.isfile(UIPATH_CONFIG_FILE):
raise ValueError(
f"File '{UIPATH_CONFIG_FILE}' not found. Please run 'uipath init'."
)

with open(UIPATH_CONFIG_FILE, "r", encoding="utf-8") as f:
uipath_config = json.loads(f.read())

entrypoints: dict[str, str] = uipath_config.get("functions", {})

if not entrypoints:
raise ValueError(
f"No entrypoints found in {UIPATH_CONFIG_FILE}. "
"Add a 'functions' section to uipath.json"
)

if len(entrypoints) > 1:
entrypoint_list = list(entrypoints.keys())
raise ValueError(
f"Multiple entrypoints found: {entrypoint_list}. "
"Please specify which entrypoint to use."
)

entrypoint_name = next(iter(entrypoints.keys()))
entrypoint_path = entrypoints[entrypoint_name]
console.info(
f"Auto-discovered entrypoint: {click.style(entrypoint_name, fg='cyan')} "
f"({entrypoint_path})"
)
return entrypoint_name


def track_evaluation_metrics(func: Callable[..., Any]) -> Callable[..., Any]:
"""Decorator to track evaluation metrics and handle errors gracefully."""

Expand Down
44 changes: 0 additions & 44 deletions src/uipath/eval/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
from pathlib import Path
from typing import Any

import click
from pydantic import ValidationError

from uipath._cli._evals._conversational_utils import UiPathLegacyEvalChatMessagesMapper
Expand Down Expand Up @@ -46,49 +45,6 @@ def discriminate_eval_set(data: dict[str, Any]) -> EvaluationSet | LegacyEvaluat
class EvalHelpers:
"""Helper functions for evaluation commands, including loading and parsing evaluation sets and evaluators."""

@staticmethod
def auto_discover_eval_set() -> str:
"""Auto-discover evaluation set from {EVAL_SETS_DIRECTORY_NAME} directory.

Returns:
Path to the evaluation set file

Raises:
ValueError: If no eval set found or multiple eval sets exist
"""
eval_sets_dir = Path(EVAL_SETS_DIRECTORY_NAME)

if not eval_sets_dir.exists():
raise ValueError(
f"No '{EVAL_SETS_DIRECTORY_NAME}' directory found. "
"Please set 'UIPATH_PROJECT_ID' env var and run 'uipath pull'."
)

eval_set_files = list(eval_sets_dir.glob("*.json"))

if not eval_set_files:
raise ValueError(
f"No evaluation set files found in '{EVAL_SETS_DIRECTORY_NAME}' directory. "
)

if len(eval_set_files) > 1:
file_names = [f.name for f in eval_set_files]
raise ValueError(
f"Multiple evaluation sets found: {file_names}. "
f"Please specify which evaluation set to use: 'uipath eval [entrypoint] <eval_set_path>'"
)

eval_set_path = str(eval_set_files[0])
logger.info(
f"Auto-discovered evaluation set: {click.style(eval_set_path, fg='cyan')}"
)

eval_set_path_obj = Path(eval_set_path)
if not eval_set_path_obj.is_file() or eval_set_path_obj.suffix != ".json":
raise ValueError("Evaluation set must be a JSON file")

return eval_set_path

@staticmethod
def load_eval_set(
eval_set_path: str, eval_ids: list[str] | None = None
Expand Down
Loading