diff --git a/README.md b/README.md index db2f840..13f824d 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,12 @@ Physical AI Runtime provides the deployment-side components for running trained - **Inference Engine** — Load exported policies from Studio with auto-detected backends - **Policy Runtime** — Control loop with observation building and action dispatch +--- + +

+ Inference demo +

+ ## Installation ```bash diff --git a/docs/assets/inference_rerun.gif b/docs/assets/inference_rerun.gif new file mode 100644 index 0000000..5ac7011 Binary files /dev/null and b/docs/assets/inference_rerun.gif differ diff --git a/examples/runtime/async_inference.py b/examples/runtime/async_inference.py index 6b07efa..1bd8ee6 100644 --- a/examples/runtime/async_inference.py +++ b/examples/runtime/async_inference.py @@ -2,102 +2,171 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""Async inference with PolicyRuntime. - -python examples/runtime/async_inference.py \ - --model ./exports/pi05_cans_openvino \ - --device GPU.0 \ - --port /dev/ttyACM0 \ - --calibration /home/max/.cache/physicalai/robots/a8d8d997-a59e-4423-9006-5d991d223887/calibrations/0b2f185a-8ab2-4956-91c2-3a2ac2dbd8c1.json \ - --overhead-camera /dev/v4l/by-id/usb-UGREEN_Camera_2K_UGREEN_Camera_2K_SN0001-video-index0 \ - --arm-camera 353322271391 \ - --front-camera /dev/v4l/by-id/usb-Innomaker_Innomaker-U20CAM-1080p-S1_SN0001-video-index0 \ - --width 640 \ - --height 480 \ - --fps 30 \ - --duration-s 60 +"""Run a trained policy on hardware with real-time Rerun visualization. + +Prerequisites:: + + uv sync --extra capture --extra robots --extra observer-rerun + +Examples: + + # SO101 with 3 cameras, Rerun viewer auto-launched + python examples/runtime/async_inference.py \\ + --robot so101 --port /dev/ttyACM0 --calibration ./cal.json \\ + --model ./exports/my_model \\ + --camera overhead:uvc:/dev/video0 \\ + --camera arm:realsense:353322271391 \\ + --camera front:uvc:/dev/video2 \\ + --rerun spawn + + # Trossen WidowXAI + python examples/runtime/async_inference.py \\ + --robot widowxai --ip 192.168.1.2 \\ + --model ./exports/my_model \\ + --camera front:uvc:/dev/video0 + + # Bimanual Trossen WidowXAI + python examples/runtime/async_inference.py \\ + --robot bimanual_widowxai --ip-left 192.168.1.2 --ip-right 192.168.1.3 \\ + --model ./exports/my_model + + # No --camera args → interactive selection + python examples/runtime/async_inference.py \\ + --robot so101 --port /dev/ttyACM0 --calibration ./cal.json \\ + --model ./exports/my_model --rerun spawn """ from __future__ import annotations import argparse +import os +import signal -import openvino as ov -import numpy as np - -from physicalai.capture import discover_all -from physicalai.capture.transport import SharedCamera +from physicalai.capture import select_cameras_interactive from physicalai.inference import InferenceModel -from physicalai.robot import SO101 from physicalai.runtime import ( ActionQueue, AsyncExecution, LerpSmoother, PolicyRuntime, + RerunCallback, ) +from utils import build_robot, parse_camera_specs + + +def main() -> None: + # Force-exit on second Ctrl+C (Rerun's blocked channels prevent clean shutdown) + def _handle_sigint(sig: int, frame: object) -> None: + # Restore default handler so next Ctrl+C kills immediately via OS signal + signal.signal(signal.SIGINT, signal.SIG_DFL) + print("\nInterrupting... press Ctrl+C again to force kill.") + raise KeyboardInterrupt + + signal.signal(signal.SIGINT, _handle_sigint) + + parser = argparse.ArgumentParser( + description="Run a trained policy on hardware", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + # Robot + robot_group = parser.add_argument_group("robot") + robot_group.add_argument("--robot", required=True, choices=("so101", "widowxai", "bimanual_widowxai")) + robot_group.add_argument("--port", help="Serial port (so101)") + robot_group.add_argument("--calibration", help="Calibration JSON path (so101)") + robot_group.add_argument("--ip", help="Robot IP (widowxai)") + robot_group.add_argument("--ip-left", help="Left arm IP (bimanual_widowxai)") + robot_group.add_argument("--ip-right", help="Right arm IP (bimanual_widowxai)") + + # Model + model_group = parser.add_argument_group("model") + model_group.add_argument("--model", required=True, help="Exported model directory") + model_group.add_argument("--device", default="GPU", help="OpenVINO device (default: GPU)") + + # Cameras + cam_group = parser.add_argument_group("cameras") + cam_group.add_argument( + "--camera", action="append", dest="cameras", metavar="NAME:DRIVER:DEVICE", + help="Camera as name:driver:device_id (repeatable). Omit for interactive selection.", + ) + cam_group.add_argument("--cam-width", type=int, default=640, help="Camera width (default: 640)") + cam_group.add_argument("--cam-height", type=int, default=480, help="Camera height (default: 480)") + cam_group.add_argument("--cam-fps", type=int, default=30, help="Camera FPS (default: 30)") + + # Runtime + rt_group = parser.add_argument_group("runtime") + rt_group.add_argument("--fps", type=float, default=30.0, help="Control loop FPS (default: 30)") + rt_group.add_argument("--duration-s", type=float, default=60.0, help="Duration in seconds") + rt_group.add_argument("--task", type=str, default=None, help="Task string for the model (e.g. 'pick up the can')") + rt_group.add_argument("--shared-camera", action="store_true", help="Use shared memory cameras (iceoryx2) — faster but incompatible with debugger") + rt_group.add_argument("--request-threshold", type=float, default=0.75, help="Request new inference when queue drops below this fraction of chunk_size (default: 0.75 = trigger when 75%% of actions remain)") + rt_group.add_argument("--lerp-frames", type=int, default=3, help="LerpSmoother blend duration in frames (default: 3)") + + # Rerun + rr_group = parser.add_argument_group("rerun") + rr_group.add_argument("--rerun", choices=("off", "spawn", "connect", "save"), default="off") + rr_group.add_argument("--rerun-addr", default="127.0.0.1:9876") + rr_group.add_argument("--rerun-save-path", default="run.rrd") + rr_group.add_argument("--rerun-no-images", action="store_true", help="Scalars only") + rr_group.add_argument("--rerun-image-decimation", type=int, default=1, help="Only send 1/N frames to Rerun") + rr_group.add_argument("--rerun-jpeg-quality", type=int, default=None, help="JPEG quality for Rerun images (0-100, default: no re-encoding)") + rr_group.add_argument("--rerun-image-max-dim", type=int, default=None, help="Max width/height for Rerun images (default: no resizing)") -def main(): - parser = argparse.ArgumentParser(description="Run policy with PolicyRuntime") - parser.add_argument("--model", required=True, help="Exported model directory") - parser.add_argument("--device", default="GPU.0", help="OpenVINO device") - parser.add_argument("--port", default="/dev/ttyACM0", help="Robot serial port") - parser.add_argument("--calibration", required=True, help="Robot calibration file") - parser.add_argument("--overhead-camera", required=True, help="Overhead camera device path") - parser.add_argument("--arm-camera", required=True, help="Arm camera serial number") - parser.add_argument("--front-camera", required=True, help="Front camera device path") - parser.add_argument("--width", type=int, default=640) - parser.add_argument("--height", type=int, default=480) - parser.add_argument("--duration-s", type=float, default=60.0) - parser.add_argument("--fps", type=float, default=30.0) args = parser.parse_args() + # ── Load model ── import openvino_tokenizers # noqa: F401 — registers OV tokenizer ops - print(f"Available devices:") - core = ov.Core() - devices = core.available_devices - for dev in devices: - print(f" {dev}: {core.get_property(dev, 'FULL_DEVICE_NAME')}") - print(f"Selected device: {args.device}") - + print(f"Loading model from {args.model} on {args.device} (this may take a minute)...", flush=True) model = InferenceModel.load(args.model, device=args.device) - robot = SO101(port=args.port, calibration=args.calibration, role="follower") - cameras = { - "overhead": SharedCamera("uvc", device=args.overhead_camera, width=args.width, height=args.height, fps=int(args.fps)), - "front": SharedCamera("uvc", device=args.front_camera, width=args.width, height=args.height, fps=int(args.fps)), - "arm": SharedCamera("realsense", serial_number=args.arm_camera, width=args.width, height=args.height, fps=int(args.fps)), - } - + print("Model loaded.") + + # ── Build robot & cameras ── + robot = build_robot(args) + if args.cameras: + cameras = parse_camera_specs(args.cameras, args.cam_width, args.cam_height, args.cam_fps, shared=args.shared_camera) + else: + cameras = select_cameras_interactive(args.cam_width, args.cam_height, args.cam_fps) + + # ── Callbacks ── + callbacks: list = [] + if args.rerun != "off": + callbacks.append( + RerunCallback( + cameras=cameras, + image_decimation=args.rerun_image_decimation, + log_images=not args.rerun_no_images, + image_jpeg_quality=args.rerun_jpeg_quality, + image_max_dim=args.rerun_image_max_dim, + mode=args.rerun, + connect_addr=args.rerun_addr, + save_path=args.rerun_save_path if args.rerun == "save" else None, + ) + ) + + # ── Run ── runtime = PolicyRuntime( robot=robot, model=model, - execution=AsyncExecution(threshold=0.3, fps=int(args.fps)), - action_queue=ActionQueue(smoother=LerpSmoother(duration_frames=5)), + execution=AsyncExecution(request_threshold=args.request_threshold, fps=int(args.fps)), + action_queue=ActionQueue(smoother=LerpSmoother(duration_frames=args.lerp_frames)), cameras=cameras, fps=args.fps, + callbacks=callbacks, + task=args.task, ) - try: - runtime.connect() - except Exception as e: - print(f"Failed to connect: {e}") - print("Available cameras:") - for driver, devices in discover_all().items(): - for dev in devices: - print(f" Driver: {driver}, Device: {dev.device_id}, Info: {dev.name}") - return - - for name, cam in cameras.items(): - print(f"Camera '{name}' connected: {cam.actual_width}x{cam.actual_height} @ {cam.actual_fps}fps") - - print("Starting policy runtime...") - try: + with runtime: + for name, cam in cameras.items(): + w = getattr(cam, "actual_width", None) + h = getattr(cam, "actual_height", None) + f = getattr(cam, "actual_fps", None) + print(f" {name}: {w}x{h} @ {f}fps" if w and h else f" {name}: connected") + print(f"Running at {args.fps} fps for {args.duration_s}s...") stats = runtime.run(duration_s=args.duration_s) - print(f"\nDone — {stats.steps} steps, {stats.inference_count} inferences, {stats.total_holds} holds") - finally: - runtime.disconnect() - print("Disconnected") + + print(f"\nDone — {stats.steps} steps, {stats.inference_count} inferences, {stats.total_holds} holds") if __name__ == "__main__": diff --git a/examples/runtime/sync_inference.py b/examples/runtime/sync_inference.py new file mode 100644 index 0000000..338f8a7 --- /dev/null +++ b/examples/runtime/sync_inference.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Run a trained policy synchronously — simplest possible control loop. + +Inference blocks the control thread. The loop pauses during each inference +call, so real-time guarantees do NOT hold. Use this to verify model behaviour +before moving to async execution with lerping. + +Examples: + + # Bimanual Trossen, no shared camera (debugger-safe) + python examples/runtime/sync_inference.py \ + --robot bimanual_widowxai --ip-left 192.168.1.2 --ip-right 192.168.1.3 \ + --model ./exports/pi05_cans_openvino \ + --camera front:uvc:/dev/video0 \ + --task "pick up the can" \ + --fps 30 --duration-s 30 + + # SO101 + python examples/runtime/sync_inference.py \ + --robot so101 --port /dev/ttyACM0 --calibration ./cal.json \ + --model ./exports/my_model \ + --camera overhead:uvc:/dev/video0 \ + --task "pick up the can" +""" + +from __future__ import annotations + +import argparse +import signal + +from physicalai.capture import select_cameras_interactive +from physicalai.inference import InferenceModel +from physicalai.runtime import ( + ActionQueue, + PolicyRuntime, + RerunCallback, +) +from physicalai.runtime.execution import SyncExecution + +from utils import build_robot, parse_camera_specs + + +def main() -> None: + def _handle_sigint(sig: int, frame: object) -> None: + signal.signal(signal.SIGINT, signal.SIG_DFL) + print("\nInterrupting... press Ctrl+C again to force kill.") + raise KeyboardInterrupt + + signal.signal(signal.SIGINT, _handle_sigint) + + parser = argparse.ArgumentParser( + description="Run a trained policy synchronously (blocking inference)", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + # Robot + robot_group = parser.add_argument_group("robot") + robot_group.add_argument("--robot", required=True, choices=("so101", "widowxai", "bimanual_widowxai")) + robot_group.add_argument("--port", help="Serial port (so101)") + robot_group.add_argument("--calibration", help="Calibration JSON path (so101)") + robot_group.add_argument("--ip", help="Robot IP (widowxai)") + robot_group.add_argument("--ip-left", help="Left arm IP (bimanual_widowxai)") + robot_group.add_argument("--ip-right", help="Right arm IP (bimanual_widowxai)") + + # Model + model_group = parser.add_argument_group("model") + model_group.add_argument("--model", required=True, help="Exported model directory") + model_group.add_argument("--device", default="GPU", help="OpenVINO device (default: GPU)") + + # Cameras + cam_group = parser.add_argument_group("cameras") + cam_group.add_argument( + "--camera", action="append", dest="cameras", metavar="NAME:DRIVER:DEVICE", + help="Camera as name:driver:device_id (repeatable). Omit for interactive selection.", + ) + cam_group.add_argument("--cam-width", type=int, default=640, help="Camera width (default: 640)") + cam_group.add_argument("--cam-height", type=int, default=480, help="Camera height (default: 480)") + cam_group.add_argument("--cam-fps", type=int, default=30, help="Camera FPS (default: 30)") + + # Runtime + rt_group = parser.add_argument_group("runtime") + rt_group.add_argument("--fps", type=float, default=30.0, help="Control loop FPS (default: 30)") + rt_group.add_argument("--duration-s", type=float, default=60.0, help="Duration in seconds") + rt_group.add_argument("--task", type=str, default=None, help="Task string for the model (e.g. 'pick up the can')") + rt_group.add_argument("--request-threshold", type=float, default=0.5, help="Request new inference when queue drops below this fraction of chunk_size (default: 0.75 = trigger when 75%% of actions remain)") + + # Rerun + rr_group = parser.add_argument_group("rerun") + rr_group.add_argument("--rerun", choices=("off", "spawn", "connect", "save"), default="off") + rr_group.add_argument("--rerun-addr", default="127.0.0.1:9876") + rr_group.add_argument("--rerun-save-path", default="run.rrd") + + args = parser.parse_args() + + # ── Load model ── + import openvino_tokenizers # noqa: F401 — registers OV tokenizer ops + + print(f"Loading model from {args.model} on {args.device}...", flush=True) + model = InferenceModel.load(args.model, device=args.device) + print("Model loaded.") + + # ── Build robot & cameras (direct, no shared memory — debugger-safe) ── + robot = build_robot(args) + if args.cameras: + cameras = parse_camera_specs(args.cameras, args.cam_width, args.cam_height, args.cam_fps, shared=False) + else: + cameras = select_cameras_interactive(args.cam_width, args.cam_height, args.cam_fps) + + # ── Callbacks ── + callbacks: list = [] + if args.rerun != "off": + callbacks.append( + RerunCallback( + cameras=cameras, + log_images=True, + mode=args.rerun, + connect_addr=args.rerun_addr, + save_path=args.rerun_save_path if args.rerun == "save" else None, + ) + ) + + # ── Run (synchronous — inference blocks the loop) ── + runtime = PolicyRuntime( + robot=robot, + model=model, + execution=SyncExecution(fps=int(args.fps), request_threshold=args.request_threshold), + action_queue=ActionQueue(), # no smoother — raw chunk playback + cameras=cameras, + fps=args.fps, + callbacks=callbacks, + task=args.task, + ) + + with runtime: + print(f"Running SYNC at {args.fps} fps for {args.duration_s}s...") + if args.task: + print(f" task: {args.task!r}") + print(" (inference blocks the loop — expect pauses)") + stats = runtime.run(duration_s=args.duration_s) + + print(f"\nDone — {stats.steps} steps, {stats.inference_count} inferences, {stats.total_holds} holds") + + +if __name__ == "__main__": + main() diff --git a/examples/runtime/utils.py b/examples/runtime/utils.py new file mode 100644 index 0000000..aa636af --- /dev/null +++ b/examples/runtime/utils.py @@ -0,0 +1,77 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers for runtime examples — robot and camera construction.""" + +from __future__ import annotations + +import argparse +import sys + +from physicalai.capture.camera import Camera +from physicalai.robot.interface import Robot + + +def build_robot(args: argparse.Namespace) -> Robot: + """Construct a robot from CLI args (--robot, --port, --ip, etc.).""" + if args.robot == "so101": + from physicalai.robot import SO101 + + if not args.port: + sys.exit("error: --port is required for so101") + if not args.calibration: + sys.exit("error: --calibration is required for so101") + return SO101(port=args.port, calibration=args.calibration, role="follower") + + if args.robot == "widowxai": + from physicalai.robot import WidowXAI + + if not args.ip: + sys.exit("error: --ip is required for widowxai") + return WidowXAI(ip=args.ip, role="follower") + + if args.robot == "bimanual_widowxai": + from physicalai.robot import BimanualWidowXAI, WidowXAI + + if not args.ip_left or not args.ip_right: + sys.exit("error: --ip-left and --ip-right are required for bimanual") + left = WidowXAI(ip=args.ip_left, role="follower") + right = WidowXAI(ip=args.ip_right, role="follower") + return BimanualWidowXAI(left, right) + + sys.exit(f"error: unknown robot type: {args.robot}") + + +def parse_camera_specs( + specs: list[str], + width: int, + height: int, + fps: int, + *, + shared: bool = True, +) -> dict[str, Camera]: + """Parse CLI camera specs into a camera dict. + + Each spec is "name:driver:device_id", e.g.: + --camera overhead:uvc:/dev/video0 + --camera arm:realsense:353322271391 + + Args: + shared: Use SharedCamera (iceoryx2 transport). Set False for + direct camera API (recommended with debugger). + """ + from physicalai.capture import create_camera + + cameras: dict[str, Camera] = {} + for spec in specs: + parts = spec.split(":", 2) + if len(parts) != 3: + sys.exit(f"error: invalid camera spec '{spec}'. Expected name:driver:device_id") + name, driver, device_id = parts + kwargs: dict = {"width": width, "height": height, "fps": fps} + if driver == "realsense": + kwargs["serial_number"] = device_id + else: + kwargs["device"] = device_id + cameras[name] = create_camera(driver, shared=shared, **kwargs) + return cameras diff --git a/pyproject.toml b/pyproject.toml index 5c4c5d0..75f25cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ abb = [] franka = [] trossen = ["trossen-arm>=1.9.0"] so101 = ["feetech-servo-sdk"] +observer-rerun = ["rerun-sdk>=0.22"] robots = [ "physicalai[ur]", "physicalai[abb]", diff --git a/src/physicalai/capture/__init__.py b/src/physicalai/capture/__init__.py index 40feb21..260dc26 100644 --- a/src/physicalai/capture/__init__.py +++ b/src/physicalai/capture/__init__.py @@ -23,7 +23,10 @@ MissingDependencyError, NotConnectedError, ) -from physicalai.capture.factory import create_camera +from physicalai.capture.factory import ( + create_camera, + select_cameras_interactive, +) from physicalai.capture.frame import Frame from physicalai.capture.mixins import DepthMixin from physicalai.capture.multi import SyncedFrames, async_read_cameras, read_cameras @@ -56,6 +59,7 @@ "create_camera", "discover_all", "read_cameras", + "select_cameras_interactive", # Concrete cameras (lazy-loaded) "IPCamera", "RealSenseCamera", diff --git a/src/physicalai/capture/factory.py b/src/physicalai/capture/factory.py index 258e84f..96abbec 100644 --- a/src/physicalai/capture/factory.py +++ b/src/physicalai/capture/factory.py @@ -1,28 +1,30 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""Factory convenience function for config-driven camera creation.""" +"""Factory convenience functions for config-driven camera creation.""" from __future__ import annotations from typing import TYPE_CHECKING +from loguru import logger + from physicalai.capture.camera import CameraType if TYPE_CHECKING: from physicalai.capture.camera import Camera -def create_camera(camera_type: str, **kwargs) -> Camera: # noqa: ANN003 +def create_camera(camera_type: str, *, shared: bool = False, **kwargs) -> Camera: # noqa: ANN003 """Create a camera by type name. - Convenience function for config-driven instantiation. Prefer - dedicated camera classes for direct usage. - Args: camera_type: Camera type — one of ``"uvc"``, ``"ip"``, ``"realsense"``, ``"basler"``, ``"genicam"``. Case-insensitive. + shared: If True, wrap the camera in a :class:`SharedCamera` + (iceoryx2 shared-memory transport). Requires the + ``transport`` extra. **kwargs: Forwarded to the camera constructor. Returns: @@ -33,6 +35,11 @@ def create_camera(camera_type: str, **kwargs) -> Camera: # noqa: ANN003 """ camera_type = camera_type.lower() + if shared: + from physicalai.capture.transport import SharedCamera # noqa: PLC0415 + + return SharedCamera(camera_type, **kwargs) + if camera_type == CameraType.UVC: from physicalai.capture.cameras.uvc import UVCCamera # noqa: PLC0415 @@ -60,3 +67,79 @@ def create_camera(camera_type: str, **kwargs) -> Camera: # noqa: ANN003 msg = f"Unknown camera type {camera_type!r}. Expected one of: {', '.join(CameraType)}" raise ValueError(msg) + + +# ─── Multi-camera construction ──────────────────────────────────────────────── + + +def select_cameras_interactive( + width: int, + height: int, + fps: int, +) -> dict[str, Camera]: + """Discover cameras and let the user pick interactively via stdin. + + Uses :func:`~physicalai.capture.discover_all` to enumerate available + devices, then presents a numbered menu. The user selects cameras + one at a time and assigns each a name. + + Args: + width: Requested frame width. + height: Requested frame height. + fps: Requested frame rate. + + Returns: + Dict mapping user-chosen names to SharedCamera instances. + Empty dict if no cameras found or none selected. + """ + from physicalai.capture.discovery import discover_all # noqa: PLC0415 + + logger.info("Discovering cameras...") + all_devices = discover_all() + + flat: list[tuple[str, str, str]] = [] + for driver, devices in all_devices.items(): + flat.extend((driver, dev.device_id, f"{driver}: {dev.name or dev.device_id}") for dev in devices) + + if not flat: + logger.warning("No cameras found. Continuing without cameras.") + return {} + + logger.info("Available cameras:") + for i, (_, _, display) in enumerate(flat): + logger.info(" [{}] {}", i, display) + + cameras: dict[str, Camera] = {} + while True: + try: + choice = input("Select camera index (or 'done' to finish): ").strip() + except (EOFError, KeyboardInterrupt): + break + if choice.lower() in {"done", "d", ""}: + break + try: + idx = int(choice) + if idx < 0 or idx >= len(flat): + logger.warning(" Invalid index. Choose 0-{}.", len(flat) - 1) + continue + except ValueError: + logger.warning(" Enter a number or 'done'.") + continue + + try: + name = input(" Name for this camera (e.g. overhead, arm, front): ").strip() + except (EOFError, KeyboardInterrupt): + break + if not name: + name = f"camera_{len(cameras)}" + + driver, device_id, _ = flat[idx] + kwargs: dict = {"width": width, "height": height, "fps": fps} + if driver == "realsense": + kwargs["serial_number"] = device_id + else: + kwargs["device"] = device_id + cameras[name] = create_camera(driver, shared=True, **kwargs) + logger.info(" Added '{}' ({}:{})", name, driver, device_id) + + return cameras diff --git a/src/physicalai/robot/interface.py b/src/physicalai/robot/interface.py index 2176ce3..d619375 100644 --- a/src/physicalai/robot/interface.py +++ b/src/physicalai/robot/interface.py @@ -42,6 +42,16 @@ class RobotObservation(Protocol): sensor_data: dict[str, np.ndarray] | None images: dict[str, Frame] | None + @property + def state(self) -> np.ndarray: + """Full robot state vector for inference (positions + sensor data if available). + + Default: ``joint_positions``. Implementations with additional + sensor data (e.g. velocities) should override to concatenate + them in the order matching training data conventions. + """ + return self.joint_positions + @runtime_checkable class Robot(Protocol): diff --git a/src/physicalai/robot/so101/so101.py b/src/physicalai/robot/so101/so101.py index 30fcc22..342bf9b 100644 --- a/src/physicalai/robot/so101/so101.py +++ b/src/physicalai/robot/so101/so101.py @@ -81,6 +81,11 @@ class SO101Observation: sensor_data: dict[str, np.ndarray] | None = None # no extra sensors available on SO-101 images: dict[str, Frame] | None = None # no built-in camera implementation + @property + def state(self) -> np.ndarray: + """State vector: joint positions (6,).""" + return self.joint_positions + class SO101(Robot): """Driver for the SO-101 robot arm (6-DOF, Feetech STS3215 servos). diff --git a/src/physicalai/robot/trossen/bimanual_widowxai.py b/src/physicalai/robot/trossen/bimanual_widowxai.py index 9c85ef3..115e6ad 100644 --- a/src/physicalai/robot/trossen/bimanual_widowxai.py +++ b/src/physicalai/robot/trossen/bimanual_widowxai.py @@ -49,6 +49,13 @@ class BimanualWidowXAIObservation: sensor_data: dict[str, np.ndarray] | None = None images: dict[str, Frame] | None = None + @property + def state(self) -> np.ndarray: + """State vector: positions (14) + velocities (14) = (28,).""" + if self.sensor_data and "velocities" in self.sensor_data: + return np.concatenate([self.joint_positions, self.sensor_data["velocities"]]) + return self.joint_positions + class BimanualWidowXAI(Robot): """Two-arm WidowX AI driver composing a left and right :class:`WidowXAI`. @@ -135,28 +142,29 @@ def get_observation(self) -> BimanualWidowXAIObservation: ) def send_action(self, action: np.ndarray, *, goal_time: float = 0.1) -> None: - """Send a 14-DOF joint position command (follower only). + """Send joint position command (follower only). Args: - action: Array of shape ``(14,)`` — left (7) then right (7) target - positions in degrees for non-gripper joints, and native scalar - values for grippers. + action: Array of shape ``(N,)`` where N >= 14. Only the first 14 + elements (left 7 + right 7 positions) are used; extra + dimensions (e.g. predicted velocities) are ignored. goal_time: Minimum time (seconds) for the arms to reach the target. Raises: RuntimeError: If called on a leader robot. - ValueError: If action shape is not ``(14,)``. + ValueError: If action has fewer than 14 elements. """ if self.role == "leader": msg = "Cannot send actions to a leader robot." raise RuntimeError(msg) n = self._left.NUM_JOINTS - expected_shape = (2 * n,) - if action.shape != expected_shape: - msg = f"Expected action shape {expected_shape}, got {action.shape}" + min_dims = 2 * n + if action.shape[0] < min_dims: + msg = f"Expected at least {min_dims} action dims, got {action.shape}" raise ValueError(msg) + action = action[:min_dims] self._left.send_action(action[:n], goal_time=goal_time) self._right.send_action(action[n:], goal_time=goal_time) diff --git a/src/physicalai/robot/trossen/widowxai.py b/src/physicalai/robot/trossen/widowxai.py index a69da5c..b57395d 100644 --- a/src/physicalai/robot/trossen/widowxai.py +++ b/src/physicalai/robot/trossen/widowxai.py @@ -58,6 +58,13 @@ class WidowXAIObservation: sensor_data: dict[str, np.ndarray] | None = None images: dict[str, Frame] | None = None + @property + def state(self) -> np.ndarray: + """State vector: positions (7) + velocities (7) = (14,).""" + if self.sensor_data and "velocities" in self.sensor_data: + return np.concatenate([self.joint_positions, self.sensor_data["velocities"]]) + return self.joint_positions + class WidowXAI(Robot): """Driver for the Trossen WidowX AI robot arm (7-DOF). @@ -221,28 +228,27 @@ def get_observation(self) -> WidowXAIObservation: ) def send_action(self, action: np.ndarray, *, goal_time: float = 0.1) -> None: - """Send a 7-DOF joint position command to follower arms. + """Send a joint position command to follower arms. Args: - action: Array of shape ``(7,)`` with target joint positions in degrees - for non-gripper joints, and native gripper scalar value. + action: Array of shape ``(N,)`` where N >= 7. Only the first 7 + elements (joint positions) are used; extra dimensions + (e.g. predicted velocities) are ignored. goal_time: Minimum time (seconds) for the arm to reach the target. - The backend control loop typically sets this to ``1 / fps``. - Not part of the :class:`~physicalai.robot.Robot` protocol. Raises: RuntimeError: If called on a leader arm. - ValueError: If action shape is not ``(7,)``. + ValueError: If action has fewer than 7 elements. """ if self._role == "leader": msg = "Cannot send actions to a leader arm." raise RuntimeError(msg) - expected_shape = (self.NUM_JOINTS,) - if action.shape != expected_shape: - msg = f"Expected action shape {expected_shape}, got {action.shape}" + if action.shape[0] < self.NUM_JOINTS: + msg = f"Expected at least {self.NUM_JOINTS} action dims, got {action.shape}" raise ValueError(msg) + action = action[: self.NUM_JOINTS] driver = self._require_driver() target_radians = np.asarray(action, dtype=np.float32).copy() diff --git a/src/physicalai/runtime/__init__.py b/src/physicalai/runtime/__init__.py index 4a4ec70..18923e4 100644 --- a/src/physicalai/runtime/__init__.py +++ b/src/physicalai/runtime/__init__.py @@ -9,9 +9,18 @@ from physicalai.runtime import SyncExecution, AsyncExecution, Execution, WorkerDiedError from physicalai.runtime import ActionQueue from physicalai.runtime import ChunkSmoother, LerpSmoother, ReplaceSmoother + from physicalai.runtime import TickEvent, InferenceEvent, LifecycleEvent + from physicalai.runtime import ConsoleCallback, JsonlCallback, AsyncCallback, RerunCallback """ from physicalai.runtime._action_queue import ActionQueue # noqa: PLC2701 +from physicalai.runtime.callbacks import ( + AsyncCallback, + ConsoleCallback, + JsonlCallback, + RerunCallback, +) +from physicalai.runtime.events import InferenceEvent, LifecycleEvent, TickEvent from physicalai.runtime.execution import ( AsyncExecution, Execution, @@ -27,14 +36,21 @@ __all__ = [ "ActionQueue", + "AsyncCallback", "AsyncExecution", "ChunkSmoother", + "ConsoleCallback", "Execution", + "InferenceEvent", + "JsonlCallback", "LerpSmoother", + "LifecycleEvent", "PolicyRuntime", "ReplaceSmoother", + "RerunCallback", "RunStats", "RuntimeCallback", "SyncExecution", + "TickEvent", "WorkerDiedError", ] diff --git a/src/physicalai/runtime/_action_queue.py b/src/physicalai/runtime/_action_queue.py index c195edf..b93091f 100644 --- a/src/physicalai/runtime/_action_queue.py +++ b/src/physicalai/runtime/_action_queue.py @@ -25,8 +25,9 @@ def __init__(self, smoother: ChunkSmoother | None = None) -> None: def push_chunk(self, chunk: np.ndarray, offset: int = 0) -> None: """Push an action chunk, blending with remaining actions via the smoother.""" with self._lock: + incoming = chunk[offset:] remaining = np.stack(list(self._deque)) if self._deque else np.empty((0, chunk.shape[1]), dtype=chunk.dtype) - merged = self._smoother.merge(remaining, chunk, offset) + merged = self._smoother.merge(remaining, incoming) self._deque.clear() self._deque.extend(merged) @@ -45,6 +46,13 @@ def pop(self) -> np.ndarray | None: self._total_pops += 1 return self._deque.popleft() + def peek_remaining(self) -> np.ndarray | None: + """Return copy of remaining actions without consuming them. Thread-safe.""" + with self._lock: + if not self._deque: + return None + return np.stack(list(self._deque)) + @property def remaining(self) -> int: with self._lock: diff --git a/src/physicalai/runtime/_callback_bus.py b/src/physicalai/runtime/_callback_bus.py new file mode 100644 index 0000000..08eeda0 --- /dev/null +++ b/src/physicalai/runtime/_callback_bus.py @@ -0,0 +1,116 @@ +# Copyright (C) 2025-2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import logging +from collections import deque +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Sequence + + import numpy as np + + from physicalai.runtime.events import InferenceEvent, LifecycleEvent, TickEvent + +logger = logging.getLogger(__name__) + +_INFERENCE_QUEUE_MAXLEN = 64 + + +class _CallbackBus: + """Internal dispatch bus for runtime callbacks. + + Two dispatch modes: + - Fire-and-forget (emit_*): telemetry hooks, exceptions isolated. + - Request-response (invoke_*): action hooks, chained return values. + + Thread safety: only ``emit_inference`` is called from the inference thread. + It appends to a bounded deque (CPython atomic). All other methods run on + the control thread. + """ + + def __init__(self, callbacks: Sequence[Any]) -> None: + self._callbacks = list(callbacks) + self._inference_queue: deque[InferenceEvent] = deque(maxlen=_INFERENCE_QUEUE_MAXLEN) + + def emit_tick(self, event: TickEvent) -> None: + self._drain_inference() + for cb in self._callbacks: + fn = getattr(cb, "on_tick", None) + if fn is None: + continue + try: + fn(event) + except Exception: + logger.exception("Callback %r failed in on_tick", cb) + + def emit_inference(self, event: InferenceEvent) -> None: + """Enqueue inference event from background thread for control-thread delivery.""" + self._inference_queue.append(event) + + def emit_lifecycle(self, event: LifecycleEvent) -> None: + for cb in self._callbacks: + fn = getattr(cb, "on_lifecycle", None) + if fn is None: + continue + try: + fn(event) + except Exception: + logger.exception("Callback %r failed in on_lifecycle", cb) + + def invoke_before_send_action(self, *, action: np.ndarray, step: int) -> np.ndarray: + result = action + for cb in self._callbacks: + fn = getattr(cb, "before_send_action", None) + if fn is None: + continue + try: + modified = fn(action=result, step=step) + if modified is not None: + result = modified + except Exception: + logger.exception("Callback %r failed in before_send_action", cb) + return result + + def invoke_on_action_sent(self, *, action: np.ndarray, step: int) -> None: + for cb in self._callbacks: + fn = getattr(cb, "on_action_sent", None) + if fn is None: + continue + try: + fn(action=action, step=step) + except Exception: + logger.exception("Callback %r failed in on_action_sent", cb) + + def invoke_on_hold(self, *, step: int, holds: int) -> None: + for cb in self._callbacks: + fn = getattr(cb, "on_hold", None) + if fn is None: + continue + try: + fn(step=step, holds=holds) + except Exception: + logger.exception("Callback %r failed in on_hold", cb) + + def close(self) -> None: + for cb in self._callbacks: + close_fn = getattr(cb, "close", None) + if close_fn is not None: + try: + close_fn() + except Exception: + logger.exception("Callback %r failed in close", cb) + + def _drain_inference(self) -> None: + while self._inference_queue: + event = self._inference_queue.popleft() + for cb in self._callbacks: + fn = getattr(cb, "on_inference", None) + if fn is None: + continue + try: + fn(event) + except Exception: + logger.exception("Callback %r failed in on_inference", cb) diff --git a/src/physicalai/runtime/_telemetry.py b/src/physicalai/runtime/_telemetry.py new file mode 100644 index 0000000..580dbc6 --- /dev/null +++ b/src/physicalai/runtime/_telemetry.py @@ -0,0 +1,124 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import logging +import time +import uuid +from typing import Any + +import numpy as np + +logger = logging.getLogger(__name__) + + +def _encode_numpy(arr: np.ndarray) -> dict[str, Any]: + return { + "__np__": True, + "dtype": str(arr.dtype), + "shape": list(arr.shape), + "data": arr.tobytes(), + } + + +def _decode_numpy(obj: dict[str, Any]) -> np.ndarray: + return np.frombuffer(obj["data"], dtype=np.dtype(obj["dtype"])).reshape(obj["shape"]) + + +class TelemetryEmitter: + def __init__(self, session_id: str | None = None) -> None: + self._session_id = session_id or uuid.uuid4().hex[:8] + self._session: Any = None + self._msgpack: Any = None + self._enabled = False + try: + import msgpack # noqa: PLC0415 + import zenoh # noqa: PLC0415 + + self._msgpack = msgpack + self._session = zenoh.open(zenoh.Config()) + self._enabled = True + except ImportError: + pass + + @property + def enabled(self) -> bool: + return self._enabled + + @property + def session_id(self) -> str: + return self._session_id + + def _pack(self, payload: dict[str, Any]) -> bytes: + def _default(obj: object) -> object: + if isinstance(obj, np.ndarray): + return _encode_numpy(obj) + if isinstance(obj, np.integer): + return int(obj) + if isinstance(obj, np.floating): + return float(obj) + return obj + + return self._msgpack.packb(payload, default=_default) + + def emit_tick( + self, + *, + step: int, + timestamp: float, + joint_positions: np.ndarray | None, + action_sent: np.ndarray | None, + queue_remaining: int, + loop_duration_s: float, + sleep_time_s: float, + stale_obs: bool = False, + ) -> None: + if not self._enabled: + return + payload = { + "step": step, + "timestamp": timestamp, + "joint_positions": joint_positions, + "action_sent": action_sent, + "queue_remaining": queue_remaining, + "physicalai.runtime.loop_duration_s": loop_duration_s, + "physicalai.runtime.sleep_time_s": sleep_time_s, + "stale_obs": stale_obs, + } + self._session.put(f"physicalai/rt/{self._session_id}/tick", self._pack(payload)) + + def emit_inference( + self, + *, + latency_s: float, + offset: int, + chunk: np.ndarray, + ) -> None: + if not self._enabled: + return + payload = { + "physicalai.runtime.inference_latency_s": latency_s, + "offset": offset, + "chunk": chunk, + } + self._session.put(f"physicalai/rt/{self._session_id}/inference", self._pack(payload)) + + def emit_lifecycle(self, event: str, **metadata: Any) -> None: # noqa: ANN401 + if not self._enabled: + return + payload = { + "event": event, + "timestamp": time.time(), + **metadata, + } + self._session.put(f"physicalai/rt/{self._session_id}/lifecycle", self._pack(payload)) + + def close(self) -> None: + if self._session is not None: + try: + self._session.close() + except Exception: + logger.debug("Error closing zenoh session", exc_info=True) + self._session = None + self._enabled = False diff --git a/src/physicalai/runtime/callbacks.py b/src/physicalai/runtime/callbacks.py new file mode 100644 index 0000000..3e50fbd --- /dev/null +++ b/src/physicalai/runtime/callbacks.py @@ -0,0 +1,540 @@ +# Copyright (C) 2025-2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Shipped callback implementations for the runtime callback bus.""" + +from __future__ import annotations + +import colorsys +import json +import logging +import threading +import time +from collections import deque +from pathlib import Path +from typing import TYPE_CHECKING, Any, Literal + +if TYPE_CHECKING: + from collections.abc import Mapping + + import numpy as np + + from physicalai.capture.camera import Camera + from physicalai.runtime.events import InferenceEvent, LifecycleEvent, TickEvent + +logger = logging.getLogger(__name__) + + +class ConsoleCallback: + """Periodic one-line summary to stdout (~1 per second).""" + + def __init__(self, throttle_steps: int = 30) -> None: # noqa: D107 + self._throttle_steps = throttle_steps + self._start_time: float | None = None + + def on_tick(self, event: TickEvent) -> None: # noqa: D102 + if self._start_time is None: + self._start_time = time.monotonic() + if event.step > 0 and event.step % self._throttle_steps != 0: + return + elapsed = time.monotonic() - self._start_time + print( # noqa: T201 + f"[{elapsed:6.1f}s] step={event.step} " + f"queue={event.queue_remaining} " + f"loop={event.loop_duration_s * 1000:.1f}ms" + f"{' STALE' if event.stale_obs else ''}", + ) + + def on_lifecycle(self, event: LifecycleEvent) -> None: # noqa: D102, PLR6301 + print(f"[lifecycle] {event.event}: {event.metadata}") # noqa: T201 + + +class JsonlCallback: + """Append-only JSONL recording. Numpy arrays converted to lists.""" + + def __init__(self, path: str | Path, *, record_chunks: bool = False) -> None: # noqa: D107 + self._path = Path(path) + self._file = self._path.open("a") + self._record_chunks = record_chunks + + def on_tick(self, event: TickEvent) -> None: # noqa: D102 + self._write( + "tick", + { + "session_id": event.session_id, + "step": event.step, + "timestamp": event.timestamp, + "joint_positions": _np_to_list(event.joint_positions), + "action_sent": _np_to_list(event.action_sent), + "queue_remaining": event.queue_remaining, + "loop_duration_s": event.loop_duration_s, + "sleep_time_s": event.sleep_time_s, + "stale_obs": event.stale_obs, + }, + ) + + def on_inference(self, event: InferenceEvent) -> None: # noqa: D102 + payload: dict[str, Any] = { + "session_id": event.session_id, + "timestamp": event.timestamp, + "latency_s": event.latency_s, + "offset": event.offset, + "chunk_shape": list(event.chunk.shape), + } + if self._record_chunks: + payload["chunk"] = event.chunk.tolist() + self._write("inference", payload) + + def on_lifecycle(self, event: LifecycleEvent) -> None: # noqa: D102 + self._write( + "lifecycle", + { + "session_id": event.session_id, + "timestamp": event.timestamp, + "event": event.event, + "metadata": event.metadata, + }, + ) + + def close(self) -> None: # noqa: D102 + self._file.close() + + def _write(self, kind: str, payload: dict[str, Any]) -> None: + record = {"type": kind, **payload} + self._file.write(json.dumps(record, default=_json_default) + "\n") + self._file.flush() + + +class AsyncCallback: + """Wraps a callback so all hooks run on a dedicated background thread. + + The control loop only pays deque.append per event. On overflow, oldest + events are dropped. + """ + + _ACTION_HOOKS = ("before_send_action", "on_action_sent", "on_hold") + + def __init__(self, inner: Any, max_queue: int = 1024) -> None: # noqa: D107, ANN401 + dropped = [h for h in self._ACTION_HOOKS if hasattr(inner, h)] + if dropped: + msg = ( + f"{type(inner).__name__} defines action hooks {dropped} which " + "AsyncCallback does not forward (use synchronous attachment instead)" + ) + raise TypeError(msg) + self._inner = inner + self._queue: deque[tuple[str, Any]] = deque(maxlen=max_queue) + self._stop = threading.Event() + self._has_work = threading.Event() + self._thread = threading.Thread(target=self._worker, name="AsyncCallbackWorker", daemon=True) + self._thread.start() + + def on_tick(self, event: TickEvent) -> None: # noqa: D102 + self._enqueue("on_tick", event) + + def on_inference(self, event: InferenceEvent) -> None: # noqa: D102 + self._enqueue("on_inference", event) + + def on_lifecycle(self, event: LifecycleEvent) -> None: # noqa: D102 + self._enqueue("on_lifecycle", event) + + def close(self) -> None: # noqa: D102 + self._stop.set() + self._has_work.set() + self._thread.join(timeout=5.0) + close_fn = getattr(self._inner, "close", None) + if close_fn is not None: + close_fn() + + def _enqueue(self, method: str, event: Any) -> None: # noqa: ANN401 + self._queue.append((method, event)) + self._has_work.set() + + def _worker(self) -> None: + while not self._stop.is_set(): + self._has_work.wait() + self._has_work.clear() + while self._queue: + method, event = self._queue.popleft() + fn = getattr(self._inner, method, None) + if fn is not None: + try: + fn(event) + except Exception: + logger.exception("AsyncCallback inner %r.%s failed", self._inner, method) + + +class RerunCallback: + """In-process Rerun logging for runtime visualization. + + Requires ``physicalai[observer-rerun]``. Logs scalars and chunks every + tick / inference event, and camera frames at ``image_decimation``-th tick. + + Do NOT wrap with :class:`AsyncCallback` — Rerun's SDK already batches I/O + asynchronously. The ``AsyncCallback`` guard (rejects inners with action + hooks) is not triggered because this class defines none, but wrapping would + double the buffering with no benefit. + """ + + def __init__( # noqa: D107 + self, + *, + cameras: Mapping[str, Camera] | None = None, + image_decimation: int = 3, + log_images: bool = True, + image_jpeg_quality: int | None = None, + image_max_dim: int | None = None, + mode: Literal["spawn", "save", "connect"] = "spawn", + save_path: str | None = None, + connect_addr: str = "127.0.0.1:9876", + application_id: str = "physicalai-runtime", + ) -> None: + if mode == "save" and save_path is None: + msg = "mode='save' requires save_path" + raise ValueError(msg) + # Fail fast if rerun-sdk is not installed. + import rerun as rr # noqa: PLC0415, F401 + + self._cameras = cameras + self._image_decimation = image_decimation + self._log_images = log_images + self._image_jpeg_quality = image_jpeg_quality + self._image_max_dim = image_max_dim + self._mode = mode + self._save_path = save_path + self._connect_addr = connect_addr + self._application_id = application_id + + self._last_step: int = 0 + self._fps: int = 30 + self._pred_horizon: int = 0 + self._initialized = False + self._blueprint_updated = False + self._camera_subscribers: dict[str, Any] = {} + self._latencies: deque[float] = deque(maxlen=200) + + def on_lifecycle(self, event: LifecycleEvent) -> None: # noqa: D102 + if event.event == "start" and not self._initialized: + self._init_rerun(event.session_id, event.metadata) + self._log_lifecycle_marker(event) + + def on_tick(self, event: TickEvent) -> None: # noqa: D102 + import rerun as rr # noqa: PLC0415 + + self._last_step = event.step + rr.set_time("step", sequence=event.step) + rr.set_time("wall", timestamp=event.timestamp) + + if event.joint_positions is not None: + # One plot with N overlaid series instead of N separate plots. + rr.log("robot/joints", rr.Scalars([float(v) for v in event.joint_positions])) + + if event.action_sent is not None: + rr.log("robot/actions", rr.Scalars([float(v) for v in event.action_sent])) + + rr.log("queue/remaining", rr.Scalars(float(event.queue_remaining))) + rr.log("queue/inference", rr.Scalars(0.0)) + rr.log("runtime/loop_duration_s", rr.Scalars(event.loop_duration_s)) + rr.log("runtime/sleep_time_s", rr.Scalars(event.sleep_time_s)) + rr.log("runtime/stale_obs", rr.Scalars(float(event.stale_obs))) + + if self._log_images and event.step % self._image_decimation == 0: + self._log_camera_frames() + + def on_inference(self, event: InferenceEvent) -> None: # noqa: D102 + import numpy as np # noqa: PLC0415 + import rerun as rr # noqa: PLC0415 + + horizon = event.chunk.shape[0] + n_joints = event.chunk.shape[1] + start_step = self._last_step + 1 + + # Clear previous predictions so stale trajectories don't linger. + if self._pred_horizon > 0: + rr.set_time("step", sequence=self._last_step) + rr.log("robot/predicted", rr.Clear(recursive=False)) + + self._pred_horizon = horizon + + # Batch-log all prediction steps in one efficient send_columns call. + # send_columns bypasses the thread-local time context, so the viewer's + # "latest" cursor is not pushed to the last prediction step. + steps = np.arange(start_step, start_step + horizon, dtype=np.int64) + wall_times = event.timestamp + np.arange(horizon, dtype=np.float64) / self._fps + + # Scalars expects one float per row when logging a single series, + # but for N joints we need N values per row → use partitioned columns. + flat_scalars = event.chunk.astype(np.float64).ravel() + rr.send_columns( + "robot/predicted", + indexes=[ + rr.TimeColumn("step", sequence=steps), + rr.TimeColumn("wall", timestamp=wall_times), + ], + columns=rr.Scalars.columns(scalars=flat_scalars).partition(lengths=[n_joints] * horizon), + ) + + # Mark the inference event on the queue timeline (shows as a spike/refill). + rr.set_time("step", sequence=self._last_step) + rr.set_time("wall", timestamp=event.timestamp) + rr.log("queue/inference", rr.Scalars(float(horizon))) + + # Inference latency stats as a live-updating table. + self._latencies.append(event.latency_s) + self._log_latency_table() + + # Re-send blueprint with correct horizon on first inference. + if not self._blueprint_updated: + self._blueprint_updated = True + self._send_default_blueprint() + + def close(self) -> None: + """Release independent camera subscribers (SharedCamera only).""" + from physicalai.capture.transport._shared_camera import SharedCamera # noqa: PLC0415, PLC2701 + + for sub in self._camera_subscribers.values(): + if not isinstance(sub, SharedCamera): + continue + try: + sub.disconnect() + except Exception: + logger.exception("Error closing RerunCallback camera subscriber") + self._camera_subscribers.clear() + + def _init_rerun(self, session_id: str, metadata: dict[str, Any]) -> None: + import rerun as rr # noqa: PLC0415 + + rr.init(application_id=self._application_id, recording_id=session_id) + if self._mode == "spawn": + rr.spawn() + elif self._mode == "save": + rr.save(self._save_path) + elif self._mode == "connect": + # Rerun 0.22+ uses gRPC. Address like "127.0.0.1:9876" is wrapped + # into the canonical rerun+http://host:port/proxy URL. + addr = self._connect_addr + url = addr if addr.startswith(("rerun+http://", "rerun+https://")) else f"rerun+http://{addr}/proxy" + rr.connect_grpc(url=url) + + self._fps = metadata.get("fps", 30) + self._joint_names: list[str] = metadata.get("joint_names", []) + self._initialized = True + + self._send_series_styles() + self._open_camera_subscribers() + self._send_default_blueprint() + + @staticmethod + def _generate_joint_colors(n: int) -> list[list[int]]: + """Generate N perceptually distinct colors via evenly-spaced hues. + + Returns: + List of RGBA colors with values in [0, 255]. + """ + colors = [] + for i in range(n): + hue = i / n + r, g, b = colorsys.hsv_to_rgb(hue, 0.75, 0.85) + colors.append([int(r * 255), int(g * 255), int(b * 255), 255]) + return colors + + def _send_series_styles(self) -> None: + """Set static visual style for series: solid lines for actions, dots for predicted.""" + import rerun as rr # noqa: PLC0415 + + names = self._joint_names or None + n_joints = len(self._joint_names) if self._joint_names else 0 + joint_colors = self._generate_joint_colors(n_joints) if n_joints else None + + # Actions: distinct color per joint so you can identify each line + action_names = [f"{n} (action)" for n in self._joint_names] if self._joint_names else None + rr.log("robot/actions", rr.SeriesLines(widths=2.0, colors=joint_colors, names=action_names), static=True) + # Predicted: same joint colors at lower alpha + cross markers to distinguish from action lines + pred_names = [f"{n} (pred)" for n in self._joint_names] if self._joint_names else None + pred_colors = [[r, g, b, 100] for r, g, b, _a in joint_colors] if joint_colors else None + rr.log( + "robot/predicted", + rr.SeriesPoints(marker_sizes=4.0, colors=pred_colors, markers="cross", names=pred_names), + static=True, + ) + # Joints: same distinct colors as actions for consistency + rr.log("robot/joints", rr.SeriesLines(widths=1.5, colors=joint_colors, names=names), static=True) + # Queue: green line; inference: thin red vertical spikes + rr.log("queue/remaining", rr.SeriesLines(widths=3.0, colors=[80, 200, 120, 255], names="queue"), static=True) + rr.log("queue/inference", rr.SeriesLines(widths=1.5, colors=[220, 50, 50, 255], names="inference"), static=True) + + def _send_default_blueprint(self) -> None: + """Send a default blueprint: actions+predicted overlaid, queue, joints, cameras.""" + try: + import rerun as rr # noqa: PLC0415 + import rerun.blueprint as rrb # noqa: PLC0415 + except ImportError: + logger.debug("rerun.blueprint not available; skipping default blueprint") + return + + camera_names = list((self._cameras or {}).keys()) if self._log_images else [] + fps = int(self._fps) + horizon = self._pred_horizon or int(fps * 1.5) # best-guess until first inference + + # The viewer's cursor tracks the latest logged "step" value. + # With send_columns, predictions are at [current+1 … current+horizon], + # so the cursor sits roughly `horizon` steps ahead of the actual tick. + # We size the visible window so actions and predictions get equal space: + # lookback = 2*horizon → horizon steps of history + horizon steps of predictions. + lookback = horizon * 2 + actions_range = rrb.VisibleTimeRange( + timeline="step", + start=rrb.TimeRangeBoundary.cursor_relative(seq=-lookback), + end=rrb.TimeRangeBoundary.cursor_relative(seq=0), + ) + + latency_view = rrb.TextDocumentView( + origin="/inference/stats", + name="Inference Latency", + ) + if camera_names: + top_row = rrb.Grid( + contents=[ + *[rrb.Spatial2DView(origin=f"/camera/{n}", name=n) for n in camera_names], + latency_view, + ], + ) + else: + top_row = latency_view + + views: list[Any] = [ + top_row, + rrb.Tabs( + rrb.TimeSeriesView( + origin="/robot", + contents=["/robot/actions", "/robot/predicted"], + name="Actions vs Predicted", + time_ranges=actions_range, + ), + rrb.TimeSeriesView( + origin="/robot/joints", + name="Joint State", + time_ranges=actions_range, + ), + active_tab="Actions vs Predicted", + ), + rrb.TimeSeriesView( + origin="/queue", + name="Action Queue", + time_ranges=actions_range, + ), + ] + + blueprint = rrb.Blueprint( + rrb.Vertical(*views), + rrb.SelectionPanel(state="collapsed"), + rrb.TimePanel(state="expanded"), + ) + try: + rr.send_blueprint(blueprint, make_active=True, make_default=True) + except Exception: + logger.debug("Failed to send Rerun blueprint", exc_info=True) + + def _open_camera_subscribers(self) -> None: + if not self._log_images: + return + from physicalai.capture.transport._shared_camera import SharedCamera # noqa: PLC0415, PLC2701 + + for name, cam in (self._cameras or {}).items(): + if isinstance(cam, SharedCamera): + sub = SharedCamera( + camera_type=None, + service_name=cam.service_name, + validate_on_connect=False, + ) + sub.connect() + self._camera_subscribers[name] = sub + else: + # Direct camera — read from it on tick (no separate subscriber needed) + self._camera_subscribers[name] = cam + + def _log_camera_frames(self) -> None: + import rerun as rr # noqa: PLC0415 + + for name, sub in self._camera_subscribers.items(): + try: + frame = sub.read_latest() + data = frame.data + if self._image_max_dim is not None: + data = _downsample_to_max_dim(data, self._image_max_dim) + img = rr.Image(data) + if self._image_jpeg_quality is not None: + img = img.compress(jpeg_quality=self._image_jpeg_quality) + rr.log(f"camera/{name}", img) + except Exception: + logger.debug("RerunCallback: failed to read camera %r", name, exc_info=True) + + def _log_lifecycle_marker(self, event: LifecycleEvent) -> None: + import rerun as rr # noqa: PLC0415 + + rr.set_time("step", sequence=self._last_step) + rr.set_time("wall", timestamp=event.timestamp) + rr.log( + f"runtime/lifecycle/{event.event}", + rr.TextLog(f"{event.event}: {event.metadata}"), + ) + + def _log_latency_table(self) -> None: + import numpy as np # noqa: PLC0415 + import rerun as rr # noqa: PLC0415 + + arr = np.array(self._latencies) + p50 = float(np.percentile(arr, 50)) + p95 = float(np.percentile(arr, 95)) + p99 = float(np.percentile(arr, 99)) + last = float(arr[-1]) + n = len(arr) + # Headroom = time the queue can sustain minus inference latency. + # Positive = safe; negative = queue starved before next chunk arrives. + queue_time = self._pred_horizon / self._fps if self._pred_horizon else 0 + headroom = queue_time - p99 + + md = ( + f"| Metric | Value |\n" + f"|--------|-------|\n" + f"| **Last** | {last * 1000:.1f} ms |\n" + f"| **p50** | {p50 * 1000:.1f} ms |\n" + f"| **p95** | {p95 * 1000:.1f} ms |\n" + f"| **p99** | {p99 * 1000:.1f} ms |\n" + f"| **Queue headroom** | {headroom * 1000:.0f} ms |\n" + f"| Samples | {n} |" + ) + rr.log("inference/stats", rr.TextDocument(md, media_type=rr.MediaType.MARKDOWN)) + + +def _np_to_list(arr: np.ndarray | None) -> list[float] | None: + if arr is None: + return None + return arr.tolist() + + +def _downsample_to_max_dim(data: np.ndarray, max_dim: int) -> np.ndarray: + """Subsample image so the longer side is <= ``max_dim``. No-op if already smaller. + + Returns: + Subsampled image. Does not modify input. + """ + h, w = data.shape[:2] + longer = max(h, w) + if longer <= max_dim: + return data + stride = (longer + max_dim - 1) // max_dim # ceil-divide + return data[::stride, ::stride] + + +def _json_default(obj: object) -> Any: # noqa: ANN401 + import numpy as np # noqa: PLC0415 + + if isinstance(obj, np.ndarray): + return obj.tolist() + if isinstance(obj, np.integer): + return int(obj) + if isinstance(obj, np.floating): + return float(obj) + msg = f"Object of type {type(obj)} is not JSON serializable" + raise TypeError(msg) diff --git a/src/physicalai/runtime/events.py b/src/physicalai/runtime/events.py new file mode 100644 index 0000000..6ee1fdc --- /dev/null +++ b/src/physicalai/runtime/events.py @@ -0,0 +1,57 @@ +# Copyright (C) 2025-2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Event dataclasses for the runtime callback bus.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + import numpy as np + + +@dataclass(frozen=True, slots=True) +class TickEvent: + """Emitted once per control-loop tick. + + All timestamps are wall-clock UTC seconds (``time.time()``). + """ + + session_id: str + step: int + timestamp: float + joint_positions: np.ndarray | None + action_sent: np.ndarray | None + queue_remaining: int + loop_duration_s: float + sleep_time_s: float + stale_obs: bool + + +@dataclass(frozen=True, slots=True) +class InferenceEvent: + """Emitted when an inference call completes (sync or async). + + All timestamps are wall-clock UTC seconds (``time.time()``). + """ + + session_id: str + timestamp: float + latency_s: float + offset: int + chunk: np.ndarray + + +@dataclass(frozen=True, slots=True) +class LifecycleEvent: + """Emitted on session boundaries and error conditions. + + All timestamps are wall-clock UTC seconds (``time.time()``). + """ + + session_id: str + timestamp: float + event: str + metadata: dict[str, Any] diff --git a/src/physicalai/runtime/execution.py b/src/physicalai/runtime/execution.py index bb2c854..7207ddc 100644 --- a/src/physicalai/runtime/execution.py +++ b/src/physicalai/runtime/execution.py @@ -16,6 +16,7 @@ if TYPE_CHECKING: from physicalai.inference.model import InferenceModel from physicalai.runtime._action_queue import ActionQueue + from physicalai.runtime._callback_bus import _CallbackBus logger = logging.getLogger(__name__) @@ -29,6 +30,14 @@ class WorkerDiedError(RuntimeError): class Execution(ABC): """Decides when and where inference runs. Pushes results into ActionQueue.""" + _bus: _CallbackBus | None + _session_id: str + + def set_bus(self, bus: _CallbackBus, session_id: str) -> None: + """Inject callback bus and session ID before the control loop starts.""" + self._bus = bus + self._session_id = session_id + @abstractmethod def start(self, model: InferenceModel, action_queue: ActionQueue) -> None: """Bind to model and queue. Called once before the loop.""" @@ -59,10 +68,30 @@ def chunk_size(self) -> int: class SyncExecution(Execution): """Synchronous inference in the control thread.""" - def __init__(self) -> None: # noqa: D107 + def __init__( + self, + fps: int = 30, + *, + request_threshold: float = 0.5, + ) -> None: + """Configure synchronous execution. + + Args: + fps: Control loop frequency. + request_threshold: Re-infer when queue drops below this fraction + of chunk_size. E.g. 0.5 means re-infer after consuming half + the chunk (discards the stale tail). Set to 0.0 to drain + the entire chunk before re-inferring. + """ self._model: InferenceModel | None = None self._queue: ActionQueue | None = None self._chunk_size: int = 0 + self._fps = fps + self._threshold_frac = request_threshold + self._threshold_count: int = 0 + self._inference_count: int = 0 + self._bus: _CallbackBus | None = None + self._session_id: str = "" def start(self, model: InferenceModel, action_queue: ActionQueue) -> None: """Bind model and queue.""" @@ -79,19 +108,35 @@ def warmup(self, sample_observation: dict[str, np.ndarray]) -> None: raise RuntimeError(_NOT_STARTED) actions = self._model.predict_action_chunk(sample_observation) self._chunk_size = actions.shape[0] + self._threshold_count = max(1, int(self._chunk_size * self._threshold_frac)) self._queue.push_chunk(actions, offset=0) def maybe_request(self, observation: dict[str, np.ndarray]) -> None: - """Refill queue synchronously when empty. + """Refill queue synchronously when below threshold. Raises: RuntimeError: If start() has not been called. """ if self._model is None or self._queue is None: raise RuntimeError(_NOT_STARTED) - if self._queue.below_threshold(1): + if self._queue.below_threshold(self._threshold_count): + t0 = time.perf_counter() actions = self._model.predict_action_chunk(observation) + latency = time.perf_counter() - t0 self._queue.push_chunk(actions, offset=0) + self._inference_count += 1 + if self._bus: + from physicalai.runtime.events import InferenceEvent # noqa: PLC0415 + + self._bus.emit_inference( + InferenceEvent( + session_id=self._session_id, + timestamp=time.time(), + latency_s=latency, + offset=0, + chunk=actions, + ) + ) def stop(self) -> None: """No-op for synchronous execution.""" @@ -101,18 +146,34 @@ def chunk_size(self) -> int: """Return discovered chunk size.""" return self._chunk_size + @property + def inference_count(self) -> int: + """Number of completed inference calls.""" + return self._inference_count + class AsyncExecution(Execution): """Async inference in a background thread with health monitoring.""" - def __init__( # noqa: D107 + def __init__( self, - threshold: float = 0.5, + request_threshold: float = 0.5, fps: int = 30, watchdog_timeout_s: float = 30.0, max_consecutive_holds: int | None = None, ) -> None: - self._threshold_frac = threshold + """Configure the async execution strategy. + + Args: + request_threshold: Queue fraction at which to request new inference. + When the action queue drops below this fraction of chunk_size, + a new inference is scheduled. E.g. 0.25 means "request when + only 25% of the chunk remains in the queue." + fps: Control loop frequency (used to compute offset from latency). + watchdog_timeout_s: If inference is stuck longer than this, force-reset. + max_consecutive_holds: Max ticks with empty queue before raising. + """ + self._threshold_frac = request_threshold self._fps = fps self._watchdog_timeout_s = watchdog_timeout_s self._max_consecutive_holds = max_consecutive_holds or 3 * fps @@ -127,10 +188,13 @@ def __init__( # noqa: D107 self._obs_ready = threading.Event() self._running_inference = False self._request_time: float = 0.0 + self._pops_at_request: int = 0 self._stop_event = threading.Event() self._thread: threading.Thread | None = None self._death_cause: BaseException | None = None self._inference_count: int = 0 + self._bus: _CallbackBus | None = None + self._session_id: str = "" def start(self, model: InferenceModel, action_queue: ActionQueue) -> None: """Bind model/queue and spawn inference thread.""" @@ -174,6 +238,7 @@ def maybe_request(self, observation: dict[str, np.ndarray]) -> None: with self._lock: self._obs_slot = snapshot self._request_time = time.perf_counter() + self._pops_at_request = self._queue.total_pops self._obs_ready.set() def stop(self) -> None: @@ -238,10 +303,27 @@ def _run(self) -> None: actions = self._model.predict_action_chunk(obs) latency = time.perf_counter() - t0 - offset = int(latency * self._fps) + # Offset = actions actually sent since the observation was + # captured. This is exact (no fps estimation error). + with self._lock: + pops_since = self._queue.total_pops - self._pops_at_request + offset = min(max(pops_since, 0), len(actions) - 1) self._queue.push_chunk(actions, offset=offset) self._inference_count += 1 + if self._bus: + from physicalai.runtime.events import InferenceEvent # noqa: PLC0415 + + self._bus.emit_inference( + InferenceEvent( + session_id=self._session_id, + timestamp=time.time(), + latency_s=latency, + offset=offset, + chunk=actions, + ) + ) + with self._lock: self._running_inference = False diff --git a/src/physicalai/runtime/observer/__init__.py b/src/physicalai/runtime/observer/__init__.py new file mode 100644 index 0000000..898120e --- /dev/null +++ b/src/physicalai/runtime/observer/__init__.py @@ -0,0 +1,10 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Runtime telemetry observer — subscribes to zenoh pub-sub events.""" + +from __future__ import annotations + +from physicalai.runtime.observer._subscriber import TelemetrySubscriber # noqa: PLC2701 + +__all__ = ["TelemetrySubscriber"] diff --git a/src/physicalai/runtime/observer/__main__.py b/src/physicalai/runtime/observer/__main__.py new file mode 100644 index 0000000..781e66a --- /dev/null +++ b/src/physicalai/runtime/observer/__main__.py @@ -0,0 +1,67 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""CLI entry point for ``python -m physicalai.runtime.observer``.""" + +from __future__ import annotations + +import argparse + + +def main(argv: list[str] | None = None) -> None: + """Run the telemetry observer with optional recording. + + Raises: + SystemExit: If zenoh/msgpack dependencies are missing. + """ + parser = argparse.ArgumentParser( + prog="python -m physicalai.runtime.observer", + description="Observe runtime telemetry from a running PolicyRuntime session", + ) + parser.add_argument("--session-id", default=None, help="Filter to a specific session ID") + parser.add_argument("--record", default=None, metavar="PATH", help="Record events to JSONL file") + parser.add_argument("--no-console", action="store_true", help="Disable live console output") + args = parser.parse_args(argv) + + try: + from physicalai.runtime.observer._subscriber import TelemetrySubscriber # noqa: PLC0415, PLC2701 + except ImportError: + raise SystemExit(1) from None + + subscriber = TelemetrySubscriber(session_id=args.session_id) + + if not args.no_console: + from physicalai.runtime.observer._console import ConsoleHandler # noqa: PLC0415, PLC2701 + + subscriber.add_handler(ConsoleHandler()) + + recorder = None + if args.record: + from pathlib import Path # noqa: PLC0415 + + from physicalai.runtime.observer._recorder import RecorderHandler # noqa: PLC0415, PLC2701 + + recorder = RecorderHandler(Path(args.record)) + subscriber.add_handler(recorder) + + subscriber.start() + + try: + import signal # noqa: PLC0415 + + signal.pause() + except AttributeError: + import time # noqa: PLC0415 + + while True: + time.sleep(1) + except KeyboardInterrupt: + pass + finally: + subscriber.stop() + if recorder: + recorder.close() + + +if __name__ == "__main__": + main() diff --git a/src/physicalai/runtime/observer/_console.py b/src/physicalai/runtime/observer/_console.py new file mode 100644 index 0000000..de4a467 --- /dev/null +++ b/src/physicalai/runtime/observer/_console.py @@ -0,0 +1,49 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import sys +from typing import Any + + +class ConsoleHandler: + def __init__(self, target_fps: float = 30.0) -> None: + self._target_fps = target_fps + self._last_step = -1 + + def __call__(self, session_id: str, topic: str, payload: dict[str, Any]) -> None: + if topic == "tick": + self._handle_tick(session_id, payload) + elif topic == "inference": + self._handle_inference(session_id, payload) + elif topic == "lifecycle": + self._handle_lifecycle(session_id, payload) + + def _handle_tick(self, session_id: str, payload: dict[str, Any]) -> None: + step = payload.get("step", "?") + loop_ms = payload.get("physicalai.runtime.loop_duration_s", 0) * 1000 + queue = payload.get("queue_remaining", "?") + stale = " [STALE]" if payload.get("stale_obs") else "" + actual_fps = 1000 / loop_ms if loop_ms > 0 else 0 + line = ( + f"\r[{session_id}] step={step} " + f"fps={actual_fps:.0f}/{self._target_fps:.0f} " + f"loop={loop_ms:.1f}ms " + f"queue={queue}{stale} " + ) + sys.stdout.write(line) + sys.stdout.flush() + + @staticmethod + def _handle_inference(session_id: str, payload: dict[str, Any]) -> None: + latency_ms = payload.get("physicalai.runtime.inference_latency_s", 0) * 1000 + offset = payload.get("offset", "?") + sys.stdout.write(f"\n[{session_id}] inference: latency={latency_ms:.0f}ms offset={offset}\n") + sys.stdout.flush() + + @staticmethod + def _handle_lifecycle(session_id: str, payload: dict[str, Any]) -> None: + event = payload.get("event", "unknown") + sys.stdout.write(f"\n[{session_id}] lifecycle: {event} {payload}\n") + sys.stdout.flush() diff --git a/src/physicalai/runtime/observer/_recorder.py b/src/physicalai/runtime/observer/_recorder.py new file mode 100644 index 0000000..1473824 --- /dev/null +++ b/src/physicalai/runtime/observer/_recorder.py @@ -0,0 +1,40 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import Any + +import numpy as np + +logger = logging.getLogger(__name__) + + +class _NumpyEncoder(json.JSONEncoder): + def default(self, o: object) -> object: + if isinstance(o, np.ndarray): + return o.tolist() + if isinstance(o, (np.integer,)): + return int(o) + if isinstance(o, (np.floating,)): + return float(o) + return super().default(o) + + +class RecorderHandler: + def __init__(self, output_path: Path) -> None: + self._path = Path(output_path) + self._path.parent.mkdir(parents=True, exist_ok=True) + self._file = Path(self._path).open("a", encoding="utf-8") # noqa: SIM115 + + def __call__(self, session_id: str, topic: str, payload: dict[str, Any]) -> None: + record = {"session_id": session_id, "topic": topic, **payload} + self._file.write(json.dumps(record, cls=_NumpyEncoder) + "\n") + self._file.flush() + + def close(self) -> None: + if self._file and not self._file.closed: + self._file.close() diff --git a/src/physicalai/runtime/observer/_subscriber.py b/src/physicalai/runtime/observer/_subscriber.py new file mode 100644 index 0000000..7d08bb0 --- /dev/null +++ b/src/physicalai/runtime/observer/_subscriber.py @@ -0,0 +1,74 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from physicalai.runtime._telemetry import _decode_numpy # noqa: PLC2701 + +if TYPE_CHECKING: + from collections.abc import Callable + +logger = logging.getLogger(__name__) + +_MIN_TOPIC_PARTS = 4 + + +def _unpack_numpy(obj: object) -> object: + if isinstance(obj, dict) and obj.get("__np__"): + return _decode_numpy(obj) + if isinstance(obj, dict): + return {k: _unpack_numpy(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_unpack_numpy(v) for v in obj] + return obj + + +class TelemetrySubscriber: + def __init__(self, session_id: str | None = None) -> None: + import msgpack # noqa: PLC0415 + import zenoh # noqa: PLC0415 + + self._zenoh = zenoh + self._msgpack = msgpack + self._session = zenoh.open(zenoh.Config()) + self._handlers: list[Callable[[str, str, dict[str, Any]], None]] = [] + self._session_id = session_id + self._sub: Any = None + + def add_handler(self, handler: Callable[[str, str, dict[str, Any]], None]) -> None: + self._handlers.append(handler) + + def start(self) -> None: + prefix = f"physicalai/rt/{self._session_id}/**" if self._session_id else "physicalai/rt/**" + self._sub = self._session.declare_subscriber(prefix, self._on_event) + + def _on_event(self, sample: Any) -> None: # noqa: ANN401 + try: + key = str(sample.key_expr) + parts = key.split("/") + if len(parts) < _MIN_TOPIC_PARTS: + return + session_id = parts[2] + topic = parts[3] + payload = self._msgpack.unpackb(sample.payload.to_bytes(), raw=False) + payload = _unpack_numpy(payload) + if not isinstance(payload, dict): + return + for handler in self._handlers: + try: + handler(session_id, topic, payload) + except Exception: + logger.exception("Handler error") + except Exception: + logger.exception("Failed to decode telemetry event") + + def stop(self) -> None: + if self._sub is not None: + self._sub.undeclare() + self._sub = None + if self._session is not None: + self._session.close() + self._session = None diff --git a/src/physicalai/runtime/runtime.py b/src/physicalai/runtime/runtime.py index 801daa9..0a4a002 100644 --- a/src/physicalai/runtime/runtime.py +++ b/src/physicalai/runtime/runtime.py @@ -7,12 +7,16 @@ import logging import time +import uuid from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Protocol, Self import numpy as np +from physicalai.capture.errors import CaptureError from physicalai.runtime._action_queue import ActionQueue # noqa: PLC2701 +from physicalai.runtime._callback_bus import _CallbackBus # noqa: PLC2701 +from physicalai.runtime.events import LifecycleEvent, TickEvent from physicalai.runtime.execution import Execution, WorkerDiedError from physicalai.runtime.smoothers import LerpSmoother @@ -20,12 +24,19 @@ from collections.abc import Mapping, Sequence from physicalai.capture.camera import Camera + from physicalai.capture.frame import Frame from physicalai.inference.model import InferenceModel - from physicalai.robot.interface import Robot + from physicalai.robot.interface import Robot, RobotObservation logger = logging.getLogger(__name__) _DEFAULT_LERP_FRAMES = 5 +_MAX_OBS_RETRIES = 3 +_MAX_SEND_RETRIES = 2 +_RETRY_BACKOFF_S = 0.001 +_WARMUP_RETRIES = 5 +_WARMUP_BACKOFF_S = 1.0 +_GOAL_TIME_TICKS = 3 class RuntimeCallback(Protocol): @@ -76,6 +87,7 @@ def __init__( # noqa: D107 cameras: Mapping[str, Camera] | None = None, action_queue: ActionQueue | None = None, callbacks: Sequence[RuntimeCallback] = (), + task: str | None = None, ) -> None: if fps <= 0: msg = f"fps must be positive, got {fps}" @@ -86,9 +98,27 @@ def __init__( # noqa: D107 self._fps = fps self._cameras: Mapping[str, Camera] = cameras or {} self._action_queue = action_queue or ActionQueue(smoother=LerpSmoother(duration_frames=_DEFAULT_LERP_FRAMES)) - self._callbacks = list(callbacks) - self._goal_time = (1.0 / fps) * 3 + self._bus = _CallbackBus(callbacks) + self._goal_time = (1.0 / fps) * _GOAL_TIME_TICKS + self._task = task self._connected = False + self._last_robot_obs: RobotObservation | None = None + self._last_camera_frames: dict[str, Frame] = {} + self._consecutive_error_ticks: int = 0 + self._max_consecutive_error_ticks: int = int(3 * fps) + self._stale_obs_ticks: int = 0 + self._transient_errors: int = 0 + self._session_id: str = "" + + @property + def robot(self) -> Robot: + """The robot instance managed by this runtime.""" + return self._robot + + @property + def cameras(self) -> Mapping[str, Camera]: + """Camera instances managed by this runtime, keyed by name.""" + return self._cameras def connect(self) -> None: """Connect robot and cameras. @@ -149,7 +179,7 @@ def __enter__(self) -> Self: # noqa: D105 def __exit__(self, *exc_info: object) -> None: # noqa: D105 self.disconnect() - def run(self, *, duration_s: float | None = None) -> RunStats: + def run(self, *, duration_s: float | None = None) -> RunStats: # noqa: PLR0915 """Run the control loop. Args: @@ -165,13 +195,30 @@ def run(self, *, duration_s: float | None = None) -> RunStats: if not self._connected: msg = "PolicyRuntime.run() called before connect(). Use 'with runtime:' or call runtime.connect() first." raise RuntimeError(msg) + + self._session_id = uuid.uuid4().hex[:8] + self._execution.set_bus(self._bus, self._session_id) + self._execution.start(self._model, self._action_queue) - sample_obs = self._build_model_input() - self._execution.warmup(sample_obs) + self._bus.emit_lifecycle( + LifecycleEvent( + session_id=self._session_id, + timestamp=time.time(), + event="start", + metadata={ + "fps": self._fps, + "duration_s": duration_s, + "cameras": list(self._cameras.keys()), + "joint_names": self._robot.joint_names, + }, + ) + ) + self._warmup_with_retry() goal_time = 1.0 / self._fps step = 0 last_action: np.ndarray | None = None + stale_this_tick = False try: while True: @@ -179,8 +226,11 @@ def run(self, *, duration_s: float | None = None) -> RunStats: break loop_start = time.perf_counter() + stale_this_tick = False - obs = self._build_model_input() + obs = self._resilient_observe() + if self._consecutive_error_ticks > 0: + stale_this_tick = True self._execution.maybe_request(obs) action = self._action_queue.pop() @@ -196,14 +246,29 @@ def run(self, *, duration_s: float | None = None) -> RunStats: step += 1 continue - modified = self._invoke_callback("before_send_action", action=action, step=step) - if modified is not None: - action = modified - - self._robot.send_action(action, goal_time=self._goal_time) - self._invoke_callback("on_action_sent", action=action, step=step) - self._tick_sleep(loop_start, goal_time) - + action = self._bus.invoke_before_send_action(action=action, step=step) + + self._resilient_send(action) + self._bus.invoke_on_action_sent(action=action, step=step) + + elapsed = time.perf_counter() - loop_start + sleep_time = goal_time - elapsed + if sleep_time > 0: + time.sleep(sleep_time) + + self._bus.emit_tick( + TickEvent( + session_id=self._session_id, + step=step, + timestamp=time.time(), + joint_positions=self._last_robot_obs.joint_positions if self._last_robot_obs else None, + action_sent=action, + queue_remaining=self._action_queue.remaining, + loop_duration_s=elapsed, + sleep_time_s=max(sleep_time, 0.0), + stale_obs=stale_this_tick, + ) + ) step += 1 except KeyboardInterrupt: @@ -219,6 +284,8 @@ def run(self, *, duration_s: float | None = None) -> RunStats: total_pops=self._action_queue.total_pops, total_holds=self._action_queue.total_holds, inference_count=getattr(self._execution, "inference_count", 0), + transient_errors=self._transient_errors, + stale_obs_ticks=self._stale_obs_ticks, ) def _handle_hold(self, *, step: int) -> None: @@ -233,21 +300,19 @@ def _handle_hold(self, *, step: int) -> None: holds, holds / self._fps, ) - self._invoke_callback("on_hold", step=step, holds=holds) + self._bus.invoke_on_hold(step=step, holds=holds) @staticmethod - def _tick_sleep(loop_start: float, goal_time: float) -> None: + def _tick_sleep(loop_start: float, goal_time: float) -> tuple[float, float]: elapsed = time.perf_counter() - loop_start sleep_time = goal_time - elapsed if sleep_time > 0: time.sleep(sleep_time) + return elapsed, sleep_time def _build_model_input(self) -> dict[str, Any]: robot_obs = self._robot.get_observation() - model_input: dict[str, Any] = {} - - if robot_obs.joint_positions is not None: - model_input["state"] = np.array([robot_obs.joint_positions], dtype=np.float32) + model_input: dict[str, Any] = {"state": np.array([robot_obs.state], dtype=np.float32)} # Merge robot-embedded images and external cameras if robot_obs.images: @@ -256,8 +321,161 @@ def _build_model_input(self) -> dict[str, Any]: for name, cam in self._cameras.items(): model_input[f"images.{name}"] = cam.read_latest().data[np.newaxis] + if self._task is not None: + model_input["task"] = [self._task] + + return model_input + + def _retry_robot_obs(self) -> tuple[RobotObservation | None, ConnectionError | OSError | None]: + robot_obs: RobotObservation | None = None + last_error: ConnectionError | OSError | None = None + for attempt in range(_MAX_OBS_RETRIES): + try: + robot_obs = self._robot.get_observation() + except (ConnectionError, OSError) as exc: + last_error = exc + if attempt + 1 < _MAX_OBS_RETRIES: + time.sleep(_RETRY_BACKOFF_S) + else: + break + return robot_obs, last_error + + def _resilient_observe(self) -> dict[str, Any]: + robot_obs, last_robot_error = self._retry_robot_obs() + + if robot_obs is None: + if self._last_robot_obs is None: + self._bus.emit_lifecycle( + LifecycleEvent( + session_id=self._session_id, + timestamp=time.time(), + event="connection_lost", + metadata={"error": str(last_robot_error)}, + ) + ) + msg = "Robot observation failed and no stale observation available" + raise ConnectionError(msg) from last_robot_error + + self._consecutive_error_ticks += 1 + self._stale_obs_ticks += 1 + if self._consecutive_error_ticks >= self._max_consecutive_error_ticks: + self._bus.emit_lifecycle( + LifecycleEvent( + session_id=self._session_id, + timestamp=time.time(), + event="connection_lost", + metadata={"error": str(last_robot_error)}, + ) + ) + msg = "Exceeded max consecutive robot observation failures" + raise ConnectionError(msg) from last_robot_error + + self._bus.emit_lifecycle( + LifecycleEvent( + session_id=self._session_id, + timestamp=time.time(), + event="obs_error", + metadata={"error": str(last_robot_error), "stale": True}, + ) + ) + robot_obs = self._last_robot_obs + else: + self._consecutive_error_ticks = 0 + self._last_robot_obs = robot_obs + + camera_frames: dict[str, Frame] = {} + for name, camera in self._cameras.items(): + try: + frame = camera.read_latest() + camera_frames[name] = frame + self._last_camera_frames[name] = frame + except CaptureError as exc: + stale_frame = self._last_camera_frames.get(name) + if stale_frame is None: + raise + logger.warning( + "Camera %s read failed — using stale frame: %s", + name, + exc, + ) + camera_frames[name] = stale_frame + + model_input: dict[str, Any] = {"state": np.array([robot_obs.state], dtype=np.float32)} + if robot_obs.images: + for name, frame in robot_obs.images.items(): + model_input[f"images.{name}"] = frame.data[np.newaxis] + for name, frame in camera_frames.items(): + model_input[f"images.{name}"] = frame.data[np.newaxis] + if self._task is not None: + model_input["task"] = [self._task] return model_input + def _resilient_send(self, action: np.ndarray) -> None: + last_error: ConnectionError | OSError | None = None + + for attempt in range(_MAX_SEND_RETRIES): + try: + self._robot.send_action(action, goal_time=self._goal_time) + except (ConnectionError, OSError) as exc: + last_error = exc + if attempt + 1 < _MAX_SEND_RETRIES: + time.sleep(_RETRY_BACKOFF_S) + else: + self._consecutive_error_ticks = 0 + return + + self._transient_errors += 1 + self._consecutive_error_ticks += 1 + if self._consecutive_error_ticks >= self._max_consecutive_error_ticks: + self._bus.emit_lifecycle( + LifecycleEvent( + session_id=self._session_id, + timestamp=time.time(), + event="connection_lost", + metadata={"error": str(last_error), "source": "send"}, + ) + ) + msg = "Exceeded max consecutive send failures" + raise ConnectionError(msg) from last_error + self._bus.emit_lifecycle( + LifecycleEvent( + session_id=self._session_id, + timestamp=time.time(), + event="send_error", + metadata={"error": str(last_error)}, + ) + ) + logger.error( + "Failed to send action after %d attempts; skipping tick: %s", + _MAX_SEND_RETRIES, + last_error, + ) + + def _warmup_with_retry(self) -> None: + last_error: ConnectionError | OSError | None = None + + for attempt in range(_WARMUP_RETRIES): + try: + sample_obs = self._build_model_input() + self._execution.warmup(sample_obs) + except (ConnectionError, OSError) as exc: + last_error = exc + if attempt + 1 < _WARMUP_RETRIES: + time.sleep(_WARMUP_BACKOFF_S) + else: + return + + msg = f"Warmup failed after {_WARMUP_RETRIES} attempts" + self._bus.emit_lifecycle( + LifecycleEvent( + session_id=self._session_id, + timestamp=time.time(), + event="warmup_failed", + metadata={"error": str(last_error), "attempts": _WARMUP_RETRIES}, + ) + ) + raise ConnectionError(msg) from last_error + def _shutdown(self, step: int) -> None: self._execution.stop() @@ -266,27 +484,30 @@ def _shutdown(self, step: int) -> None: for _ in range(drain_limit): action = self._action_queue.pop() if action is not None: - self._robot.send_action(action) + try: + self._resilient_send(action) + except ConnectionError: + logger.warning("Send failed during drain; skipping remaining actions") + break time.sleep(1.0 / self._fps) + self._bus.emit_lifecycle( + LifecycleEvent( + session_id=self._session_id, + timestamp=time.time(), + event="shutdown", + metadata={ + "steps": step, + "transient_errors": self._transient_errors, + "stale_obs_ticks": self._stale_obs_ticks, + }, + ) + ) + self._bus.close() + logger.info( "Shutdown complete — %d steps, %d pops, %d holds", step, self._action_queue.total_pops, self._action_queue.total_holds, ) - - def _invoke_callback(self, method: str, **kwargs: Any) -> Any: # noqa: ANN401 - result = None - for cb in self._callbacks: - fn = getattr(cb, method, None) - if fn is not None: - try: - callback_result = fn(**kwargs) - if callback_result is not None: - result = callback_result - if method == "before_send_action": - kwargs["action"] = callback_result - except Exception: - logger.exception("Callback %s.%s raised", type(cb).__name__, method) - return result diff --git a/src/physicalai/runtime/smoothers.py b/src/physicalai/runtime/smoothers.py index 58b1d14..feb3b3f 100644 --- a/src/physicalai/runtime/smoothers.py +++ b/src/physicalai/runtime/smoothers.py @@ -19,30 +19,30 @@ class ChunkSmoother(ABC): """Merges a new action chunk into remaining actions from the previous chunk.""" @abstractmethod - def merge(self, remaining: np.ndarray, incoming: np.ndarray, offset: int = 0) -> np.ndarray: + def merge(self, remaining: np.ndarray, incoming: np.ndarray) -> np.ndarray: """Merge a previous remainder with a new incoming chunk.""" raise NotImplementedError class ReplaceSmoother(ChunkSmoother): - """Replace remaining actions with the incoming tail.""" + """Replace remaining actions with the incoming chunk.""" @override - def merge(self, remaining: np.ndarray, incoming: np.ndarray, offset: int = 0) -> np.ndarray: - """Return the incoming chunk after skipping the offset.""" + def merge(self, remaining: np.ndarray, incoming: np.ndarray) -> np.ndarray: + """Return the incoming chunk (remaining is discarded).""" _validate_inputs(remaining, incoming) - return incoming[offset:] + return incoming class LerpSmoother(ChunkSmoother): """Blend overlapping actions and append the incoming tail.""" def __init__(self, duration_frames: int = 5) -> None: - """Create a smoother with a fallback lerp window.""" + """Create a smoother with a lerp window.""" self.duration_frames = duration_frames @override - def merge(self, remaining: np.ndarray, incoming: np.ndarray, offset: int = 0) -> np.ndarray: + def merge(self, remaining: np.ndarray, incoming: np.ndarray) -> np.ndarray: """Merge chunks using queue-mixer-style linear interpolation. Returns: @@ -50,10 +50,8 @@ def merge(self, remaining: np.ndarray, incoming: np.ndarray, offset: int = 0) -> """ _validate_inputs(remaining, incoming) - lerp_dur = max(offset, 1) if offset > 0 else self.duration_frames - incoming = incoming[offset:] n_remain = len(remaining) - lerp_dur = min(n_remain, lerp_dur) + lerp_dur = min(n_remain, self.duration_frames) weights = np.maximum(1.0 - np.arange(n_remain) / max(lerp_dur, 1), 0.0) weights = weights[:, np.newaxis] diff --git a/tests/unit/robot/test_bimanual_widowxai.py b/tests/unit/robot/test_bimanual_widowxai.py index ffa8667..7d7a0a6 100644 --- a/tests/unit/robot/test_bimanual_widowxai.py +++ b/tests/unit/robot/test_bimanual_widowxai.py @@ -267,7 +267,7 @@ def test_follower_splits_action(self, mock_trossen_arm: MagicMock) -> None: def test_wrong_shape_raises(self, mock_trossen_arm: MagicMock) -> None: robot = _make_bimanual(mock_trossen_arm, role="follower") - with pytest.raises(ValueError, match="Expected action shape"): + with pytest.raises(ValueError, match="Expected at least"): robot.send_action(np.zeros(7, dtype=np.float32)) def test_leader_raises(self, mock_trossen_arm: MagicMock) -> None: diff --git a/tests/unit/robot/test_widowxai.py b/tests/unit/robot/test_widowxai.py index ba85d9b..a57cee9 100644 --- a/tests/unit/robot/test_widowxai.py +++ b/tests/unit/robot/test_widowxai.py @@ -255,7 +255,7 @@ def test_send_action_wrong_shape_raises(self, mock_trossen_arm: MagicMock) -> No """ValueError on wrong action shape.""" robot = _create_robot(mock_trossen_arm, role="follower") - with pytest.raises(ValueError, match="Expected action shape"): + with pytest.raises(ValueError, match="Expected at least"): robot.send_action(np.zeros(3, dtype=np.float32)) # type: ignore[union-attr] def test_send_action_accepts_degrees_and_sends_radians(self, mock_trossen_arm: MagicMock) -> None: diff --git a/tests/unit/runtime/test_action_queue.py b/tests/unit/runtime/test_action_queue.py index ec60296..82ea17a 100644 --- a/tests/unit/runtime/test_action_queue.py +++ b/tests/unit/runtime/test_action_queue.py @@ -56,6 +56,35 @@ def test_remaining_property(self) -> None: queue.pop() assert queue.remaining == 1 + def test_peek_remaining_returns_copy(self) -> None: + queue = ActionQueue() + chunk = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32) + queue.push_chunk(chunk) + + peeked = queue.peek_remaining() + + assert peeked is not None + np.testing.assert_array_equal(peeked, chunk) + assert queue.remaining == 2 + + action = queue.pop() + assert action is not None + np.testing.assert_array_equal(action, chunk[0]) + + def test_peek_remaining_empty_returns_none(self) -> None: + queue = ActionQueue() + assert queue.peek_remaining() is None + + def test_peek_remaining_shape(self) -> None: + queue = ActionQueue() + chunk = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], dtype=np.float32) + queue.push_chunk(chunk) + + peeked = queue.peek_remaining() + + assert peeked is not None + assert peeked.shape == (2, 3) + def test_below_threshold(self) -> None: queue = ActionQueue() assert queue.below_threshold(1) is True diff --git a/tests/unit/runtime/test_execution.py b/tests/unit/runtime/test_execution.py index 08e354e..e1d7b87 100644 --- a/tests/unit/runtime/test_execution.py +++ b/tests/unit/runtime/test_execution.py @@ -74,6 +74,20 @@ def test_stop_is_noop(self) -> None: ex = SyncExecution() ex.stop() + def test_inference_count_increments(self) -> None: + chunk = np.random.randn(4, 2).astype(np.float32) + model = _make_mock_model(chunk) + queue = ActionQueue() + ex = SyncExecution() + obs = {"state": np.zeros(2)} + + ex.start(model, queue) + ex.warmup(obs) + for _ in range(4): + queue.pop() + ex.maybe_request(obs) + assert ex.inference_count == 1 + class TestAsyncExecution: def test_start_spawns_thread(self) -> None: @@ -103,7 +117,7 @@ def test_maybe_request_submits_when_below_threshold(self) -> None: chunk = np.random.randn(10, 2).astype(np.float32) model = _make_mock_model(chunk) queue = ActionQueue() - ex = AsyncExecution(threshold=0.5, fps=10) + ex = AsyncExecution(request_threshold=0.5, fps=10) ex.start(model, queue) obs = {"state": np.zeros(2)} @@ -124,7 +138,7 @@ def test_defensive_copy_of_observation(self) -> None: chunk = np.random.randn(4, 2).astype(np.float32) model = _make_mock_model(chunk) queue = ActionQueue() - ex = AsyncExecution(threshold=0.5, fps=10) + ex = AsyncExecution(request_threshold=0.5, fps=10) ex.start(model, queue) obs = {"state": np.zeros(2)} @@ -151,7 +165,7 @@ def test_worker_death_raises_error(self) -> None: ValueError("model exploded"), ] queue = ActionQueue() - ex = AsyncExecution(threshold=0.5, fps=10) + ex = AsyncExecution(request_threshold=0.5, fps=10) ex.start(model, queue) obs = {"state": np.zeros(2)} @@ -208,7 +222,6 @@ def test_watchdog_triggers_force_reset(self) -> None: model = _make_mock_model(chunk) call_count = 0 - original_side_effect = None def slow_predict(obs: dict) -> np.ndarray: nonlocal call_count @@ -219,7 +232,7 @@ def slow_predict(obs: dict) -> np.ndarray: model.predict_action_chunk.side_effect = slow_predict queue = ActionQueue() - ex = AsyncExecution(threshold=0.5, fps=10, watchdog_timeout_s=0.1) + ex = AsyncExecution(request_threshold=0.5, fps=10, watchdog_timeout_s=0.1) ex.start(model, queue) obs = {"state": np.zeros(2)} diff --git a/tests/unit/runtime/test_fault_tolerance.py b/tests/unit/runtime/test_fault_tolerance.py new file mode 100644 index 0000000..3169a6a --- /dev/null +++ b/tests/unit/runtime/test_fault_tolerance.py @@ -0,0 +1,299 @@ +# Copyright (C) 2025-2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import Any +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest + +from physicalai.capture import Frame +from physicalai.capture.errors import CaptureError +from physicalai.runtime._action_queue import ActionQueue +from physicalai.runtime.execution import SyncExecution +from physicalai.runtime.runtime import ( + PolicyRuntime, + _MAX_OBS_RETRIES, + _MAX_SEND_RETRIES, + _WARMUP_RETRIES, +) + + +@dataclass +class FakeRobotObservation: + joint_positions: np.ndarray + timestamp: float + sensor_data: dict[str, np.ndarray] | None + images: dict | None + + @property + def state(self) -> np.ndarray: + return self.joint_positions + + +def _make_obs(positions: np.ndarray | None = None) -> FakeRobotObservation: + if positions is None: + positions = np.array([0.1, 0.2, 0.3], dtype=np.float32) + return FakeRobotObservation( + joint_positions=positions, + timestamp=time.monotonic(), + sensor_data=None, + images=None, + ) + + +def _make_mock_robot(obs: FakeRobotObservation | None = None) -> MagicMock: + robot = MagicMock() + robot.get_observation.return_value = obs or _make_obs() + return robot + + +def _make_mock_model(chunk_size: int = 10, action_dim: int = 3) -> MagicMock: + model = MagicMock() + model.predict_action_chunk.return_value = np.random.randn(chunk_size, action_dim).astype(np.float32) + return model + + +def _make_runtime( + robot: MagicMock | None = None, + model: MagicMock | None = None, + cameras: dict | None = None, + fps: float = 10.0, +) -> PolicyRuntime: + return PolicyRuntime( + robot=robot or _make_mock_robot(), + model=model or _make_mock_model(), + execution=SyncExecution(), + fps=fps, + cameras=cameras or {}, + ) + + +class TestResilientObserve: + def test_transient_observe_error_retries_then_succeeds(self) -> None: + obs = _make_obs() + robot = _make_mock_robot() + robot.get_observation.side_effect = [ConnectionError("flake"), obs] + + rt = _make_runtime(robot=robot) + + with patch("physicalai.runtime.runtime.time") as mock_time: + mock_time.sleep = MagicMock() + mock_time.perf_counter.return_value = 0.0 + mock_time.time.return_value = 0.0 + result = rt._resilient_observe() + + assert result is not None + assert robot.get_observation.call_count == 2 + assert rt._stale_obs_ticks == 0 + + def test_sustained_observe_error_uses_stale_fallback(self) -> None: + robot = _make_mock_robot() + robot.get_observation.side_effect = ConnectionError("down") + + rt = _make_runtime(robot=robot) + rt._last_robot_obs = _make_obs() + + with patch("physicalai.runtime.runtime.time") as mock_time: + mock_time.sleep = MagicMock() + mock_time.perf_counter.return_value = 0.0 + mock_time.time.return_value = 0.0 + result = rt._resilient_observe() + + assert result is not None + assert robot.get_observation.call_count == _MAX_OBS_RETRIES + assert rt._stale_obs_ticks == 1 + assert rt._consecutive_error_ticks == 1 + + def test_max_consecutive_errors_raises(self) -> None: + robot = _make_mock_robot() + robot.get_observation.side_effect = ConnectionError("down") + + rt = _make_runtime(robot=robot, fps=10.0) + rt._last_robot_obs = _make_obs() + rt._consecutive_error_ticks = rt._max_consecutive_error_ticks - 1 + + with patch("physicalai.runtime.runtime.time") as mock_time: + mock_time.sleep = MagicMock() + mock_time.time.return_value = 0.0 + with pytest.raises(ConnectionError, match="Exceeded max consecutive"): + rt._resilient_observe() + + def test_no_stale_obs_raises_immediately(self) -> None: + robot = _make_mock_robot() + robot.get_observation.side_effect = OSError("USB gone") + + rt = _make_runtime(robot=robot) + assert rt._last_robot_obs is None + + with patch("physicalai.runtime.runtime.time") as mock_time: + mock_time.sleep = MagicMock() + mock_time.time.return_value = 0.0 + with pytest.raises(ConnectionError, match="no stale observation"): + rt._resilient_observe() + + def test_fatal_error_propagates(self) -> None: + robot = _make_mock_robot() + robot.get_observation.side_effect = ValueError("bad joint config") + + rt = _make_runtime(robot=robot) + + with pytest.raises(ValueError, match="bad joint config"): + rt._resilient_observe() + + assert robot.get_observation.call_count == 1 + + +class TestResilientObserveCameras: + def test_camera_capture_error_uses_stale_frame(self) -> None: + stale_frame = Frame(data=np.zeros((480, 640, 3), dtype=np.uint8), timestamp=0.0, sequence=0) + camera = MagicMock() + camera.read_latest.side_effect = CaptureError("timeout") + + rt = _make_runtime(cameras={"cam0": camera}) + rt._last_camera_frames["cam0"] = stale_frame + + with patch("physicalai.runtime.runtime.time") as mock_time: + mock_time.sleep = MagicMock() + mock_time.perf_counter.return_value = 0.0 + mock_time.time.return_value = 0.0 + result = rt._resilient_observe() + + assert "images.cam0" in result + np.testing.assert_array_equal(result["images.cam0"], stale_frame.data[np.newaxis]) + + def test_camera_first_read_fails_raises(self) -> None: + camera = MagicMock() + camera.read_latest.side_effect = CaptureError("no device") + + rt = _make_runtime(cameras={"cam0": camera}) + + with patch("physicalai.runtime.runtime.time") as mock_time: + mock_time.sleep = MagicMock() + mock_time.perf_counter.return_value = 0.0 + mock_time.time.return_value = 0.0 + with pytest.raises(CaptureError, match="no device"): + rt._resilient_observe() + + +class TestResilientSend: + def test_resilient_send_retries(self) -> None: + robot = _make_mock_robot() + robot.send_action.side_effect = [ConnectionError("flake"), None] + + rt = _make_runtime(robot=robot) + action = np.zeros(3, dtype=np.float32) + + with patch("physicalai.runtime.runtime.time") as mock_time: + mock_time.sleep = MagicMock() + mock_time.time.return_value = 0.0 + rt._resilient_send(action) + + assert robot.send_action.call_count == 2 + assert rt._transient_errors == 0 + assert rt._consecutive_error_ticks == 0 + + def test_resilient_send_all_retries_fail_skips_tick(self) -> None: + robot = _make_mock_robot() + robot.send_action.side_effect = OSError("USB gone") + + rt = _make_runtime(robot=robot) + action = np.zeros(3, dtype=np.float32) + + with patch("physicalai.runtime.runtime.time") as mock_time: + mock_time.sleep = MagicMock() + mock_time.time.return_value = 0.0 + rt._resilient_send(action) + + assert robot.send_action.call_count == _MAX_SEND_RETRIES + assert rt._transient_errors == 1 + + +class TestWarmupWithRetry: + def test_warmup_retries_on_connection_error(self) -> None: + obs = _make_obs() + robot = _make_mock_robot(obs) + robot.get_observation.side_effect = [ConnectionError(), ConnectionError(), obs] + model = _make_mock_model() + + execution = MagicMock() + rt = PolicyRuntime( + robot=robot, + model=model, + execution=execution, + fps=10.0, + ) + + with patch("physicalai.runtime.runtime.time") as mock_time: + mock_time.sleep = MagicMock() + mock_time.perf_counter.return_value = 0.0 + mock_time.time.return_value = 0.0 + rt._warmup_with_retry() + + assert execution.warmup.called + + def test_warmup_exhausted_raises(self) -> None: + robot = _make_mock_robot() + robot.get_observation.side_effect = ConnectionError("down") + + rt = _make_runtime(robot=robot) + + with patch("physicalai.runtime.runtime.time") as mock_time: + mock_time.sleep = MagicMock() + mock_time.time.return_value = 0.0 + with pytest.raises(ConnectionError, match=f"Warmup failed after {_WARMUP_RETRIES}"): + rt._warmup_with_retry() + + +class TestShutdownDrain: + def test_shutdown_drain_uses_resilient_send(self) -> None: + robot = _make_mock_robot() + robot.send_action.side_effect = OSError("USB gone during drain") + model = _make_mock_model(chunk_size=20) + + rt = _make_runtime(robot=robot, model=model) + rt._action_queue.push_chunk(np.ones((5, 3), dtype=np.float32)) + + with patch("physicalai.runtime.runtime.time") as mock_time: + mock_time.sleep = MagicMock() + mock_time.perf_counter.return_value = 0.0 + mock_time.time.return_value = 0.0 + rt._shutdown(step=10) + + assert rt._transient_errors > 0 + + +class TestRunStatsWithFaults: + def test_run_stats_includes_fault_metrics(self) -> None: + obs = _make_obs() + robot = _make_mock_robot(obs) + + warmup_call = [0] + + def get_obs_with_loop_errors(): + warmup_call[0] += 1 + if warmup_call[0] <= 1: + return obs + if warmup_call[0] <= 1 + _MAX_OBS_RETRIES: + raise ConnectionError("flake") + return obs + + robot.get_observation.side_effect = get_obs_with_loop_errors + robot.send_action.return_value = None + + rt = _make_runtime(robot=robot) + rt._last_robot_obs = obs + rt._connected = True + + with patch("physicalai.runtime.runtime.time") as mock_time: + mock_time.perf_counter.return_value = 0.0 + mock_time.sleep = MagicMock() + mock_time.time.return_value = 0.0 + stats = rt.run(duration_s=0.3) + + assert stats.stale_obs_ticks >= 1 + assert stats.steps == 3 diff --git a/tests/unit/runtime/test_observer.py b/tests/unit/runtime/test_observer.py new file mode 100644 index 0000000..c11b33b --- /dev/null +++ b/tests/unit/runtime/test_observer.py @@ -0,0 +1,73 @@ +# Copyright (C) 2025-2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import tempfile +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import numpy as np + +from physicalai.runtime.observer._console import ConsoleHandler +from physicalai.runtime.observer._recorder import RecorderHandler + + +class TestConsoleHandler: + def test_formats_tick(self, capsys: Any) -> None: + handler = ConsoleHandler(target_fps=30.0) + handler( + "sess1", + "tick", + { + "step": 10, + "physicalai.runtime.loop_duration_s": 0.033, + "queue_remaining": 5, + "stale_obs": False, + }, + ) + captured = capsys.readouterr() + assert "step=10" in captured.out + assert "queue=5" in captured.out + + def test_formats_lifecycle(self, capsys: Any) -> None: + handler = ConsoleHandler() + handler("sess1", "lifecycle", {"event": "start", "fps": 30}) + captured = capsys.readouterr() + assert "start" in captured.out + + +class TestRecorderHandler: + def test_writes_jsonl(self) -> None: + with tempfile.NamedTemporaryFile(suffix=".jsonl", delete=False, mode="w") as f: + path = Path(f.name) + + recorder = RecorderHandler(path) + recorder("sess1", "tick", {"step": 0, "value": 1.0}) + recorder("sess1", "lifecycle", {"event": "start"}) + recorder.close() + + lines = path.read_text().strip().split("\n") + assert len(lines) == 2 + record = json.loads(lines[0]) + assert record["session_id"] == "sess1" + assert record["topic"] == "tick" + assert record["step"] == 0 + + path.unlink() + + def test_handles_numpy_arrays(self) -> None: + with tempfile.NamedTemporaryFile(suffix=".jsonl", delete=False, mode="w") as f: + path = Path(f.name) + + recorder = RecorderHandler(path) + recorder("sess1", "tick", {"action": np.array([1.0, 2.0, 3.0])}) + recorder.close() + + line = path.read_text().strip() + record = json.loads(line) + assert record["action"] == [1.0, 2.0, 3.0] + + path.unlink() diff --git a/tests/unit/runtime/test_rerun_callback.py b/tests/unit/runtime/test_rerun_callback.py new file mode 100644 index 0000000..aa8390a --- /dev/null +++ b/tests/unit/runtime/test_rerun_callback.py @@ -0,0 +1,378 @@ +# Copyright (C) 2025-2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import sys +from typing import Any +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest + +from physicalai.runtime.events import InferenceEvent, LifecycleEvent, TickEvent + + +@pytest.fixture() +def mock_rerun() -> MagicMock: + """Provide a mock rerun module injected into sys.modules.""" + rr = MagicMock() + rr.__name__ = "rerun" + rr.Scalars = MagicMock(side_effect=lambda *a, **kw: ("Scalars", a, kw)) + rr.Scalars.columns = MagicMock(return_value=MagicMock()) + rr.Image = MagicMock(side_effect=lambda d: ("Image", d)) + rr.TextLog = MagicMock(side_effect=lambda t: ("TextLog", t)) + rr.StateChange = MagicMock(side_effect=lambda *a, **kw: ("StateChange", a, kw)) + rr.Clear = MagicMock(side_effect=lambda *a, **kw: ("Clear", a, kw)) + rr.TimeColumn = MagicMock(side_effect=lambda *a, **kw: ("TimeColumn", a, kw)) + return rr + + +@pytest.fixture() +def _patch_rerun(mock_rerun: MagicMock) -> Any: + """Patch rerun in sys.modules so RerunCallback can import it.""" + with patch.dict(sys.modules, {"rerun": mock_rerun}): + yield + + +@pytest.fixture() +def make_callback(_patch_rerun: Any, mock_rerun: MagicMock) -> Any: + """Factory that creates a RerunCallback with the mocked rerun module.""" + from physicalai.runtime.callbacks import RerunCallback + + def _factory(**kwargs: Any) -> RerunCallback: + defaults: dict[str, Any] = {"mode": "spawn"} + defaults.update(kwargs) + return RerunCallback(**defaults) + + return _factory + + +def _lifecycle_start(session_id: str = "sess-1", fps: int = 30) -> LifecycleEvent: + return LifecycleEvent( + session_id=session_id, + timestamp=1000.0, + event="start", + metadata={"fps": fps, "cameras": []}, + ) + + +def _tick(step: int = 0, dof: int = 7) -> TickEvent: + return TickEvent( + session_id="sess-1", + step=step, + timestamp=1000.0 + step * (1 / 30), + joint_positions=np.arange(dof, dtype=np.float64), + action_sent=np.ones(dof, dtype=np.float64), + queue_remaining=5, + loop_duration_s=0.033, + sleep_time_s=0.0, + stale_obs=False, + ) + + +def _inference(horizon: int = 50, dof: int = 7) -> InferenceEvent: + return InferenceEvent( + session_id="sess-1", + timestamp=1001.0, + latency_s=0.05, + offset=0, + chunk=np.zeros((horizon, dof), dtype=np.float32), + ) + + +@pytest.mark.usefixtures("_patch_rerun") +class TestRerunCallbackConstruction: + def test_construction_succeeds_with_mocked_rerun(self, make_callback: Any) -> None: + cb = make_callback() + assert cb is not None + + def test_mode_save_requires_save_path(self, make_callback: Any) -> None: + with pytest.raises(ValueError, match="save_path"): + make_callback(mode="save") + + def test_mode_save_with_path_succeeds(self, make_callback: Any) -> None: + cb = make_callback(mode="save", save_path="/tmp/test.rrd") + assert cb is not None + + def test_missing_rerun_raises_import_error(self) -> None: + with patch.dict(sys.modules, {"rerun": None}): + from physicalai.runtime.callbacks import RerunCallback + + with pytest.raises((ImportError, ModuleNotFoundError)): + RerunCallback(mode="spawn") + + +@pytest.mark.usefixtures("_patch_rerun") +class TestRerunCallbackLifecycle: + def test_init_rerun_spawn(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start("my-session")) + + mock_rerun.init.assert_called_once_with(application_id="physicalai-runtime", recording_id="my-session") + mock_rerun.spawn.assert_called_once() + mock_rerun.save.assert_not_called() + + def test_init_rerun_save(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback(mode="save", save_path="/tmp/out.rrd") + cb.on_lifecycle(_lifecycle_start()) + + mock_rerun.save.assert_called_once_with("/tmp/out.rrd") + mock_rerun.spawn.assert_not_called() + + def test_lifecycle_marker_logged(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start()) + + mock_rerun.log.assert_called() + lifecycle_calls = [c for c in mock_rerun.log.call_args_list if "lifecycle" in str(c)] + assert len(lifecycle_calls) > 0 + + def test_second_start_does_not_reinitialize(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start()) + cb.on_lifecycle(_lifecycle_start()) + + assert mock_rerun.init.call_count == 1 + + def test_fps_extracted_from_metadata(self, make_callback: Any) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start(fps=60)) + assert cb._fps == 60 + + +@pytest.mark.usefixtures("_patch_rerun") +class TestRerunCallbackTick: + def test_tick_logs_joint_scalars(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start()) + mock_rerun.reset_mock() + + cb.on_tick(_tick(step=1, dof=3)) + + log_calls = mock_rerun.log.call_args_list + joint_calls = [c for c in log_calls if c.args[0] == "robot/joints"] + assert len(joint_calls) == 1 + + def test_tick_logs_action_scalars(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start()) + mock_rerun.reset_mock() + + cb.on_tick(_tick(step=1, dof=4)) + + log_calls = mock_rerun.log.call_args_list + action_calls = [c for c in log_calls if c.args[0] == "robot/actions"] + assert len(action_calls) == 1 + + def test_tick_logs_runtime_metrics(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start()) + mock_rerun.reset_mock() + + cb.on_tick(_tick(step=1)) + + log_calls = mock_rerun.log.call_args_list + paths = [c.args[0] for c in log_calls] + assert "queue/remaining" in paths + assert "runtime/loop_duration_s" in paths + assert "runtime/sleep_time_s" in paths + assert "runtime/stale_obs" in paths + + def test_tick_updates_last_step(self, make_callback: Any) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start()) + cb.on_tick(_tick(step=42)) + assert cb._last_step == 42 + + def test_tick_sets_timelines(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start()) + mock_rerun.reset_mock() + + event = _tick(step=5) + cb.on_tick(event) + + mock_rerun.set_time.assert_any_call("step", sequence=5) + mock_rerun.set_time.assert_any_call("wall", timestamp=event.timestamp) + + def test_none_joint_positions_skipped(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start()) + mock_rerun.reset_mock() + + event = TickEvent( + session_id="sess-1", + step=1, + timestamp=1000.0, + joint_positions=None, + action_sent=None, + queue_remaining=5, + loop_duration_s=0.033, + sleep_time_s=0.0, + stale_obs=False, + ) + cb.on_tick(event) + + log_calls = mock_rerun.log.call_args_list + joint_calls = [c for c in log_calls if c.args[0] == "robot/joints"] + assert len(joint_calls) == 0 + + +@pytest.mark.usefixtures("_patch_rerun") +class TestRerunCallbackInference: + def test_send_columns_called_with_prediction_steps(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start(fps=30)) + + cb.on_tick(_tick(step=10)) + mock_rerun.reset_mock() + + cb.on_inference(_inference(horizon=5, dof=3)) + + # Predictions are logged via send_columns, not individual log calls. + mock_rerun.send_columns.assert_called_once() + call_args = mock_rerun.send_columns.call_args + assert call_args.args[0] == "robot/predicted" + + def test_inference_logs_queue_spike(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start()) + cb.on_tick(_tick(step=0)) + mock_rerun.reset_mock() + + cb.on_inference(_inference(horizon=5, dof=3)) + + log_calls = mock_rerun.log.call_args_list + queue_calls = [c for c in log_calls if c.args[0] == "queue/inference"] + assert len(queue_calls) == 1 + + def test_inference_resets_time_to_current_step(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start(fps=10)) + cb.on_tick(_tick(step=5)) + mock_rerun.reset_mock() + + cb.on_inference(_inference(horizon=3, dof=1)) + + # After send_columns, time is reset to current step for queue/inference log. + set_time_calls = mock_rerun.set_time.call_args_list + step_values = [c.kwargs["sequence"] for c in set_time_calls if c.args[0] == "step" and "sequence" in c.kwargs] + assert 5 in step_values + + def test_stale_predictions_cleared(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start(fps=30)) + cb.on_tick(_tick(step=5)) + cb.on_inference(_inference(horizon=5, dof=3)) + mock_rerun.reset_mock() + + # Second inference should clear old predictions first. + cb.on_tick(_tick(step=10)) + cb.on_inference(_inference(horizon=5, dof=3)) + + log_calls = mock_rerun.log.call_args_list + clear_calls = [c for c in log_calls if c.args[0] == "robot/predicted"] + assert len(clear_calls) >= 1 # At least the Clear call + + def test_pred_horizon_tracked(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start(fps=30)) + cb.on_tick(_tick(step=0)) + cb.on_inference(_inference(horizon=50, dof=7)) + assert cb._pred_horizon == 50 + + +@pytest.mark.usefixtures("_patch_rerun") +class TestRerunCallbackImageDecimation: + def test_decimation_skips_non_nth_ticks(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback(image_decimation=3) + cb.on_lifecycle(_lifecycle_start()) + + mock_sub = MagicMock() + mock_frame = MagicMock() + mock_frame.data = np.zeros((480, 640, 3), dtype=np.uint8) + mock_sub.read_latest.return_value = mock_frame + cb._camera_subscribers = {"top": mock_sub} + + image_logged_at: list[int] = [] + for step in range(6): + mock_rerun.reset_mock() + cb.on_tick(_tick(step=step)) + log_calls = mock_rerun.log.call_args_list + if any("camera/top" in str(c.args[0]) for c in log_calls): + image_logged_at.append(step) + + assert image_logged_at == [0, 3] + + def test_decimation_default_is_3(self, make_callback: Any) -> None: + cb = make_callback() + assert cb._image_decimation == 3 + + +@pytest.mark.usefixtures("_patch_rerun") +class TestRerunCallbackCameraSubscribers: + def test_non_shared_camera_warns(self, make_callback: Any, mock_rerun: MagicMock, caplog: Any) -> None: + fake_camera = MagicMock() + fake_camera.__class__.__name__ = "FakeCamera" + + cb = make_callback(cameras={"top": fake_camera}) + cb.on_lifecycle(_lifecycle_start()) + + assert cb._camera_subscribers == {} + + def test_shared_camera_subscriber_logs_frames(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback(image_decimation=1) + cb.on_lifecycle(_lifecycle_start()) + + mock_sub = MagicMock() + mock_frame = MagicMock() + mock_frame.data = np.zeros((480, 640, 3), dtype=np.uint8) + mock_sub.read_latest.return_value = mock_frame + cb._camera_subscribers = {"top": mock_sub} + + mock_rerun.reset_mock() + cb.on_tick(_tick(step=0)) + + log_calls = mock_rerun.log.call_args_list + image_calls = [c for c in log_calls if "camera/top" in str(c.args[0])] + assert len(image_calls) == 1 + + def test_close_disconnects_subscribers(self, make_callback: Any) -> None: + cb = make_callback() + mock_sub = MagicMock() + cb._camera_subscribers = {"cam1": mock_sub, "cam2": MagicMock()} + + cb.close() + + mock_sub.disconnect.assert_called_once() + assert cb._camera_subscribers == {} + + def test_close_handles_disconnect_error(self, make_callback: Any) -> None: + cb = make_callback() + mock_sub = MagicMock() + mock_sub.disconnect.side_effect = RuntimeError("connection lost") + cb._camera_subscribers = {"cam1": mock_sub} + + cb.close() + assert cb._camera_subscribers == {} + + +@pytest.mark.usefixtures("_patch_rerun") +class TestRerunCallbackLifecycleMarker: + def test_lifecycle_logs_text(self, make_callback: Any, mock_rerun: MagicMock) -> None: + cb = make_callback() + cb.on_lifecycle(_lifecycle_start()) + mock_rerun.reset_mock() + + event = LifecycleEvent( + session_id="sess-1", + timestamp=2000.0, + event="shutdown", + metadata={"reason": "done"}, + ) + cb.on_lifecycle(event) + + log_calls = mock_rerun.log.call_args_list + lifecycle_calls = [c for c in log_calls if "runtime/lifecycle/shutdown" in str(c.args[0])] + assert len(lifecycle_calls) == 1 diff --git a/tests/unit/runtime/test_runtime.py b/tests/unit/runtime/test_runtime.py index 45b7bf8..4ba88a9 100644 --- a/tests/unit/runtime/test_runtime.py +++ b/tests/unit/runtime/test_runtime.py @@ -28,6 +28,10 @@ class FakeRobotObservation: sensor_data: dict[str, np.ndarray] | None images: dict | None + @property + def state(self) -> np.ndarray: + return self.joint_positions + def _make_mock_robot(joint_positions: np.ndarray | None = None) -> MagicMock: robot = MagicMock() @@ -87,6 +91,7 @@ def test_full_loop_with_duration(self) -> None: with patch("physicalai.runtime.runtime.time") as mock_time: mock_time.perf_counter.return_value = 0.0 mock_time.sleep = MagicMock() + mock_time.time.return_value = 0.0 stats = runtime.run(duration_s=0.5) assert stats.steps == 5 @@ -112,6 +117,7 @@ def test_hold_fallback_when_queue_empty(self) -> None: with patch("physicalai.runtime.runtime.time") as mock_time: mock_time.perf_counter.return_value = 0.0 mock_time.sleep = MagicMock() + mock_time.time.return_value = 0.0 stats = runtime.run(duration_s=0.4) assert stats.steps == 4 @@ -141,6 +147,7 @@ def test_worker_died_error_propagation(self) -> None: with patch("physicalai.runtime.runtime.time") as mock_time, pytest.raises(WorkerDiedError, match="dead"): mock_time.perf_counter.return_value = 0.0 mock_time.sleep = MagicMock() + mock_time.time.return_value = 0.0 runtime.run(duration_s=1.0) def test_shutdown_does_not_disconnect(self) -> None: @@ -158,6 +165,7 @@ def test_shutdown_does_not_disconnect(self) -> None: with patch("physicalai.runtime.runtime.time") as mock_time: mock_time.perf_counter.return_value = 0.0 mock_time.sleep = MagicMock() + mock_time.time.return_value = 0.0 runtime.run(duration_s=0.1) robot.disconnect.assert_not_called() @@ -197,6 +205,7 @@ def test_before_send_action_called(self) -> None: with patch("physicalai.runtime.runtime.time") as mock_time: mock_time.perf_counter.return_value = 0.0 mock_time.sleep = MagicMock() + mock_time.time.return_value = 0.0 runtime.run(duration_s=0.2) assert callback.before_send_action.call_count == 2 @@ -219,6 +228,7 @@ def test_callback_raises_does_not_crash_loop(self) -> None: with patch("physicalai.runtime.runtime.time") as mock_time: mock_time.perf_counter.return_value = 0.0 mock_time.sleep = MagicMock() + mock_time.time.return_value = 0.0 stats = runtime.run(duration_s=0.3) assert stats.steps == 3 @@ -245,6 +255,7 @@ def test_on_hold_called_when_queue_empty(self) -> None: with patch("physicalai.runtime.runtime.time") as mock_time: mock_time.perf_counter.return_value = 0.0 mock_time.sleep = MagicMock() + mock_time.time.return_value = 0.0 runtime.run(duration_s=0.3) assert callback.on_hold.call_count >= 1 diff --git a/tests/unit/runtime/test_smoothers.py b/tests/unit/runtime/test_smoothers.py index d5c02b0..052604d 100644 --- a/tests/unit/runtime/test_smoothers.py +++ b/tests/unit/runtime/test_smoothers.py @@ -7,37 +7,24 @@ class TestReplaceSmoother: - def test_merge_drops_remaining_and_returns_incoming_offset(self) -> None: + def test_merge_returns_incoming(self) -> None: smoother = ReplaceSmoother() remaining = np.array([[1.0, 1.0], [1.0, 1.0]], dtype=np.float32) incoming = np.array([[2.0, 2.0], [3.0, 3.0], [4.0, 4.0]], dtype=np.float32) - result = smoother.merge(remaining, incoming, offset=1) + result = smoother.merge(remaining, incoming) - np.testing.assert_array_equal( - result, - np.array([[3.0, 3.0], [4.0, 4.0]], dtype=np.float32), - ) + np.testing.assert_array_equal(result, incoming) - def test_offset_zero_returns_all_incoming(self) -> None: + def test_empty_remaining_returns_incoming(self) -> None: smoother = ReplaceSmoother() - remaining = np.array([[1.0, 1.0]], dtype=np.float32) + remaining = np.empty((0, 2), dtype=np.float32) incoming = np.array([[2.0, 2.0], [3.0, 3.0]], dtype=np.float32) - result = smoother.merge(remaining, incoming, offset=0) + result = smoother.merge(remaining, incoming) np.testing.assert_array_equal(result, incoming) - def test_offset_beyond_incoming_returns_empty_array(self) -> None: - smoother = ReplaceSmoother() - remaining = np.array([[1.0, 1.0]], dtype=np.float32) - incoming = np.array([[2.0, 2.0]], dtype=np.float32) - - result = smoother.merge(remaining, incoming, offset=5) - - assert result.shape == (0, 2) - assert result.dtype == incoming.dtype - class TestLerpSmoother: def test_lerp_weights_match_queue_mixer_formula(self) -> None: @@ -48,7 +35,7 @@ def test_lerp_weights_match_queue_mixer_formula(self) -> None: dtype=np.float32, ) - result = smoother.merge(remaining, incoming, offset=0) + result = smoother.merge(remaining, incoming) expected = np.array( [[10.0, 10.0], [50.0, 50.0], [90.0, 90.0], [130.0, 130.0]], @@ -56,63 +43,63 @@ def test_lerp_weights_match_queue_mixer_formula(self) -> None: ) np.testing.assert_array_equal(result, expected) - def test_offset_aware_duration(self) -> None: + def test_incoming_shorter_than_remaining(self) -> None: smoother = LerpSmoother(duration_frames=99) remaining = np.array([[1.0, 1.0], [2.0, 2.0], [3.0, 3.0]], dtype=np.float32) - incoming = np.array([[4.0, 4.0], [5.0, 5.0], [6.0, 6.0], [7.0, 7.0]], dtype=np.float32) + incoming = np.array([[6.0, 6.0], [7.0, 7.0]], dtype=np.float32) - result = smoother.merge(remaining, incoming, offset=2) + result = smoother.merge(remaining, incoming) - expected = np.array([[1.0, 1.0], [4.5, 4.5]], dtype=np.float32) - np.testing.assert_array_equal(result, expected) + # lerp_dur = min(n_remain=3, duration_frames=99) = 3 + # weights = [1.0, 2/3, 1/3]; n_blend = min(3,2) = 2 + # blended[0] = 1.0*1 + 0.0*6 = 1.0 + # blended[1] = (2/3)*2 + (1/3)*7 = 4/3 + 7/3 = 11/3 + expected = np.array([[1.0, 1.0], [11.0 / 3, 11.0 / 3]], dtype=np.float32) + np.testing.assert_allclose(result, expected, rtol=1e-6) - def test_edge_cases_empty_remaining_single_element_and_offset_beyond_chunk(self) -> None: + def test_empty_remaining_returns_incoming(self) -> None: smoother = LerpSmoother(duration_frames=5) - empty_result = smoother.merge( + result = smoother.merge( np.empty((0, 2), dtype=np.float32), np.array([[1.0, 1.0]], dtype=np.float32), - offset=0, ) - np.testing.assert_array_equal(empty_result, np.array([[1.0, 1.0]], dtype=np.float32)) + np.testing.assert_array_equal(result, np.array([[1.0, 1.0]], dtype=np.float32)) - single_result = smoother.merge( - np.array([[1.0, 1.0]], dtype=np.float32), - np.array([[2.0, 2.0]], dtype=np.float32), - offset=0, - ) - np.testing.assert_array_equal(single_result, np.array([[1.0, 1.0]], dtype=np.float32)) + def test_single_remaining_single_incoming(self) -> None: + smoother = LerpSmoother(duration_frames=5) - offset_beyond_result = smoother.merge( + result = smoother.merge( np.array([[1.0, 1.0]], dtype=np.float32), np.array([[2.0, 2.0]], dtype=np.float32), - offset=5, ) - assert offset_beyond_result.shape == (0, 2) + # weight[0] = 1.0 -> blended = 1.0*1 + 0.0*2 = 1.0 + np.testing.assert_array_equal(result, np.array([[1.0, 1.0]], dtype=np.float32)) def test_exact_numerical_blend_values(self) -> None: smoother = LerpSmoother(duration_frames=5) remaining = np.array([[1.0, 1.0], [1.0, 1.0], [1.0, 1.0]], dtype=np.float32) incoming = np.array( - [[2.0, 2.0], [2.0, 2.0], [2.0, 2.0], [2.0, 2.0]], + [[2.0, 2.0], [2.0, 2.0], [2.0, 2.0]], dtype=np.float32, ) - result = smoother.merge(remaining, incoming, offset=1) + result = smoother.merge(remaining, incoming) + # lerp_dur = min(3, 5) = 3; weights = [1.0, 2/3, 1/3] expected = np.array( - [[1.0, 1.0], [2.0, 2.0], [2.0, 2.0]], + [[1.0, 1.0], [4.0 / 3, 4.0 / 3], [5.0 / 3, 5.0 / 3]], dtype=np.float32, ) - np.testing.assert_array_equal(result, expected) + np.testing.assert_allclose(result, expected, rtol=1e-6) def test_merge_is_stateless_for_same_arguments(self) -> None: smoother = LerpSmoother(duration_frames=5) remaining = np.array([[1.0, 1.0], [2.0, 2.0]], dtype=np.float32) incoming = np.array([[3.0, 3.0], [4.0, 4.0], [5.0, 5.0]], dtype=np.float32) - first = smoother.merge(remaining, incoming, offset=1) - second = smoother.merge(remaining, incoming, offset=1) + first = smoother.merge(remaining, incoming) + second = smoother.merge(remaining, incoming) np.testing.assert_array_equal(first, second) @@ -123,4 +110,4 @@ def test_input_validation_mismatched_action_dim_raises_value_error() -> None: incoming = np.array([[2.0, 2.0, 2.0]], dtype=np.float32) with pytest.raises(ValueError, match="action_dim"): - smoother.merge(remaining, incoming, offset=0) + smoother.merge(remaining, incoming) diff --git a/tests/unit/runtime/test_telemetry.py b/tests/unit/runtime/test_telemetry.py new file mode 100644 index 0000000..ebcac3d --- /dev/null +++ b/tests/unit/runtime/test_telemetry.py @@ -0,0 +1,297 @@ +# Copyright (C) 2025-2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import tempfile +import time +from pathlib import Path +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest + +from physicalai.runtime._callback_bus import _CallbackBus +from physicalai.runtime._telemetry import TelemetryEmitter, _decode_numpy, _encode_numpy +from physicalai.runtime.callbacks import AsyncCallback, ConsoleCallback, JsonlCallback +from physicalai.runtime.events import InferenceEvent, LifecycleEvent, TickEvent + + +class TestNumpyEncoding: + def test_encode_numpy_float32(self) -> None: + arr = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32) + encoded = _encode_numpy(arr) + assert encoded["__np__"] is True + assert encoded["dtype"] == "float32" + assert encoded["shape"] == [2, 2] + assert isinstance(encoded["data"], bytes) + + def test_encode_preserves_shape(self) -> None: + arr = np.zeros((3, 4, 5), dtype=np.float64) + encoded = _encode_numpy(arr) + assert encoded["shape"] == [3, 4, 5] + assert encoded["dtype"] == "float64" + + def test_roundtrip(self) -> None: + arr = np.array([1.5, 2.5, 3.5], dtype=np.float32) + decoded = _decode_numpy(_encode_numpy(arr)) + np.testing.assert_array_equal(arr, decoded) + + +class TestTelemetryEmitterNoOp: + def test_emitter_noop_without_zenoh(self) -> None: + with patch.dict("sys.modules", {"zenoh": None, "msgpack": None}): + e = TelemetryEmitter.__new__(TelemetryEmitter) + e._session_id = "test" + e._session = None + e._msgpack = None + e._enabled = False + + assert not e.enabled + + def test_noop_emit_methods(self) -> None: + e = TelemetryEmitter.__new__(TelemetryEmitter) + e._session_id = "test" + e._session = None + e._msgpack = None + e._enabled = False + + e.emit_lifecycle("test_event", foo="bar") + e.emit_tick( + step=0, + timestamp=0.0, + joint_positions=None, + action_sent=None, + queue_remaining=0, + loop_duration_s=0.033, + sleep_time_s=0.001, + ) + e.emit_inference(latency_s=0.1, offset=3, chunk=np.zeros((5, 3))) + e.close() + + +class TestTelemetryEmitterWithMock: + def _make_emitter(self) -> tuple[TelemetryEmitter, MagicMock]: + mock_session = MagicMock() + mock_msgpack = MagicMock() + mock_msgpack.packb.return_value = b"\x80" + + e = TelemetryEmitter.__new__(TelemetryEmitter) + e._session_id = "abc123" + e._session = mock_session + e._msgpack = mock_msgpack + e._enabled = True + return e, mock_session + + def test_emit_tick_publishes(self) -> None: + e, session = self._make_emitter() + e.emit_tick( + step=42, + timestamp=1.0, + joint_positions=np.zeros(3), + action_sent=np.ones(3), + queue_remaining=5, + loop_duration_s=0.033, + sleep_time_s=0.001, + ) + session.put.assert_called_once() + topic = session.put.call_args[0][0] + assert topic == "physicalai/rt/abc123/tick" + + def test_emit_lifecycle_publishes(self) -> None: + e, session = self._make_emitter() + e.emit_lifecycle("start", fps=30) + session.put.assert_called_once() + topic = session.put.call_args[0][0] + assert topic == "physicalai/rt/abc123/lifecycle" + + def test_emit_inference_publishes(self) -> None: + e, session = self._make_emitter() + e.emit_inference(latency_s=0.05, offset=2, chunk=np.zeros((10, 6))) + session.put.assert_called_once() + topic = session.put.call_args[0][0] + assert topic == "physicalai/rt/abc123/inference" + + def test_close_closes_session(self) -> None: + e, session = self._make_emitter() + e.close() + session.close.assert_called_once() + assert not e.enabled + + +class TestCallbackBus: + def _make_tick_event(self, step: int = 0) -> TickEvent: + return TickEvent( + session_id="test", + step=step, + timestamp=0.0, + joint_positions=None, + action_sent=np.zeros(3), + queue_remaining=5, + loop_duration_s=0.03, + sleep_time_s=0.003, + stale_obs=False, + ) + + def _make_inference_event(self) -> InferenceEvent: + return InferenceEvent( + session_id="test", + timestamp=0.0, + latency_s=0.1, + offset=3, + chunk=np.zeros((10, 3)), + ) + + def _make_lifecycle_event(self, event: str = "start") -> LifecycleEvent: + return LifecycleEvent( + session_id="test", + timestamp=0.0, + event=event, + metadata={"fps": 30}, + ) + + def test_emit_tick_dispatches_to_callback(self) -> None: + cb = MagicMock() + bus = _CallbackBus([cb]) + event = self._make_tick_event() + bus.emit_tick(event) + cb.on_tick.assert_called_once_with(event) + + def test_emit_lifecycle_dispatches(self) -> None: + cb = MagicMock() + bus = _CallbackBus([cb]) + event = self._make_lifecycle_event() + bus.emit_lifecycle(event) + cb.on_lifecycle.assert_called_once_with(event) + + def test_emit_inference_queues_for_drain(self) -> None: + cb = MagicMock() + bus = _CallbackBus([cb]) + event = self._make_inference_event() + bus.emit_inference(event) + cb.on_inference.assert_not_called() + bus.emit_tick(self._make_tick_event()) + cb.on_inference.assert_called_once_with(event) + + def test_invoke_before_send_action_chains(self) -> None: + cb1 = MagicMock() + cb1.before_send_action.return_value = np.ones(3) + cb2 = MagicMock() + cb2.before_send_action.return_value = None + + bus = _CallbackBus([cb1, cb2]) + original = np.zeros(3) + result = bus.invoke_before_send_action(action=original, step=0) + + np.testing.assert_array_equal(result, np.ones(3)) + cb2.before_send_action.assert_called_once() + passed = cb2.before_send_action.call_args[1]["action"] + np.testing.assert_array_equal(passed, np.ones(3)) + + def test_invoke_on_hold_dispatches(self) -> None: + cb = MagicMock() + bus = _CallbackBus([cb]) + bus.invoke_on_hold(step=5, holds=3) + cb.on_hold.assert_called_once_with(step=5, holds=3) + + def test_callback_exception_isolated(self) -> None: + bad_cb = MagicMock() + bad_cb.on_tick.side_effect = RuntimeError("oops") + good_cb = MagicMock() + bus = _CallbackBus([bad_cb, good_cb]) + bus.emit_tick(self._make_tick_event()) + good_cb.on_tick.assert_called_once() + + def test_close_calls_close_on_callbacks(self) -> None: + cb = MagicMock() + bus = _CallbackBus([cb]) + bus.close() + cb.close.assert_called_once() + + def test_missing_methods_skipped(self) -> None: + class MinimalCallback: + pass + + bus = _CallbackBus([MinimalCallback()]) + bus.emit_tick(self._make_tick_event()) + bus.emit_lifecycle(self._make_lifecycle_event()) + bus.invoke_on_hold(step=0, holds=1) + + +class TestConsoleCallback: + def test_throttles_output(self, capsys: pytest.CaptureFixture[str]) -> None: + cb = ConsoleCallback(throttle_steps=5) + for i in range(10): + cb.on_tick( + TickEvent( + session_id="t", + step=i, + timestamp=0.0, + joint_positions=None, + action_sent=np.zeros(3), + queue_remaining=5, + loop_duration_s=0.03, + sleep_time_s=0.003, + stale_obs=False, + ) + ) + captured = capsys.readouterr() + lines = [l for l in captured.out.strip().split("\n") if l] + assert len(lines) == 2 + + +class TestJsonlCallback: + def test_writes_events(self, tmp_path: Path) -> None: + path = tmp_path / "test.jsonl" + cb = JsonlCallback(path) + + cb.on_tick( + TickEvent( + session_id="s1", + step=0, + timestamp=1.0, + joint_positions=np.array([0.1, 0.2]), + action_sent=np.array([0.3, 0.4]), + queue_remaining=5, + loop_duration_s=0.03, + sleep_time_s=0.003, + stale_obs=False, + ) + ) + cb.on_lifecycle( + LifecycleEvent( + session_id="s1", + timestamp=1.0, + event="start", + metadata={"fps": 30}, + ) + ) + cb.close() + + lines = path.read_text().strip().split("\n") + assert len(lines) == 2 + tick_record = json.loads(lines[0]) + assert tick_record["type"] == "tick" + assert tick_record["step"] == 0 + lifecycle_record = json.loads(lines[1]) + assert lifecycle_record["type"] == "lifecycle" + assert lifecycle_record["event"] == "start" + + +class TestAsyncCallback: + def test_dispatches_events_asynchronously(self) -> None: + inner = MagicMock(spec=["on_tick", "on_inference", "on_lifecycle", "close"]) + cb = AsyncCallback(inner, max_queue=64) + event = LifecycleEvent(session_id="t", timestamp=0.0, event="start", metadata={}) + cb.on_lifecycle(event) + time.sleep(0.1) + inner.on_lifecycle.assert_called_once_with(event) + cb.close() + + def test_close_joins_thread(self) -> None: + inner = MagicMock(spec=["on_tick", "on_inference", "on_lifecycle", "close"]) + cb = AsyncCallback(inner) + cb.close() + assert not cb._thread.is_alive() + inner.close.assert_called_once() diff --git a/uv.lock b/uv.lock index dae1041..45f86be 100644 --- a/uv.lock +++ b/uv.lock @@ -89,6 +89,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/39/e7eaf1799466a4aef85b6a4fe7bd175ad2b1c6345066aa33f1f58d4b18d0/asttokens-3.0.1-py3-none-any.whl", hash = "sha256:15a3ebc0f43c2d0a50eeafea25e19046c68398e487b9f1f5b517f7c0f40f976a", size = 27047, upload-time = "2025-11-15T16:43:16.109Z" }, ] +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "babel" version = "2.18.0" @@ -1431,6 +1440,9 @@ notebooks = [ { name = "matplotlib" }, { name = "matplotlib-inline" }, ] +observer-rerun = [ + { name = "rerun-sdk" }, +] realsense = [ { name = "pyrealsense2", marker = "sys_platform != 'darwin'" }, { name = "pyrealsense2-macosx", marker = "sys_platform == 'darwin'" }, @@ -1499,11 +1511,12 @@ requires-dist = [ { name = "pytest", marker = "extra == 'tests'" }, { name = "pyturbojpeg", marker = "sys_platform == 'linux'", specifier = "<2" }, { name = "pyyaml", specifier = ">=6.0" }, + { name = "rerun-sdk", marker = "extra == 'observer-rerun'", specifier = ">=0.22" }, { name = "safetensors", specifier = ">=0.4.3,<1.0.0" }, { name = "transformers", specifier = ">=5.3.0,<5.4.0" }, { name = "trossen-arm", marker = "extra == 'trossen'", specifier = ">=1.9.0" }, ] -provides-extras = ["tests", "notebooks", "realsense", "basler", "transport", "genicam", "capture", "ur", "abb", "franka", "trossen", "so101", "robots"] +provides-extras = ["tests", "notebooks", "realsense", "basler", "transport", "genicam", "capture", "ur", "abb", "franka", "trossen", "so101", "observer-rerun", "robots"] [package.metadata.requires-dev] dev = [ @@ -1702,6 +1715,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] +[[package]] +name = "pyarrow" +version = "24.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/a9/9686d9f07837f91f775e8932659192e02c74f9d8920524b480b85212cc68/pyarrow-24.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:6233c9ed9ab9d1db47de57d9753256d9dcffbf42db341576099f0fd9f6bf4810", size = 34981559, upload-time = "2026-04-21T10:47:22.17Z" }, + { url = "https://files.pythonhosted.org/packages/80/b6/0ddf0e9b6ead3474ab087ae598c76b031fc45532bf6a63f3a553440fb258/pyarrow-24.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f7616236ec1bc2b15bfdec22a71ab38851c86f8f05ff64f379e1278cf20c634a", size = 36663654, upload-time = "2026-04-21T10:47:28.315Z" }, + { url = "https://files.pythonhosted.org/packages/7c/3b/926382efe8ce27ba729071d3566ade6dfb86bdf112f366000196b2f5780a/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1617043b99bd33e5318ae18eb2919af09c71322ef1ca46566cdafc6e6712fb66", size = 45679394, upload-time = "2026-04-21T10:47:34.821Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7a/829f7d9dfd37c207206081d6dad474d81dde29952401f07f2ba507814818/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6165461f55ef6314f026de6638d661188e3455d3ec49834556a0ebbdbace18bb", size = 48863122, upload-time = "2026-04-21T10:47:42.056Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e8/f88ce625fe8babaae64e8db2d417c7653adb3019b08aae85c5ed787dc816/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b13dedfe76a0ad2d1d859b0811b53827a4e9d93a0bcb05cf59333ab4980cc7e", size = 49376032, upload-time = "2026-04-21T10:47:48.967Z" }, + { url = "https://files.pythonhosted.org/packages/36/7a/82c363caa145fff88fb475da50d3bf52bb024f61917be5424c3392eaf878/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:25ea65d868eb04015cd18e6df2fbe98f07e5bda2abefabcb88fce39a947716f6", size = 51929490, upload-time = "2026-04-21T10:47:55.981Z" }, + { url = "https://files.pythonhosted.org/packages/66/1c/e3e72c8014ad2743ca64a701652c733cc5cbcee15c0463a32a8c55518d9e/pyarrow-24.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:295f0a7f2e242dabd513737cf076007dc5b2d59237e3eca37b05c0c6446f3826", size = 27355660, upload-time = "2026-04-21T10:48:01.718Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d3/a1abf004482026ddc17f4503db227787fa3cfe41ec5091ff20e4fea55e57/pyarrow-24.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:02b001b3ed4723caa44f6cd1af2d5c86aa2cf9971dacc2ffa55b21237713dfba", size = 34976759, upload-time = "2026-04-21T10:48:07.258Z" }, + { url = "https://files.pythonhosted.org/packages/4f/4a/34f0a36d28a2dd32225301b79daad44e243dc1a2bb77d43b60749be255c4/pyarrow-24.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:04920d6a71aabd08a0417709efce97d45ea8e6fb733d9ca9ecffb13c67839f68", size = 36658471, upload-time = "2026-04-21T10:48:13.347Z" }, + { url = "https://files.pythonhosted.org/packages/1f/78/543b94712ae8bb1a6023bcc1acf1a740fbff8286747c289cd9468fced2a5/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a964266397740257f16f7bb2e4f08a0c81454004beab8ff59dd531b73610e9f2", size = 45675981, upload-time = "2026-04-21T10:48:20.201Z" }, + { url = "https://files.pythonhosted.org/packages/84/9f/8fb7c222b100d314137fa40ec050de56cd8c6d957d1cfff685ce72f15b17/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f066b179d68c413374294bc1735f68475457c933258df594443bb9d88ddc2a0", size = 48859172, upload-time = "2026-04-21T10:48:27.541Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d3/1ea72538e6c8b3b475ed78d1049a2c518e655761ea50fe1171fc855fcab7/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1183baeb14c5f587b1ec52831e665718ce632caab84b7cd6b85fd44f96114495", size = 49385733, upload-time = "2026-04-21T10:48:34.7Z" }, + { url = "https://files.pythonhosted.org/packages/c3/be/c3d8b06a1ba35f2260f8e1f771abbee7d5e345c0937aab90675706b1690a/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:806f24b4085453c197a5078218d1ee08783ebbba271badd153d1ae22a3ee804f", size = 51934335, upload-time = "2026-04-21T10:48:42.099Z" }, + { url = "https://files.pythonhosted.org/packages/9c/62/89e07a1e7329d2cde3e3c6994ba0839a24977a2beda8be6005ea3d860b99/pyarrow-24.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:e4505fc6583f7b05ab854934896bcac8253b04ac1171a77dfb73efef92076d91", size = 27271748, upload-time = "2026-04-21T10:49:42.532Z" }, + { url = "https://files.pythonhosted.org/packages/17/1a/cff3a59f80b5b1658549d46611b67163f65e0664431c076ad728bf9d5af4/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1a4e45017efbf115032e4475ee876d525e0e36c742214fbe405332480ecd6275", size = 35238554, upload-time = "2026-04-21T10:48:48.526Z" }, + { url = "https://files.pythonhosted.org/packages/a8/99/cce0f42a327bfef2c420fb6078a3eb834826e5d6697bf3009fe11d2ad051/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:7986f1fa71cee060ad00758bcc79d3a93bab8559bf978fab9e53472a2e25a17b", size = 36782301, upload-time = "2026-04-21T10:48:55.181Z" }, + { url = "https://files.pythonhosted.org/packages/2a/66/8e560d5ff6793ca29aca213c53eec0dd482dd46cb93b2819e5aab52e4252/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d3e0b61e8efb24ed38898e5cdc5fffa9124be480008d401a1f8071500494ae42", size = 45721929, upload-time = "2026-04-21T10:49:03.676Z" }, + { url = "https://files.pythonhosted.org/packages/27/0c/a26e25505d030716e078d9f16eb74973cbf0b33b672884e9f9da1c83b871/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:55a3bc1e3df3b5567b7d27ef551b2283f0c68a5e86f1cd56abc569da4f31335b", size = 48825365, upload-time = "2026-04-21T10:49:11.714Z" }, + { url = "https://files.pythonhosted.org/packages/5f/eb/771f9ecb0c65e73fe9dccdd1717901b9594f08c4515d000c7c62df573811/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:641f795b361874ac9da5294f8f443dfdbee355cf2bd9e3b8d97aaac2306b9b37", size = 49451819, upload-time = "2026-04-21T10:49:21.474Z" }, + { url = "https://files.pythonhosted.org/packages/48/da/61ae89a88732f5a785646f3ec6125dbb640fa98a540eb2b9889caa561403/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8adc8e6ce5fccf5dc707046ae4914fd537def529709cc0d285d37a7f9cd442ca", size = 51909252, upload-time = "2026-04-21T10:49:31.164Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1a/8dd5cafab7b66573fa91c03d06d213356ad4edd71813aa75e08ce2b3a844/pyarrow-24.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:9b18371ad2f44044b81a8d23bc2d8a9b6a6226dca775e8e16cfee640473d6c5d", size = 27388127, upload-time = "2026-04-21T10:49:37.334Z" }, + { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" }, + { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" }, + { url = "https://files.pythonhosted.org/packages/29/88/f4e9145da0417b3d2c12035a8492b35ff4a3dbc653e614fcfb51d9dedb38/pyarrow-24.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:38be1808cdd068605b787e6ca9119b27eb275a0234e50212c3492331680c3b1e", size = 28001155, upload-time = "2026-04-21T10:51:22.337Z" }, + { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" }, + { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" }, + { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" }, + { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" }, + { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" }, + { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" }, + { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -2155,6 +2211,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "rerun-sdk" +version = "0.32.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "numpy" }, + { name = "pillow" }, + { name = "psutil" }, + { name = "pyarrow" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/65/d593a2628f2b2b718231dd15b971a81159a72a6e4c498db454e8b96c3404/rerun_sdk-0.32.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:db554a1af3a6a5562f0c87ad166e5470c0acc54b61685aa59226e4e57cbe3dcf", size = 125141695, upload-time = "2026-05-15T16:37:31.344Z" }, + { url = "https://files.pythonhosted.org/packages/03/89/f1b1ab2222d14fc6a9aa6e60d3842b3f6dc0e07ce4457a6fd2d8c5150fd6/rerun_sdk-0.32.1-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:99a9a38f919f1bc16caa933303379284c8ec57b4326edb1f328325a82e5a9f7b", size = 134641179, upload-time = "2026-05-15T16:37:36.766Z" }, + { url = "https://files.pythonhosted.org/packages/51/7b/c4d591443ef9065781187be32ff4054aa8c283d44c2f89b2777799b40a32/rerun_sdk-0.32.1-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2bdca8a581f510e36e261b7d0e5d60dbede9d6e69ba3f3a6d44d11a21a13d18c", size = 138948354, upload-time = "2026-05-15T16:37:41.675Z" }, + { url = "https://files.pythonhosted.org/packages/29/20/b633908f0702a649eb95877d4603f58bf570e4be8fe4db25de5dc8184b91/rerun_sdk-0.32.1-cp310-abi3-win_amd64.whl", hash = "sha256:8ea2f2f78fbeae8a8e49fb7607399be340a8fb072a6f6a422331270512605e02", size = 119826167, upload-time = "2026-05-15T16:37:46.242Z" }, +] + [[package]] name = "rich" version = "15.0.0"