diff --git a/orchestrator.py b/orchestrator.py index 5e7adb2..3360c2f 100644 --- a/orchestrator.py +++ b/orchestrator.py @@ -304,11 +304,11 @@ def run_shell_command(command: str) -> str: # noqa: S603 tells the linter we have explicitly sandboxed this input result = subprocess.run( # noqa: S603 - args, - capture_output=True, + args, + capture_output=True, text=True, - encoding="utf-8", - timeout=60, + encoding="utf-8", + timeout=60, shell=False, ) @@ -368,7 +368,7 @@ def get_active_artifacts(): # SHIFT-LEFT: Match any project file path anywhere in the document. # Pattern matches common project paths: docs/, src/, public/, tests/ with typical extensions. paths = re.findall(r"(?:docs|src|public|tests)[a-zA-Z0-9_./-]+\.[a-zA-Z0-9]+", content) - + for path in set(paths): # Deduplicate identical paths if "current_run.md" not in path: artifacts.append(path) @@ -526,6 +526,43 @@ def extract_routing_queue(response_text): return None +def auto_lint_file(filepath): + """Zero-Cost Pre-Audit: Automatically lints files immediately after they are written.""" + abs_path = os.path.join(BASE_DIR, filepath) if not os.path.isabs(filepath) else filepath + ext = os.path.splitext(abs_path)[1] + + args = [] + if ext == ".py": + args = ["uv", "run", "ruff", "check", "--no-cache", abs_path] + elif ext in [".ts", ".tsx", ".js", ".jsx"]: + # Forge uses Biome for JS/TS + args = ["npx", "biome", "check", abs_path] + else: + return None # No auto-linter for this file type + + # Cross-Platform Executable Resolution (Windows Support) + if os.name == "nt": + executable = shutil.which(args[0]) + if executable: + args[0] = executable + + try: + # noqa: S603 tells the linter we explicitly control the args array + result = subprocess.run( # noqa: S603 + args, capture_output=True, text=True, encoding="utf-8", timeout=30, shell=False + ) + if result.returncode != 0: + # Wrap the long string in parentheses to comply with the 100-char limit + return ( + f"[⚠️ AUTO-LINT FAILED on {filepath}]:\n" + f"{result.stdout}\n{result.stderr}\n" + "Fix this syntax error before proceeding." + ) + return f"[✅ AUTO-LINT PASSED for {filepath}]" + except Exception as e: + return f"[⚠️ AUTO-LINT EXECUTION ERROR on {filepath}]: {e}" + + def execute_autonomous_actions(response_text): """Scans the AI's response for a JSON payload and executes the sandbox tools.""" # Look for a JSON block explicitly tagged for the OS @@ -548,6 +585,12 @@ def execute_autonomous_actions(response_text): result = write_file(path, content) execution_logs.append(result) + # --- SHIFT-LEFT: FORGE AUTO-LINTING --- + if "SUCCESS" in result: + lint_result = auto_lint_file(path) + if lint_result: + execution_logs.append(lint_result) + # 1.5 Execute File Appends (Scalpel) if "append_to_file" in payload: for file_data in payload["append_to_file"]: @@ -557,6 +600,12 @@ def execute_autonomous_actions(response_text): result = append_file(path, content) execution_logs.append(result) + # --- SHIFT-LEFT: FORGE AUTO-LINTING --- + if "SUCCESS" in result: + lint_result = auto_lint_file(path) + if lint_result: + execution_logs.append(lint_result) + # 2. Execute Shell Commands (Testing/Linting) if "run_commands" in payload: for cmd in payload["run_commands"]: diff --git a/pyproject.toml b/pyproject.toml index 479320a..0cf6342 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dev = [ "pytest-cov>=5.0.0", "ruff>=0.3.0", "httpx>=0.27.0", + "pytest-mock>=3.15.1", ] [tool.ruff] @@ -34,9 +35,9 @@ ignore = [] "tests/**/*.py" = ["S101"] [tool.pytest.ini_options] -addopts = "-v --strict-markers --cov=orchestrator --cov-report=term-missing --cov-fail-under=40" +addopts = "-v -m \"not eval\" --strict-markers --cov=orchestrator --cov-report=term-missing --cov-fail-under=40" pythonpath = "." -testpaths = ["tests/api"] +testpaths = ["tests/api", "tests/evals"] markers = [ "core: core system functionality", "integration: testing 3rd party APIs", diff --git a/tests/evals/test_orchestrator.py b/tests/evals/test_orchestrator.py index 72969f3..9b407cf 100644 --- a/tests/evals/test_orchestrator.py +++ b/tests/evals/test_orchestrator.py @@ -206,3 +206,44 @@ def test_log_jsonl_telemetry(tmp_path, monkeypatch) -> None: assert data["agent"] == "Engineering" assert data["response"] == "response_text" assert data["prompt_tokens"] == 10 + + +def test_auto_lint_file_python_success(mocker): + """Ensure the Forge auto-linter correctly triggers ruff for Python files and passes.""" + from orchestrator import auto_lint_file + + mock_run = mocker.patch("subprocess.run") + # Simulate a successful ruff check (exit code 0) + mock_run.return_value = mocker.MagicMock(returncode=0) + + result = auto_lint_file("src/api/main.py") + + assert "✅ AUTO-LINT PASSED" in result + mock_run.assert_called_once() + + # Prove it specifically chose the Python linter + called_command = mock_run.call_args[0][0] + assert "ruff" in called_command + assert "check" in called_command + + +def test_auto_lint_file_typescript_failure(mocker): + """Ensure the Forge auto-linter triggers biome for TS files and catches syntax errors.""" + from orchestrator import auto_lint_file + + mock_run = mocker.patch("subprocess.run") + # Simulate a failed biome check (exit code 1) + mock_run.return_value = mocker.MagicMock( + returncode=1, stdout="Expected an identifier, but found '}'", stderr="" + ) + + result = auto_lint_file("src/web/components/ui/button.tsx") + + assert "⚠️ AUTO-LINT FAILED" in result + assert "Expected an identifier" in result + mock_run.assert_called_once() + + # Prove it specifically chose the Frontend linter + called_command = mock_run.call_args[0][0] + assert "biome" in called_command + assert "check" in called_command