Skip to content

Commit bdb6686

Browse files
author
Test User
committed
test: add lifecycle testing to remove live API calls from standard unit tests
1 parent 8cc7b4f commit bdb6686

8 files changed

Lines changed: 718 additions & 0 deletions

File tree

‎.github/workflows/lifecycle.yml‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
name: Full Lifecycle Tests
2+
3+
# Expensive — makes real LLM API calls.
4+
# NOT triggered by push or PR. Manual dispatch or weekly schedule only.
5+
6+
on:
7+
workflow_dispatch:
8+
inputs:
9+
mode:
10+
description: 'Test mode'
11+
required: false
12+
default: 'cli'
13+
type: choice
14+
options: [cli, api, web, all]
15+
model:
16+
description: 'LLM model (haiku = cheaper, sonnet = stronger)'
17+
required: false
18+
default: 'haiku'
19+
type: choice
20+
options: [haiku, sonnet]
21+
schedule:
22+
# Weekly — Sunday 3am UTC
23+
- cron: '0 3 * * 0'
24+
25+
env:
26+
PYTHON_VERSION: '3.11'
27+
NODE_VERSION: '20'
28+
29+
jobs:
30+
lifecycle:
31+
name: Lifecycle (${{ inputs.mode || 'cli' }} / ${{ inputs.model || 'haiku' }})
32+
runs-on: ubuntu-latest
33+
timeout-minutes: 60
34+
35+
steps:
36+
- name: Checkout code
37+
uses: actions/checkout@v4
38+
39+
- name: Set up Python
40+
uses: actions/setup-python@v5
41+
with:
42+
python-version: ${{ env.PYTHON_VERSION }}
43+
44+
- name: Install uv
45+
uses: astral-sh/setup-uv@v4
46+
with:
47+
enable-cache: true
48+
49+
- name: Install dependencies
50+
run: |
51+
uv venv
52+
uv sync --extra dev
53+
uv pip install -e .
54+
55+
- name: Configure git
56+
run: |
57+
git config --global user.name "Lifecycle Test"
58+
git config --global user.email "lifecycle@codeframe.test"
59+
60+
- name: Resolve test selection
61+
id: tests
62+
run: |
63+
MODE="${{ inputs.mode || 'cli' }}"
64+
case "$MODE" in
65+
cli) echo "path=tests/lifecycle/test_cli_lifecycle.py" >> "$GITHUB_OUTPUT" ;;
66+
api) echo "path=tests/lifecycle/test_api_lifecycle.py" >> "$GITHUB_OUTPUT" ;;
67+
web) echo "path=tests/lifecycle/test_web_lifecycle.py" >> "$GITHUB_OUTPUT" ;;
68+
all) echo "path=tests/lifecycle/" >> "$GITHUB_OUTPUT" ;;
69+
esac
70+
71+
- name: Run lifecycle tests
72+
env:
73+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
74+
CODEFRAME_LIFECYCLE_MODEL: ${{ inputs.model || 'haiku' }}
75+
run: |
76+
uv run pytest ${{ steps.tests.outputs.path }} \
77+
-m lifecycle \
78+
-v \
79+
--tb=long \
80+
--no-header \
81+
-p no:warnings \
82+
--timeout=1800
83+
84+
- name: Upload test artifacts on failure
85+
if: failure()
86+
uses: actions/upload-artifact@v4
87+
with:
88+
name: lifecycle-failure-${{ github.run_id }}
89+
path: |
90+
/tmp/pytest-*/
91+
retention-days: 7

‎scripts/lifecycle‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
#!/usr/bin/env bash
2+
# lifecycle — Run CodeFRAME full lifecycle tests (real LLM calls)
3+
#
4+
# Usage:
5+
# scripts/lifecycle [options]
6+
#
7+
# Options:
8+
# --mode cli|api|web|all Which test mode (default: cli)
9+
# --model haiku|sonnet LLM model to use (default: haiku, cheaper)
10+
# --verbose / -v Pass -s to pytest (show live output)
11+
# --no-cleanup Keep temp directories after test (for inspection)
12+
# --dry-run Show what would run without running it
13+
# -h, --help Show this help
14+
15+
set -euo pipefail
16+
17+
# ── Defaults ────────────────────────────────────────────────────────────────
18+
MODE="cli"
19+
MODEL="haiku"
20+
VERBOSE=""
21+
NO_CLEANUP=""
22+
DRY_RUN=""
23+
24+
# ── Argument parsing ─────────────────────────────────────────────────────────
25+
while [[ $# -gt 0 ]]; do
26+
case "$1" in
27+
--mode) MODE="$2"; shift 2 ;;
28+
--model) MODEL="$2"; shift 2 ;;
29+
--verbose|-v) VERBOSE="-s"; shift ;;
30+
--no-cleanup) NO_CLEANUP="1"; shift ;;
31+
--dry-run) DRY_RUN="1"; shift ;;
32+
-h|--help)
33+
sed -n '2,16p' "$0" # print the header comment block
34+
exit 0
35+
;;
36+
*) echo "Unknown option: $1" >&2; exit 1 ;;
37+
esac
38+
done
39+
40+
# ── Validate args ────────────────────────────────────────────────────────────
41+
case "$MODE" in
42+
cli|api|web|all) ;;
43+
*) echo "Error: --mode must be cli, api, web, or all" >&2; exit 1 ;;
44+
esac
45+
46+
case "$MODEL" in
47+
haiku|sonnet) ;;
48+
*) echo "Error: --model must be haiku or sonnet" >&2; exit 1 ;;
49+
esac
50+
51+
# ── Check API key ────────────────────────────────────────────────────────────
52+
if [[ -z "${ANTHROPIC_API_KEY:-}" ]]; then
53+
echo "Error: ANTHROPIC_API_KEY is not set." >&2
54+
echo "" >&2
55+
echo "Export it first:" >&2
56+
echo " export ANTHROPIC_API_KEY=sk-ant-..." >&2
57+
echo "" >&2
58+
echo "Note: These tests make real API calls and cost money." >&2
59+
echo "Use --model haiku to minimize cost (~\$0.50–1.00 per full run)." >&2
60+
exit 1
61+
fi
62+
63+
# ── Resolve test path ────────────────────────────────────────────────────────
64+
case "$MODE" in
65+
cli) TEST_PATH="tests/lifecycle/test_cli_lifecycle.py" ;;
66+
api) TEST_PATH="tests/lifecycle/test_api_lifecycle.py" ;;
67+
web) TEST_PATH="tests/lifecycle/test_web_lifecycle.py" ;;
68+
all) TEST_PATH="tests/lifecycle/" ;;
69+
esac
70+
71+
# ── Cost warning ─────────────────────────────────────────────────────────────
72+
COST_HINT="~\$0.50–1.00"
73+
if [[ "$MODEL" == "sonnet" ]]; then
74+
COST_HINT="~\$1.00–3.00"
75+
fi
76+
if [[ "$MODE" == "all" ]]; then
77+
COST_HINT="~\$1.50–5.00"
78+
fi
79+
80+
# ── Summary ──────────────────────────────────────────────────────────────────
81+
echo ""
82+
echo " CodeFRAME Lifecycle Test"
83+
echo " ─────────────────────────────"
84+
echo " Mode: $MODE"
85+
echo " Model: $MODEL (set via CODEFRAME_LIFECYCLE_MODEL)"
86+
echo " Path: $TEST_PATH"
87+
echo " Cost: $COST_HINT (real API calls)"
88+
echo ""
89+
90+
if [[ -n "$DRY_RUN" ]]; then
91+
echo "[dry-run] Would run:"
92+
echo " CODEFRAME_LIFECYCLE_MODEL=$MODEL uv run pytest $TEST_PATH -m lifecycle -v $VERBOSE"
93+
echo ""
94+
exit 0
95+
fi
96+
97+
# ── Confirm (skip if non-interactive / piped) ─────────────────────────────────
98+
if [[ -t 0 && -z "${LIFECYCLE_NO_CONFIRM:-}" ]]; then
99+
read -rp " Proceed? [y/N] " confirm
100+
echo ""
101+
if [[ "$confirm" != [yY] && "$confirm" != [yY][eE][sS] ]]; then
102+
echo "Aborted."
103+
exit 0
104+
fi
105+
fi
106+
107+
# ── Build pytest args ─────────────────────────────────────────────────────────
108+
PYTEST_ARGS=(
109+
"$TEST_PATH"
110+
-m lifecycle
111+
-v
112+
--tb=long
113+
--no-header
114+
--timeout=1800
115+
)
116+
117+
if [[ -n "$VERBOSE" ]]; then
118+
PYTEST_ARGS+=(-s)
119+
fi
120+
121+
if [[ -n "$NO_CLEANUP" ]]; then
122+
PYTEST_ARGS+=(--basetemp=/tmp/lifecycle-keep)
123+
fi
124+
125+
# ── Run ───────────────────────────────────────────────────────────────────────
126+
echo " Starting... (this may take 10–30 minutes)"
127+
echo ""
128+
129+
export CODEFRAME_LIFECYCLE_MODEL="$MODEL"
130+
131+
uv run pytest "${PYTEST_ARGS[@]}"

‎tests/lifecycle/conftest.py‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
"""
2+
Shared fixtures for lifecycle tests.
3+
4+
These tests are EXPENSIVE — they make real LLM API calls.
5+
They are excluded from normal pytest runs and must be invoked explicitly:
6+
7+
uv run pytest tests/lifecycle/ -m lifecycle
8+
# or via the convenience script:
9+
scripts/lifecycle [--mode cli|api|web|all] [--model haiku|sonnet]
10+
"""
11+
12+
import os
13+
import shutil
14+
import subprocess
15+
import sys
16+
from pathlib import Path
17+
18+
import pytest
19+
20+
SAMPLE_PROJECT_DIR = Path(__file__).parent / "sample_project"
21+
22+
23+
def pytest_configure(config):
24+
config.addinivalue_line(
25+
"markers",
26+
"lifecycle: full end-to-end lifecycle tests (real LLM calls — run explicitly)",
27+
)
28+
29+
30+
# ---------------------------------------------------------------------------
31+
# Guard: skip entire suite if no API key
32+
# ---------------------------------------------------------------------------
33+
34+
def pytest_collection_modifyitems(config, items):
35+
if not os.getenv("ANTHROPIC_API_KEY"):
36+
skip = pytest.mark.skip(reason="ANTHROPIC_API_KEY not set — lifecycle tests require real API")
37+
for item in items:
38+
if "lifecycle" in str(item.fspath):
39+
item.add_marker(skip)
40+
41+
42+
# ---------------------------------------------------------------------------
43+
# Fixtures
44+
# ---------------------------------------------------------------------------
45+
46+
@pytest.fixture(scope="session")
47+
def sample_prd_path() -> Path:
48+
"""Path to the sample project PRD."""
49+
return SAMPLE_PROJECT_DIR / "PRD.md"
50+
51+
52+
@pytest.fixture
53+
def target_project_dir(tmp_path) -> Path:
54+
"""
55+
A fresh temporary directory with git initialized,
56+
ready for `cf init` and agent execution.
57+
"""
58+
project = tmp_path / "csv-stats"
59+
project.mkdir()
60+
61+
# Initialize git repo (cf init requires it)
62+
subprocess.run(["git", "init", str(project)], check=True, capture_output=True)
63+
subprocess.run(["git", "config", "user.email", "lifecycle@test.com"],
64+
cwd=project, check=True, capture_output=True)
65+
subprocess.run(["git", "config", "user.name", "Lifecycle Test"],
66+
cwd=project, check=True, capture_output=True)
67+
68+
yield project
69+
70+
# Optional: keep on failure for inspection (controlled by --no-cleanup flag)
71+
# cleanup happens automatically via tmp_path
72+
73+
74+
@pytest.fixture
75+
def cf(target_project_dir):
76+
"""
77+
Helper to invoke `cf` CLI commands against the target project directory.
78+
Returns the completed process (stdout/stderr captured, no exception on nonzero).
79+
"""
80+
def run(*args, cwd=None, timeout=60, **kwargs):
81+
return subprocess.run(
82+
["uv", "run", "cf", *args],
83+
cwd=cwd or target_project_dir,
84+
capture_output=True,
85+
text=True,
86+
timeout=timeout,
87+
env={**os.environ},
88+
)
89+
return run
90+
91+
92+
@pytest.fixture
93+
def initialized_workspace(target_project_dir, cf, sample_prd_path):
94+
"""
95+
A workspace with:
96+
- cf init complete
97+
- PRD added
98+
- Tasks generated
99+
Ready for `cf work batch run --all-ready --execute`.
100+
"""
101+
result = cf("init", str(target_project_dir), "--tech-stack", "Python with uv")
102+
assert result.returncode == 0, f"cf init failed:\n{result.stderr}"
103+
104+
# Copy PRD into the target dir so cf prd add has a local path
105+
prd_dest = target_project_dir / "PRD.md"
106+
shutil.copy(sample_prd_path, prd_dest)
107+
108+
result = cf("prd", "add", "PRD.md")
109+
assert result.returncode == 0, f"cf prd add failed:\n{result.stderr}"
110+
111+
result = cf("tasks", "generate")
112+
assert result.returncode == 0, f"cf tasks generate failed:\n{result.stderr}"
113+
114+
return target_project_dir

‎tests/lifecycle/sample_project/PRD.md‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# CSV Statistics Tool
2+
3+
## Overview
4+
5+
A command-line Python script that reads a CSV file and prints summary statistics.
6+
This is a self-contained tool with no external dependencies beyond Python stdlib.
7+
8+
## Requirements
9+
10+
### Script: `csv_stats.py`
11+
12+
**Usage:**
13+
```
14+
python csv_stats.py <path/to/file.csv>
15+
```
16+
17+
**Output format (exact):**
18+
```
19+
Rows: 42
20+
Columns: name, age, score, city
21+
age:
22+
min: 18.00
23+
max: 65.00
24+
mean: 34.21
25+
score:
26+
min: 55.00
27+
max: 99.00
28+
mean: 78.45
29+
```
30+
31+
Rules:
32+
- List all column names on the `Columns:` line, comma-separated
33+
- For each **numeric** column, print its name followed by min/max/mean (2 decimal places)
34+
- Skip non-numeric columns (no output for them beyond listing in Columns)
35+
- Skip blank/missing values when computing stats
36+
- Non-numeric columns are silently skipped in the stats section
37+
38+
**Error handling:**
39+
- File not found → print `Error: file not found: <path>` to stderr, exit code 1
40+
- Not a valid CSV (unparseable) → print `Error: not a valid CSV file` to stderr, exit code 1
41+
- Empty file (no rows after header) → print `Rows: 0` and column names, skip stats
42+
43+
### Test file: `test_csv_stats.py`
44+
45+
Must include at least 4 pytest tests:
46+
1. Happy path: numeric columns produce correct min/max/mean output
47+
2. Mixed columns: non-numeric columns are skipped in stats
48+
3. Missing values: blanks are excluded from calculations
49+
4. File not found: exits with code 1 and correct error message
50+
51+
### Code quality
52+
- Must pass `ruff check .` with zero errors
53+
- No third-party imports (stdlib only: csv, sys, pathlib, statistics, etc.)

0 commit comments

Comments
 (0)