diff --git a/.github/DISCUSSION_TEMPLATE/alpha-feedback.yml b/.github/DISCUSSION_TEMPLATE/alpha-feedback.yml new file mode 100644 index 0000000..cce23a6 --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/alpha-feedback.yml @@ -0,0 +1,46 @@ +title: "[Alpha]: " +labels: ["alpha", "feedback"] +body: + - type: markdown + attributes: + value: | + Thanks for testing RecallForge early. Please avoid sharing private indexed content. Summaries and redacted snippets are enough. + - type: input + id: tester_profile + attributes: + label: Tester profile + description: Machine/backend/MCP client, for example "M3 Pro, MLX, Claude Desktop". + validations: + required: true + - type: textarea + id: workflow + attributes: + label: Workflow tested + description: What did you ingest and what did you ask RecallForge to recall? + validations: + required: true + - type: checkboxes + id: checklist + attributes: + label: Checklist + options: + - label: Installed RecallForge and captured `recallforge --version`. + - label: Started the MCP server in a real client. + - label: Ingested at least one text item and one media/document item. + - label: Re-ran an ingest or update and checked that stale memories did not show up. + - type: textarea + id: quality + attributes: + label: Retrieval quality notes + description: What worked, what missed, and what result surprised you? + - type: textarea + id: friction + attributes: + label: Friction and docs gaps + description: Where did setup, terminology, or the workflow feel confusing? + - type: textarea + id: crash_report + attributes: + label: Optional opt-in crash report + description: Attach or paste reviewed output from `recallforge crash-report` only if useful. + render: json diff --git a/.github/DISCUSSION_TEMPLATE/beta-feedback.yml b/.github/DISCUSSION_TEMPLATE/beta-feedback.yml new file mode 100644 index 0000000..676ee6e --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/beta-feedback.yml @@ -0,0 +1,48 @@ +title: "[Beta]: " +labels: ["beta", "feedback"] +body: + - type: markdown + attributes: + value: | + Beta feedback should focus on release readiness: install, MCP startup, workflow quality, and safe defaults. + - type: input + id: install_source + attributes: + label: Install source + placeholder: PyPI, release candidate wheel, source checkout + validations: + required: true + - type: input + id: environment + attributes: + label: Environment + placeholder: OS, Python version, backend, MCP client + validations: + required: true + - type: textarea + id: workflows + attributes: + label: Workflows tested + description: List the 3-5 real workflows you tried and whether they succeeded. + validations: + required: true + - type: textarea + id: flags + attributes: + label: Feature flags + description: Paste `recallforge flags --json` if you changed any defaults. + render: json + - type: textarea + id: quality + attributes: + label: Quality and performance notes + description: Note slow spots, bad matches, missing docs, or release-blocking behavior. + - type: checkboxes + id: release_readiness + attributes: + label: Release readiness + options: + - label: Install was understandable. + - label: MCP startup worked in my client. + - label: Default feature flags felt safe. + - label: I would trust this build for a small local-memory workflow. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..3ff8cc5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,61 @@ +name: Bug report +description: Report a reproducible RecallForge bug. +title: "[Bug]: " +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for reporting a bug. For alpha/beta workflow feedback that is not a reproducible bug, use Discussions instead. + - type: textarea + id: summary + attributes: + label: What happened? + description: Describe the bug and what you expected instead. + validations: + required: true + - type: textarea + id: steps + attributes: + label: Steps to reproduce + description: Include commands, MCP client, backend, and a tiny fixture if possible. + placeholder: | + 1. Run `recallforge ...` + 2. Ingest ... + 3. Search ... + validations: + required: true + - type: dropdown + id: surface + attributes: + label: Surface + options: + - CLI + - MCP stdio + - MCP HTTP/SSE + - Python API + - Storage/indexing + - Search/retrieval + - Packaging/install + validations: + required: true + - type: input + id: version + attributes: + label: RecallForge version + placeholder: recallforge --version + validations: + required: true + - type: textarea + id: crash_report + attributes: + label: Optional crash report + description: Run `recallforge crash-report --include-env` and paste or attach the reviewed JSON if helpful. Do not include private indexed content. + render: json + - type: checkboxes + id: privacy + attributes: + label: Privacy check + options: + - label: I reviewed logs/crash reports for secrets and private indexed content before attaching them. + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..4218eb6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Alpha and beta feedback + url: https://github.com/brianmeyer/recallforge/discussions + about: Use Discussions for workflow reports, install notes, and non-blocking product feedback. + - name: Release and testing guide + url: https://github.com/brianmeyer/recallforge/blob/master/docs/ALPHA_BETA_TESTING.md + about: Read the alpha/beta checklist and opt-in crash-report policy before filing feedback. diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e55bd7..a3f0726 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to RecallForge will be documented in this file. - Added staged background reindex promotion so document, video, audio, and conversation replacements stay hidden until their parent/child memory batches are complete. - Added index-version-aware query caching for repeated text/media embeddings and generated expansion branches. - Added MCP progress notifications for long-running search, ingest, batch, memory write, and FTS rebuild tool calls when clients provide a progress token. +- Added alpha/beta testing scaffolding with GitHub templates, feature-flag inspection, and local-only opt-in crash report generation. - Added deterministic memory graph enrichment with entity/relation side tables and new `memory_graph_entities` / `memory_graph_related` MCP tools. - Replaced the tiny UAT video clips with compact episodic-memory fixtures, richer transcript sidecars, related artifact metadata, and regression coverage for the video corpus. - Added `memory_add_conversation` so conversation threads ingest as canonical parent memories with turn-level child memories and standard memory rollups. diff --git a/README.md b/README.md index b969bd9..e85910d 100644 --- a/README.md +++ b/README.md @@ -265,6 +265,15 @@ Full references: [`docs/MEMORY_POLICY.md`](docs/MEMORY_POLICY.md), and [`docs/RUNTIME_BUDGETS.md`](docs/RUNTIME_BUDGETS.md) +For alpha/beta testers: + +```bash +recallforge flags +recallforge crash-report --output recallforge-crash-report.json --message "what happened" +``` + +Testing guidance, feature-flag defaults, and the opt-in crash-report policy live in [docs/ALPHA_BETA_TESTING.md](docs/ALPHA_BETA_TESTING.md). + ## Project structure ``` @@ -280,6 +289,8 @@ src/recallforge/ ├── documents.py # PDF/DOCX/PPTX extraction ├── video.py # Frame/transcript extraction ├── audio.py # Transcript-first audio ingest +├── feature_flags.py # Alpha/beta feature flag registry +├── diagnostics.py # Local-only crash report helpers ├── watch_folder.py # Folder monitoring with dedup └── cli.py # CLI interface ``` diff --git a/docs/ALPHA_BETA_TESTING.md b/docs/ALPHA_BETA_TESTING.md new file mode 100644 index 0000000..e4e405c --- /dev/null +++ b/docs/ALPHA_BETA_TESTING.md @@ -0,0 +1,126 @@ +# RecallForge Alpha and Beta Testing Program + +This program keeps early testing useful without turning a local-first memory tool into silent telemetry. Alpha and beta users should share feedback intentionally through GitHub Discussions or Issues. + +## Goals + +- Recruit 5-10 alpha users who use RecallForge with real local notes, documents, images, audio, or videos. +- Validate install, MCP setup, ingest, search, memory rollups, and local runtime safety. +- Move successful alpha workflows into a broader beta with structured feedback. +- Keep crash reporting opt-in, inspectable, and manually shared. + +## Channels + +- Alpha feedback: `https://github.com/brianmeyer/recallforge/discussions` +- Bug reports: `https://github.com/brianmeyer/recallforge/issues/new/choose` +- Security issues: use the repository security policy or private maintainer contact, not public Discussions. + +GitHub discussion category forms live under `.github/DISCUSSION_TEMPLATE/`. To activate them, enable GitHub Discussions and create categories whose slugs match the template filenames, such as `alpha-feedback` and `beta-feedback`. + +## Tester Cohorts + +Alpha users should cover: + +- Apple Silicon local agents using MLX +- CPU/CUDA users through the torch backend +- Claude Desktop or another MCP host +- text-heavy personal notes +- multimodal folders with images, PDFs, audio transcripts, and short videos + +Beta users should add: + +- larger folders and watch-folder workflows +- repeated reindexing +- HTTP/SSE MCP clients +- cross-modal query workflows +- release-candidate install checks from PyPI + +## Feature Flags + +Run this to see the supported feature flags and current values: + +```bash +recallforge flags +recallforge flags --json +``` + +Recommended alpha defaults: + +- Keep `RECALLFORGE_ENABLE_MEDIA_RERANKING=0` unless a tester is explicitly validating capped media reranking. +- Keep `RECALLFORGE_ENABLE_RAW_VIDEO_QUERY_EMBEDDING=0` unless validating raw video-query behavior. +- Keep `RECALLFORGE_ENABLE_MLX_NATIVE_VIDEO_PROCESSING=0` unless validating native MLX video decoding. +- Use `RECALLFORGE_TRACE=1` only while collecting a diagnostic reproduction. + +The canonical environment variable reference is `docs/ENV_VARS.md`. + +## Opt-In Crash Reports + +RecallForge does not send crash reports automatically. A tester can create a local JSON report and review it before attaching it to a Discussion or Issue: + +```bash +recallforge crash-report --output recallforge-crash-report.json --message "Search crashed after a video query" +``` + +To include allowlisted `RECALLFORGE_*` values with home paths redacted: + +```bash +recallforge crash-report --include-env --output recallforge-crash-report.json +``` + +Crash reports include: + +- RecallForge version +- Python implementation/version +- OS family, release, and machine architecture +- effective feature flag values +- optional allowlisted `RECALLFORGE_*` environment values +- a user-provided message + +Crash reports do not include: + +- indexed content +- search queries +- arbitrary environment variables +- automatic network upload +- API keys or tokens unless a user manually adds them after generation + +## Alpha Checklist + +1. Install from PyPI or the current release branch. +2. Run `recallforge --version`. +3. Run `recallforge flags --json` and attach the output if testing experimental flags. +4. Start MCP with `recallforge serve --mode embed`. +5. Ingest a tiny folder with text plus one media file. +6. Search text to text, text to media, and media to text. +7. Reindex the same folder and confirm old/stale results do not appear. +8. Share feedback through the alpha discussion template. + +## Beta Checklist + +1. Install from the release candidate wheel or PyPI package. +2. Run the MCP server through the real client host used day to day. +3. Ingest a representative local folder. +4. Run at least five saved workflows: + - exact lookup + - broad semantic lookup + - image query + - video or transcript-backed query + - conversation or memory rollup lookup +5. Test one runtime feature flag intentionally, then return to default. +6. Attach an opt-in crash report only if the run fails. +7. Record quality and latency notes in the beta discussion template. + +## Exit Criteria + +Alpha is complete when: + +- at least five users complete the alpha checklist +- no release-blocking install or MCP startup issue remains open +- crash reports, if any, are reproducible or explicitly accepted as known limitations + +Beta is complete when: + +- at least ten workflow reports are collected across at least three machine profiles +- PyPI install and MCP startup are boring +- docs explain the safest defaults and opt-in flags +- known limitations are documented before release diff --git a/docs/ENV_VARS.md b/docs/ENV_VARS.md index 1125425..1593edd 100644 --- a/docs/ENV_VARS.md +++ b/docs/ENV_VARS.md @@ -2,6 +2,8 @@ This is the canonical reference for all `RECALLFORGE_*` environment variables used in the codebase. +Run `recallforge flags` to inspect the feature flags that are especially relevant to alpha/beta testers and experimental local workflows. + ## Runtime selection - `RECALLFORGE_BACKEND` @@ -77,6 +79,20 @@ This is the canonical reference for all `RECALLFORGE_*` environment variables us - `RECALLFORGE_ENABLE_RAW_VIDEO_QUERY_EMBEDDING` Enable raw video query embedding. On MLX, RecallForge now defaults to safer caption/transcript-first retrieval unless you explicitly enable this. +## Feature flag surfaces + +These variables are intentionally visible through `recallforge flags`: + +- `RECALLFORGE_ENABLE_MEDIA_RERANKING` +- `RECALLFORGE_ENABLE_RAW_VIDEO_QUERY_EMBEDDING` +- `RECALLFORGE_ENABLE_MLX_NATIVE_VIDEO_PROCESSING` +- `RECALLFORGE_MEDIA_RERANK_REQUIRE_AMBIGUITY` +- `RECALLFORGE_MLX_HEAVY_OP_CONCURRENCY` +- `RECALLFORGE_MCP_MAX_CONCURRENCY` +- `RECALLFORGE_TRACE` + +Alpha/beta users should keep risky media and native-video flags disabled unless they are specifically testing that behavior. + ## Server behavior - `RECALLFORGE_TRACE` diff --git a/docs/RELEASE.md b/docs/RELEASE.md index dcc79ab..b3a00fc 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -15,6 +15,7 @@ Use this checklist before cutting a version. - `docs/ARCHITECTURE.md` - `docs/mcp-tools.md` 3. Confirm CLI help and MCP tool lists still match the docs. +4. Review [ALPHA_BETA_TESTING.md](ALPHA_BETA_TESTING.md) and confirm the alpha/beta defaults still match the release candidate. ## 2. Required local validation @@ -123,7 +124,17 @@ When you omit `--output`, the benchmark now keeps profile-specific filenames for - `ingest`, `index_audio`, watch-folder indexing, CLI indexing, and `content_type="audio"` filters cover the shipped audio path. - Dedicated raw-audio embeddings or transcription are still future work; release checks should verify sidecar transcript retrieval rather than microphone/audio decoding. -## 4. Tag and publish +## 4. Alpha/beta release readiness + +Before inviting alpha or beta users: + +1. Confirm GitHub Discussions is enabled. +2. Create discussion categories with slugs matching `.github/DISCUSSION_TEMPLATE/alpha-feedback.yml` and `.github/DISCUSSION_TEMPLATE/beta-feedback.yml`. +3. Run `recallforge flags --json` from the release candidate and check that risky flags default off. +4. Run `recallforge crash-report --include-env` and inspect the JSON for secret-free, manually shareable diagnostics. +5. Point testers to [ALPHA_BETA_TESTING.md](ALPHA_BETA_TESTING.md). + +## 5. Tag and publish 1. Commit the release changes. 2. Create the release tag: @@ -141,7 +152,7 @@ If your checkout uses a conventional `origin` remote instead of `recallforge`, p gh run list --repo brianmeyer/recallforge --workflow publish.yml --limit 5 ``` -## 5. Post-release smoke check +## 6. Post-release smoke check Verify the released package from PyPI: @@ -153,6 +164,6 @@ recallforge serve --http --mode embed --host 127.0.0.1 --port 7433 Then hit `http://127.0.0.1:7433/health` and confirm the process reports healthy model state. -## 6. Git cleanup +## 7. Git cleanup After the release PR is merged and the tag is published, follow the routine in [`docs/GIT_HYGIENE.md`](GIT_HYGIENE.md) to prune remote-tracking refs, remove local merged branches, and clear generated build artifacts. diff --git a/docs/research/recallforge-memory-mcp-roadmap.md b/docs/research/recallforge-memory-mcp-roadmap.md index ad2bfdb..6072a6e 100644 --- a/docs/research/recallforge-memory-mcp-roadmap.md +++ b/docs/research/recallforge-memory-mcp-roadmap.md @@ -134,12 +134,10 @@ Why this must stay late: Goal: - Prove RecallForge as a memory MCP, not just a benchmark pipeline. -Current Linear fit: -- `REC-33` - Shipped Linear work: - `REC-153` - `REC-61` +- `REC-33` What this phase delivers: - memory-level evaluation @@ -148,6 +146,7 @@ What this phase delivers: - real episodic corpora coverage - MCP progress notifications for long-running search, ingest, batch, and rebuild workflows - alpha and beta validation with real workflows +- explicit feature flags and local-only opt-in crash reports Why this comes last: - launch should validate the staged architecture in practice, not just synthetic retrieval quality @@ -158,7 +157,7 @@ Why this comes last: - Keep `Retrieval and Ranking` for cheap broad retrieval work like `REC-169`, `REC-148`, `REC-72`, `REC-71`, `REC-146` - Add a milestone such as `Memory Policy and Enrichment` for `REC-84`, `REC-83`, `REC-75`, `REC-76`, `REC-78` - Keep `Research Queue` for gated expensive-stage work like `REC-130`, `REC-115`, `REC-147`, `REC-168` -- Keep `Benchmark Integrity` and `Launch and Distribution` for `REC-33` and any future public validation work +- Keep `Benchmark Integrity` and `Launch and Distribution` for future public validation work ## Architecture Principle diff --git a/src/recallforge/cli.py b/src/recallforge/cli.py index 684157e..1d9f61b 100644 --- a/src/recallforge/cli.py +++ b/src/recallforge/cli.py @@ -6,6 +6,8 @@ recallforge index Index a file or directory recallforge search Run a search query recallforge status Show system status + recallforge flags Show feature flags + recallforge crash-report Create a local opt-in crash report """ import argparse @@ -183,6 +185,35 @@ def main(): default=None, help="Path to storage directory", ) + + # feature flags command + flags_parser = subparsers.add_parser("flags", help="Show feature flags") + flags_parser.add_argument( + "--json", + action="store_true", + help="Print feature flags as JSON", + ) + + # crash-report command + crash_parser = subparsers.add_parser( + "crash-report", + help="Create a local-only opt-in crash report", + ) + crash_parser.add_argument( + "--output", "-o", + default=None, + help="Write report JSON to this path. Defaults to stdout.", + ) + crash_parser.add_argument( + "--message", + default="", + help="Optional human-readable note about what happened.", + ) + crash_parser.add_argument( + "--include-env", + action="store_true", + help="Include allowlisted RECALLFORGE_* environment values with home paths redacted.", + ) # collections command collections_parser = subparsers.add_parser("collections", help="Manage collections") @@ -318,6 +349,10 @@ def main(): return cmd_search(args) elif args.command == "status": return cmd_status(args) + elif args.command == "flags": + return cmd_flags(args) + elif args.command == "crash-report": + return cmd_crash_report(args) elif args.command == "collections": return cmd_collections(args) elif args.command == "watch": @@ -589,6 +624,51 @@ def cmd_status(args): return 0 +def cmd_flags(args): + """Show effective feature flags.""" + from .feature_flags import list_feature_flags + + flags = list_feature_flags() + if args.json: + print(json.dumps({"feature_flags": flags}, indent=2)) + return 0 + + print("RecallForge Feature Flags") + print("=" * 40) + for flag in flags: + value = flag["value"] + default = flag["default"] + enabled = flag["enabled"] + enabled_text = "n/a" if enabled is None else ("on" if enabled else "off") + print(f"{flag['name']}={value} (default: {default}, {enabled_text})") + print(f" stage: {flag['stage']}") + print(f" {flag['description']}") + print() + return 0 + + +def cmd_crash_report(args): + """Create a local-only diagnostic crash report.""" + from .diagnostics import collect_crash_report, write_crash_report + + if args.output: + output = write_crash_report( + args.output, + message=args.message, + include_env=args.include_env, + ) + print(f"Wrote local crash report: {output}") + print("Review it before attaching it to a GitHub Discussion or Issue.") + return 0 + + report = collect_crash_report( + message=args.message, + include_env=args.include_env, + ) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + def cmd_collections(args): """Manage collections.""" from . import get_storage diff --git a/src/recallforge/diagnostics.py b/src/recallforge/diagnostics.py new file mode 100644 index 0000000..71fbed4 --- /dev/null +++ b/src/recallforge/diagnostics.py @@ -0,0 +1,123 @@ +""" +diagnostics.py - Local-only diagnostic report helpers for RecallForge. + +No network transport lives here. Users must explicitly generate and share a +report, which keeps crash reporting opt-in and inspectable. +""" + +from __future__ import annotations + +import json +import os +import platform +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Mapping, Optional + +from . import __version__ +from .feature_flags import list_feature_flags + + +_ALLOWLIST_ENV_KEYS = { + "RECALLFORGE_BACKEND", + "RECALLFORGE_MODE", + "RECALLFORGE_MLX_QUANTIZE", + "RECALLFORGE_DISABLE_MLX", + "RECALLFORGE_MLX_HEAVY_OP_CONCURRENCY", + "RECALLFORGE_MLX_VIDEO_SAMPLE_FPS", + "RECALLFORGE_MLX_VIDEO_MAX_FRAMES", + "RECALLFORGE_MLX_VIDEO_FALLBACK_MAX_FRAMES", + "RECALLFORGE_MLX_MIN_PIXELS", + "RECALLFORGE_MLX_MAX_PIXELS", + "RECALLFORGE_ENABLE_MLX_NATIVE_VIDEO_PROCESSING", + "RECALLFORGE_CAPTIONER_IDLE_SECONDS", + "RECALLFORGE_STORAGE", + "RECALLFORGE_STORE_PATH", + "RECALLFORGE_OVERFETCH_FACTOR", + "RECALLFORGE_MAX_CANDIDATES", + "RECALLFORGE_RERANK_TOP_K", + "RECALLFORGE_ENABLE_MEDIA_RERANKING", + "RECALLFORGE_MEDIA_QUERY_RERANK_TOP_K", + "RECALLFORGE_MEDIA_RESULT_RERANK_TOP_K", + "RECALLFORGE_MEDIA_RERANK_REQUIRE_AMBIGUITY", + "RECALLFORGE_MEDIA_RERANK_MIN_RRF_MARGIN", + "RECALLFORGE_ENABLE_RAW_VIDEO_QUERY_EMBEDDING", + "RECALLFORGE_TRACE", + "RECALLFORGE_MCP_MAX_CONCURRENCY", + "RECALLFORGE_BM25_FALLBACK_MAX_ROWS", + "RECALLFORGE_BULK_FLUSH_DOCS", + "RECALLFORGE_BULK_FLUSH_EMBEDDINGS", +} + + +def _redact_path(value: str) -> str: + """Redact the user's home directory from path-like environment values.""" + home = str(Path.home()) + if home and value.startswith(home): + return "~" + value[len(home):] + return value + + +def sanitized_recallforge_env(environ: Optional[Mapping[str, str]] = None) -> dict[str, str]: + """Return allowlisted RecallForge env vars with home paths redacted.""" + env = os.environ if environ is None else environ + sanitized: dict[str, str] = {} + for key in sorted(_ALLOWLIST_ENV_KEYS): + if key not in env: + continue + value = str(env[key]) + if key.endswith("_PATH") or key == "RECALLFORGE_STORE_PATH": + value = _redact_path(value) + sanitized[key] = value + return sanitized + + +def collect_crash_report( + *, + message: str = "", + include_env: bool = False, + environ: Optional[Mapping[str, str]] = None, +) -> dict: + """Build a local-only crash report payload for manual sharing.""" + env = os.environ if environ is None else environ + report = { + "schema_version": 1, + "created_at": datetime.now(timezone.utc).isoformat(), + "recallforge_version": __version__, + "python": { + "version": platform.python_version(), + "implementation": platform.python_implementation(), + "executable": Path(sys.executable).name, + }, + "platform": { + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + }, + "feature_flags": list_feature_flags(env), + "user_message": message.strip(), + "privacy": { + "network_sent": False, + "sharing": "manual", + "notes": "Generated locally. Review before attaching to GitHub Discussions or Issues.", + }, + } + if include_env: + report["environment"] = sanitized_recallforge_env(env) + return report + + +def write_crash_report( + output_path: str | os.PathLike[str], + *, + message: str = "", + include_env: bool = False, + environ: Optional[Mapping[str, str]] = None, +) -> Path: + """Write a crash report JSON file and return its resolved path.""" + output = Path(output_path).expanduser() + output.parent.mkdir(parents=True, exist_ok=True) + report = collect_crash_report(message=message, include_env=include_env, environ=environ) + output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return output.resolve() diff --git a/src/recallforge/feature_flags.py b/src/recallforge/feature_flags.py new file mode 100644 index 0000000..f95b32c --- /dev/null +++ b/src/recallforge/feature_flags.py @@ -0,0 +1,110 @@ +""" +feature_flags.py - Central registry for RecallForge beta/experimental flags. + +Feature flags are environment-variable backed so CLI, MCP, tests, and local +agent hosts all see the same behavior without a separate config service. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Mapping, Optional + + +@dataclass(frozen=True) +class FeatureFlag: + """Documented environment-backed feature flag.""" + + name: str + default: str + description: str + stage: str = "beta" + choices: tuple[str, ...] = () + restart_required: bool = True + + def value_from(self, environ: Mapping[str, str]) -> str: + return str(environ.get(self.name, self.default)) + + def enabled_from(self, environ: Mapping[str, str]) -> Optional[bool]: + if not self.choices: + return None + value = self.value_from(environ).strip().lower() + if value in {"1", "true", "yes", "on"}: + return True + if value in {"0", "false", "no", "off"}: + return False + return None + + def as_dict(self, environ: Mapping[str, str]) -> dict: + value = self.value_from(environ) + return { + "name": self.name, + "value": value, + "default": self.default, + "enabled": self.enabled_from(environ), + "stage": self.stage, + "choices": list(self.choices), + "restart_required": self.restart_required, + "description": self.description, + } + + +FEATURE_FLAGS: tuple[FeatureFlag, ...] = ( + FeatureFlag( + name="RECALLFORGE_ENABLE_MEDIA_RERANKING", + default="0", + description="Enable capped multimodal reranking for image/video-involved searches.", + stage="beta", + choices=("0", "1"), + restart_required=False, + ), + FeatureFlag( + name="RECALLFORGE_ENABLE_RAW_VIDEO_QUERY_EMBEDDING", + default="0", + description="Enable raw video query embedding instead of safer caption/transcript-first retrieval.", + stage="experimental", + choices=("0", "1"), + ), + FeatureFlag( + name="RECALLFORGE_ENABLE_MLX_NATIVE_VIDEO_PROCESSING", + default="0", + description="Enable qwen-vl-utils native video decoding on MLX.", + stage="experimental", + choices=("0", "1"), + ), + FeatureFlag( + name="RECALLFORGE_MEDIA_RERANK_REQUIRE_AMBIGUITY", + default="1", + description="Only run media reranking when cheap RRF results are close enough to need it.", + stage="beta", + choices=("0", "1"), + restart_required=False, + ), + FeatureFlag( + name="RECALLFORGE_MLX_HEAVY_OP_CONCURRENCY", + default="1", + description="Concurrency ceiling for heavy MLX multimodal operations.", + stage="safety", + ), + FeatureFlag( + name="RECALLFORGE_MCP_MAX_CONCURRENCY", + default="2", + description="Maximum number of blocking MCP tool operations run concurrently.", + stage="safety", + ), + FeatureFlag( + name="RECALLFORGE_TRACE", + default="0", + description="Enable structured trace logging for MCP tool handlers.", + stage="diagnostics", + choices=("0", "1"), + restart_required=False, + ), +) + + +def list_feature_flags(environ: Optional[Mapping[str, str]] = None) -> list[dict]: + """Return the effective feature flag registry for display or diagnostics.""" + env = os.environ if environ is None else environ + return [flag.as_dict(env) for flag in FEATURE_FLAGS] diff --git a/tests/test_alpha_beta_program.py b/tests/test_alpha_beta_program.py new file mode 100644 index 0000000..5d70f4a --- /dev/null +++ b/tests/test_alpha_beta_program.py @@ -0,0 +1,99 @@ +""" +test_alpha_beta_program.py - Launch-program utilities for REC-33. +""" + +import io +import json +import os +import sys +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "src")) + +from recallforge.cli import main +from recallforge.diagnostics import collect_crash_report, sanitized_recallforge_env, write_crash_report +from recallforge.feature_flags import list_feature_flags + + +class TestFeatureFlags(unittest.TestCase): + def test_feature_flag_registry_includes_beta_flags(self): + flags = list_feature_flags( + { + "RECALLFORGE_ENABLE_MEDIA_RERANKING": "1", + "RECALLFORGE_TRACE": "0", + } + ) + names = {flag["name"] for flag in flags} + self.assertIn("RECALLFORGE_ENABLE_MEDIA_RERANKING", names) + self.assertIn("RECALLFORGE_ENABLE_RAW_VIDEO_QUERY_EMBEDDING", names) + media_flag = next(flag for flag in flags if flag["name"] == "RECALLFORGE_ENABLE_MEDIA_RERANKING") + self.assertTrue(media_flag["enabled"]) + concurrency_flag = next(flag for flag in flags if flag["name"] == "RECALLFORGE_MLX_HEAVY_OP_CONCURRENCY") + self.assertIsNone(concurrency_flag["enabled"]) + + def test_flags_cli_outputs_json(self): + stdout = io.StringIO() + with patch.object(sys, "argv", ["recallforge", "flags", "--json"]), redirect_stdout(stdout): + code = main() + self.assertEqual(code, 0) + payload = json.loads(stdout.getvalue()) + self.assertIn("feature_flags", payload) + self.assertTrue(payload["feature_flags"]) + + +class TestCrashReports(unittest.TestCase): + def test_sanitized_env_allowlists_and_redacts_home_path(self): + home_store = str(Path.home() / ".recallforge") + env = { + "RECALLFORGE_STORE_PATH": home_store, + "RECALLFORGE_MODE": "embed", + "SECRET_TOKEN": "do-not-include", + } + sanitized = sanitized_recallforge_env(env) + self.assertEqual(sanitized["RECALLFORGE_STORE_PATH"], "~/.recallforge") + self.assertEqual(sanitized["RECALLFORGE_MODE"], "embed") + self.assertNotIn("SECRET_TOKEN", sanitized) + + def test_collect_crash_report_is_local_only(self): + report = collect_crash_report( + message="search crashed after video query", + include_env=True, + environ={"RECALLFORGE_TRACE": "1"}, + ) + self.assertEqual(report["privacy"]["network_sent"], False) + self.assertEqual(report["privacy"]["sharing"], "manual") + self.assertEqual(report["environment"]["RECALLFORGE_TRACE"], "1") + self.assertIn("search crashed", report["user_message"]) + + def test_crash_report_cli_writes_json_file(self): + with tempfile.TemporaryDirectory() as tmp: + output = Path(tmp) / "crash.json" + stdout = io.StringIO() + with patch.object( + sys, + "argv", + ["recallforge", "crash-report", "--output", str(output), "--message", "boom"], + ), redirect_stdout(stdout): + code = main() + + self.assertEqual(code, 0) + self.assertTrue(output.exists()) + payload = json.loads(output.read_text(encoding="utf-8")) + self.assertEqual(payload["user_message"], "boom") + self.assertFalse(payload["privacy"]["network_sent"]) + + def test_write_crash_report_returns_resolved_path(self): + with tempfile.TemporaryDirectory() as tmp: + output = Path(tmp) / "nested" / "report.json" + written = write_crash_report(output, message="local failure") + self.assertTrue(written.exists()) + payload = json.loads(written.read_text(encoding="utf-8")) + self.assertEqual(payload["user_message"], "local failure") + + +if __name__ == "__main__": + unittest.main()