From dafe266bc1aa966563431c041974b555f33aa8c1 Mon Sep 17 00:00:00 2001 From: Siyao Zheng Date: Mon, 18 May 2026 18:03:55 +0800 Subject: [PATCH 1/5] feat(codex): add optional Codex adapter Signed-off-by: Siyao Zheng --- .agents/plugins/marketplace.json | 20 + CHANGELOG.md | 18 + codex-plugin/.codex-plugin/plugin.json | 30 + codex-plugin/.mcp.json | 15 + codex-plugin/README.md | 312 ++++ codex-plugin/hooks/hooks.codex.json | 90 + codex-plugin/scripts/codex-security.test.mjs | 358 ++++ codex-plugin/scripts/gateway.mjs | 37 + codex-plugin/scripts/import-codex-history.mjs | 373 ++++ codex-plugin/scripts/lib.mjs | 1503 +++++++++++++++++ codex-plugin/scripts/mcp-server.mjs | 227 +++ codex-plugin/scripts/offload-store.mjs | 675 ++++++++ codex-plugin/scripts/permission-request.mjs | 15 + codex-plugin/scripts/post-compact.mjs | 7 + codex-plugin/scripts/post-tool-use.mjs | 22 + codex-plugin/scripts/pre-compact.mjs | 6 + codex-plugin/scripts/pre-tool-use.mjs | 6 + codex-plugin/scripts/query.mjs | 114 ++ codex-plugin/scripts/session-start.mjs | 6 + codex-plugin/scripts/stop.mjs | 6 + codex-plugin/scripts/user-prompt-submit.mjs | 8 + codex-plugin/skills/tdai-memory/SKILL.md | 35 + codex-plugin/tdai-gateway.example.json | 47 + package.json | 8 +- scripts/build-optional-bin-scripts.mjs | 33 + src/adapters/standalone/llm-runner.test.ts | 51 + src/adapters/standalone/llm-runner.ts | 93 +- src/cli/README.md | 9 +- src/cli/commands/seed.ts | 10 +- src/core/seed/seed-runtime.ts | 45 +- src/core/seed/types.ts | 6 + src/core/tdai-core.ts | 2 + src/core/tools/conversation-search.ts | 204 ++- src/core/tools/memory-search.ts | 217 ++- src/core/tools/search-prefix-filter.test.ts | 85 + src/core/types.ts | 2 + src/gateway/auth.test.ts | 331 ++++ src/gateway/cli.ts | 107 ++ src/gateway/server.ts | 175 +- src/gateway/types.ts | 27 + src/offload/backend-client.ts | 6 +- src/utils/pipeline-manager.ts | 178 +- src/utils/sanitize.ts | 12 + tsdown.config.ts | 2 +- vitest.config.ts | 2 +- 45 files changed, 5352 insertions(+), 183 deletions(-) create mode 100644 .agents/plugins/marketplace.json create mode 100644 codex-plugin/.codex-plugin/plugin.json create mode 100644 codex-plugin/.mcp.json create mode 100644 codex-plugin/README.md create mode 100644 codex-plugin/hooks/hooks.codex.json create mode 100644 codex-plugin/scripts/codex-security.test.mjs create mode 100644 codex-plugin/scripts/gateway.mjs create mode 100644 codex-plugin/scripts/import-codex-history.mjs create mode 100644 codex-plugin/scripts/lib.mjs create mode 100644 codex-plugin/scripts/mcp-server.mjs create mode 100644 codex-plugin/scripts/offload-store.mjs create mode 100644 codex-plugin/scripts/permission-request.mjs create mode 100644 codex-plugin/scripts/post-compact.mjs create mode 100644 codex-plugin/scripts/post-tool-use.mjs create mode 100644 codex-plugin/scripts/pre-compact.mjs create mode 100644 codex-plugin/scripts/pre-tool-use.mjs create mode 100644 codex-plugin/scripts/query.mjs create mode 100644 codex-plugin/scripts/session-start.mjs create mode 100644 codex-plugin/scripts/stop.mjs create mode 100644 codex-plugin/scripts/user-prompt-submit.mjs create mode 100644 codex-plugin/skills/tdai-memory/SKILL.md create mode 100644 codex-plugin/tdai-gateway.example.json create mode 100644 scripts/build-optional-bin-scripts.mjs create mode 100644 src/adapters/standalone/llm-runner.test.ts create mode 100644 src/core/tools/search-prefix-filter.test.ts create mode 100644 src/gateway/auth.test.ts create mode 100644 src/gateway/cli.ts diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json new file mode 100644 index 0000000..e873e66 --- /dev/null +++ b/.agents/plugins/marketplace.json @@ -0,0 +1,20 @@ +{ + "name": "tencentdb-agent-memory-local", + "interface": { + "displayName": "TencentDB Agent Memory Local" + }, + "plugins": [ + { + "name": "memory-tencentdb-codex", + "source": { + "source": "local", + "path": "./codex-plugin" + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL" + }, + "category": "Productivity" + } + ] +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f2d346..268b7d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,24 @@ --- +## [Unreleased] + +### ✨ 新功能 + +- **Codex adapter**：新增独立 `codex-plugin/` 适配层，覆盖 Codex CLI 与 Codex App 的生命周期 hook、MCP 检索工具、历史 JSONL 导入、工具输出 offload 与本地 Gateway 自动启动，并包含 Codex App 的额外适配与验证。 + +### 🔒 安全增强 + +- Gateway 默认要求 tokenized POST；无 token 时仅保留 loopback GET 探活，loopback tokenless POST 必须显式启用开发开关。 +- Codex adapter 默认拒绝非 loopback Gateway URL，MCP 默认不暴露跨项目检索或完整 offload 内容。 +- Gateway token 文件改为私有权限、owner 校验、atomic create；并发 autostart 不再可能生成互相覆盖的 token。 +- Codex hook 诊断写入私有 `hook.log`，日志内容先经过敏感字段 redaction。 + +### 🐛 修复 + +- scoped memory/conversation search 会扩展候选窗口到 store 记录总数，避免当前项目结果被其他项目的前 500 个候选挤掉。 +- `prepack` 不再因为已缺失的历史可选 bin-script 源目录而失败；存在对应 `tsconfig.json` 时仍会构建这些脚本。 + ## [0.3.4] - 2026-05-12 ### 🐛 修复 diff --git a/codex-plugin/.codex-plugin/plugin.json b/codex-plugin/.codex-plugin/plugin.json new file mode 100644 index 0000000..799a3a6 --- /dev/null +++ b/codex-plugin/.codex-plugin/plugin.json @@ -0,0 +1,30 @@ +{ + "name": "memory-tencentdb-codex", + "version": "0.1.0", + "description": "Codex adapter for TencentDB Agent Memory: auto-recall, auto-capture, hook context injection, MCP search/offload tools, compaction flush, and seed through the TDAI Gateway.", + "author": { + "name": "TencentDB Agent Memory Codex adapter" + }, + "license": "MIT", + "homepage": "https://github.com/Tencent/TencentDB-Agent-Memory", + "repository": "https://github.com/Tencent/TencentDB-Agent-Memory", + "interface": { + "displayName": "TencentDB Agent Memory", + "shortDescription": "Automatic recall, capture, flush, offload, and MCP search for Codex sessions through TencentDB Agent Memory.", + "longDescription": "Adds Codex hooks that recall relevant memory before each prompt, inject model-visible memory hints, capture completed turns, track tool and permission activity, offload large tool results into JSONL/ref/Mermaid artifacts, flush session memory after compaction or every N turns, expose memory and offload lookup as MCP tools, seed historical conversations, and manage a local TencentDB Agent Memory Gateway. The adapter supports Codex CLI and Codex App, with additional Codex App adaptation and validation.", + "developerName": "TencentDB Agent Memory Codex adapter", + "category": "Coding", + "capabilities": [ + "Interactive", + "Read", + "Write" + ], + "defaultPrompt": [ + "Use TencentDB Agent Memory to recall prior context before answering and capture important session details after the turn." + ], + "brandColor": "#2563EB" + }, + "skills": "./skills/", + "mcpServers": "./.mcp.json", + "hooks": "./hooks/hooks.codex.json" +} diff --git a/codex-plugin/.mcp.json b/codex-plugin/.mcp.json new file mode 100644 index 0000000..d4ec3a9 --- /dev/null +++ b/codex-plugin/.mcp.json @@ -0,0 +1,15 @@ +{ + "mcpServers": { + "tdai-memory": { + "command": "node", + "args": [ + "${CLAUDE_PLUGIN_ROOT}/scripts/mcp-server.mjs" + ], + "env": { + "TDAI_CODEX_AUTOSTART": "true" + }, + "startup_timeout_sec": 20, + "tool_timeout_sec": 60 + } + } +} diff --git a/codex-plugin/README.md b/codex-plugin/README.md new file mode 100644 index 0000000..c87f69a --- /dev/null +++ b/codex-plugin/README.md @@ -0,0 +1,312 @@ +# TencentDB Agent Memory Codex Adapter + +Codex adapter for the **memory-tencentdb** four-layer memory system +(L0 conversation capture -> L1 episodic extraction -> L2 scene blocks -> L3 +persona synthesis). + +The heavy lifting runs in the Node.js **Gateway** sidecar used by the other +TencentDB Agent Memory integrations. This adapter is a thin Codex plugin layer: +it translates Codex hooks and MCP tool calls into the Gateway API, keeps +Codex-specific state under the adapter data directory, and leaves the OpenClaw +and Hermes paths unchanged. + +The adapter targets Codex as a host, including Codex CLI and Codex App. +It also includes extra Codex App adaptation and validation for App session +history, archived JSONL import, plugin-cache loading, and App-observed hook +behavior. + +## Architecture + +```text +Codex (CLI and App) + +-- Hooks + | +-- SessionStart -> scripts/session-start.mjs + | +-- UserPromptSubmit -> scripts/user-prompt-submit.mjs + | +-- PreToolUse -> scripts/pre-tool-use.mjs + | +-- PermissionRequest -> scripts/permission-request.mjs + | +-- PostToolUse -> scripts/post-tool-use.mjs + | +-- PreCompact -> scripts/pre-compact.mjs + | +-- PostCompact -> scripts/post-compact.mjs + | +-- Stop -> scripts/stop.mjs + +-- MCP + +-- scripts/mcp-server.mjs + +-- tdai_memory_search + +-- tdai_conversation_search + +-- tdai_offload_lookup + | + v HTTP (127.0.0.1:8420 by default) + memory-tencentdb Gateway + +-- POST /recall + +-- POST /capture + +-- POST /search/memories + +-- POST /search/conversations + +-- POST /session/end + +-- POST /seed +``` + +The Codex-specific integration lives in this directory. The shared changes +outside `codex-plugin/` are limited to host-neutral Gateway and seed support +used by sidecar clients: a lightweight root metadata endpoint, optional +`started_at` capture metadata, and opt-in full-pipeline waiting for `/seed`. + +## Lifecycle Mapping + +| Codex surface | Gateway or local path | Behavior | +| --- | --- | --- | +| `SessionStart` | `/recall`, `/search/memories`, selective `/search/conversations` | Restores project/session context and returns Codex `additionalContext` when useful context exists. | +| `UserPromptSubmit` | Local turn state, `/recall`, `/search/memories`, selective `/search/conversations`, local L0 JSONL fallback | Starts a pending turn, recalls relevant memory, and injects bounded context; if Gateway recall/search has no useful context, scans project-scoped local L0 JSONL as a last resort. | +| `PreToolUse` | Local turn state | Records tool intent and returns a compact memory/offload hint. | +| `PermissionRequest` | Local turn state | Records permission activity for the current turn. | +| `PostToolUse` | Local turn state, context-offload files | Records tool results and can replace large tool output with compact hook feedback plus a lookup reference. | +| `PreCompact` | `/capture` | Captures pending turn state before compaction. | +| `PostCompact` | `/session/end` | Flushes pending Gateway pipeline work after compaction. | +| `Stop` | `/capture`, periodic `/session/end` | Captures the completed Codex turn and flushes every `TDAI_CODEX_FLUSH_EVERY_N_TURNS` captured turns. | +| MCP `tdai_memory_search` | `/search/memories` | Searches L1 structured memory. | +| MCP `tdai_conversation_search` | `/search/conversations` | Searches L0 raw conversation history. | +| MCP `tdai_offload_lookup` | Local context-offload index | Retrieves exact redacted tool results by `node_id`, `tool_call_id`, or query. | + +## Reliability Features + +- **Gateway supervision** - the adapter can auto-discover and start the Gateway + from a local TencentDB Agent Memory checkout, then poll `/health` before use. +- **Circuit breaker** - repeated Gateway failures pause calls for a short + cooldown instead of slowing every hook invocation. +- **Bounded prompt injection** - empty Gateway search responses are not injected, + recall output is capped by `TDAI_CODEX_CONTEXT_MAX_CHARS`, and tool hints are + intentionally compact. +- **Injected-context cleanup** - adapter-controlled capture, import, transcript, + and Gateway L0/L1 write paths strip TencentDB/Codex injected blocks before + persistence to avoid recall feedback loops. +- **Local L0 fallback** - when Gateway recall/search is unavailable or empty, + automatic prompt recall can stream recent local L0 JSONL and filter by the + current Codex project session-key prefix. +- **Short-term offload lookup** - large `PostToolUse` output can be stored under + local JSONL/ref/Mermaid artifacts and retrieved later even if the Gateway is + temporarily unavailable. + +## Installation Location + +This directory (`codex-plugin/`) is the source of truth for the Codex adapter. +Codex loads it as a local plugin or from a local marketplace/cache copy. + +The plugin manifest is: + +```text +codex-plugin/.codex-plugin/plugin.json +``` + +It declares the Codex skill, bundled hook config, and bundled MCP server config: + +```text +codex-plugin/hooks/hooks.codex.json +codex-plugin/.mcp.json +``` + +Codex can load these hooks as plugin-bundled hooks when `plugin_hooks` is +enabled, or as user-level hooks from `~/.codex/hooks.json`. When mirroring the +hook file into a user-level config, replace the plugin-root variable with the +installed adapter path because user-level hooks do not receive plugin-specific +environment variables. The bundled MCP config exposes memory search and offload +lookup tools; the manual `codex mcp add` command below is a fallback for local +development or older Codex builds. + +## Setup + +From the TencentDB-Agent-Memory repository root: + +```bash +npm install +``` + +Optional Codex adapter environment: + +```bash +export TDAI_CODEX_TDAI_ROOT="/path/to/TencentDB-Agent-Memory" +export TDAI_CODEX_DATA_DIR="$HOME/.memory-tencentdb/codex-memory-tdai" +export TDAI_CODEX_GATEWAY_URL="http://127.0.0.1:8420" +export TDAI_CODEX_AUTOSTART=true +export TDAI_CODEX_FLUSH_EVERY_N_TURNS=5 +export TDAI_CODEX_TOOL_OFFLOAD=true +``` + +When the adapter autostarts the Gateway it keeps the service on loopback by +default, creates a private bearer token under +`$TDAI_CODEX_DATA_DIR/codex-adapter/gateway-token`, and sends that token on +Gateway requests. The token is passed to the daemon through `TDAI_TOKEN_PATH` +instead of a generated token environment variable. Set `TDAI_CODEX_GATEWAY_TOKEN` +or `TDAI_TOKEN_PATH` if you want to manage the token yourself. Autostart refuses +non-loopback hosts unless `TDAI_CODEX_ALLOW_NON_LOOPBACK=true` is set explicitly. + +By default autostart uses the package bin +(`npx --yes --ignore-scripts --package @tencentdb-agent-memory/memory-tencentdb tdai-memory-gateway`), +so the copied Codex plugin does not need to import package dependencies from the +plugin directory and daemon launch does not run npm lifecycle scripts. For +source-tree development, set `TDAI_CODEX_TDAI_ROOT` to use +`npx tsx src/gateway/server.ts` from a local checkout, or set +`TDAI_CODEX_GATEWAY_PACKAGE` to override the package spec, including a pinned +version or tarball during release validation. Package-bin launch does not +hydrate additional shell-only LLM secrets unless +`TDAI_CODEX_HYDRATE_ENV_FOR_PACKAGE_GATEWAY=true` is set explicitly. + +The Gateway also rejects non-loopback browser origins by default and blocks +credential-bearing `/seed config_override` keys, so imported Codex history cannot +redirect configured LLM, embedding, TCVDB, or backend credentials to a different +network endpoint. + +When no Gateway token is configured, unauthenticated loopback access is limited +to `GET` routes such as `/health`. Tokenless `POST` routes require the explicit +loopback-only development flag `TDAI_GATEWAY_AUTH_DISABLED=true`; non-loopback +tokenless access is always rejected. + +Adapter requests also refuse non-loopback `TDAI_CODEX_GATEWAY_URL` values unless +`TDAI_CODEX_ALLOW_NON_LOOPBACK=true` is set explicitly. This prevents hooks from +sending local bearer tokens or captured memory to an unexpected remote URL. + +For L1/L2/L3 extraction, configure an OpenAI-compatible LLM for the Gateway: + +```bash +export TDAI_LLM_BASE_URL="https://api.openai.com/v1" +export TDAI_LLM_API_KEY="..." +export TDAI_LLM_MODEL="gpt-4o-mini" +``` + +The example Gateway config is `tdai-gateway.example.json`. Copy it to: + +```bash +$TDAI_CODEX_DATA_DIR/tdai-gateway.json +``` + +or use environment variables only. During autostart the adapter sets +`TDAI_GATEWAY_CONFIG=$TDAI_CODEX_DATA_DIR/tdai-gateway.json`, because the Gateway +normally discovers config files from the current working directory or its +default data directory unless this variable is explicit. + +## Register MCP Tools + +The plugin bundles `codex-plugin/.mcp.json`, so normal Codex plugin installation +can register the MCP server from the plugin manifest. For local development, +or if a Codex build does not load plugin-bundled MCP config, register it +manually: + +```bash +codex mcp add tdai-memory \ + --env TDAI_CODEX_TDAI_ROOT="/path/to/TencentDB-Agent-Memory" \ + --env TDAI_CODEX_DATA_DIR="$HOME/.memory-tencentdb/codex-memory-tdai" \ + --env TDAI_CODEX_GATEWAY_URL="http://127.0.0.1:8420" \ + --env TDAI_CODEX_AUTOSTART="true" \ + -- node "/path/to/TencentDB-Agent-Memory/codex-plugin/scripts/mcp-server.mjs" +``` + +MCP search tools are scoped to the current Codex project path by default. Pass +`all_projects: true` only when you intentionally want cross-project memory or +offload lookup. + +For model-facing MCP safety, cross-project search and exact offload content are +not exposed by default. To opt in from outside the model context, set: + +```bash +export TDAI_CODEX_MCP_ALLOW_ALL_PROJECTS=true +export TDAI_CODEX_MCP_ALLOW_OFFLOAD_CONTENT=true +``` + +## Diagnostics + +```bash +node codex-plugin/scripts/gateway.mjs status +node codex-plugin/scripts/gateway.mjs start +node codex-plugin/scripts/query.mjs status +node codex-plugin/scripts/query.mjs memory "previous decision" +node codex-plugin/scripts/query.mjs conversation "continue where we left off" +node codex-plugin/scripts/query.mjs remember "This project uses X as the source of truth." +node codex-plugin/scripts/query.mjs flush +node codex-plugin/scripts/query.mjs seed ./historical-conversations.json +node codex-plugin/scripts/query.mjs import-codex-history --dry-run --since 30d +node codex-plugin/scripts/query.mjs import-codex-history --yes --since 30d --cwd "/path/to/project" +node codex-plugin/scripts/query.mjs offload list --all --limit 10 +node codex-plugin/scripts/query.mjs offload node Cxxxxxx_N001 --content +node codex-plugin/scripts/query.mjs offload canvas +node codex-plugin/scripts/mcp-server.mjs +``` + +Logs: + +```text +$TDAI_CODEX_DATA_DIR/codex-adapter/logs/gateway.stdout.log +$TDAI_CODEX_DATA_DIR/codex-adapter/logs/gateway.stderr.log +$TDAI_CODEX_DATA_DIR/codex-adapter/logs/hook.log +``` + +## Import Existing Codex History + +The Gateway supports historical seeding through `POST /seed`. The Codex adapter +adds a host-specific importer that converts local Codex JSONL rollouts into +that seed format. + +By default it reads: + +```text +~/.codex/sessions/**/*.jsonl +~/.codex/archived_sessions/*.jsonl +``` + +The importer is opt-in and runs as a dry run unless `--yes` is provided: + +```bash +node codex-plugin/scripts/import-codex-history.mjs --dry-run --since 30d +node codex-plugin/scripts/import-codex-history.mjs --yes --since 30d --cwd "/path/to/project" +``` + +It skips Codex-generated context scaffolding such as `AGENTS.md` injections and +imports only paired user/assistant rounds. Use `--no-archived` to exclude +archived sessions, `--limit` for a small trial import, and `--out` to inspect +the generated `/seed` payload before writing. + +By default, a real import requests `wait_for_full_pipeline`, so Gateway `/seed` +records L0, waits for L1, flushes L2 scene extraction, and waits for L3 persona +generation before returning. Use `--no-full-pipeline` when the faster L0/L1-only +seed behavior is preferred. + +## Short-Term Context Offload + +Codex does not expose OpenClaw's `slots.contextEngine`, so the adapter uses the +official Codex hook surface as the equivalent control point: + +1. `PostToolUse` evaluates tool-result size against mild, aggressive, and + emergency thresholds. +2. When offload is triggered, the full redacted result is written under + `$TDAI_CODEX_DATA_DIR/codex-adapter/context-offload//refs/`. +3. A structured `offload-.jsonl` entry is appended with `node_id`, + `tool_call_id`, summary, score, policy, and `result_ref`. +4. The deterministic L2 canvas at `mmds/001-codex-tool-offload.mmd` is rebuilt + and injected on later `SessionStart` / `UserPromptSubmit` hooks. +5. The model can drill down by calling `tdai_offload_lookup`; humans can use + `query.mjs offload node ... --content`. + +Thresholds are configurable: + +```bash +export TDAI_CODEX_TOOL_OFFLOAD_MIN_CHARS=20000 +export TDAI_CODEX_TOOL_OFFLOAD_AGGRESSIVE_MIN_CHARS=80000 +export TDAI_CODEX_TOOL_OFFLOAD_EMERGENCY_MIN_CHARS=250000 +export TDAI_CODEX_TOOL_OFFLOAD_PREVIEW_CHARS=2000 +export TDAI_CODEX_TOOL_OFFLOAD_AGGRESSIVE_PREVIEW_CHARS=800 +export TDAI_CODEX_TOOL_OFFLOAD_EMERGENCY_PREVIEW_CHARS=240 +``` + +## Codex Host Notes + +OpenClaw- or Claude Code-only interfaces such as host-specific slot APIs are not +applicable to Codex; this adapter uses Codex hook, MCP, JSONL history, and +context-injection surfaces instead. Codex can gate plugin-scoped hooks or omit +optional transcript fields in some builds; the adapter provides Codex-native +fallbacks through user-level hooks, local session state, tool-event summaries, +and history import. + +## Security Notes + +- Adapter-owned session state, gateway tokens, and offloaded tool-result files + are written with private owner-only permissions on POSIX filesystems. +- Tokenized Gateways require `Authorization: Bearer ...` for all routes. A + tokenless Gateway exposes only loopback `GET` probes by default; loopback + tokenless `POST` routes require explicit development opt-in, and non-loopback + tokenless access is rejected. diff --git a/codex-plugin/hooks/hooks.codex.json b/codex-plugin/hooks/hooks.codex.json new file mode 100644 index 0000000..1ae77b4 --- /dev/null +++ b/codex-plugin/hooks/hooks.codex.json @@ -0,0 +1,90 @@ +{ + "hooks": { + "SessionStart": [ + { + "matcher": "startup|resume|clear", + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/session-start.mjs", + "statusMessage": "tdai-memory: loading Codex memory context" + } + ] + } + ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/user-prompt-submit.mjs", + "statusMessage": "tdai-memory: recalling relevant memory" + } + ] + } + ], + "PreToolUse": [ + { + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/pre-tool-use.mjs" + } + ] + } + ], + "PermissionRequest": [ + { + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/permission-request.mjs" + } + ] + } + ], + "PostToolUse": [ + { + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/post-tool-use.mjs" + } + ] + } + ], + "PreCompact": [ + { + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/pre-compact.mjs", + "statusMessage": "tdai-memory: preserving turn before compaction" + } + ] + } + ], + "PostCompact": [ + { + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/post-compact.mjs", + "statusMessage": "tdai-memory: flushing memory after compaction" + } + ] + } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/stop.mjs", + "statusMessage": "tdai-memory: capturing completed Codex turn" + } + ] + } + ] + } +} diff --git a/codex-plugin/scripts/codex-security.test.mjs b/codex-plugin/scripts/codex-security.test.mjs new file mode 100644 index 0000000..cdeb94d --- /dev/null +++ b/codex-plugin/scripts/codex-security.test.mjs @@ -0,0 +1,358 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + beginTurn, + configuredGatewayTokenPath, + captureCurrentTurn, + debug, + ensureGatewayAuthToken, + healthCheck, + hookLogPath, + httpPost, + loadSessionState, + recallForPrompt, + readGatewayAuthToken, + sanitizeMemoryText, + sessionKeyFromPayload, +} from "./lib.mjs"; +import { + lookupCodexOffload, + recordCodexToolOffload, +} from "./offload-store.mjs"; + +let tmpDir; +let originalDataDir; +let originalAutostart; +let originalGatewayUrl; +let originalAllowNonLoopback; +let originalCodexGatewayToken; +let originalGatewayToken; +let originalTokenPath; + +beforeEach(() => { + originalDataDir = process.env.TDAI_CODEX_DATA_DIR; + originalAutostart = process.env.TDAI_CODEX_AUTOSTART; + originalGatewayUrl = process.env.TDAI_CODEX_GATEWAY_URL; + originalAllowNonLoopback = process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK; + originalCodexGatewayToken = process.env.TDAI_CODEX_GATEWAY_TOKEN; + originalGatewayToken = process.env.TDAI_GATEWAY_TOKEN; + originalTokenPath = process.env.TDAI_TOKEN_PATH; + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-codex-security-")); + process.env.TDAI_CODEX_DATA_DIR = tmpDir; +}); + +afterEach(() => { + if (originalDataDir === undefined) { + delete process.env.TDAI_CODEX_DATA_DIR; + } else { + process.env.TDAI_CODEX_DATA_DIR = originalDataDir; + } + if (originalAutostart === undefined) { + delete process.env.TDAI_CODEX_AUTOSTART; + } else { + process.env.TDAI_CODEX_AUTOSTART = originalAutostart; + } + if (originalGatewayUrl === undefined) { + delete process.env.TDAI_CODEX_GATEWAY_URL; + } else { + process.env.TDAI_CODEX_GATEWAY_URL = originalGatewayUrl; + } + if (originalAllowNonLoopback === undefined) { + delete process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK; + } else { + process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK = originalAllowNonLoopback; + } + if (originalCodexGatewayToken === undefined) { + delete process.env.TDAI_CODEX_GATEWAY_TOKEN; + } else { + process.env.TDAI_CODEX_GATEWAY_TOKEN = originalCodexGatewayToken; + } + if (originalGatewayToken === undefined) { + delete process.env.TDAI_GATEWAY_TOKEN; + } else { + process.env.TDAI_GATEWAY_TOKEN = originalGatewayToken; + } + if (originalTokenPath === undefined) { + delete process.env.TDAI_TOKEN_PATH; + } else { + process.env.TDAI_TOKEN_PATH = originalTokenPath; + } + if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true }); + vi.restoreAllMocks(); +}); + +describe("Codex adapter security defaults", () => { + it("strips injected memory blocks and redacts common secrets", () => { + const githubToken = ["github", "pat", "1234567890abcdefghijklmnopqrstuvwxyz"].join("_"); + const awsAccessKey = `AKIA${"1234567890ABCDEF"}`; + const keyKind = "PRIVATE KEY"; + const privateKeyBlock = [ + `-----BEGIN ${keyKind}-----`, + "secret material", + `-----END ${keyKind}-----`, + ].join("\n"); + const cleaned = sanitizeMemoryText(` +keep this +private injected context +${githubToken} +${awsAccessKey} +${privateKeyBlock} +`); + + expect(cleaned).toContain("keep this"); + expect(cleaned).not.toContain("private injected context"); + expect(cleaned).not.toContain(githubToken); + expect(cleaned).not.toContain(awsAccessKey); + expect(cleaned).not.toContain("secret material"); + expect(cleaned).toContain("[REDACTED_GITHUB_TOKEN]"); + expect(cleaned).toContain("[REDACTED_AWS_ACCESS_KEY]"); + expect(cleaned).toContain("[REDACTED_PRIVATE_KEY]"); + }); + + it("redacts JSON-style credential fields", () => { + const cleaned = sanitizeMemoryText(JSON.stringify({ + apiKey: "plain-secret-123", + password: "hunter2", + token: "abc123xyz", + authorization: "Basic abc123", + nested: { + clientSecret: "client-secret-value", + accessToken: "access-token-value", + }, + })); + + expect(cleaned).not.toContain("plain-secret-123"); + expect(cleaned).not.toContain("hunter2"); + expect(cleaned).not.toContain("abc123xyz"); + expect(cleaned).not.toContain("Basic abc123"); + expect(cleaned).not.toContain("client-secret-value"); + expect(cleaned).not.toContain("access-token-value"); + expect(cleaned.match(/\[REDACTED\]/g)?.length).toBeGreaterThanOrEqual(6); + }); + + it("redacts env-style credential fields with prefixes", () => { + const cleaned = sanitizeMemoryText([ + "CLIENT_SECRET=client-secret-value", + "ACCESS_TOKEN=access-token-value", + "DB_PASSWORD=hunter2", + ].join("\n")); + + expect(cleaned).not.toContain("client-secret-value"); + expect(cleaned).not.toContain("access-token-value"); + expect(cleaned).not.toContain("hunter2"); + expect(cleaned).toContain("CLIENT_SECRET=[REDACTED]"); + expect(cleaned).toContain("ACCESS_TOKEN=[REDACTED]"); + expect(cleaned).toContain("DB_PASSWORD=[REDACTED]"); + }); + + it("redacts full Authorization and Proxy-Authorization header values", () => { + const cleaned = sanitizeMemoryText([ + "Authorization: Basic dXNlcjpwYXNz", + "Proxy-Authorization: Token proxy-secret-value", + ].join("\n")); + + expect(cleaned).not.toContain("dXNlcjpwYXNz"); + expect(cleaned).not.toContain("proxy-secret-value"); + expect(cleaned).toContain("Authorization=[REDACTED]"); + expect(cleaned).toContain("Proxy-Authorization=[REDACTED]"); + }); + + it("writes redacted diagnostics to hook.log without throwing", () => { + debug("Gateway failed with Authorization: Bearer diagnostic-secret-value"); + + const log = fs.readFileSync(hookLogPath(), "utf-8"); + expect(log).toContain("Gateway failed"); + expect(log).toContain("Authorization=[REDACTED]"); + expect(log).not.toContain("diagnostic-secret-value"); + }); + + it("writes Codex state and offload files with private permissions", async () => { + const payload = { cwd: process.cwd(), session_id: "perm-test", prompt: "hello" }; + await beginTurn(payload); + await recordCodexToolOffload({ + sessionKey: sessionKeyFromPayload(payload), + sessionId: "perm-test", + cwd: process.cwd(), + toolName: "test-tool", + toolUseId: "tool-1", + inputSummary: "input", + redactedOutput: "output".repeat(100), + storedText: "stored output", + policy: { name: "mild", score: 8 }, + }); + + const sessionDir = path.join(tmpDir, "codex-adapter", "sessions"); + const sessionFile = path.join(sessionDir, fs.readdirSync(sessionDir)[0]); + const offloadBase = path.join(tmpDir, "codex-adapter", "context-offload"); + const offloadRoot = path.join(offloadBase, fs.readdirSync(offloadBase)[0]); + const refFile = path.join(offloadRoot, "refs", fs.readdirSync(path.join(offloadRoot, "refs"))[0]); + + expect(mode(sessionDir)).toBe("700"); + expect(mode(sessionFile)).toBe("600"); + expect(mode(offloadRoot)).toBe("700"); + expect(mode(refFile)).toBe("600"); + }); + + it("scopes offload lookup by project cwd unless explicitly omitted", async () => { + const cwdA = path.join(tmpDir, "project-a"); + const cwdB = path.join(tmpDir, "project-b"); + fs.mkdirSync(cwdA); + fs.mkdirSync(cwdB); + + await recordCodexToolOffload(offloadParams(cwdA, "session-a", "tool-a")); + await recordCodexToolOffload(offloadParams(cwdB, "session-b", "tool-b")); + + const scoped = await lookupCodexOffload({ cwd: cwdA, limit: 10 }); + expect(scoped.matches).toHaveLength(1); + expect(scoped.matches[0].tool_call_id).toBe("tool-a"); + + const all = await lookupCodexOffload({ limit: 10 }); + expect(all.matches.map((entry) => entry.tool_call_id).sort()).toEqual(["tool-a", "tool-b"]); + }); + + it("falls back to project-scoped local L0 JSONL search when Gateway recall is unavailable", async () => { + process.env.TDAI_CODEX_AUTOSTART = "false"; + process.env.TDAI_CODEX_GATEWAY_URL = "http://127.0.0.1:9"; + const cwdA = path.join(tmpDir, "project-a"); + const cwdB = path.join(tmpDir, "project-b"); + fs.mkdirSync(cwdA); + fs.mkdirSync(cwdB); + + const conversationsDir = path.join(tmpDir, "conversations"); + fs.mkdirSync(conversationsDir); + fs.writeFileSync(path.join(conversationsDir, "2026-05-18.jsonl"), [ + JSON.stringify({ + sessionKey: sessionKeyFromPayload({ cwd: cwdA, session_id: "a" }), + sessionId: "a", + recordedAt: "2026-05-18T01:00:00.000Z", + role: "assistant", + content: "The project decision was to use SQLite for local recall.", + }), + JSON.stringify({ + sessionKey: sessionKeyFromPayload({ cwd: cwdB, session_id: "b" }), + sessionId: "b", + recordedAt: "2026-05-18T02:00:00.000Z", + role: "assistant", + content: "The project decision was to use a remote service.", + }), + ].join("\n") + "\n"); + + const context = await recallForPrompt( + { cwd: cwdA, session_id: "a" }, + "previous project decision SQLite", + "prompt", + ); + + expect(context).toContain('source="local-jsonl-direct"'); + expect(context).toContain("SQLite for local recall"); + expect(context).not.toContain("remote service"); + }); + + it("keeps a pending turn when capture cannot reach the Gateway", async () => { + process.env.TDAI_CODEX_AUTOSTART = "false"; + process.env.TDAI_CODEX_GATEWAY_URL = "http://127.0.0.1:9"; + const payload = { cwd: process.cwd(), session_id: "capture-failure", prompt: "keep me pending" }; + const sessionKey = sessionKeyFromPayload(payload); + + await beginTurn(payload); + const result = await captureCurrentTurn(payload, "stop"); + const state = await loadSessionState(sessionKey); + + expect(result).toEqual({ captured: false, reason: "gateway_unavailable" }); + expect(state.currentTurn.userPrompt).toBe("keep me pending"); + expect(state.turns || []).toHaveLength(0); + }); + + it("writes explicit env token to the token file used by the spawned Gateway", async () => { + const customTokenPath = path.join(tmpDir, "custom-token"); + const defaultTokenPath = path.join(tmpDir, "codex-adapter", "gateway-token"); + fs.mkdirSync(path.dirname(defaultTokenPath), { recursive: true }); + fs.writeFileSync(defaultTokenPath, "stale-default-token\n", { mode: 0o600 }); + + process.env.TDAI_TOKEN_PATH = customTokenPath; + process.env.TDAI_CODEX_GATEWAY_TOKEN = "explicit-env-token"; + delete process.env.TDAI_GATEWAY_TOKEN; + + await expect(ensureGatewayAuthToken()).resolves.toBe("explicit-env-token"); + expect(fs.readFileSync(customTokenPath, "utf-8").trim()).toBe("explicit-env-token"); + await expect(readGatewayAuthToken()).resolves.toBe("explicit-env-token"); + }); + + it("treats a custom token path as authoritative over a stale default token file", async () => { + const customTokenPath = path.join(tmpDir, "custom-token"); + const defaultTokenPath = path.join(tmpDir, "codex-adapter", "gateway-token"); + fs.mkdirSync(path.dirname(defaultTokenPath), { recursive: true }); + fs.writeFileSync(defaultTokenPath, "stale-default-token\n", { mode: 0o600 }); + + process.env.TDAI_TOKEN_PATH = customTokenPath; + delete process.env.TDAI_CODEX_GATEWAY_TOKEN; + delete process.env.TDAI_GATEWAY_TOKEN; + + const token = await ensureGatewayAuthToken(); + expect(token).not.toBe("stale-default-token"); + expect(fs.readFileSync(customTokenPath, "utf-8").trim()).toBe(token); + await expect(readGatewayAuthToken()).resolves.toBe(token); + }); + + it("expands tilde token paths consistently for adapter and spawned Gateway env", async () => { + const tokenFileName = `.tdai-codex-token-test-${process.pid}-${Date.now()}`; + const expandedTokenPath = path.join(os.homedir(), tokenFileName); + process.env.TDAI_TOKEN_PATH = `~/${tokenFileName}`; + process.env.TDAI_CODEX_GATEWAY_TOKEN = "tilde-env-token"; + delete process.env.TDAI_GATEWAY_TOKEN; + + try { + expect(configuredGatewayTokenPath()).toBe(expandedTokenPath); + await expect(ensureGatewayAuthToken()).resolves.toBe("tilde-env-token"); + expect(fs.readFileSync(expandedTokenPath, "utf-8").trim()).toBe("tilde-env-token"); + await expect(readGatewayAuthToken()).resolves.toBe("tilde-env-token"); + } finally { + fs.rmSync(expandedTokenPath, { force: true }); + } + }); + + it("creates generated Gateway tokens atomically across concurrent autostarts", async () => { + const tokenPath = path.join(tmpDir, "concurrent-token"); + process.env.TDAI_TOKEN_PATH = tokenPath; + delete process.env.TDAI_CODEX_GATEWAY_TOKEN; + delete process.env.TDAI_GATEWAY_TOKEN; + + const tokens = await Promise.all(Array.from({ length: 12 }, () => ensureGatewayAuthToken())); + const unique = new Set(tokens); + + expect(unique.size).toBe(1); + expect(fs.readFileSync(tokenPath, "utf-8").trim()).toBe(tokens[0]); + }); + + it("does not send auth or payloads to non-loopback Gateway URLs unless explicitly enabled", async () => { + process.env.TDAI_CODEX_GATEWAY_URL = "https://attacker.example"; + process.env.TDAI_CODEX_GATEWAY_TOKEN = "secret-token"; + delete process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK; + const fetchMock = vi.spyOn(globalThis, "fetch"); + + await expect(healthCheck()).resolves.toBe(false); + await expect(httpPost("/capture", { user_content: "secret" })).resolves.toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +function offloadParams(cwd, sessionId, toolUseId) { + const payload = { cwd, session_id: sessionId }; + return { + sessionKey: sessionKeyFromPayload(payload), + sessionId, + cwd, + toolName: "test-tool", + toolUseId, + inputSummary: "input", + redactedOutput: "output".repeat(100), + storedText: "stored output", + policy: { name: "mild", score: 8 }, + }; +} + +function mode(filePath) { + return (fs.statSync(filePath).mode & 0o777).toString(8); +} diff --git a/codex-plugin/scripts/gateway.mjs b/codex-plugin/scripts/gateway.mjs new file mode 100644 index 0000000..423e214 --- /dev/null +++ b/codex-plugin/scripts/gateway.mjs @@ -0,0 +1,37 @@ +#!/usr/bin/env node +import { + ensureGateway, + gatewayUrl, + healthCheck, + resolveTdaiRoot, + startGatewayDetached, + stopGateway, + tdaiDataDir +} from "./lib.mjs"; + +const command = process.argv[2] || "status"; + +if (command === "status") { + const healthy = await healthCheck(); + console.log(JSON.stringify({ + healthy, + gatewayUrl: gatewayUrl(), + tdaiRoot: resolveTdaiRoot(), + dataDir: tdaiDataDir() + }, null, 2)); +} else if (command === "start") { + const ok = await ensureGateway(); + console.log(ok ? "gateway ready" : "gateway unavailable"); + process.exit(ok ? 0 : 1); +} else if (command === "start-detached") { + const ok = await startGatewayDetached(); + console.log(ok ? "gateway start requested" : "gateway start failed"); + process.exit(ok ? 0 : 1); +} else if (command === "stop") { + const ok = await stopGateway(); + console.log(ok ? "gateway stop requested" : "no gateway pid found"); + process.exit(ok ? 0 : 1); +} else { + console.error("Usage: node scripts/gateway.mjs [status|start|start-detached|stop]"); + process.exit(2); +} diff --git a/codex-plugin/scripts/import-codex-history.mjs b/codex-plugin/scripts/import-codex-history.mjs new file mode 100644 index 0000000..220997c --- /dev/null +++ b/codex-plugin/scripts/import-codex-history.mjs @@ -0,0 +1,373 @@ +#!/usr/bin/env node +import fs from "node:fs/promises"; +import fsSync from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { + ensureGateway, + expandHome, + httpPost, + sanitizeMemoryText, + sha1 +} from "./lib.mjs"; + +const DEFAULT_SESSIONS_DIR = "~/.codex/sessions"; +const DEFAULT_ARCHIVED_DIR = "~/.codex/archived_sessions"; +const DEFAULT_FULL_PIPELINE_TIMEOUT_MS = 900_000; +const DEFAULT_SEED_TIMEOUT_MS = 960_000; + +export async function importCodexHistoryCli(args = process.argv.slice(2)) { + const opts = parseArgs(args); + if (opts.help) return usage(0); + + const roots = [opts.sessionsDir]; + if (opts.includeArchived) roots.push(opts.archivedDir); + + const files = []; + for (const root of roots) { + files.push(...await collectJsonlFiles(root, root === opts.archivedDir ? "archived" : "active")); + } + files.sort((a, b) => a.file.localeCompare(b.file)); + + const selected = []; + const skipped = { + byDate: 0, + byCwd: 0, + empty: 0, + parseError: 0 + }; + + for (const entry of files) { + const parsed = await parseCodexRollout(entry, opts); + if (!parsed.ok) { + skipped[parsed.reason] = (skipped[parsed.reason] || 0) + 1; + continue; + } + selected.push(parsed.session); + if (opts.limit && selected.length >= opts.limit) break; + } + + const seedData = { sessions: selected }; + const summary = summarize(files, selected, skipped, opts); + + if (opts.out) { + await fs.mkdir(path.dirname(opts.out), { recursive: true }); + await fs.writeFile(opts.out, `${JSON.stringify(seedData, null, 2)}\n`, "utf-8"); + summary.output = opts.out; + } + + if (opts.dryRun || !opts.yes) { + summary.mode = "dry-run"; + summary.next = "Re-run with --yes to import these rounds through Gateway /seed."; + console.log(JSON.stringify(summary, null, 2)); + return; + } + + if (selected.length === 0) { + summary.mode = "import"; + summary.imported = false; + summary.reason = "no_sessions_with_user_assistant_rounds"; + console.log(JSON.stringify(summary, null, 2)); + return; + } + + const ready = await ensureGateway(); + if (!ready) { + console.error("TDAI Gateway unavailable"); + process.exit(1); + } + + const result = await httpPost("/seed", { + data: seedData, + strict_round_role: true, + auto_fill_timestamps: false, + wait_for_full_pipeline: opts.fullPipeline, + full_pipeline_timeout_ms: opts.fullPipelineTimeoutMs + }, Number(process.env.TDAI_CODEX_SEED_TIMEOUT_MS || DEFAULT_SEED_TIMEOUT_MS)); + + console.log(JSON.stringify({ + ...summary, + mode: "import", + imported: !!result, + seedResult: result + }, null, 2)); +} + +async function collectJsonlFiles(root, kind) { + const dir = path.resolve(expandHome(root)); + if (!fsSync.existsSync(dir)) return []; + const found = []; + async function walk(current) { + const entries = await fs.readdir(current, { withFileTypes: true }); + for (const entry of entries) { + const full = path.join(current, entry.name); + if (entry.isDirectory()) { + await walk(full); + } else if (entry.isFile() && entry.name.endsWith(".jsonl")) { + found.push({ file: full, kind }); + } + } + } + await walk(dir); + return found; +} + +async function parseCodexRollout(entry, opts) { + let text; + try { + text = await fs.readFile(entry.file, "utf-8"); + } catch { + return { ok: false, reason: "parseError" }; + } + + let sessionId = path.basename(entry.file, ".jsonl"); + let sessionCwd = ""; + let sessionTimestamp = 0; + let source = ""; + const messages = []; + + for (const line of text.split(/\r?\n/)) { + if (!line.trim()) continue; + let row; + try { + row = JSON.parse(line); + } catch { + continue; + } + + const payload = row.payload || {}; + if (row.type === "session_meta") { + sessionId = payload.id || sessionId; + sessionCwd = payload.cwd || sessionCwd; + sessionTimestamp = timestampMs(payload.timestamp || row.timestamp) || sessionTimestamp; + source = sourceLabel(payload.source); + continue; + } + + if (row.type !== "response_item" || payload.type !== "message") continue; + if (payload.role !== "user" && payload.role !== "assistant") continue; + + const content = sanitizeMemoryText(contentToText(payload.content)); + if (shouldSkipMessage(payload.role, content)) continue; + + messages.push({ + role: payload.role, + content, + timestamp: timestampMs(row.timestamp) || sessionTimestamp || Date.now() + }); + } + + if (opts.since && sessionTimestamp && sessionTimestamp < opts.since) { + return { ok: false, reason: "byDate" }; + } + + if (opts.cwd) { + const wanted = path.resolve(expandHome(opts.cwd)); + const actual = sessionCwd ? path.resolve(expandHome(sessionCwd)) : ""; + if (actual !== wanted) return { ok: false, reason: "byCwd" }; + } + + const conversations = pairMessages(messages); + if (conversations.length === 0) return { ok: false, reason: "empty" }; + + const cwdLabel = sessionCwd || "unknown-cwd"; + return { + ok: true, + session: { + sessionKey: `codex-import:${sha1(cwdLabel).slice(0, 10)}:${safeKey(sessionId)}`, + sessionId, + conversations, + metadata: { + source: "codex-jsonl", + codexSource: source || undefined, + codexCwd: sessionCwd || undefined, + codexArchiveKind: entry.kind, + codexPath: entry.file + } + } + }; +} + +function pairMessages(messages) { + const rounds = []; + let pendingUser = null; + + for (const msg of messages) { + if (msg.role === "user") { + if (pendingUser) { + pendingUser.content = `${pendingUser.content}\n\n${msg.content}`; + pendingUser.timestamp = Math.min(pendingUser.timestamp, msg.timestamp); + } else { + pendingUser = { ...msg }; + } + continue; + } + + if (msg.role === "assistant" && pendingUser) { + rounds.push([ + pendingUser, + msg + ]); + pendingUser = null; + } + } + + return rounds; +} + +function contentToText(content) { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + return content.map((part) => { + if (!part || typeof part !== "object") return ""; + if (typeof part.text === "string") return part.text; + if (typeof part.content === "string") return part.content; + return ""; + }).filter(Boolean).join("\n"); +} + +function shouldSkipMessage(role, text) { + const value = String(text || "").trim(); + if (!value) return true; + if (value.includes("")) return true; + if (value.includes("")) return true; + if (value.includes("")) return true; + if (value.includes("")) return true; + if (role === "user" && value.startsWith("# AGENTS.md instructions")) return true; + if (role === "user" && value.startsWith("")) return true; + if (role === "user" && value.startsWith("")) return true; + return false; +} + +function summarize(files, sessions, skipped, opts) { + const rounds = sessions.reduce((sum, session) => sum + session.conversations.length, 0); + const messages = sessions.reduce((sum, session) => ( + sum + session.conversations.reduce((inner, round) => inner + round.length, 0) + ), 0); + const byKind = sessions.reduce((acc, session) => { + const kind = session.metadata?.codexArchiveKind || "unknown"; + acc[kind] = (acc[kind] || 0) + 1; + return acc; + }, {}); + return { + source: "codex-jsonl", + sessionsDir: opts.sessionsDir, + archivedDir: opts.includeArchived ? opts.archivedDir : null, + includeArchived: opts.includeArchived, + waitForFullPipeline: opts.fullPipeline, + fullPipelineTimeoutMs: opts.fullPipelineTimeoutMs, + cwd: opts.cwd || null, + since: opts.since ? new Date(opts.since).toISOString() : null, + filesScanned: files.length, + sessionsPrepared: sessions.length, + roundsPrepared: rounds, + messagesPrepared: messages, + sessionsByKind: byKind, + skipped + }; +} + +function parseArgs(args) { + const opts = { + sessionsDir: path.resolve(expandHome(process.env.CODEX_SESSIONS_DIR || DEFAULT_SESSIONS_DIR)), + archivedDir: path.resolve(expandHome(process.env.CODEX_ARCHIVED_SESSIONS_DIR || DEFAULT_ARCHIVED_DIR)), + includeArchived: true, + fullPipeline: true, + fullPipelineTimeoutMs: positiveNumber(process.env.TDAI_CODEX_FULL_PIPELINE_TIMEOUT_MS, DEFAULT_FULL_PIPELINE_TIMEOUT_MS), + dryRun: false, + yes: false, + cwd: "", + since: 0, + limit: 0, + out: "" + }; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === "--help" || arg === "-h") opts.help = true; + else if (arg === "--dry-run") opts.dryRun = true; + else if (arg === "--yes" || arg === "-y") opts.yes = true; + else if (arg === "--no-archived") opts.includeArchived = false; + else if (arg === "--no-full-pipeline") opts.fullPipeline = false; + else if (arg === "--full-pipeline-timeout-ms") opts.fullPipelineTimeoutMs = positiveNumber(next(args, ++i, arg), 0); + else if (arg === "--sessions-dir") opts.sessionsDir = path.resolve(expandHome(next(args, ++i, arg))); + else if (arg === "--archived-dir") opts.archivedDir = path.resolve(expandHome(next(args, ++i, arg))); + else if (arg === "--cwd") opts.cwd = next(args, ++i, arg); + else if (arg === "--since") opts.since = parseSince(next(args, ++i, arg)); + else if (arg === "--limit") opts.limit = Math.max(0, Number(next(args, ++i, arg)) || 0); + else if (arg === "--out") opts.out = path.resolve(expandHome(next(args, ++i, arg))); + else throw new Error(`Unknown option: ${arg}`); + } + + return opts; +} + +function next(args, index, flag) { + if (index >= args.length) throw new Error(`${flag} requires a value`); + return args[index]; +} + +function parseSince(value) { + const raw = String(value || "").trim(); + const days = raw.match(/^(\d+)d$/i); + if (days) return Date.now() - Number(days[1]) * 24 * 60 * 60 * 1000; + const parsed = Date.parse(raw); + if (!Number.isFinite(parsed)) throw new Error(`Invalid --since value: ${value}`); + return parsed; +} + +function positiveNumber(value, fallback) { + const n = Number(value); + return Number.isFinite(n) && n > 0 ? n : fallback; +} + +function timestampMs(value) { + if (typeof value === "number") return value < 10_000_000_000 ? value * 1000 : value; + if (typeof value === "string") { + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : 0; + } + return 0; +} + +function sourceLabel(value) { + if (!value) return ""; + if (typeof value === "string") return value; + if (value.subagent) return "subagent"; + return "object"; +} + +function safeKey(value) { + return String(value).replace(/[^a-zA-Z0-9_.:-]/g, "_").slice(0, 120); +} + +function usage(code = 0) { + const message = `Usage: node scripts/import-codex-history.mjs [options] + +Import local Codex JSONL history into TencentDB Agent Memory through Gateway /seed. +By default this is a dry run unless --yes is provided. + +Options: + --dry-run Show the import plan without writing to the Gateway. + --yes, -y Actually import prepared rounds through /seed. + --sessions-dir Active Codex sessions directory. Default: ${DEFAULT_SESSIONS_DIR} + --archived-dir Archived Codex sessions directory. Default: ${DEFAULT_ARCHIVED_DIR} + --no-archived Do not include archived Codex JSONL files. + --no-full-pipeline Only seed through the Gateway's default L0/L1 path. + --full-pipeline-timeout-ms + Max wait for the final L1/L2/L3 flush. Default: ${DEFAULT_FULL_PIPELINE_TIMEOUT_MS} + --cwd Import only sessions whose session_meta.cwd matches this path. + --since Import only sessions newer than an ISO date or relative day window. + --limit Import at most n prepared sessions. + --out Write the generated Gateway /seed JSON to a file. +`; + (code === 0 ? console.log : console.error)(message); + process.exit(code); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { + importCodexHistoryCli().catch((err) => { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); + }); +} diff --git a/codex-plugin/scripts/lib.mjs b/codex-plugin/scripts/lib.mjs new file mode 100644 index 0000000..2ffcb5f --- /dev/null +++ b/codex-plugin/scripts/lib.mjs @@ -0,0 +1,1503 @@ +import crypto from "node:crypto"; +import { execFile, spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import fsSync from "node:fs"; +import { fileURLToPath } from "node:url"; +import os from "node:os"; +import path from "node:path"; +import { createInterface } from "node:readline"; +import { promisify } from "node:util"; +import { + buildCodexOffloadContext, + maxStoreChars as offloadMaxStoreChars, + previewForPolicy, + recordCodexToolOffload, + selectToolOffloadPolicy +} from "./offload-store.mjs"; + +const DEFAULT_GATEWAY_URL = "http://127.0.0.1:8420"; +const DEFAULT_CONTEXT_MAX_CHARS = 12000; +const DEFAULT_RECALL_TIMEOUT_MS = 5000; +const DEFAULT_CAPTURE_TIMEOUT_MS = 8000; +const DEFAULT_HEALTH_TIMEOUT_MS = 700; +const DEFAULT_START_TIMEOUT_MS = 12000; +const DEFAULT_SESSION_END_TIMEOUT_MS = 8000; +const DEFAULT_BREAKER_FAILURE_THRESHOLD = 5; +const DEFAULT_BREAKER_COOLDOWN_MS = 60_000; +const DEFAULT_FLUSH_EVERY_N_TURNS = 5; +const DEFAULT_TOOL_OFFLOAD_CONTEXT_CHARS = 6_000; +const DEFAULT_GATEWAY_PACKAGE = "@tencentdb-agent-memory/memory-tencentdb"; +const PRIVATE_DIR_MODE = 0o700; +const PRIVATE_FILE_MODE = 0o600; +const execFileAsync = promisify(execFile); +const INJECTED_MEMORY_TAGS = [ + "tdai-codex-memory-context", + "structured-memory-results", + "tdai-recall-context", + "raw-conversation-results", + "tdai-codex-context-offload", + "tdai-codex-tool-memory-hint", + "tdai-codex-tool-output-offload", + "relevant-memories", + "user-persona", + "relevant-scenes", + "scene-navigation", + "memory-tools-guide", + "current_task_context", + "history_task_context", +]; + +export function pluginRoot() { + const configured = process.env.PLUGIN_ROOT || process.env.CLAUDE_PLUGIN_ROOT; + return configured + ? path.resolve(configured) + : path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +} + +export function expandHome(value) { + if (!value) return value; + if (value === "~") return os.homedir(); + if (value.startsWith("~/")) return path.join(os.homedir(), value.slice(2)); + return value; +} + +export function tdaiDataDir() { + const configured = + process.env.TDAI_CODEX_DATA_DIR || + process.env.TDAI_DATA_DIR || + path.join(os.homedir(), ".memory-tencentdb", "codex-memory-tdai"); + return path.resolve(expandHome(configured)); +} + +export function hookLogPath() { + return path.join(tdaiDataDir(), "codex-adapter", "logs", "hook.log"); +} + +export function gatewayStdoutLogPath() { + return path.join(tdaiDataDir(), "codex-adapter", "logs", "gateway.stdout.log"); +} + +export function gatewayStderrLogPath() { + return path.join(tdaiDataDir(), "codex-adapter", "logs", "gateway.stderr.log"); +} + +export function gatewayUrl() { + return (process.env.TDAI_CODEX_GATEWAY_URL || DEFAULT_GATEWAY_URL).replace(/\/+$/, ""); +} + +export function gatewayHostPort() { + const url = new URL(gatewayUrl()); + return { + host: process.env.TDAI_GATEWAY_HOST || url.hostname || "127.0.0.1", + port: process.env.TDAI_GATEWAY_PORT || url.port || "8420" + }; +} + +export function resolveTdaiRoot() { + const root = pluginRoot(); + const candidates = [ + process.env.TDAI_CODEX_TDAI_ROOT, + process.env.TDAI_INSTALL_DIR, + path.join(root, ".."), + path.join(root, "vendor", "TencentDB-Agent-Memory"), + path.join(process.cwd(), "TencentDB-Agent-Memory") + ].filter(Boolean); + + for (const candidate of candidates) { + const resolved = path.resolve(expandHome(candidate)); + const pkg = path.join(resolved, "package.json"); + const gateway = path.join(resolved, "src", "gateway", "server.ts"); + if (!fsSync.existsSync(pkg) || !fsSync.existsSync(gateway)) continue; + try { + const parsed = JSON.parse(fsSync.readFileSync(pkg, "utf-8")); + if (parsed.name === "@tencentdb-agent-memory/memory-tencentdb") { + return resolved; + } + } catch { + return resolved; + } + } + return null; +} + +export async function readHookInput() { + let input = ""; + for await (const chunk of process.stdin) input += chunk; + if (!input.trim()) return {}; + try { + return JSON.parse(input); + } catch { + return {}; + } +} + +export function cwdFromPayload(payload) { + return path.resolve( + payload.cwd || + payload.project || + process.env.CLAUDE_PROJECT_DIR || + process.cwd() + ); +} + +export function sessionIdFromPayload(payload) { + return String(payload.session_id || payload.sessionId || payload.session || "unknown-session"); +} + +export function promptFromPayload(payload) { + return String( + payload.prompt ?? + payload.user_prompt ?? + payload.userPrompt ?? + payload.input ?? + "" + ); +} + +export function sessionKeyFromPayload(payload) { + const cwd = cwdFromPayload(payload); + const sessionId = sessionIdFromPayload(payload); + return `codex:${sha1(cwd).slice(0, 10)}:${safeKey(sessionId)}`; +} + +export function sessionKeyPrefixesForCwd(cwd) { + const cwdHash = sha1(path.resolve(expandHome(cwd))).slice(0, 10); + return [ + `codex:${cwdHash}:`, + `codex-import:${cwdHash}:` + ]; +} + +export function projectLabel(payload) { + const cwd = cwdFromPayload(payload); + return `${path.basename(cwd)} (${cwd})`; +} + +export function sha1(value) { + return crypto.createHash("sha1").update(String(value)).digest("hex"); +} + +function safeKey(value) { + return String(value).replace(/[^a-zA-Z0-9_.:-]/g, "_").slice(0, 120); +} + +function statePath(sessionKey) { + return path.join(tdaiDataDir(), "codex-adapter", "sessions", `${sha1(sessionKey)}.json`); +} + +async function ensureStateDir() { + await ensurePrivateDir(path.join(tdaiDataDir(), "codex-adapter", "sessions")); +} + +function gatewayCircuitPath() { + return path.join(tdaiDataDir(), "codex-adapter", "gateway-circuit.json"); +} + +export async function loadSessionState(sessionKey) { + await ensureStateDir(); + const file = statePath(sessionKey); + try { + return JSON.parse(await fs.readFile(file, "utf-8")); + } catch { + return { sessionKey, turns: [] }; + } +} + +export async function saveSessionState(sessionKey, state) { + await ensureStateDir(); + const file = statePath(sessionKey); + const tmp = `${file}.${process.pid}.tmp`; + await writePrivateFile(tmp, `${JSON.stringify(state, null, 2)}\n`); + await fs.rename(tmp, file); + await chmodPrivateFile(file); +} + +export async function beginTurn(payload) { + const sessionKey = sessionKeyFromPayload(payload); + const state = await loadSessionState(sessionKey); + const prompt = sanitizeMemoryText(promptFromPayload(payload)); + const now = Date.now(); + + if (state.currentTurn && !state.currentTurn.captured) { + state.turns = state.turns || []; + state.turns.push({ + ...state.currentTurn, + abandonedAt: now, + abandonedReason: "new_user_prompt_before_stop" + }); + } + + state.currentTurn = { + turnId: `turn_${now}_${crypto.randomBytes(3).toString("hex")}`, + sessionKey, + sessionId: sessionIdFromPayload(payload), + cwd: cwdFromPayload(payload), + project: projectLabel(payload), + userPrompt: prompt, + startedAt: now, + events: [ + { + phase: "user_prompt", + timestamp: now, + content: sanitizeMemoryText(prompt) + } + ], + captured: false + }; + + await saveSessionState(sessionKey, state); + return state.currentTurn; +} + +export async function appendToolEvent(payload, phase, extra = {}) { + const sessionKey = sessionKeyFromPayload(payload); + const state = await loadSessionState(sessionKey); + const now = Date.now(); + if (!state.currentTurn) { + state.currentTurn = { + turnId: `turn_${now}_${crypto.randomBytes(3).toString("hex")}`, + sessionKey, + sessionId: sessionIdFromPayload(payload), + cwd: cwdFromPayload(payload), + project: projectLabel(payload), + userPrompt: "", + startedAt: now, + events: [], + captured: false + }; + } + + state.currentTurn.events.push({ + phase, + timestamp: now, + toolName: payload.tool_name || payload.toolName || "", + toolInput: compact(payload.tool_input ?? payload.toolInput ?? payload.input, 2500), + toolOutput: compact(toolOutputFromPayload(payload), 4000), + ...sanitizeEventDetail(extra) + }); + await saveSessionState(sessionKey, state); +} + +export async function appendLifecycleEvent(payload, phase, detail = {}, options = {}) { + const sessionKey = sessionKeyFromPayload(payload); + const state = await loadSessionState(sessionKey); + const now = Date.now(); + if (!state.currentTurn) { + if (options.createTurn === false) { + return { appended: false, reason: "no_pending_turn" }; + } + state.currentTurn = { + turnId: `turn_${now}_${crypto.randomBytes(3).toString("hex")}`, + sessionKey, + sessionId: sessionIdFromPayload(payload), + cwd: cwdFromPayload(payload), + project: projectLabel(payload), + userPrompt: "", + startedAt: now, + events: [], + captured: false + }; + } + state.currentTurn.events.push({ + phase, + timestamp: now, + ...sanitizeEventDetail(detail) + }); + await saveSessionState(sessionKey, state); + return { appended: true }; +} + +export async function captureCurrentTurn(payload, reason = "stop") { + const sessionKey = sessionKeyFromPayload(payload); + const state = await loadSessionState(sessionKey); + const turn = state.currentTurn; + if (!turn || turn.captured) return { captured: false, reason: "no_pending_turn" }; + + const gatewayReady = await ensureGateway(); + if (!gatewayReady) return { captured: false, reason: "gateway_unavailable" }; + + const userContent = buildUserContent(turn, payload, reason); + const assistantContent = + await extractAssistantFromTranscript(payload.transcript_path, turn.startedAt) + || buildAssistantSummary(turn, reason); + + const messages = [ + { role: "user", content: userContent, timestamp: turn.startedAt }, + { role: "assistant", content: assistantContent, timestamp: Date.now() } + ]; + + const response = await httpPost("/capture", { + user_content: userContent, + assistant_content: assistantContent, + session_key: sessionKey, + session_id: turn.sessionId, + started_at: Math.max(0, turn.startedAt - 1), + messages + }, DEFAULT_CAPTURE_TIMEOUT_MS); + + if (!response) { + turn.lastCaptureFailure = { + reason, + failedAt: Date.now() + }; + await saveSessionState(sessionKey, state); + return { + captured: false, + reason: "capture_failed" + }; + } + + turn.captured = true; + turn.capturedAt = Date.now(); + turn.captureReason = reason; + turn.captureResponse = { + l0_recorded: response.l0_recorded, + scheduler_notified: response.scheduler_notified + }; + + state.turns = state.turns || []; + state.turns.push(turn); + delete state.currentTurn; + await saveSessionState(sessionKey, state); + + return { + captured: true, + l0Recorded: response?.l0_recorded ?? 0, + schedulerNotified: response?.scheduler_notified ?? false, + turnCount: state.turns.length + }; +} + +export async function maybeFlushCapturedTurns(payload, captureResult, reason = "periodic_turn_flush") { + if (!captureResult?.captured) return { flushed: false, reason: "no_capture" }; + const interval = numericEnv("TDAI_CODEX_FLUSH_EVERY_N_TURNS", DEFAULT_FLUSH_EVERY_N_TURNS); + if (!Number.isFinite(interval) || interval <= 0) return { flushed: false, reason: "disabled" }; + + const sessionKey = sessionKeyFromPayload(payload); + const state = await loadSessionState(sessionKey); + const turnCount = Array.isArray(state.turns) ? state.turns.length : 0; + if (turnCount === 0 || turnCount % interval !== 0 || state.lastPeriodicFlushTurnCount === turnCount) { + return { flushed: false, reason: "not_due", turnCount, interval }; + } + + const result = await sessionEnd(payload, reason); + state.lastPeriodicFlushTurnCount = turnCount; + state.lastPeriodicFlushAt = Date.now(); + state.lastPeriodicFlushResult = result; + await saveSessionState(sessionKey, state); + return { ...result, turnCount, interval }; +} + +function buildUserContent(turn, payload, reason) { + const prompt = sanitizeMemoryText(turn.userPrompt || promptFromPayload(payload)) || "[Codex turn without captured user prompt]"; + return [ + `Codex project: ${turn.project || projectLabel(payload)}`, + `Codex session: ${turn.sessionId || sessionIdFromPayload(payload)}`, + `Capture reason: ${reason}`, + "", + "User request:", + prompt + ].join("\n"); +} + +function buildAssistantSummary(turn, reason) { + const lines = [ + `Codex turn completed. Capture reason: ${reason}.`, + `Project: ${turn.project || turn.cwd || ""}`, + "" + ]; + + const events = Array.isArray(turn.events) ? turn.events : []; + const toolEvents = events.filter((event) => event.phase === "pre_tool_use" || event.phase === "post_tool_use"); + if (toolEvents.length === 0) { + lines.push("No tool events were available from Codex hooks for this turn."); + } else { + lines.push("Captured tool activity:"); + for (const event of toolEvents.slice(-20)) { + lines.push(`- ${event.phase}: ${event.toolName || "(unknown tool)"}`); + if (event.toolInput) lines.push(indentBlock(`input: ${event.toolInput}`, " ")); + if (event.toolOutput) lines.push(indentBlock(`output: ${event.toolOutput}`, " ")); + } + } + + return truncate(sanitizeMemoryText(lines.join("\n")), 10000); +} + +export async function recallForPrompt(payload, prompt, mode = "prompt") { + const sessionKey = sessionKeyFromPayload(payload); + const cwd = cwdFromPayload(payload); + const cleanPrompt = sanitizeMemoryText(prompt || ""); + const offloadContextPromise = buildCodexOffloadContext({ + sessionKey, + sessionId: sessionIdFromPayload(payload), + maxChars: numericEnv("TDAI_CODEX_TOOL_OFFLOAD_CONTEXT_CHARS", DEFAULT_TOOL_OFFLOAD_CONTEXT_CHARS) + }); + + const gatewayReady = await ensureGateway(); + const query = [ + `Codex project cwd: ${cwd}`, + `Recall mode: ${mode}`, + "", + cleanPrompt.trim() + ? `Current user request:\n${cleanPrompt.trim()}` + : "Session startup/resume: recover project state, active decisions, pending tasks, and user preferences." + ].join("\n"); + + const [recall, memories, conversations] = gatewayReady + ? await Promise.all([ + httpPost("/recall", { query, session_key: sessionKey }, DEFAULT_RECALL_TIMEOUT_MS), + httpPost("/search/memories", { + query, + limit: numericEnv("TDAI_CODEX_MEMORY_LIMIT", 8), + session_key_prefixes: sessionKeyPrefixesForCwd(cwd) + }, DEFAULT_RECALL_TIMEOUT_MS), + shouldSearchConversations(cleanPrompt, mode) + ? httpPost("/search/conversations", { + query, + limit: numericEnv("TDAI_CODEX_CONVERSATION_LIMIT", 5), + session_key_prefixes: sessionKeyPrefixesForCwd(cwd) + }, DEFAULT_RECALL_TIMEOUT_MS) + : Promise.resolve(null) + ]) + : [null, null, null]; + const directConversations = !hasUsefulGatewayText(memories?.results) && + !recall?.context?.trim() && + !hasUsefulGatewayText(conversations?.results) && + cleanPrompt.trim() + ? await searchL0JsonlDirect({ + query: cleanPrompt, + cwd, + limit: numericEnv("TDAI_CODEX_DIRECT_L0_LIMIT", 3) + }) + : ""; + + const parts = []; + if (hasUsefulGatewayText(memories?.results)) { + parts.push(`\n${memories.results.trim()}\n`); + } + if (recall?.context?.trim()) { + parts.push(`\n${recall.context.trim()}\n`); + } + if (hasUsefulGatewayText(conversations?.results)) { + parts.push(`\n${conversations.results.trim()}\n`); + } + if (directConversations) { + parts.push(`\n${directConversations}\n`); + } + const offloadContext = await offloadContextPromise; + if (offloadContext) { + parts.push(offloadContext); + } + + if (parts.length === 0) return ""; + + const context = ` +Use retrieved memory as operating context; verify drift-prone facts; persist durable new decisions. +${parts.join("\n\n")} +`; + + return truncate(context, numericEnv("TDAI_CODEX_CONTEXT_MAX_CHARS", DEFAULT_CONTEXT_MAX_CHARS)); +} + +export function hookAdditionalContext(hookEventName, context) { + if (!context?.trim()) return ""; + return `${JSON.stringify({ + hookSpecificOutput: { + hookEventName, + additionalContext: context + } + })}\n`; +} + +export function toolMemoryHint(payload) { + const toolName = payload.tool_name || payload.toolName || ""; + if (!toolName) return ""; + const input = compact(payload.tool_input ?? payload.toolInput ?? payload.input, 1200); + return ` +Use injected memory/MCP search for prior decisions or exact history; use tdai_offload_lookup for offload node_id refs. +Tool: ${escapeText(toolName)}${input ? `\nInput: ${escapeText(input)}` : ""} +`; +} + +export async function maybeOffloadToolOutput(payload) { + if (process.env.TDAI_CODEX_TOOL_OFFLOAD === "false") return null; + + const rawOutput = toolOutputFromPayload(payload); + const rendered = renderToolValue(rawOutput); + if (!rendered.trim()) return null; + + const redacted = redact(rendered); + const policy = selectToolOffloadPolicy(redacted.length); + if (!policy) return null; + + const sessionKey = sessionKeyFromPayload(payload); + const sessionId = sessionIdFromPayload(payload); + const cwd = cwdFromPayload(payload); + const toolName = payload.tool_name || payload.toolName || "unknown-tool"; + const toolUseId = payload.tool_use_id || payload.toolUseId || `${Date.now()}-${crypto.randomBytes(3).toString("hex")}`; + const maxStoreChars = offloadMaxStoreChars(); + const storedText = truncate(redacted, maxStoreChars); + const inputSummary = compact(payload.tool_input ?? payload.toolInput ?? payload.input, 5000); + const recorded = await recordCodexToolOffload({ + sessionKey, + sessionId, + cwd, + toolName, + toolUseId, + inputSummary, + redactedOutput: redacted, + storedText, + policy + }); + + const preview = previewForPolicy(redacted, policy); + const nodeId = recorded.entry.node_id || "pending"; + const outputPath = path.join(recorded.paths.root, recorded.entry.result_ref); + const summary = [ + "TencentDB Agent Memory offloaded this large tool result to keep Codex context compact.", + `Tool: ${toolName}`, + `Tool use id: ${toolUseId}`, + `Node id: ${nodeId}`, + `Offload policy: ${policy.name}`, + `Original output size after redaction: ${redacted.length} characters.`, + `Stored output path: ${outputPath}`, + `Offload JSONL: ${recorded.paths.offloadJsonl}`, + `Mermaid canvas: ${recorded.paths.canvasPath}`, + "", + "Use tdai_offload_lookup with the node id or tool use id for exact audit details.", + "Use tdai_memory_search / tdai_conversation_search for later long-term recall.", + "", + "Preview:", + preview + ].join("\n"); + + return { + outputPath, + nodeId, + toolUseId, + policy: policy.name, + offloadJsonlPath: recorded.paths.offloadJsonl, + canvasPath: recorded.paths.canvasPath, + originalChars: redacted.length, + storedChars: storedText.length, + summary: truncate(summary, numericEnv("TDAI_CODEX_TOOL_OFFLOAD_SUMMARY_MAX_CHARS", 7000)) + }; +} + +export function postToolOffloadHookOutput(offload) { + if (!offload) return ""; + return `${JSON.stringify({ + decision: "block", + reason: offload.summary, + hookSpecificOutput: { + hookEventName: "PostToolUse", + additionalContext: ` +${escapeText(offload.nodeId || "")} +${escapeText(offload.policy || "")} +${escapeText(offload.summary)} +` + } + })}\n`; +} + +function shouldSearchConversations(prompt, mode) { + if (mode === "session-start") return true; + if (!prompt) return false; + return /(继续|上次|之前|刚才|resume|continue|previous|last time|where were we|做到哪)/i.test(prompt); +} + +async function searchL0JsonlDirect(params) { + const { query, cwd, limit } = params; + const keywords = queryKeywords(query); + if (keywords.length === 0) return ""; + + const conversationsDir = path.join(tdaiDataDir(), "conversations"); + let entries; + try { + entries = await fs.readdir(conversationsDir, { withFileTypes: true }); + } catch { + return ""; + } + + const files = (await Promise.all(entries + .filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")) + .map(async (entry) => { + const file = path.join(conversationsDir, entry.name); + try { + const stat = await fs.stat(file); + return { file, mtimeMs: stat.mtimeMs }; + } catch { + return { file, mtimeMs: 0 }; + } + }))) + .sort((a, b) => b.mtimeMs - a.mtimeMs) + .slice(0, numericEnv("TDAI_CODEX_DIRECT_L0_MAX_FILES", 30)); + + const prefixes = sessionKeyPrefixesForCwd(cwd); + const matches = []; + const seen = new Set(); + + for (const { file } of files) { + let stream; + try { + stream = fsSync.createReadStream(file, { encoding: "utf-8" }); + } catch { + continue; + } + + const rl = createInterface({ input: stream, crlfDelay: Infinity }); + try { + for await (const line of rl) { + if (!line.trim()) continue; + let row; + try { + row = JSON.parse(line); + } catch { + continue; + } + const sessionKey = typeof row.sessionKey === "string" ? row.sessionKey : ""; + if (!prefixes.some((prefix) => sessionKey.startsWith(prefix))) continue; + const content = sanitizeMemoryText(row.content || row.message_text || ""); + if (!content) continue; + const lower = content.toLowerCase(); + const hits = keywords.filter((keyword) => lower.includes(keyword)).length; + if (hits === 0) continue; + const fingerprint = `${row.role || ""}:${content.slice(0, 180)}`; + if (seen.has(fingerprint)) continue; + seen.add(fingerprint); + matches.push({ + role: row.role || "unknown", + content: truncate(content, 2000), + recordedAt: row.recordedAt || row.recorded_at || "", + hits + }); + } + } finally { + rl.close(); + stream.destroy(); + } + } + + if (matches.length === 0) return ""; + matches.sort((a, b) => + rolePriority(b.role) - rolePriority(a.role) || + b.hits - a.hits || + b.content.length - a.content.length + ); + + const lines = [`Found ${Math.min(matches.length, limit)} matching local L0 conversation message(s):`, ""]; + for (const match of matches.slice(0, limit)) { + lines.push("---"); + lines.push(`**[${match.role}]** ${match.recordedAt}`); + lines.push(""); + lines.push(match.content); + lines.push(""); + } + return lines.join("\n"); +} + +function queryKeywords(value) { + const cjkStop = new Set([ + "之前", "前聊", "聊的", "还记", "记得", "得么", "得吗", + "一下", "怎么", "什么", "关于", "知道", "以前", "上次", + "如何", "为何", "为啥", "哪里", "哪些", "为什", + "请问", "请帮", "帮我", "麻烦" + ]); + const keywords = []; + for (const segment of String(value || "").toLowerCase().replace(/[^\w一-鿿]/g, " ").split(/\s+/)) { + if (!segment) continue; + if (/[一-鿿]/.test(segment)) { + for (let i = 0; i <= segment.length - 2; i++) { + const gram = segment.slice(i, i + 2); + if (!cjkStop.has(gram)) keywords.push(gram); + } + } else if (segment.length >= 2) { + keywords.push(segment); + } + } + return [...new Set(keywords)].slice(0, 40); +} + +function rolePriority(role) { + return role === "assistant" ? 1 : 0; +} + +function hasUsefulGatewayText(value) { + const text = String(value || "").trim(); + if (!text) return false; + return !/^No matching (memories|conversations) found\.?$/i.test(text); +} + +export async function ensureGateway() { + if (await healthCheck()) return true; + if (process.env.TDAI_CODEX_AUTOSTART === "false") return false; + + const started = await startGatewayDetached(); + if (!started) return false; + + const deadline = Date.now() + numericEnv("TDAI_CODEX_START_TIMEOUT_MS", DEFAULT_START_TIMEOUT_MS); + while (Date.now() < deadline) { + await delay(500); + if (await healthCheck()) return true; + } + return false; +} + +export async function healthCheck() { + if (!isAllowedGatewayEndpoint()) return false; + try { + const headers = await gatewayAuthHeaders(); + const res = await fetch(`${gatewayUrl()}/health`, { + headers, + signal: AbortSignal.timeout(DEFAULT_HEALTH_TIMEOUT_MS) + }); + return res.ok; + } catch { + return false; + } +} + +export async function startGatewayDetached() { + const { host, port } = gatewayHostPort(); + if (!isLoopbackHost(host) && process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK !== "true") { + debug(`Refusing to autostart gateway on non-loopback host=${host}. Set TDAI_CODEX_ALLOW_NON_LOOPBACK=true to override.`); + return false; + } + + const logDir = path.dirname(hookLogPath()); + await ensurePrivateDir(logDir); + const lock = await acquireGatewaySpawnLock(logDir); + if (!lock) { + const deadline = Date.now() + numericEnv("TDAI_CODEX_START_TIMEOUT_MS", DEFAULT_START_TIMEOUT_MS); + while (Date.now() < deadline) { + await delay(500); + if (await healthCheck()) return true; + } + debug("Gateway spawn lock contention timed out"); + return false; + } + + try { + await ensureGatewayAuthToken(); + if (await healthCheck()) return true; + const outFd = fsSync.openSync(gatewayStdoutLogPath(), "a", PRIVATE_FILE_MODE); + const errFd = fsSync.openSync(gatewayStderrLogPath(), "a", PRIVATE_FILE_MODE); + const pidFile = path.join(logDir, "gateway.pid"); + const pidMetadataFile = path.join(logDir, "gateway.pid.json"); + const launch = gatewayLaunchSpec(); + + const env = { + ...process.env, + TDAI_DATA_DIR: tdaiDataDir(), + TDAI_GATEWAY_CONFIG: process.env.TDAI_GATEWAY_CONFIG || path.join(tdaiDataDir(), "tdai-gateway.json"), + TDAI_GATEWAY_HOST: host, + TDAI_GATEWAY_PORT: port, + TDAI_TOKEN_PATH: configuredGatewayTokenPath(), + TDAI_CODEX_PARENT_PID: String(process.ppid || process.pid) + }; + delete env.TDAI_GATEWAY_TOKEN; + delete env.TDAI_CODEX_GATEWAY_TOKEN; + + if (launch.mode !== "package-bin" || process.env.TDAI_CODEX_HYDRATE_ENV_FOR_PACKAGE_GATEWAY === "true") { + await hydrateLoginShellEnv(env, [ + "DEEPSEEK_API_KEY", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "TDAI_LLM_API_KEY", + "TDAI_LLM_BASE_URL", + "TDAI_LLM_MODEL" + ]); + } + + const child = spawn(launch.command, launch.args, { + cwd: launch.cwd, + detached: true, + env, + stdio: ["ignore", outFd, errFd] + }); + const spawnError = await detectSpawnError(child); + if (spawnError || !child.pid) { + debug(`Gateway spawn failed (${launch.mode}): ${spawnError?.message || "child has no pid"}`); + return false; + } + child.unref(); + await writePrivateFile(pidFile, `${child.pid}\n`); + await writePrivateFile(pidMetadataFile, `${JSON.stringify({ + pid: child.pid, + root: launch.root, + command: [launch.command, ...launch.args], + launchMode: launch.mode, + startedAt: new Date().toISOString() + }, null, 2)}\n`); + debug(`Started TDAI gateway pid=${child.pid} mode=${launch.mode}`); + + const deadline = Date.now() + numericEnv("TDAI_CODEX_START_TIMEOUT_MS", DEFAULT_START_TIMEOUT_MS); + while (Date.now() < deadline) { + await delay(500); + if (await healthCheck()) return true; + } + debug(`Gateway did not become healthy after spawn mode=${launch.mode}`); + return false; + } finally { + await lock.release(); + } +} + +function detectSpawnError(child) { + return new Promise((resolve) => { + const timer = setTimeout(() => resolve(null), 50); + child.once("error", (err) => { + clearTimeout(timer); + resolve(err); + }); + }); +} + +function gatewayLaunchSpec() { + const override = process.env.TDAI_CODEX_GATEWAY_COMMAND || process.env.TDAI_GATEWAY_COMMAND; + if (override) { + return { + command: override, + args: [], + cwd: tdaiDataDir(), + root: "", + mode: "override" + }; + } + + const explicitRoot = process.env.TDAI_CODEX_TDAI_ROOT || process.env.TDAI_INSTALL_DIR; + const root = explicitRoot ? resolveTdaiRoot() : null; + if (root && fsSync.existsSync(path.join(root, "node_modules", "tsx"))) { + return { + command: "npx", + args: ["tsx", "src/gateway/server.ts"], + cwd: root, + root, + mode: "local-checkout" + }; + } + + return { + command: "npx", + args: ["--yes", "--ignore-scripts", "--package", gatewayPackageSpec(), "tdai-memory-gateway"], + cwd: tdaiDataDir(), + root: "", + mode: "package-bin" + }; +} + +function gatewayPackageSpec() { + return process.env.TDAI_CODEX_GATEWAY_PACKAGE || DEFAULT_GATEWAY_PACKAGE; +} + +async function acquireGatewaySpawnLock(logDir) { + const lockPath = path.join(logDir, "gateway.spawn.lock"); + const tryCreate = async () => { + try { + const handle = await fs.open(lockPath, "wx", PRIVATE_FILE_MODE); + await handle.writeFile(`${process.pid}\n`); + await handle.close(); + return { + release: async () => { + await fs.rm(lockPath, { force: true }); + } + }; + } catch (err) { + if (err?.code === "EEXIST") return null; + throw err; + } + }; + + const first = await tryCreate(); + if (first) return first; + + try { + const stat = await fs.stat(lockPath); + if (Date.now() - stat.mtimeMs > 60_000) { + await fs.rm(lockPath, { force: true }); + return tryCreate(); + } + } catch { + return tryCreate(); + } + return null; +} + +export async function stopGateway() { + const logDir = path.join(tdaiDataDir(), "codex-adapter", "logs"); + const pidInfo = await readGatewayPidInfo(logDir); + if (!pidInfo) return false; + + if (!(await pidLooksLikeGateway(pidInfo.pid))) { + debug(`Refusing to stop pid=${pidInfo.pid}: process does not look like TDAI gateway`); + return false; + } + + try { + process.kill(pidInfo.pid, "SIGTERM"); + await removeGatewayPidFiles(logDir); + return true; + } catch { + return false; + } +} + +async function readGatewayPidInfo(logDir) { + const pidMetadataFile = path.join(logDir, "gateway.pid.json"); + try { + const metadata = JSON.parse(await fs.readFile(pidMetadataFile, "utf-8")); + const pid = Number(metadata.pid); + if (Number.isFinite(pid) && pid > 0) return { ...metadata, pid }; + } catch { + // Fall back to the legacy plain PID file below. + } + + try { + const pid = Number((await fs.readFile(path.join(logDir, "gateway.pid"), "utf-8")).trim()); + if (Number.isFinite(pid) && pid > 0) return { pid }; + } catch { + return null; + } + return null; +} + +async function pidLooksLikeGateway(pid) { + try { + process.kill(pid, 0); + } catch { + return false; + } + + try { + const { stdout } = await execFileAsync("ps", ["-p", String(pid), "-o", "command="], { + timeout: 1000, + maxBuffer: 4096 + }); + const command = stdout.trim(); + return command.includes("tdai-memory-gateway") || + (command.includes("tsx") && command.includes("src/gateway/server.ts")); + } catch (err) { + debug(`Could not inspect pid=${pid}: ${err instanceof Error ? err.message : String(err)}`); + return false; + } +} + +async function removeGatewayPidFiles(logDir) { + await Promise.all([ + fs.rm(path.join(logDir, "gateway.pid"), { force: true }), + fs.rm(path.join(logDir, "gateway.pid.json"), { force: true }) + ]); +} + +export async function httpPost(route, body, timeoutMs = DEFAULT_RECALL_TIMEOUT_MS) { + if (!isAllowedGatewayEndpoint()) return null; + if (await isGatewayCircuitOpen()) return null; + try { + const headers = { + "Content-Type": "application/json", + ...await gatewayAuthHeaders() + }; + const res = await fetch(`${gatewayUrl()}${route}`, { + method: "POST", + headers, + body: JSON.stringify(body), + signal: AbortSignal.timeout(timeoutMs) + }); + if (!res.ok) { + debug(`Gateway ${route} returned ${res.status}: ${await res.text().catch(() => "")}`); + await recordGatewayFailure(route); + return null; + } + const json = await res.json(); + await recordGatewaySuccess(); + return json; + } catch (err) { + debug(`Gateway ${route} failed: ${err instanceof Error ? err.message : String(err)}`); + await recordGatewayFailure(route); + return null; + } +} + +function isAllowedGatewayEndpoint() { + let url; + try { + url = new URL(gatewayUrl()); + } catch { + debug(`Refusing invalid Gateway URL: ${gatewayUrl()}`); + return false; + } + if (isLoopbackHost(url.hostname)) return true; + if (process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK === "true") return true; + debug(`Refusing non-loopback Gateway URL=${url.origin}. Set TDAI_CODEX_ALLOW_NON_LOOPBACK=true to override.`); + return false; +} + +async function isGatewayCircuitOpen() { + if (process.env.TDAI_CODEX_CIRCUIT_BREAKER === "false") return false; + const state = await readGatewayCircuit(); + const openedUntil = Number(state.openedUntil || 0); + if (openedUntil > Date.now()) { + debug(`Gateway circuit breaker open for ${Math.ceil((openedUntil - Date.now()) / 1000)}s`); + return true; + } + if (openedUntil) { + await writeGatewayCircuit({ failureCount: 0, openedUntil: 0, lastRoute: state.lastRoute || "" }); + } + return false; +} + +async function recordGatewayFailure(route) { + if (process.env.TDAI_CODEX_CIRCUIT_BREAKER === "false") return; + const state = await readGatewayCircuit(); + const threshold = numericEnv("TDAI_CODEX_BREAKER_FAILURES", DEFAULT_BREAKER_FAILURE_THRESHOLD); + const cooldownMs = numericEnv("TDAI_CODEX_BREAKER_COOLDOWN_MS", DEFAULT_BREAKER_COOLDOWN_MS); + const failureCount = Number(state.failureCount || 0) + 1; + const openedUntil = failureCount >= threshold ? Date.now() + cooldownMs : Number(state.openedUntil || 0); + await writeGatewayCircuit({ + failureCount, + openedUntil, + lastRoute: route, + lastFailureAt: Date.now() + }); + if (openedUntil) { + debug(`Gateway circuit breaker opened after ${failureCount} failures; cooldown=${cooldownMs}ms`); + } +} + +async function recordGatewaySuccess() { + if (process.env.TDAI_CODEX_CIRCUIT_BREAKER === "false") return; + const state = await readGatewayCircuit(); + if (!state.failureCount && !state.openedUntil) return; + await writeGatewayCircuit({ failureCount: 0, openedUntil: 0, lastSuccessAt: Date.now() }); +} + +async function readGatewayCircuit() { + try { + return JSON.parse(await fs.readFile(gatewayCircuitPath(), "utf-8")); + } catch { + return { failureCount: 0, openedUntil: 0 }; + } +} + +async function writeGatewayCircuit(state) { + const file = gatewayCircuitPath(); + await ensurePrivateDir(path.dirname(file)); + const tmp = `${file}.${process.pid}.tmp`; + await writePrivateFile(tmp, `${JSON.stringify(state, null, 2)}\n`); + await fs.rename(tmp, file); + await chmodPrivateFile(file); +} + +async function extractAssistantFromTranscript(transcriptPath, sinceMs) { + if (!transcriptPath || !fsSync.existsSync(transcriptPath)) return ""; + try { + const stat = await fs.stat(transcriptPath); + const maxBytes = numericEnv("TDAI_CODEX_TRANSCRIPT_TAIL_BYTES", 2_000_000); + const fh = await fs.open(transcriptPath, "r"); + const start = Math.max(0, stat.size - maxBytes); + const buffer = Buffer.alloc(stat.size - start); + await fh.read(buffer, 0, buffer.length, start); + await fh.close(); + + const candidates = []; + for (const line of buffer.toString("utf-8").split(/\r?\n/)) { + if (!line.trim()) continue; + let parsed; + try { + parsed = JSON.parse(line); + } catch { + continue; + } + const ts = timestampFromAny(parsed); + if (sinceMs && ts && ts < sinceMs - 1000) continue; + const text = extractAssistantText(parsed).trim(); + if (text.length > 20) candidates.push(text); + } + return truncate(sanitizeMemoryText(candidates.at(-1) || ""), 10000); + } catch { + return ""; + } +} + +function timestampFromAny(value) { + if (!value || typeof value !== "object") return 0; + const direct = value.timestamp || value.created_at || value.createdAt; + if (typeof direct === "number") return direct; + if (typeof direct === "string") { + const parsed = Date.parse(direct); + return Number.isFinite(parsed) ? parsed : 0; + } + return timestampFromAny(value.item || value.message || value.payload || value.response); +} + +function extractAssistantText(value) { + if (!value) return ""; + if (typeof value === "string") return ""; + if (Array.isArray(value)) return value.map(extractAssistantText).filter(Boolean).join("\n"); + if (typeof value !== "object") return ""; + + const role = value.role || value.author?.role || value.message?.role || value.item?.role; + const type = value.type; + if ((role === "assistant" || type === "assistant_message") && value.content) { + return contentToText(value.content); + } + if (type === "message" && role === "assistant" && value.content) { + return contentToText(value.content); + } + if (type === "response_item" && value.item) return extractAssistantText(value.item); + if (value.item) return extractAssistantText(value.item); + if (value.message) return extractAssistantText(value.message); + if (value.payload) return extractAssistantText(value.payload); + if (value.response) return extractAssistantText(value.response); + return ""; +} + +function contentToText(content) { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content.map((part) => { + if (!part || typeof part !== "object") return ""; + if (typeof part.text === "string") return part.text; + if (typeof part.content === "string") return part.content; + if (part.type === "output_text" && typeof part.text === "string") return part.text; + return ""; + }).filter(Boolean).join("\n"); + } + if (content && typeof content === "object" && typeof content.text === "string") { + return content.text; + } + return ""; +} + +export async function rememberText(payload, text) { + const gatewayReady = await ensureGateway(); + if (!gatewayReady) return { remembered: false, reason: "gateway_unavailable" }; + const now = Date.now(); + const sessionKey = sessionKeyFromPayload(payload); + const content = [ + `Codex project: ${projectLabel(payload)}`, + "Explicit memory note:", + sanitizeMemoryText(text) + ].join("\n"); + const response = await httpPost("/capture", { + user_content: "Explicit memory note saved from Codex.", + assistant_content: content, + session_key: sessionKey, + session_id: sessionIdFromPayload(payload), + started_at: Math.max(0, now - 1), + messages: [ + { role: "user", content: "Remember this.", timestamp: now }, + { role: "assistant", content, timestamp: now + 1 } + ] + }, DEFAULT_CAPTURE_TIMEOUT_MS); + return { remembered: !!response, response }; +} + +export async function sessionEnd(payload, reason = "session_end") { + const gatewayReady = await ensureGateway(); + if (!gatewayReady) return { flushed: false, reason: "gateway_unavailable" }; + const sessionKey = sessionKeyFromPayload(payload); + const response = await httpPost("/session/end", { + session_key: sessionKey, + reason + }, DEFAULT_SESSION_END_TIMEOUT_MS); + return { + flushed: !!response?.flushed, + response + }; +} + +export function compact(value, maxChars) { + if (value === undefined || value === null || value === "") return ""; + let str; + try { + str = typeof value === "string" ? value : JSON.stringify(value); + } catch { + str = String(value); + } + return truncate(sanitizeMemoryText(str), maxChars); +} + +export function redactText(value) { + return redact(value); +} + +export function sanitizeMemoryText(value) { + return redact(stripInjectedMemoryTags(value)); +} + +export function stripInjectedMemoryTags(value) { + let cleaned = String(value ?? ""); + for (const tag of INJECTED_MEMORY_TAGS) { + cleaned = stripTagBlock(cleaned, tag); + cleaned = stripHtmlEscapedTagBlock(cleaned, tag); + } + cleaned = cleaned.replace( + /^\s*\{?\s*"hookSpecificOutput"\s*:\s*\{\s*"hookEventName"\s*:\s*"[^"]+"\s*,\s*"additionalContext"\s*:\s*""\s*\}\s*\}?\s*$/gm, + "", + ); + return cleaned.replace(/\n{3,}/g, "\n\n").trim(); +} + +export function toolOutputFromPayload(payload) { + return ( + payload.tool_response ?? + payload.toolResponse ?? + payload.tool_output ?? + payload.tool_result ?? + payload.toolResult ?? + payload.output ?? + "" + ); +} + +function renderToolValue(value) { + if (value === undefined || value === null || value === "") return ""; + if (typeof value === "string") return value; + try { + return JSON.stringify(value, null, 2); + } catch { + return String(value); + } +} + +function previewHeadTail(value, chars) { + const str = String(value ?? ""); + if (str.length <= chars * 2 + 200) return str; + return [ + str.slice(0, chars), + `\n[...omitted ${str.length - chars * 2} chars; full redacted output is stored on disk...]\n`, + str.slice(-chars) + ].join(""); +} + +function truncate(value, maxChars) { + const str = String(value ?? ""); + if (str.length <= maxChars) return str; + return `${str.slice(0, maxChars)}\n[...truncated ${str.length - maxChars} chars]`; +} + +function redact(value) { + const sensitiveJsonKey = "[A-Za-z0-9_-]*(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|secret|token|password|authorization)[A-Za-z0-9_-]*"; + return String(value ?? "") + .replace(/-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/g, "[REDACTED_PRIVATE_KEY]") + .replace(/(^|[\r\n])(\s*(?:proxy-)?authorization\s*[:=]\s*)[^\r\n]*/gi, "$1$2[REDACTED]") + .replace(/Bearer\s+[A-Za-z0-9._~+/-]+=*/g, "Bearer [REDACTED]") + .replace(/\b(sk-[A-Za-z0-9_-]{16,})\b/g, "[REDACTED_API_KEY]") + .replace(/\b(github_pat_[A-Za-z0-9_]{20,})\b/g, "[REDACTED_GITHUB_TOKEN]") + .replace(/\b(gh[pousr]_[A-Za-z0-9_]{30,})\b/g, "[REDACTED_GITHUB_TOKEN]") + .replace(/\b(xox[baprs]-[A-Za-z0-9-]{10,})\b/g, "[REDACTED_SLACK_TOKEN]") + .replace(/\b((?:AKIA|ASIA)[0-9A-Z]{16})\b/g, "[REDACTED_AWS_ACCESS_KEY]") + .replace(new RegExp(`(["'])(${sensitiveJsonKey})\\1\\s*:\\s*"[^"]*"`, "gi"), "$1$2$1: \"[REDACTED]\"") + .replace(new RegExp(`(["'])(${sensitiveJsonKey})\\1\\s*:\\s*'[^']*'`, "gi"), "$1$2$1: '[REDACTED]'") + .replace(new RegExp(`(["'])(${sensitiveJsonKey})\\1\\s*:\\s*[^,}\\]\\s]+`, "gi"), "$1$2$1: [REDACTED]") + .replace(new RegExp(`\\b(${sensitiveJsonKey})\\b\\s*[:=]\\s*['"]?[^'\"\\s,}]+`, "gi"), "$1=[REDACTED]"); +} + +function stripTagBlock(value, tag) { + const complete = new RegExp(`<${tag}\\b[^>]*>[\\s\\S]*?<\\/${tag}>`, "gi"); + const dangling = new RegExp(`<${tag}\\b[^>]*>[\\s\\S]*$`, "gi"); + return value.replace(complete, "").replace(dangling, ""); +} + +function stripHtmlEscapedTagBlock(value, tag) { + const complete = new RegExp(`<${tag}\\b[\\s\\S]*?>[\\s\\S]*?</${tag}>`, "gi"); + const dangling = new RegExp(`<${tag}\\b[\\s\\S]*?>[\\s\\S]*$`, "gi"); + return value.replace(complete, "").replace(dangling, ""); +} + +function sanitizeEventDetail(value) { + if (typeof value === "string") return sanitizeMemoryText(value); + if (Array.isArray(value)) return value.map(sanitizeEventDetail); + if (!value || typeof value !== "object") return value; + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [key, sanitizeEventDetail(item)]), + ); +} + +function indentBlock(text, prefix) { + return String(text).split("\n").map((line) => `${prefix}${line}`).join("\n"); +} + +function numericEnv(name, fallback) { + const raw = process.env[name]; + if (!raw) return fallback; + const parsed = Number(raw); + return Number.isFinite(parsed) ? parsed : fallback; +} + +async function gatewayAuthHeaders() { + const token = await readGatewayAuthToken(); + return token ? { Authorization: `Bearer ${token}` } : {}; +} + +export async function readGatewayAuthToken() { + const direct = process.env.TDAI_CODEX_GATEWAY_TOKEN || process.env.TDAI_GATEWAY_TOKEN; + if (direct) return direct.trim(); + + const tokenPath = configuredGatewayTokenPath(); + try { + const token = (await fs.readFile(tokenPath, "utf-8")).trim(); + if (token) return token; + } catch { + // Missing token files are expected before the adapter autostarts Gateway. + } + return ""; +} + +export async function ensureGatewayAuthToken() { + const tokenPath = configuredGatewayTokenPath(); + const existing = (process.env.TDAI_CODEX_GATEWAY_TOKEN || process.env.TDAI_GATEWAY_TOKEN || "").trim(); + if (existing) { + await writePrivateFile(tokenPath, `${existing}\n`); + return existing; + } + + await ensurePrivateDir(path.dirname(tokenPath)); + + try { + const existingFileToken = (await fs.readFile(tokenPath, "utf-8")).trim(); + if (existingFileToken) return existingFileToken; + } catch { + // Missing token files are expected before first autostart. + } + + const token = crypto.randomBytes(32).toString("base64url"); + let handle; + try { + handle = await fs.open(tokenPath, "wx", PRIVATE_FILE_MODE); + await handle.writeFile(`${token}\n`); + await handle.close(); + handle = null; + await chmodPrivateFile(tokenPath); + return token; + } catch (err) { + if (handle) { + try { + await handle.close(); + } catch { + // Best effort cleanup after failed atomic create. + } + } + if (err?.code !== "EEXIST") throw err; + const racedToken = (await fs.readFile(tokenPath, "utf-8")).trim(); + if (racedToken) return racedToken; + await writePrivateFile(tokenPath, `${token}\n`); + return token; + } +} + +export function configuredGatewayTokenPath() { + return path.resolve(expandHome(process.env.TDAI_TOKEN_PATH || gatewayTokenPath())); +} + +function gatewayTokenPath() { + return path.join(tdaiDataDir(), "codex-adapter", "gateway-token"); +} + +function isLoopbackHost(host) { + const normalized = String(host || "").trim().toLowerCase(); + return normalized === "localhost" || + normalized === "127.0.0.1" || + normalized === "::1" || + normalized === "[::1]"; +} + +async function ensurePrivateDir(dir) { + await fs.mkdir(dir, { recursive: true, mode: PRIVATE_DIR_MODE }); + try { + await fs.chmod(dir, PRIVATE_DIR_MODE); + } catch { + // Best effort: chmod can fail on some mounted filesystems. + } +} + +async function writePrivateFile(file, content) { + await ensurePrivateDir(path.dirname(file)); + await fs.writeFile(file, content, { encoding: "utf-8", mode: PRIVATE_FILE_MODE }); + await chmodPrivateFile(file); +} + +async function chmodPrivateFile(file) { + try { + await fs.chmod(file, PRIVATE_FILE_MODE); + } catch { + // Best effort: chmod can fail on some mounted filesystems. + } +} + +function escapeText(value) { + return String(value).replace(/[&<>]/g, (ch) => ({ "&": "&", "<": "<", ">": ">" }[ch])); +} + +function escapeAttr(value) { + return escapeText(value).replace(/"/g, """); +} + +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function hydrateLoginShellEnv(env, names) { + const missing = names.filter((name) => !env[name]); + if (missing.length === 0) return; + try { + const script = missing.map((name) => `printf '%s=%s\\n' ${shellQuote(name)} "${"$"}${name}"`).join("; "); + const output = await captureCommand("zsh", ["-lc", script], 1500); + for (const line of output.split(/\r?\n/)) { + const idx = line.indexOf("="); + if (idx <= 0) continue; + const name = line.slice(0, idx); + const value = line.slice(idx + 1); + if (value && !env[name]) env[name] = value; + } + } catch (err) { + debug(`Unable to hydrate login-shell env: ${err instanceof Error ? err.message : String(err)}`); + } +} + +function captureCommand(command, args, timeoutMs) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + const timeout = setTimeout(() => { + child.kill("SIGTERM"); + reject(new Error(`${command} timed out`)); + }, timeoutMs); + child.stdout.on("data", (chunk) => { stdout += chunk; }); + child.stderr.on("data", (chunk) => { stderr += chunk; }); + child.on("error", (err) => { + clearTimeout(timeout); + reject(err); + }); + child.on("close", (code) => { + clearTimeout(timeout); + if (code === 0) resolve(stdout); + else reject(new Error(`${command} exited ${code}: ${stderr.slice(0, 200)}`)); + }); + }); +} + +function shellQuote(value) { + return `'${String(value).replace(/'/g, "'\\''")}'`; +} + +export function debug(message) { + const line = `[${new Date().toISOString()}] ${truncate(redact(String(message)), 2000)}\n`; + try { + const file = hookLogPath(); + fsSync.mkdirSync(path.dirname(file), { recursive: true, mode: PRIVATE_DIR_MODE }); + fsSync.appendFileSync(file, line, { mode: PRIVATE_FILE_MODE }); + try { + fsSync.chmodSync(file, PRIVATE_FILE_MODE); + } catch { + // Best effort: Windows and some filesystems do not expose POSIX modes. + } + } catch { + // Hook diagnostics must never make memory capture/recall fail. + } + if (process.env.TDAI_CODEX_DEBUG === "true") { + process.stderr.write(`[tdai-codex] ${line}`); + } +} diff --git a/codex-plugin/scripts/mcp-server.mjs b/codex-plugin/scripts/mcp-server.mjs new file mode 100644 index 0000000..b93440d --- /dev/null +++ b/codex-plugin/scripts/mcp-server.mjs @@ -0,0 +1,227 @@ +#!/usr/bin/env node +import { + cwdFromPayload, + ensureGateway, + httpPost, + sessionKeyPrefixesForCwd +} from "./lib.mjs"; +import { + formatLookupText, + lookupCodexOffload +} from "./offload-store.mjs"; + +const PROTOCOL_VERSION = "2024-11-05"; +const allProjectsEnabled = process.env.TDAI_CODEX_MCP_ALLOW_ALL_PROJECTS === "true"; +const offloadContentEnabled = process.env.TDAI_CODEX_MCP_ALLOW_OFFLOAD_CONTENT === "true"; + +const tools = [ + { + name: "tdai_memory_search", + description: "Search TencentDB Agent Memory L1 structured memories for user preferences, prior decisions, durable facts, instructions, or scene summaries. Use before asking the user to repeat context.", + inputSchema: { + type: "object", + properties: { + query: { type: "string", description: "Search query. Include current project/path when relevant." }, + limit: { type: "number", description: "Maximum number of results, 1-20.", default: 5 }, + type: { type: "string", enum: ["persona", "episodic", "instruction"], description: "Optional memory type filter." }, + scene: { type: "string", description: "Optional scene name filter." }, + ...(allProjectsEnabled ? { + all_projects: { type: "boolean", description: "Search across all projects instead of the current Codex project.", default: false } + } : {}) + }, + required: ["query"] + } + }, + { + name: "tdai_conversation_search", + description: "Search TencentDB Agent Memory L0 raw conversation history for exact prior wording, timelines, paths, commands, or evidence snippets.", + inputSchema: { + type: "object", + properties: { + query: { type: "string", description: "Search query. Use exact phrases, paths, or markers when available." }, + limit: { type: "number", description: "Maximum number of messages, 1-20.", default: 5 }, + session_key: { type: "string", description: "Optional session key filter." }, + ...(allProjectsEnabled ? { + all_projects: { type: "boolean", description: "Search across all projects instead of the current Codex project.", default: false } + } : {}) + }, + required: ["query"] + } + }, + { + name: "tdai_offload_lookup", + description: "Look up Codex short-term context offload entries by node_id, tool_call_id, or query. Use this to retrieve the exact redacted tool result behind an injected Mermaid canvas node.", + inputSchema: { + type: "object", + properties: { + node_id: { type: "string", description: "Mermaid node id from the injected context offload canvas." }, + tool_call_id: { type: "string", description: "Original Codex tool call id." }, + query: { type: "string", description: "Optional text query over tool names, summaries, refs, and cwd." }, + limit: { type: "number", description: "Maximum number of entries, 1-20.", default: 5 }, + ...(offloadContentEnabled ? { + include_content: { type: "boolean", description: "Include stored redacted tool output content.", default: false } + } : {}), + ...(allProjectsEnabled ? { + all_projects: { type: "boolean", description: "Search across all projects instead of the current Codex project.", default: false } + } : {}) + } + } + } +]; + +let buffer = ""; + +process.stdin.setEncoding("utf-8"); +process.stdin.on("data", (chunk) => { + buffer += chunk; + let idx; + while ((idx = buffer.indexOf("\n")) >= 0) { + const line = buffer.slice(0, idx).trim(); + buffer = buffer.slice(idx + 1); + if (!line) continue; + handleLine(line).catch((err) => { + writeError(null, -32603, err instanceof Error ? err.message : String(err)); + }); + } +}); + +async function handleLine(line) { + let message; + try { + message = JSON.parse(line); + } catch { + writeError(null, -32700, "Parse error"); + return; + } + + if (message.id === undefined) return; + + try { + switch (message.method) { + case "initialize": + writeResult(message.id, { + protocolVersion: message.params?.protocolVersion || PROTOCOL_VERSION, + capabilities: { tools: {} }, + serverInfo: { name: "tdai-memory-codex", version: "0.1.0" } + }); + break; + case "ping": + writeResult(message.id, {}); + break; + case "tools/list": + writeResult(message.id, { tools }); + break; + case "tools/call": + writeResult(message.id, await callTool(message.params || {})); + break; + default: + writeError(message.id, -32601, `Method not found: ${message.method}`); + } + } catch (err) { + writeError(message.id, -32603, err instanceof Error ? err.message : String(err)); + } +} + +async function callTool(params) { + const name = params.name; + const args = params.arguments || {}; + + if (name === "tdai_offload_lookup") { + const allProjects = allProjectsEnabled && args.all_projects === true; + const includeContent = offloadContentEnabled && args.include_content === true; + if (args.all_projects === true && !allProjectsEnabled) { + return textResult("Cross-project memory/offload lookup is disabled for MCP. Set TDAI_CODEX_MCP_ALLOW_ALL_PROJECTS=true outside the model context to enable it.", true); + } + if (args.include_content === true && !offloadContentEnabled) { + return textResult("Exact offload content lookup is disabled for MCP. Set TDAI_CODEX_MCP_ALLOW_OFFLOAD_CONTENT=true outside the model context to enable it.", true); + } + const result = await lookupCodexOffload({ + nodeId: optionalString(args.node_id), + toolCallId: optionalString(args.tool_call_id), + query: optionalString(args.query), + cwd: allProjects ? "" : currentProjectCwd(), + includeContent, + limit: clampLimit(args.limit) + }); + return textResult(formatLookupText(result)); + } + + const ready = await ensureGateway(); + if (!ready) return textResult("TencentDB Agent Memory Gateway is unavailable.", true); + + if (name === "tdai_memory_search") { + const allProjects = allProjectsEnabled && args.all_projects === true; + if (args.all_projects === true && !allProjectsEnabled) { + return textResult("Cross-project memory search is disabled for MCP. Set TDAI_CODEX_MCP_ALLOW_ALL_PROJECTS=true outside the model context to enable it.", true); + } + const query = scopedQuery(args.query, allProjects); + const result = await httpPost("/search/memories", { + query, + limit: clampLimit(args.limit), + type: optionalString(args.type), + scene: optionalString(args.scene), + session_key_prefixes: allProjects ? undefined : currentProjectPrefixes() + }); + return textResult(result?.results || "No matching memories found."); + } + + if (name === "tdai_conversation_search") { + const allProjects = allProjectsEnabled && args.all_projects === true; + if (args.all_projects === true && !allProjectsEnabled) { + return textResult("Cross-project conversation search is disabled for MCP. Set TDAI_CODEX_MCP_ALLOW_ALL_PROJECTS=true outside the model context to enable it.", true); + } + const query = scopedQuery(args.query, allProjects); + const body = { + query, + limit: clampLimit(args.limit), + session_key_prefixes: allProjects ? undefined : currentProjectPrefixes() + }; + const sessionKey = optionalString(args.session_key); + if (sessionKey) body.session_key = sessionKey; + const result = await httpPost("/search/conversations", body); + return textResult(result?.results || "No matching conversations found."); + } + + return textResult(`Unknown tool: ${name}`, true); +} + +function optionalString(value) { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function clampLimit(value) { + const parsed = Number(value || 5); + if (!Number.isFinite(parsed)) return 5; + return Math.max(1, Math.min(20, Math.floor(parsed))); +} + +function currentProjectCwd() { + return cwdFromPayload({ + cwd: process.env.CLAUDE_PROJECT_DIR || process.env.CODEX_PROJECT_DIR || process.cwd() + }); +} + +function scopedQuery(query, allProjects) { + const text = String(query || ""); + if (allProjects) return text; + return `Codex project cwd: ${currentProjectCwd()}\n${text}`; +} + +function currentProjectPrefixes() { + return sessionKeyPrefixesForCwd(currentProjectCwd()); +} + +function textResult(text, isError = false) { + return { + content: [{ type: "text", text: String(text || "") }], + isError + }; +} + +function writeResult(id, result) { + process.stdout.write(`${JSON.stringify({ jsonrpc: "2.0", id, result })}\n`); +} + +function writeError(id, code, message) { + process.stdout.write(`${JSON.stringify({ jsonrpc: "2.0", id, error: { code, message } })}\n`); +} diff --git a/codex-plugin/scripts/offload-store.mjs b/codex-plugin/scripts/offload-store.mjs new file mode 100644 index 0000000..b26b2e3 --- /dev/null +++ b/codex-plugin/scripts/offload-store.mjs @@ -0,0 +1,675 @@ +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import fsSync from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const DEFAULT_TOOL_OFFLOAD_MIN_CHARS = 20_000; +const DEFAULT_TOOL_OFFLOAD_AGGRESSIVE_MIN_CHARS = 80_000; +const DEFAULT_TOOL_OFFLOAD_EMERGENCY_MIN_CHARS = 250_000; +const DEFAULT_TOOL_OFFLOAD_PREVIEW_CHARS = 2_000; +const DEFAULT_TOOL_OFFLOAD_AGGRESSIVE_PREVIEW_CHARS = 800; +const DEFAULT_TOOL_OFFLOAD_EMERGENCY_PREVIEW_CHARS = 240; +const DEFAULT_TOOL_OFFLOAD_MAX_STORE_CHARS = 2_000_000; +const DEFAULT_TOOL_OFFLOAD_L2_NULL_THRESHOLD = 1; +const DEFAULT_TOOL_OFFLOAD_CONTEXT_CHARS = 6_000; +const DEFAULT_TOOL_OFFLOAD_LOOKUP_CONTENT_CHARS = 20_000; +const CANVAS_FILE = "001-codex-tool-offload.mmd"; +const NODE_INDEX_FILE = "node-index.json"; +const PRIVATE_DIR_MODE = 0o700; +const PRIVATE_FILE_MODE = 0o600; + +export function selectToolOffloadPolicy(charCount) { + const mildMin = numericEnv( + "TDAI_CODEX_TOOL_OFFLOAD_MILD_MIN_CHARS", + numericEnv("TDAI_CODEX_TOOL_OFFLOAD_MIN_CHARS", DEFAULT_TOOL_OFFLOAD_MIN_CHARS), + ); + const aggressiveMin = numericEnv( + "TDAI_CODEX_TOOL_OFFLOAD_AGGRESSIVE_MIN_CHARS", + Math.max(DEFAULT_TOOL_OFFLOAD_AGGRESSIVE_MIN_CHARS, mildMin * 4), + ); + const emergencyMin = numericEnv( + "TDAI_CODEX_TOOL_OFFLOAD_EMERGENCY_MIN_CHARS", + Math.max(DEFAULT_TOOL_OFFLOAD_EMERGENCY_MIN_CHARS, aggressiveMin * 3), + ); + + if (charCount >= emergencyMin) { + return { + name: "emergency", + minChars: emergencyMin, + previewChars: numericEnv("TDAI_CODEX_TOOL_OFFLOAD_EMERGENCY_PREVIEW_CHARS", DEFAULT_TOOL_OFFLOAD_EMERGENCY_PREVIEW_CHARS), + score: 10, + }; + } + if (charCount >= aggressiveMin) { + return { + name: "aggressive", + minChars: aggressiveMin, + previewChars: numericEnv("TDAI_CODEX_TOOL_OFFLOAD_AGGRESSIVE_PREVIEW_CHARS", DEFAULT_TOOL_OFFLOAD_AGGRESSIVE_PREVIEW_CHARS), + score: 9, + }; + } + if (charCount >= mildMin) { + return { + name: "mild", + minChars: mildMin, + previewChars: numericEnv("TDAI_CODEX_TOOL_OFFLOAD_PREVIEW_CHARS", DEFAULT_TOOL_OFFLOAD_PREVIEW_CHARS), + score: 8, + }; + } + return null; +} + +export function maxStoreChars() { + return numericEnv("TDAI_CODEX_TOOL_OFFLOAD_MAX_STORE_CHARS", DEFAULT_TOOL_OFFLOAD_MAX_STORE_CHARS); +} + +export async function recordCodexToolOffload(params) { + const { + sessionKey, + sessionId, + cwd, + toolName, + toolUseId, + inputSummary, + redactedOutput, + storedText, + policy, + } = params; + const paths = pathsForSession(sessionKey, sessionId); + await ensureOffloadDirs(paths); + + const entries = await readEntries(paths.offloadJsonl); + const existing = entries.find((entry) => entry.tool_call_id === toolUseId); + if (existing) { + const canvas = await maybeRebuildCanvas(paths, entries, { force: false }); + return { + entry: existing, + paths, + canvas, + duplicated: true, + }; + } + + const now = new Date(); + const fileStem = `${safeKey(toolName)}-${safeKey(toolUseId)}-${now.getTime()}`; + const resultRef = `refs/${fileStem}.md`; + const refPath = path.join(paths.root, resultRef); + const summary = summarizeToolResult(toolName, inputSummary, redactedOutput.length, policy); + const entry = { + timestamp: now.toISOString(), + node_id: null, + tool_call: `${toolName}: ${singleLine(inputSummary, 220) || "(no input captured)"}`, + summary, + result_ref: resultRef, + tool_call_id: toolUseId, + session_key: sessionKey, + score: policy.score, + codex: { + session_id: sessionId, + cwd, + tool_name: toolName, + policy: policy.name, + original_chars_redacted: redactedOutput.length, + stored_chars: storedText.length, + }, + }; + + await writePrivateFile(refPath, buildRefMarkdown({ + entry, + cwd, + inputSummary, + redactedOutput, + storedText, + policy, + })); + + entries.push(entry); + await writeEntries(paths.offloadJsonl, entries); + const canvas = await maybeRebuildCanvas(paths, entries, { force: false }); + const refreshed = await readEntries(paths.offloadJsonl); + const updatedEntry = refreshed.find((candidate) => candidate.tool_call_id === toolUseId) || entry; + if (updatedEntry.node_id !== entry.node_id) { + await writePrivateFile(refPath, buildRefMarkdown({ + entry: updatedEntry, + cwd, + inputSummary, + redactedOutput, + storedText, + policy, + })); + } + + return { + entry: updatedEntry, + paths, + canvas, + duplicated: false, + }; +} + +export async function buildCodexOffloadContext(params) { + const { sessionKey, sessionId } = params; + const maxChars = params.maxChars ?? numericEnv("TDAI_CODEX_TOOL_OFFLOAD_CONTEXT_CHARS", DEFAULT_TOOL_OFFLOAD_CONTEXT_CHARS); + const paths = pathsForSession(sessionKey, sessionId); + const entries = await readEntries(paths.offloadJsonl); + if (entries.length === 0) return ""; + + let canvas = ""; + try { + canvas = await fs.readFile(paths.canvasPath, "utf-8"); + } catch { + const rebuilt = await maybeRebuildCanvas(paths, entries, { force: true }); + canvas = rebuilt?.content || ""; + } + + const recentLimit = numericEnv("TDAI_CODEX_TOOL_OFFLOAD_CONTEXT_RECENT", 8); + const recent = entries.slice(-recentLimit).map((entry) => { + return [ + `- node_id=${entry.node_id || "pending"}`, + `tool_call_id=${entry.tool_call_id}`, + `tool=${entry.codex?.tool_name || toolNameFromCall(entry.tool_call)}`, + `policy=${entry.codex?.policy || "unknown"}`, + `ref=${path.join(paths.root, entry.result_ref)}`, + `summary=${singleLine(entry.summary, 220)}`, + ].join(" "); + }).join("\n"); + + const block = ` + +Large Codex tool results are stored outside the prompt and represented by node ids. +Use tdai_offload_lookup with a node_id or tool_call_id when exact stored output is needed. + + +\`\`\`mermaid +${stripMermaidFence(canvas).trim()} +\`\`\` + + +${escapeText(recent)} + +`; + + return truncate(block, maxChars); +} + +export async function lookupCodexOffload(params = {}) { + const roots = await listSessionRoots(params.sessionKey ? sessionRootFromKey(params.sessionKey) : null); + const nodeId = optionalLower(params.nodeId); + const toolCallId = optionalLower(params.toolCallId); + const query = optionalLower(params.query); + const cwd = normalizePath(params.cwd); + const includeContent = params.includeContent === true; + const contentMaxChars = params.contentMaxChars ?? DEFAULT_TOOL_OFFLOAD_LOOKUP_CONTENT_CHARS; + const limit = clampLimit(params.limit, 10, 50); + const matches = []; + + for (const root of roots) { + const files = await listOffloadJsonlFiles(root); + for (const file of files) { + const entries = await readEntries(file); + for (const entry of entries) { + if (nodeId && optionalLower(entry.node_id) !== nodeId) continue; + if (toolCallId && optionalLower(entry.tool_call_id) !== toolCallId) continue; + if (query && !entryMatchesQuery(entry, query)) continue; + if (cwd && normalizePath(entry.codex?.cwd) !== cwd) continue; + + const resultPath = entry.result_ref ? path.join(root, entry.result_ref) : ""; + const item = { + node_id: entry.node_id, + tool_call_id: entry.tool_call_id, + tool_call: entry.tool_call, + summary: entry.summary, + score: entry.score, + policy: entry.codex?.policy, + timestamp: entry.timestamp, + session_key: entry.session_key, + result_ref: entry.result_ref, + result_path: resultPath, + offload_jsonl: file, + canvas_path: path.join(root, "mmds", CANVAS_FILE), + }; + if (includeContent && resultPath) { + item.content = await readFileIfExists(resultPath, contentMaxChars); + } + matches.push(item); + if (matches.length >= limit) { + return { matches, total: matches.length, truncated: true }; + } + } + } + } + + return { matches, total: matches.length, truncated: false }; +} + +export async function offloadCli(args, context = {}) { + const command = args[0] || "list"; + if (command === "help" || command === "--help" || command === "-h") return offloadUsage(0); + + if (command === "list") { + const opts = parseLookupArgs(args.slice(1)); + const result = await lookupCodexOffload({ + sessionKey: opts.all ? "" : context.sessionKey, + query: opts.query, + includeContent: false, + limit: opts.limit, + }); + console.log(formatLookupText(result)); + return; + } + + if (command === "node") { + const id = args[1]; + if (!id) return offloadUsage(2); + const opts = parseLookupArgs(args.slice(2)); + let result = await lookupCodexOffload({ + nodeId: id, + query: opts.query, + includeContent: opts.content, + limit: opts.limit || 5, + }); + if (result.matches.length === 0) { + result = await lookupCodexOffload({ + toolCallId: id, + query: opts.query, + includeContent: opts.content, + limit: opts.limit || 5, + }); + } + console.log(opts.json ? JSON.stringify(result, null, 2) : formatLookupText(result)); + return; + } + + if (command === "canvas") { + const paths = pathsForSession(context.sessionKey, context.sessionId); + const content = await readFileIfExists(paths.canvasPath, 200_000); + if (!content) { + console.error(`No Codex offload canvas found for session: ${context.sessionKey}`); + process.exit(1); + } + console.log(content); + return; + } + + return offloadUsage(2); +} + +export function formatLookupText(result) { + if (!result.matches.length) return "No matching Codex offload entries found."; + return result.matches.map((entry) => { + const parts = [ + `node_id: ${entry.node_id || "pending"}`, + `tool_call_id: ${entry.tool_call_id}`, + `tool_call: ${entry.tool_call}`, + `policy: ${entry.policy || "unknown"}`, + `score: ${entry.score ?? ""}`, + `timestamp: ${entry.timestamp}`, + `result_path: ${entry.result_path}`, + `canvas_path: ${entry.canvas_path}`, + "", + entry.summary, + ]; + if (entry.content) { + parts.push("", "content:", entry.content); + } + return parts.join("\n"); + }).join("\n\n---\n\n"); +} + +function pathsForSession(sessionKey, sessionId) { + const root = sessionRootFromKey(sessionKey); + return { + root, + refsDir: path.join(root, "refs"), + mmdsDir: path.join(root, "mmds"), + offloadJsonl: path.join(root, `offload-${safeKey(sessionId || "unknown-session")}.jsonl`), + canvasPath: path.join(root, "mmds", CANVAS_FILE), + nodeIndexPath: path.join(root, NODE_INDEX_FILE), + }; +} + +function sessionRootFromKey(sessionKey) { + return path.join(tdaiDataDir(), "codex-adapter", "context-offload", sha1(sessionKey || "all").slice(0, 16)); +} + +async function ensureOffloadDirs(paths) { + await ensurePrivateDir(paths.root); + await ensurePrivateDir(paths.refsDir); + await ensurePrivateDir(paths.mmdsDir); +} + +async function maybeRebuildCanvas(paths, entries, options = {}) { + if (process.env.TDAI_CODEX_TOOL_OFFLOAD_L2_CANVAS === "false") return null; + const nullCount = entries.filter((entry) => !entry.node_id || entry.node_id === "wait").length; + const threshold = numericEnv("TDAI_CODEX_TOOL_OFFLOAD_L2_NULL_THRESHOLD", DEFAULT_TOOL_OFFLOAD_L2_NULL_THRESHOLD); + if (!options.force && nullCount < threshold && fsSync.existsSync(paths.canvasPath)) { + return { + path: paths.canvasPath, + content: await readFileIfExists(paths.canvasPath, 200_000), + rebuilt: false, + }; + } + + const updated = assignNodeIds(entries, paths.root); + const content = buildMermaidCanvas(updated, paths); + await ensurePrivateDir(paths.mmdsDir); + await writePrivateFile(paths.canvasPath, `${content}\n`); + await writePrivateFile(paths.nodeIndexPath, `${JSON.stringify({ + version: 1, + generated_at: new Date().toISOString(), + offload_jsonl: paths.offloadJsonl, + canvas_path: paths.canvasPath, + nodes: updated.map((entry) => ({ + node_id: entry.node_id, + tool_call_id: entry.tool_call_id, + result_ref: entry.result_ref, + result_path: path.join(paths.root, entry.result_ref), + summary: entry.summary, + })), + }, null, 2)}\n`); + await writeEntries(paths.offloadJsonl, updated); + return { path: paths.canvasPath, content, rebuilt: true }; +} + +function assignNodeIds(entries, root) { + const prefix = `C${sha1(root).slice(0, 6)}`; + const used = new Set(entries.map((entry) => entry.node_id).filter(Boolean)); + let next = 1; + return entries.map((entry) => { + if (entry.node_id && entry.node_id !== "wait") return entry; + let nodeId; + do { + nodeId = `${prefix}_N${String(next).padStart(3, "0")}`; + next += 1; + } while (used.has(nodeId)); + used.add(nodeId); + return { ...entry, node_id: nodeId }; + }); +} + +function buildMermaidCanvas(entries, paths) { + const lines = [ + "%% TencentDB Agent Memory Codex context offload canvas", + `%% generated_at: ${new Date().toISOString()}`, + `%% offload_jsonl: ${paths.offloadJsonl}`, + "flowchart TD", + ]; + + if (entries.length === 0) { + lines.push(" EMPTY[\"No offloaded tool results yet\"]"); + return lines.join("\n"); + } + + for (const entry of entries) { + const label = [ + toolNameFromCall(entry.tool_call), + `status: done`, + `policy: ${entry.codex?.policy || "unknown"}`, + `summary: ${singleLine(entry.summary, 130)}`, + `ref: ${path.basename(entry.result_ref || "")}`, + ].join("
"); + lines.push(` ${entry.node_id}["${escapeMermaidLabel(label)}"]`); + } + + for (let i = 1; i < entries.length; i++) { + lines.push(` ${entries[i - 1].node_id} --> ${entries[i].node_id}`); + } + + lines.push(" classDef offloaded fill:#eef6ff,stroke:#3b82f6,color:#0f172a;"); + lines.push(` class ${entries.map((entry) => entry.node_id).join(",")} offloaded;`); + return lines.join("\n"); +} + +async function readEntries(filePath) { + if (!filePath || !fsSync.existsSync(filePath)) return []; + const content = await fs.readFile(filePath, "utf-8"); + const entries = []; + for (const line of content.split(/\r?\n/)) { + if (!line.trim()) continue; + try { + const parsed = JSON.parse(line); + if (parsed && typeof parsed === "object" && parsed.tool_call_id) entries.push(parsed); + } catch { + // Ignore corrupt lines rather than breaking hook execution. + } + } + return entries; +} + +async function writeEntries(filePath, entries) { + await ensurePrivateDir(path.dirname(filePath)); + const tmp = `${filePath}.${process.pid}.tmp`; + await writePrivateFile(tmp, entries.map((entry) => JSON.stringify(entry)).join("\n") + "\n"); + await fs.rename(tmp, filePath); + await chmodPrivateFile(filePath); +} + +async function listSessionRoots(singleRoot) { + if (singleRoot) return fsSync.existsSync(singleRoot) ? [singleRoot] : []; + const base = path.join(tdaiDataDir(), "codex-adapter", "context-offload"); + if (!fsSync.existsSync(base)) return []; + const entries = await fs.readdir(base, { withFileTypes: true }); + return entries.filter((entry) => entry.isDirectory()).map((entry) => path.join(base, entry.name)).sort(); +} + +async function listOffloadJsonlFiles(root) { + try { + const entries = await fs.readdir(root, { withFileTypes: true }); + return entries + .filter((entry) => entry.isFile() && entry.name.startsWith("offload-") && entry.name.endsWith(".jsonl")) + .map((entry) => path.join(root, entry.name)) + .sort(); + } catch { + return []; + } +} + +function buildRefMarkdown(params) { + const { entry, cwd, inputSummary, storedText, redactedOutput, policy } = params; + return [ + "# Codex Tool Result Offload", + "", + `- node_id: ${entry.node_id || "pending_until_l2_canvas"}`, + `- tool_call_id: ${entry.tool_call_id}`, + `- tool_name: ${entry.codex.tool_name}`, + `- session_key: ${entry.session_key}`, + `- cwd: ${cwd}`, + `- captured_at: ${entry.timestamp}`, + `- policy: ${policy.name}`, + `- original_chars_redacted: ${redactedOutput.length}`, + `- stored_chars: ${storedText.length}`, + "", + "## Summary", + "", + entry.summary, + "", + "## Tool Input", + "", + "```text", + inputSummary || "(no input captured)", + "```", + "", + "## Tool Output", + "", + "```text", + storedText, + "```", + ].join("\n"); +} + +function summarizeToolResult(toolName, inputSummary, charCount, policy) { + const input = singleLine(inputSummary, 120); + return [ + `${toolName} produced a ${policy.name} offloaded result (${charCount} redacted characters).`, + input ? `Input: ${input}.` : "", + "Use result_ref for the exact redacted output.", + ].filter(Boolean).join(" "); +} + +function parseLookupArgs(args) { + const opts = { query: "", limit: 10, content: false, json: false, all: false }; + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === "--query") opts.query = args[++i] || ""; + else if (arg === "--limit") opts.limit = clampLimit(Number(args[++i] || 10), 10, 50); + else if (arg === "--content" || arg === "--full") opts.content = true; + else if (arg === "--json") opts.json = true; + else if (arg === "--all") opts.all = true; + else throw new Error(`Unknown offload option: ${arg}`); + } + return opts; +} + +function offloadUsage(code) { + const message = `Usage: + node scripts/query.mjs offload list [--all] [--query ] [--limit ] + node scripts/query.mjs offload node [--content] [--json] + node scripts/query.mjs offload canvas +`; + (code === 0 ? console.log : console.error)(message); + process.exit(code); +} + +function entryMatchesQuery(entry, query) { + return [ + entry.node_id, + entry.tool_call_id, + entry.tool_call, + entry.summary, + entry.result_ref, + entry.codex?.tool_name, + entry.codex?.cwd, + ].filter(Boolean).join("\n").toLowerCase().includes(query); +} + +async function readFileIfExists(filePath, maxChars) { + try { + const content = await fs.readFile(filePath, "utf-8"); + return truncate(content, maxChars); + } catch { + return ""; + } +} + +function tdaiDataDir() { + const configured = + process.env.TDAI_CODEX_DATA_DIR || + process.env.TDAI_DATA_DIR || + path.join(os.homedir(), ".memory-tencentdb", "codex-memory-tdai"); + return path.resolve(expandHome(configured)); +} + +function expandHome(value) { + if (!value) return value; + if (value === "~") return os.homedir(); + if (value.startsWith("~/")) return path.join(os.homedir(), value.slice(2)); + return value; +} + +function sha1(value) { + return crypto.createHash("sha1").update(String(value)).digest("hex"); +} + +function safeKey(value) { + return String(value).replace(/[^a-zA-Z0-9_.:-]/g, "_").slice(0, 120); +} + +function optionalLower(value) { + return typeof value === "string" && value.trim() ? value.trim().toLowerCase() : ""; +} + +function normalizePath(value) { + return typeof value === "string" && value.trim() + ? path.resolve(expandHome(value.trim())) + : ""; +} + +function clampLimit(value, fallback, max) { + const parsed = Number(value || fallback); + if (!Number.isFinite(parsed)) return fallback; + return Math.max(1, Math.min(max, Math.floor(parsed))); +} + +function numericEnv(name, fallback) { + const raw = process.env[name]; + if (!raw) return fallback; + const parsed = Number(raw); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function singleLine(value, maxChars) { + const flattened = String(value || "").replace(/\s+/g, " ").trim(); + if (!maxChars || flattened.length <= maxChars) return flattened; + return `${flattened.slice(0, maxChars)}...`; +} + +function toolNameFromCall(value) { + return String(value || "tool").split(":")[0].trim() || "tool"; +} + +function previewHeadTail(value, chars) { + const str = String(value ?? ""); + if (str.length <= chars * 2 + 200) return str; + return [ + str.slice(0, chars), + `\n[...omitted ${str.length - chars * 2} chars; full redacted output is stored on disk...]\n`, + str.slice(-chars), + ].join(""); +} + +export function previewForPolicy(value, policy) { + return previewHeadTail(value, policy?.previewChars ?? DEFAULT_TOOL_OFFLOAD_PREVIEW_CHARS); +} + +function stripMermaidFence(value) { + return String(value || "") + .replace(/^```mermaid\s*/i, "") + .replace(/```\s*$/i, ""); +} + +function truncate(value, maxChars) { + const str = String(value ?? ""); + if (!maxChars || str.length <= maxChars) return str; + return `${str.slice(0, maxChars)}\n[...truncated ${str.length - maxChars} chars]`; +} + +async function ensurePrivateDir(dir) { + await fs.mkdir(dir, { recursive: true, mode: PRIVATE_DIR_MODE }); + try { + await fs.chmod(dir, PRIVATE_DIR_MODE); + } catch { + // Best effort on filesystems that do not support POSIX modes. + } +} + +async function writePrivateFile(file, content) { + await ensurePrivateDir(path.dirname(file)); + await fs.writeFile(file, content, { encoding: "utf-8", mode: PRIVATE_FILE_MODE }); + await chmodPrivateFile(file); +} + +async function chmodPrivateFile(file) { + try { + await fs.chmod(file, PRIVATE_FILE_MODE); + } catch { + // Best effort on filesystems that do not support POSIX modes. + } +} + +function escapeText(value) { + return String(value).replace(/[&<>]/g, (ch) => ({ "&": "&", "<": "<", ">": ">" }[ch])); +} + +function escapeAttr(value) { + return escapeText(value).replace(/"/g, """); +} + +function escapeMermaidLabel(value) { + return String(value) + .replace(/\\/g, "\\\\") + .replace(/"/g, """) + .replace(/\[/g, "[") + .replace(/\]/g, "]") + .replace(/\n/g, "
"); +} diff --git a/codex-plugin/scripts/permission-request.mjs b/codex-plugin/scripts/permission-request.mjs new file mode 100644 index 0000000..c2f2300 --- /dev/null +++ b/codex-plugin/scripts/permission-request.mjs @@ -0,0 +1,15 @@ +#!/usr/bin/env node +import { appendLifecycleEvent, compact, readHookInput } from "./lib.mjs"; + +const payload = await readHookInput(); +await appendLifecycleEvent(payload, "permission_request", { + toolName: payload.tool_name || payload.toolName || payload.tool?.name || "", + permission: compact( + payload.permission || + payload.permission_request || + payload.permissionRequest || + payload.request || + payload, + 3000 + ) +}); diff --git a/codex-plugin/scripts/post-compact.mjs b/codex-plugin/scripts/post-compact.mjs new file mode 100644 index 0000000..457f46c --- /dev/null +++ b/codex-plugin/scripts/post-compact.mjs @@ -0,0 +1,7 @@ +#!/usr/bin/env node +import { appendLifecycleEvent, readHookInput, sessionEnd } from "./lib.mjs"; + +const payload = await readHookInput(); +const reason = payload.reason || "context_compaction"; +await appendLifecycleEvent(payload, "post_compact", { reason }, { createTurn: false }); +await sessionEnd(payload, "post_compact"); diff --git a/codex-plugin/scripts/post-tool-use.mjs b/codex-plugin/scripts/post-tool-use.mjs new file mode 100644 index 0000000..6b4c0e6 --- /dev/null +++ b/codex-plugin/scripts/post-tool-use.mjs @@ -0,0 +1,22 @@ +#!/usr/bin/env node +import { + appendToolEvent, + maybeOffloadToolOutput, + postToolOffloadHookOutput, + readHookInput +} from "./lib.mjs"; + +const payload = await readHookInput(); +const offload = await maybeOffloadToolOutput(payload); +await appendToolEvent(payload, "post_tool_use", offload ? { + toolOutputOffloaded: true, + toolOutputNodeId: offload.nodeId, + toolOutputPolicy: offload.policy, + toolOutputPath: offload.outputPath, + toolOutputJsonlPath: offload.offloadJsonlPath, + toolOutputCanvasPath: offload.canvasPath, + toolOutputOriginalChars: offload.originalChars, + toolOutputStoredChars: offload.storedChars, + toolOutputSummary: offload.summary +} : {}); +process.stdout.write(postToolOffloadHookOutput(offload)); diff --git a/codex-plugin/scripts/pre-compact.mjs b/codex-plugin/scripts/pre-compact.mjs new file mode 100644 index 0000000..61cde94 --- /dev/null +++ b/codex-plugin/scripts/pre-compact.mjs @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import { appendLifecycleEvent, captureCurrentTurn, readHookInput } from "./lib.mjs"; + +const payload = await readHookInput(); +await appendLifecycleEvent(payload, "pre_compact", { reason: payload.reason || "context_compaction" }); +await captureCurrentTurn(payload, "pre_compact"); diff --git a/codex-plugin/scripts/pre-tool-use.mjs b/codex-plugin/scripts/pre-tool-use.mjs new file mode 100644 index 0000000..8818e42 --- /dev/null +++ b/codex-plugin/scripts/pre-tool-use.mjs @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import { appendToolEvent, hookAdditionalContext, readHookInput, toolMemoryHint } from "./lib.mjs"; + +const payload = await readHookInput(); +await appendToolEvent(payload, "pre_tool_use"); +process.stdout.write(hookAdditionalContext("PreToolUse", toolMemoryHint(payload))); diff --git a/codex-plugin/scripts/query.mjs b/codex-plugin/scripts/query.mjs new file mode 100644 index 0000000..03d731b --- /dev/null +++ b/codex-plugin/scripts/query.mjs @@ -0,0 +1,114 @@ +#!/usr/bin/env node +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { + cwdFromPayload, + ensureGateway, + expandHome, + gatewayStderrLogPath, + gatewayStdoutLogPath, + gatewayUrl, + healthCheck, + hookLogPath, + httpPost, + projectLabel, + rememberText, + sessionEnd, + sessionIdFromPayload, + sessionKeyFromPayload, + sessionKeyPrefixesForCwd +} from "./lib.mjs"; + +const command = process.argv[2] || "status"; +const args = process.argv.slice(3); +const payload = { + cwd: process.env.CLAUDE_PROJECT_DIR || process.cwd(), + session_id: process.env.TDAI_CODEX_MANUAL_SESSION_ID || "manual" +}; + +if (command === "status") { + console.log(JSON.stringify({ + healthy: await healthCheck(), + gatewayUrl: gatewayUrl(), + logs: { + hook: hookLogPath(), + gatewayStdout: gatewayStdoutLogPath(), + gatewayStderr: gatewayStderrLogPath(), + }, + sessionKey: sessionKeyFromPayload(payload), + project: projectLabel(payload) + }, null, 2)); +} else if (command === "memory") { + const query = args.join(" ").trim(); + if (!query) usage(2); + const result = await httpPost("/search/memories", { + query: `Codex project cwd: ${cwdFromPayload(payload)}\n${query}`, + limit: 10, + session_key_prefixes: sessionKeyPrefixesForCwd(cwdFromPayload(payload)) + }); + console.log(result?.results || ""); +} else if (command === "conversation") { + const query = args.join(" ").trim(); + if (!query) usage(2); + const result = await httpPost("/search/conversations", { + query, + limit: 10, + session_key_prefixes: sessionKeyPrefixesForCwd(cwdFromPayload(payload)) + }); + console.log(result?.results || ""); +} else if (command === "remember") { + const text = args.join(" ").trim() || (await readTextFromStdin()); + if (!text) usage(2); + const result = await rememberText(payload, text); + console.log(JSON.stringify(result, null, 2)); +} else if (command === "flush") { + const result = await sessionEnd(payload, "manual_flush"); + console.log(JSON.stringify(result, null, 2)); +} else if (command === "seed") { + const file = args[0]; + if (!file) usage(2); + const ok = await ensureGateway(); + if (!ok) { + console.error("TDAI Gateway unavailable"); + process.exit(1); + } + const fullPipelineTimeoutMs = positiveNumber(process.env.TDAI_CODEX_FULL_PIPELINE_TIMEOUT_MS, 900000); + const seedTimeoutMs = positiveNumber(process.env.TDAI_CODEX_SEED_TIMEOUT_MS, 960000); + const dataPath = path.resolve(expandHome(file)); + const data = JSON.parse(await readFile(dataPath, "utf-8")); + const result = await httpPost("/seed", { + data, + session_key: sessionKeyFromPayload(payload), + wait_for_full_pipeline: process.env.TDAI_CODEX_SEED_FULL_PIPELINE !== "false", + full_pipeline_timeout_ms: fullPipelineTimeoutMs + }, seedTimeoutMs); + console.log(JSON.stringify(result, null, 2)); +} else if (command === "import-codex-history") { + const { importCodexHistoryCli } = await import("./import-codex-history.mjs"); + await importCodexHistoryCli(args); +} else if (command === "offload") { + const { offloadCli } = await import("./offload-store.mjs"); + await offloadCli(args, { + sessionKey: sessionKeyFromPayload(payload), + sessionId: sessionIdFromPayload(payload) + }); +} else { + usage(2); +} + +function usage(code = 0) { + console.error("Usage: node scripts/query.mjs [status|memory |conversation |remember |flush|seed |import-codex-history [options]|offload ]"); + process.exit(code); +} + +async function readTextFromStdin() { + if (process.stdin.isTTY) return ""; + let input = ""; + for await (const chunk of process.stdin) input += chunk; + return input.trim(); +} + +function positiveNumber(value, fallback) { + const n = Number(value); + return Number.isFinite(n) && n > 0 ? n : fallback; +} diff --git a/codex-plugin/scripts/session-start.mjs b/codex-plugin/scripts/session-start.mjs new file mode 100644 index 0000000..9d4acc6 --- /dev/null +++ b/codex-plugin/scripts/session-start.mjs @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import { hookAdditionalContext, readHookInput, recallForPrompt } from "./lib.mjs"; + +const payload = await readHookInput(); +const context = await recallForPrompt(payload, "", "session-start"); +process.stdout.write(hookAdditionalContext("SessionStart", context)); diff --git a/codex-plugin/scripts/stop.mjs b/codex-plugin/scripts/stop.mjs new file mode 100644 index 0000000..49c3f74 --- /dev/null +++ b/codex-plugin/scripts/stop.mjs @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import { captureCurrentTurn, maybeFlushCapturedTurns, readHookInput } from "./lib.mjs"; + +const payload = await readHookInput(); +const capture = await captureCurrentTurn(payload, "stop"); +await maybeFlushCapturedTurns(payload, capture, "periodic_stop_flush"); diff --git a/codex-plugin/scripts/user-prompt-submit.mjs b/codex-plugin/scripts/user-prompt-submit.mjs new file mode 100644 index 0000000..fbbdd43 --- /dev/null +++ b/codex-plugin/scripts/user-prompt-submit.mjs @@ -0,0 +1,8 @@ +#!/usr/bin/env node +import { beginTurn, hookAdditionalContext, promptFromPayload, readHookInput, recallForPrompt } from "./lib.mjs"; + +const payload = await readHookInput(); +const prompt = promptFromPayload(payload); +await beginTurn(payload); +const context = await recallForPrompt(payload, prompt, "prompt"); +process.stdout.write(hookAdditionalContext("UserPromptSubmit", context)); diff --git a/codex-plugin/skills/tdai-memory/SKILL.md b/codex-plugin/skills/tdai-memory/SKILL.md new file mode 100644 index 0000000..c903511 --- /dev/null +++ b/codex-plugin/skills/tdai-memory/SKILL.md @@ -0,0 +1,35 @@ +--- +name: tdai-memory +description: Use TencentDB Agent Memory from Codex for manual inspection, explicit remembering, and diagnostics. Automatic hooks are the primary path; use this skill only when the user asks about memory state, recall, or saving a specific note. +--- + +# TDAI Memory For Codex + +This plugin is designed to be automatic: + +- `SessionStart` injects project memory and recent raw-conversation hints. +- `UserPromptSubmit` starts a turn, recalls relevant L1/L2/L3 memory, and injects it as context. +- If Gateway recall/search has no useful context, prompt recall falls back to a project-scoped local L0 JSONL scan. +- `PreToolUse` injects a lightweight model-visible memory hint and collects tool activity. +- `PermissionRequest` / `PostToolUse` collect permission and tool result activity. +- Large `PostToolUse` results are offloaded into `context-offload//offload-*.jsonl`, `refs/*.md`, and `mmds/001-codex-tool-offload.mmd`; later prompts inject the compact canvas. +- `PreCompact` captures pending turn state before compaction. +- `PostCompact` flushes session-scoped memory pipeline work through `/session/end`. +- `Stop` captures the completed turn through the TencentDB Agent Memory Gateway and flushes every `TDAI_CODEX_FLUSH_EVERY_N_TURNS` turns. +- Adapter-controlled capture and import paths strip injected TencentDB/Codex memory tags before persistence, matching the original `before_message_write` cleanup goal even though Codex does not expose that exact hook. +- `tdai_memory_search` / `tdai_conversation_search` / `tdai_offload_lookup` are available as Codex MCP tools when `scripts/mcp-server.mjs` is registered. + +Manual commands are only for inspection or explicit notes: + +```bash +node "${PLUGIN_ROOT}/scripts/query.mjs" status +node "${PLUGIN_ROOT}/scripts/query.mjs" memory "query terms" +node "${PLUGIN_ROOT}/scripts/query.mjs" conversation "query terms" +node "${PLUGIN_ROOT}/scripts/query.mjs" remember "durable note to save" +node "${PLUGIN_ROOT}/scripts/query.mjs" flush +node "${PLUGIN_ROOT}/scripts/query.mjs" seed ./historical-conversations.json +node "${PLUGIN_ROOT}/scripts/query.mjs" offload list --all --limit 10 +node "${PLUGIN_ROOT}/scripts/query.mjs" offload node Cxxxxxx_N001 --content +``` + +When memory is retrieved, use it as operating context. Verify path- and data-dependent claims against the current filesystem before acting. If current evidence contradicts memory, trust the current evidence and save the correction. diff --git a/codex-plugin/tdai-gateway.example.json b/codex-plugin/tdai-gateway.example.json new file mode 100644 index 0000000..89e7cf9 --- /dev/null +++ b/codex-plugin/tdai-gateway.example.json @@ -0,0 +1,47 @@ +{ + "server": { + "host": "127.0.0.1", + "port": 8420 + }, + "data": { + "baseDir": "${TDAI_DATA_DIR}" + }, + "llm": { + "baseUrl": "${TDAI_LLM_BASE_URL}", + "apiKey": "${TDAI_LLM_API_KEY}", + "model": "${TDAI_LLM_MODEL}", + "maxTokens": 4096, + "timeoutMs": 120000 + }, + "memory": { + "capture": { + "enabled": true, + "l0l1RetentionDays": 0 + }, + "recall": { + "enabled": true, + "maxResults": 8, + "strategy": "hybrid", + "timeoutMs": 5000 + }, + "pipeline": { + "everyNConversations": 3, + "enableWarmup": true, + "l1IdleTimeoutSeconds": 300 + }, + "extraction": { + "enabled": true, + "enableDedup": true, + "maxMemoriesPerSession": 20 + }, + "persona": { + "triggerEveryN": 30, + "maxScenes": 20 + }, + "embedding": { + "provider": "none", + "enabled": false + }, + "storeBackend": "sqlite" + } +} diff --git a/package.json b/package.json index 2d74158..e514876 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "bin": { "migrate-sqlite-to-tcvdb": "./bin/migrate-sqlite-to-tcvdb.mjs", "export-tencent-vdb": "./bin/export-tencent-vdb.mjs", - "read-local-memory": "./bin/read-local-memory.mjs" + "read-local-memory": "./bin/read-local-memory.mjs", + "tdai-memory-gateway": "./dist/src/gateway/cli.mjs" }, "exports": { ".": { @@ -18,7 +19,7 @@ "scripts": { "build": "npm run build:plugin && npm run build:scripts", "build:plugin": "tsdown", - "build:scripts": "npm run build:migrate-sqlite-to-vdb && npm run build:export-tencent-vdb && npm run build:read-local-memory", + "build:scripts": "node scripts/build-optional-bin-scripts.mjs", "prepack": "npm run build", "build:migrate-sqlite-to-vdb": "tsc -p scripts/migrate-sqlite-to-tcvdb/tsconfig.json --noEmitOnError false", "migrate-sqlite-to-tcvdb": "node ./bin/migrate-sqlite-to-tcvdb.mjs", @@ -38,6 +39,7 @@ "scripts/migrate-sqlite-to-tcvdb/dist/", "scripts/export-tencent-vdb/dist/", "scripts/read-local-memory/dist/", + "scripts/build-optional-bin-scripts.mjs", "scripts/memory-tencentdb-ctl.sh", "scripts/install_hermes_memory_tencentdb.sh", "scripts/README.memory-tencentdb-ctl.md", @@ -45,10 +47,12 @@ "scripts/openclaw-after-tool-call-messages.patch.sh", "scripts/setup-offload.sh", "hermes-plugin/", + "codex-plugin/", "openclaw.plugin.json", "README.md", "CHANGELOG.md", "LICENSE", + "!codex-plugin/**/*.test.mjs", "!src/**/*.test.ts", "!src/**/*.spec.ts", "!src/**/__tests__/" diff --git a/scripts/build-optional-bin-scripts.mjs b/scripts/build-optional-bin-scripts.mjs new file mode 100644 index 0000000..9960b59 --- /dev/null +++ b/scripts/build-optional-bin-scripts.mjs @@ -0,0 +1,33 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; + +const jobs = [ + { + name: "migrate-sqlite-to-tcvdb", + tsconfig: "scripts/migrate-sqlite-to-tcvdb/tsconfig.json", + args: ["-p", "scripts/migrate-sqlite-to-tcvdb/tsconfig.json", "--noEmitOnError", "false"], + }, + { + name: "export-tencent-vdb", + tsconfig: "scripts/export-tencent-vdb/tsconfig.json", + args: ["--project", "scripts/export-tencent-vdb/tsconfig.json"], + }, + { + name: "read-local-memory", + tsconfig: "scripts/read-local-memory/tsconfig.json", + args: ["--project", "scripts/read-local-memory/tsconfig.json"], + }, +]; + +for (const job of jobs) { + if (!existsSync(job.tsconfig)) { + console.warn(`[build:scripts] skipping ${job.name}: missing ${job.tsconfig}`); + continue; + } + + const result = spawnSync("tsc", job.args, { stdio: "inherit" }); + if (result.status !== 0) { + process.exit(result.status ?? 1); + } +} diff --git a/src/adapters/standalone/llm-runner.test.ts b/src/adapters/standalone/llm-runner.test.ts new file mode 100644 index 0000000..2bbaea5 --- /dev/null +++ b/src/adapters/standalone/llm-runner.test.ts @@ -0,0 +1,51 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + resolveSandboxedExistingPath, + resolveSandboxedPath, + resolveSandboxedWritablePath, +} from "./llm-runner.js"; + +describe("StandaloneLLMRunner file-tool sandbox", () => { + it("rejects sibling-prefix traversal outside workspaceDir", () => { + const root = path.resolve("/tmp/scene_blocks"); + + expect(resolveSandboxedPath(root, "../scene_blocks_backup/file.md")).toBeNull(); + expect(resolveSandboxedPath(root, "../scene_blocks2/file.md")).toBeNull(); + expect(resolveSandboxedPath(root, "inside/file.md")).toBe(path.join(root, "inside/file.md")); + }); + + it("rejects symlink escapes for existing and writable paths", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-llm-sandbox-")); + const outside = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-llm-outside-")); + try { + fs.writeFileSync(path.join(outside, "secret.txt"), "secret"); + fs.symlinkSync(outside, path.join(root, "outside-link"), "dir"); + + await expect(resolveSandboxedExistingPath(root, "outside-link/secret.txt")).resolves.toBeNull(); + await expect(resolveSandboxedWritablePath(root, "outside-link/new.txt")).resolves.toBeNull(); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + fs.rmSync(outside, { recursive: true, force: true }); + } + }); + + it("rejects existing file symlinks for writable paths", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-llm-sandbox-")); + const outside = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-llm-outside-")); + try { + const outsideFile = path.join(outside, "secret.txt"); + fs.writeFileSync(outsideFile, "secret"); + fs.symlinkSync(outsideFile, path.join(root, "out.md")); + + await expect(resolveSandboxedExistingPath(root, "out.md")).resolves.toBeNull(); + await expect(resolveSandboxedWritablePath(root, "out.md")).resolves.toBeNull(); + expect(fs.readFileSync(outsideFile, "utf-8")).toBe("secret"); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + fs.rmSync(outside, { recursive: true, force: true }); + } + }); +}); diff --git a/src/adapters/standalone/llm-runner.ts b/src/adapters/standalone/llm-runner.ts index 2f7c9e1..372d83f 100644 --- a/src/adapters/standalone/llm-runner.ts +++ b/src/adapters/standalone/llm-runner.ts @@ -16,6 +16,7 @@ * All file paths are resolved relative to `workspaceDir`, enforcing sandbox boundaries. */ +import { constants as fsConstants } from "node:fs"; import fsPromises from "node:fs/promises"; import path from "node:path"; import { generateText, tool, stepCountIs, jsonSchema } from "ai"; @@ -55,14 +56,90 @@ export interface StandaloneLLMConfig { // Sandboxed tool execution helpers // ============================ -function resolveSandboxedPath(workspaceDir: string, relativePath: string): string | null { - const resolved = path.resolve(workspaceDir, relativePath); - if (!resolved.startsWith(path.resolve(workspaceDir))) { +export function resolveSandboxedPath(workspaceDir: string, relativePath: string): string | null { + const root = path.resolve(workspaceDir); + const resolved = path.resolve(root, relativePath); + const relative = path.relative(root, resolved); + if (relative.startsWith("..") || path.isAbsolute(relative)) { return null; } return resolved; } +export async function resolveSandboxedExistingPath(workspaceDir: string, relativePath: string): Promise { + const resolved = resolveSandboxedPath(workspaceDir, relativePath); + if (!resolved) return null; + + try { + const linkStat = await fsPromises.lstat(resolved); + if (linkStat.isSymbolicLink()) return null; + const [realRoot, realResolved] = await Promise.all([ + fsPromises.realpath(workspaceDir), + fsPromises.realpath(resolved), + ]); + return isWithinPath(realRoot, realResolved) ? realResolved : null; + } catch { + return null; + } +} + +export async function resolveSandboxedWritablePath(workspaceDir: string, relativePath: string): Promise { + const resolved = resolveSandboxedPath(workspaceDir, relativePath); + if (!resolved) return null; + + try { + const linkStat = await fsPromises.lstat(resolved); + if (linkStat.isSymbolicLink()) return null; + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "ENOENT") return null; + } + + const parent = await nearestExistingParent(path.dirname(resolved)); + if (!parent) return null; + + try { + const [realRoot, realParent] = await Promise.all([ + fsPromises.realpath(workspaceDir), + fsPromises.realpath(parent), + ]); + return isWithinPath(realRoot, realParent) ? resolved : null; + } catch { + return null; + } +} + +async function nearestExistingParent(dir: string): Promise { + let current = path.resolve(dir); + while (true) { + try { + const stat = await fsPromises.stat(current); + return stat.isDirectory() ? current : null; + } catch { + const parent = path.dirname(current); + if (parent === current) return null; + current = parent; + } + } +} + +async function writeSandboxedUtf8File(filePath: string, content: string): Promise { + const flags = fsConstants.O_WRONLY | + fsConstants.O_CREAT | + fsConstants.O_TRUNC | + ((fsConstants as Record).O_NOFOLLOW ?? 0); + const handle = await fsPromises.open(filePath, flags, 0o600); + try { + await handle.writeFile(content, "utf-8"); + } finally { + await handle.close(); + } +} + +function isWithinPath(root: string, target: string): boolean { + const relative = path.relative(path.resolve(root), path.resolve(target)); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + // ============================ // Tool definitions (Vercel AI SDK `tool()` format) // ============================ @@ -79,7 +156,7 @@ function createSandboxedTools(workspaceDir: string, logger?: Logger) { required: ["path"], }), execute: (async (args: { path: string }) => { - const resolved = resolveSandboxedPath(workspaceDir, args.path); + const resolved = await resolveSandboxedExistingPath(workspaceDir, args.path); if (!resolved) return JSON.stringify({ error: `Path "${args.path}" escapes workspace boundary.` }); try { return await fsPromises.readFile(resolved, "utf-8"); @@ -102,11 +179,11 @@ function createSandboxedTools(workspaceDir: string, logger?: Logger) { required: ["path", "content"], }), execute: (async (args: { path: string; content: string }) => { - const resolved = resolveSandboxedPath(workspaceDir, args.path); + const resolved = await resolveSandboxedWritablePath(workspaceDir, args.path); if (!resolved) return JSON.stringify({ error: `Path "${args.path}" escapes workspace boundary.` }); try { await fsPromises.mkdir(path.dirname(resolved), { recursive: true }); - await fsPromises.writeFile(resolved, args.content, "utf-8"); + await writeSandboxedUtf8File(resolved, args.content); return JSON.stringify({ success: true }); } catch (err) { const msg = err instanceof Error ? err.message : String(err); @@ -128,7 +205,7 @@ function createSandboxedTools(workspaceDir: string, logger?: Logger) { required: ["path", "old_str", "new_str"], }), execute: (async (args: { path: string; old_str: string; new_str: string }) => { - const resolved = resolveSandboxedPath(workspaceDir, args.path); + const resolved = await resolveSandboxedExistingPath(workspaceDir, args.path); if (!resolved) return JSON.stringify({ error: `Path "${args.path}" escapes workspace boundary.` }); if (!args.old_str) return JSON.stringify({ error: "old_str cannot be empty." }); try { @@ -137,7 +214,7 @@ function createSandboxedTools(workspaceDir: string, logger?: Logger) { return JSON.stringify({ error: `old_str not found in file "${args.path}".` }); } const updated = existing.replace(args.old_str, args.new_str); - await fsPromises.writeFile(resolved, updated, "utf-8"); + await writeSandboxedUtf8File(resolved, updated); return JSON.stringify({ success: true }); } catch (err) { const msg = err instanceof Error ? err.message : String(err); diff --git a/src/cli/README.md b/src/cli/README.md index f314e6e..1252692 100644 --- a/src/cli/README.md +++ b/src/cli/README.md @@ -4,7 +4,8 @@ ## seed — 导入历史对话数据 -将历史对话 JSON 文件导入到记忆管线中，完整执行 L0→L1→L2→L3 流程。适用于： +将历史对话 JSON 文件导入到记忆管线中。默认等待 L0→L1；如果需要 seed 返回前立即产出 +L2 场景块和 L3 persona，使用 `--wait-for-full-pipeline` 完整等待 L0→L1→L2→L3。适用于： - 将已有对话数据灌入记忆系统 - 批量测试记忆提取效果 @@ -25,6 +26,8 @@ openclaw memory-tdai seed --input [options] | `--session-key ` | — | 回退 session key（当输入数据缺少时使用） | | `--config ` | — | 配置覆盖文件（JSON，与 openclaw.json 插件配置深度合并） | | `--strict-round-role` | — | 严格校验每轮对话必须包含 user 和 assistant 消息 | +| `--wait-for-full-pipeline` | — | 返回前等待最终 L1→L2→L3 flush 完成 | +| `--full-pipeline-timeout-ms ` | — | 最终 L1→L2→L3 flush 的最长等待时间（默认 900000） | | `--yes` | — | 跳过交互确认（如时间戳自动填充确认） | ### 示例 @@ -42,6 +45,9 @@ openclaw memory-tdai seed --input data.json --config seed-config.json # 跳过所有确认 openclaw memory-tdai seed --input data.json --yes +# 返回前完整等待 L1/L2/L3 +openclaw memory-tdai seed --input data.json --wait-for-full-pipeline --yes + # 严格模式 + 自定义配置 openclaw memory-tdai seed --input data.json --config seed-config.json --strict-round-role --yes ``` @@ -167,6 +173,7 @@ Seed 完成后，`manifest.json` 会记录本次运行信息： "sessions": 3, "rounds": 42, "messages": 128, + "fullPipelineFlushed": true, "startedAt": "2026-04-01T22:00:00.000Z", "completedAt": "2026-04-01T22:05:30.000Z" } diff --git a/src/cli/commands/seed.ts b/src/cli/commands/seed.ts index 611af05..ee68944 100644 --- a/src/cli/commands/seed.ts +++ b/src/cli/commands/seed.ts @@ -25,12 +25,14 @@ const TAG = "[memory-tdai] [seed-cmd]"; export function registerSeedCommand(parent: Command, ctx: SeedCliContext): void { parent .command("seed") - .description("Seed historical conversation data into the memory pipeline (L0 → L1)") + .description("Seed historical conversation data into the memory pipeline (L0 → L1, optionally L2/L3)") .requiredOption("--input ", "Path to input JSON file") .option("--output-dir ", "Output directory for pipeline data (default: auto-generated)") .option("--session-key ", "Fallback session key when input lacks one") .option("--config ", "Path to memory-tdai config override file (JSON, deep-merged on top of current plugin config)") .option("--strict-round-role", "Require each round to have both user and assistant messages", false) + .option("--wait-for-full-pipeline", "Wait for final L1→L2→L3 processing before returning", false) + .option("--full-pipeline-timeout-ms ", "Max wait time for final L1→L2→L3 processing", "900000") .option("--yes", "Skip interactive confirmations (e.g. timestamp auto-fill)", false) .addHelpText("after", ` Examples: @@ -47,6 +49,8 @@ Examples: strictRoundRole: rawOpts.strictRoundRole === true, yes: rawOpts.yes === true, configFile: rawOpts.config as string | undefined, + waitForFullPipeline: rawOpts.waitForFullPipeline === true, + fullPipelineTimeoutMs: Number(rawOpts.fullPipelineTimeoutMs) || undefined, }; await runSeedCommand(opts, ctx); @@ -66,6 +70,7 @@ async function runSeedCommand(opts: SeedCommandOptions, ctx: SeedCliContext): Pr logger.info(`${TAG} sessionKey: ${opts.sessionKey ?? "(from input)"}`); logger.info(`${TAG} config: ${opts.configFile ?? "(default)"}`); logger.info(`${TAG} strict: ${opts.strictRoundRole}`); + logger.info(`${TAG} full: ${opts.waitForFullPipeline === true}`); logger.info(`${TAG} yes: ${opts.yes}`); // 0. Load config override file and deep-merge with base plugin config @@ -149,6 +154,8 @@ async function runSeedCommand(opts: SeedCommandOptions, ctx: SeedCliContext): Pr openclawConfig: ctx.config, pluginConfig: mergedPluginConfig, inputFile: opts.input, + waitForFullPipeline: opts.waitForFullPipeline === true, + fullPipelineFlushTimeoutMs: opts.fullPipelineTimeoutMs, logger, onProgress: (progress) => { const pct = ((progress.currentRound / progress.totalRounds) * 100).toFixed(0); @@ -168,6 +175,7 @@ async function runSeedCommand(opts: SeedCommandOptions, ctx: SeedCliContext): Pr console.log(`║ Rounds: ${String(summary.roundsProcessed).padStart(11)} ║`); console.log(`║ Messages: ${String(summary.messagesProcessed).padStart(11)} ║`); console.log(`║ L0 recorded: ${String(summary.l0RecordedCount).padStart(11)} ║`); + console.log(`║ Full flush: ${String(summary.fullPipelineFlushed ? "yes" : "no").padStart(11)} ║`); console.log(`║ Duration: ${(summary.durationMs / 1000).toFixed(1).padStart(10)}s ║`); console.log("╚══════════════════════════════════════════╝"); console.log(`\n📁 Output: ${summary.outputDir}\n`); diff --git a/src/core/seed/seed-runtime.ts b/src/core/seed/seed-runtime.ts index 46e6647..34ec09a 100644 --- a/src/core/seed/seed-runtime.ts +++ b/src/core/seed/seed-runtime.ts @@ -5,15 +5,14 @@ * L1 runner, L2 runner, L3 runner, and persister wiring — keeping this * module focused on seed-specific concerns: * - Synchronous per-round L0 capture with progress reporting - * - waitForL1Idle polling (L1 only — see FIXME below) + * - waitForL1Idle polling at batch boundaries + * - Optional final full-pipeline flush for callers that need L2/L3 artifacts * - Ctrl+C graceful shutdown * - * FIXME: Currently we only wait for L1 to become idle before destroying the - * pipeline. L2 (scene extraction) and L3 (persona generation) may still be - * in-flight when `pipeline.destroy()` is called. This is intentional for now - * to avoid excessively long seed runs, but means seed output may not include - * the latest L2/L3 artifacts. Re-evaluate adding a full L1+L2+L3 idle wait - * once pipeline-manager exposes reliable L2/L3 idle signals. + * By default, seed preserves the historical CLI behavior and waits for L1 at + * batch boundaries. Callers such as the Codex history importer can opt into a + * final L1→L2→L3 flush before shutdown when they need higher-level artifacts + * to be immediately available for recall injection. */ import path from "node:path"; @@ -47,6 +46,10 @@ export interface SeedRuntimeOptions { pluginConfig?: Record; /** Original input file path (for manifest traceability). */ inputFile?: string; + /** Wait for a final L1→L2→L3 flush before returning. */ + waitForFullPipeline?: boolean; + /** Max time for the final L1→L2→L3 flush. */ + fullPipelineFlushTimeoutMs?: number; /** Logger instance. */ logger: PipelineLogger; /** Progress callback (called after each round). */ @@ -202,9 +205,9 @@ async function waitForL1Idle( /** * Execute the seed pipeline: feed normalized input through L0 → L1. * - * L2/L3 runners are wired but their completion is **not** awaited — see the - * module-level FIXME. The pipeline is destroyed after L1 idle, so L2/L3 may - * be interrupted mid-run. + * L2/L3 runners are wired. Their completion is awaited only when + * `waitForFullPipeline` is true; otherwise seed preserves the faster historical + * behavior and returns after L1 drains. * * This is the core runtime called by `src/cli/commands/seed.ts` after * all input validation and user confirmation are complete. @@ -232,6 +235,7 @@ export async function executeSeed( let pipeline: PipelineInstance | undefined; let totalL0Recorded = 0; let roundsProcessed = 0; + let fullPipelineFlushed = false; try { // Create and start pipeline (returns both the pipeline instance and the @@ -364,6 +368,25 @@ export async function executeSeed( { pollIntervalMs: 1_000, stableRounds: 3, maxWaitMs: 300_000 }, ); } + + if (!interrupted && opts.waitForFullPipeline) { + onProgress?.({ + currentRound: roundsProcessed, + totalRounds: input.totalRounds, + sessionKey: "*", + stage: "l1_l2_l3_flushing", + }); + + logger.info(`${TAG} Final full pipeline flush requested (L1→L2→L3)...`); + await pipeline.scheduler.flushPendingWork({ + reason: "seed", + timeoutMs: opts.fullPipelineFlushTimeoutMs ?? 900_000, + pollIntervalMs: 100, + stableRounds: 3, + armFollowUpL2Timers: false, + }); + fullPipelineFlushed = true; + } } finally { process.removeListener("SIGINT", onSigint); @@ -384,6 +407,7 @@ export async function executeSeed( roundsProcessed, messagesProcessed: input.totalMessages, l0RecordedCount: totalL0Recorded, + fullPipelineFlushed, durationMs, outputDir: opts.outputDir, }; @@ -407,6 +431,7 @@ export async function executeSeed( sessions: summary.sessionsProcessed, rounds: summary.roundsProcessed, messages: summary.messagesProcessed, + fullPipelineFlushed: summary.fullPipelineFlushed, startedAt: new Date(startTime).toISOString(), completedAt: new Date().toISOString(), }; diff --git a/src/core/seed/types.ts b/src/core/seed/types.ts index 2cb4ab8..eb09477 100644 --- a/src/core/seed/types.ts +++ b/src/core/seed/types.ts @@ -111,6 +111,10 @@ export interface SeedCommandOptions { yes: boolean; /** Path to memory-tdai config override file (JSON, deep-merged on top of current plugin config). */ configFile?: string; + /** Wait for final L1→L2→L3 processing before returning. */ + waitForFullPipeline?: boolean; + /** Max wait time for final L1→L2→L3 processing. */ + fullPipelineTimeoutMs?: number; } // ============================ @@ -135,6 +139,8 @@ export interface SeedSummary { roundsProcessed: number; messagesProcessed: number; l0RecordedCount: number; + /** True when the caller requested and completed a final L1→L2→L3 flush. */ + fullPipelineFlushed?: boolean; durationMs: number; outputDir: string; } diff --git a/src/core/tdai-core.ts b/src/core/tdai-core.ts index 977d4a2..1a53732 100644 --- a/src/core/tdai-core.ts +++ b/src/core/tdai-core.ts @@ -293,6 +293,7 @@ export class TdaiCore { limit: params.limit ?? 5, type: params.type, scene: params.scene, + sessionKeyPrefixes: params.sessionKeyPrefixes, vectorStore: this.vectorStore, embeddingService: this.embeddingService, logger: this.logger, @@ -314,6 +315,7 @@ export class TdaiCore { query: params.query, limit: params.limit ?? 5, sessionKey: params.sessionKey, + sessionKeyPrefixes: params.sessionKeyPrefixes, vectorStore: this.vectorStore, embeddingService: this.embeddingService, logger: this.logger, diff --git a/src/core/tools/conversation-search.ts b/src/core/tools/conversation-search.ts index ac4f3b1..423a702 100644 --- a/src/core/tools/conversation-search.ts +++ b/src/core/tools/conversation-search.ts @@ -46,6 +46,8 @@ export interface ConversationSearchResult { } const TAG = "[memory-tdai][tdai_conversation_search]"; +const FILTERED_SEARCH_INITIAL_CANDIDATES = 50; +const FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES = 500; // ============================ // RRF (Reciprocal Rank Fusion) @@ -90,6 +92,7 @@ export async function executeConversationSearch(params: { query: string; limit: number; sessionKey?: string; + sessionKeyPrefixes?: string[]; vectorStore?: IMemoryStore; embeddingService?: EmbeddingService; logger?: Logger; @@ -98,14 +101,17 @@ export async function executeConversationSearch(params: { query, limit, sessionKey: sessionFilter, + sessionKeyPrefixes, vectorStore, embeddingService, logger, } = params; + const normalizedSessionPrefixes = normalizeSessionPrefixes(sessionKeyPrefixes); logger?.debug?.( `${TAG} CALLED: query="${query.slice(0, 100)}", limit=${limit}, ` + `sessionFilter=${sessionFilter ?? "(none)"}, ` + + `sessionPrefixFilter=${normalizedSessionPrefixes.join("|") || "(none)"}, ` + `vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` + `embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`, ); @@ -137,12 +143,96 @@ export async function executeConversationSearch(params: { }; } - // ── Over-retrieve for later filtering and RRF merging ── - const candidateK = sessionFilter ? limit * 4 : limit * 3; + const hasSessionScope = !!sessionFilter || normalizedSessionPrefixes.length > 0; + let candidateK = hasSessionScope + ? Math.max(limit * 6, FILTERED_SEARCH_INITIAL_CANDIDATES) + : limit * 3; + const maxCandidateK = hasSessionScope + ? await scopedSearchMaxCandidates({ + count: () => vectorStore.countL0(), + candidateK, + logger, + }) + : candidateK; + + while (true) { + const search = await collectConversationCandidates({ + query, + candidateK, + hasFts, + hasEmbedding, + vectorStore, + embeddingService, + logger, + }); + + if (search.results.length === 0) { + logger?.debug?.(`${TAG} Both search paths returned 0 results`); + return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" }; + } + + const filtered = filterConversationResults(search.results, { + sessionFilter, + sessionPrefixes: normalizedSessionPrefixes, + logger, + }); + const trimmed = filtered.slice(0, limit); + + if ( + trimmed.length >= limit || + !hasSessionScope || + !search.mayHaveMore || + candidateK >= maxCandidateK + ) { + logger?.debug?.( + `${TAG} RESULT (strategy=${search.strategy}, candidateK=${candidateK}): returning ${trimmed.length} messages ` + + `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`, + ); + + return { + results: trimmed, + total: trimmed.length, + strategy: search.strategy, + }; + } + + candidateK = Math.min(candidateK * 2, maxCandidateK); + logger?.debug?.(`${TAG} Expanding scoped search window to candidateK=${candidateK}`); + } +} + +async function scopedSearchMaxCandidates(params: { + count: () => number | Promise; + candidateK: number; + logger?: Logger; +}): Promise { + const { count, candidateK, logger } = params; + try { + const total = await count(); + if (Number.isFinite(total) && total > 0) { + return Math.max(candidateK, Math.floor(total)); + } + } catch (err) { + logger?.warn?.( + `${TAG} Scoped search could not count records; falling back to ${FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES} candidates: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + return Math.max(candidateK, FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES); +} + +async function collectConversationCandidates(params: { + query: string; + candidateK: number; + hasFts: boolean; + hasEmbedding: boolean; + vectorStore: IMemoryStore; + embeddingService?: EmbeddingService; + logger?: Logger; +}): Promise<{ results: ConversationSearchResultItem[]; strategy: string; mayHaveMore: boolean }> { + const { query, candidateK, hasFts, hasEmbedding, vectorStore, embeddingService, logger } = params; - // ── Run available search strategies in parallel ── const [ftsItems, vecItems] = await Promise.all([ - // FTS5 keyword search on L0 (async (): Promise => { if (!hasFts) return []; try { @@ -154,14 +244,7 @@ export async function executeConversationSearch(params: { logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`); const ftsResults = await vectorStore.searchL0Fts(ftsQuery, candidateK); logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`); - return ftsResults.map((r) => ({ - id: r.record_id, - session_key: r.session_key, - role: r.role, - content: r.message_text, - score: r.score, - recorded_at: r.recorded_at, - })); + return ftsResults.map(conversationResultItemFromStore); } catch (err) { logger?.warn?.( `${TAG} [hybrid-fts] FTS5 search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, @@ -169,8 +252,6 @@ export async function executeConversationSearch(params: { return []; } })(), - - // Vector embedding search on L0 (async (): Promise => { if (!hasEmbedding) return []; try { @@ -181,14 +262,7 @@ export async function executeConversationSearch(params: { ); const vecResults: L0SearchResult[] = await vectorStore.searchL0Vector(queryEmbedding, candidateK, query); logger?.debug?.(`${TAG} [hybrid-vec] Vector search returned ${vecResults.length} candidates`); - return vecResults.map((r) => ({ - id: r.record_id, - session_key: r.session_key, - role: r.role, - content: r.message_text, - score: r.score, - recorded_at: r.recorded_at, - })); + return vecResults.map(conversationResultItemFromStore); } catch (err) { logger?.warn?.( `${TAG} [hybrid-vec] Embedding search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, @@ -198,56 +272,70 @@ export async function executeConversationSearch(params: { })(), ]); - // ── Determine effective strategy ── const ftsOk = ftsItems.length > 0; const vecOk = vecItems.length > 0; - let strategy: string; - - if (ftsOk && vecOk) { - strategy = "hybrid"; - } else if (vecOk) { - strategy = "embedding"; - } else if (ftsOk) { - strategy = "fts"; - } else { - logger?.debug?.(`${TAG} Both search paths returned 0 results`); - return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" }; - } + const strategy = ftsOk && vecOk ? "hybrid" : vecOk ? "embedding" : ftsOk ? "fts" : "none"; + const results = strategy === "hybrid" + ? rrfMergeL0(ftsItems, vecItems) + : ftsOk ? ftsItems : vecItems; - // ── Merge results ── - let results: ConversationSearchResultItem[]; if (strategy === "hybrid") { - results = rrfMergeL0(ftsItems, vecItems); logger?.debug?.( `${TAG} [hybrid] RRF merged: fts=${ftsItems.length}, vec=${vecItems.length} → ${results.length} unique`, ); - } else { - // Single-source: use whichever list has results (already sorted by score) - results = ftsOk ? ftsItems : vecItems; } - // ── Apply session key filter ── - if (sessionFilter) { - const preFilterCount = results.length; - results = results.filter((r) => r.session_key === sessionFilter); - logger?.debug?.(`${TAG} After session filter "${sessionFilter}": ${results.length}/${preFilterCount}`); - } - - // ── Trim to requested limit ── - const trimmed = results.slice(0, limit); - - logger?.debug?.( - `${TAG} RESULT (strategy=${strategy}): returning ${trimmed.length} messages ` + - `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`, - ); - return { - results: trimmed, - total: trimmed.length, + results, strategy, + mayHaveMore: (hasFts && ftsItems.length >= candidateK) || (hasEmbedding && vecItems.length >= candidateK), + }; +} + +function conversationResultItemFromStore(r: L0SearchResult): ConversationSearchResultItem { + return { + id: r.record_id, + session_key: r.session_key, + role: r.role, + content: r.message_text, + score: r.score, + recorded_at: r.recorded_at, }; } +function filterConversationResults( + results: ConversationSearchResultItem[], + filters: { + sessionFilter?: string; + sessionPrefixes: string[]; + logger?: Logger; + }, +): ConversationSearchResultItem[] { + const { sessionFilter, sessionPrefixes, logger } = filters; + let filtered = results; + if (sessionFilter) { + const preFilterCount = filtered.length; + filtered = filtered.filter((r) => r.session_key === sessionFilter); + logger?.debug?.(`${TAG} After session filter "${sessionFilter}": ${filtered.length}/${preFilterCount}`); + } + if (sessionPrefixes.length > 0) { + const preFilterCount = filtered.length; + filtered = filtered.filter((r) => + sessionPrefixes.some((prefix) => r.session_key.startsWith(prefix)), + ); + logger?.debug?.(`${TAG} After session-prefix filter: ${filtered.length}/${preFilterCount}`); + } + return filtered; +} + +function normalizeSessionPrefixes(prefixes: string[] | undefined): string[] { + if (!Array.isArray(prefixes)) return []; + return prefixes + .map((prefix) => typeof prefix === "string" ? prefix.trim() : "") + .filter(Boolean) + .slice(0, 20); +} + // ============================ // Tool response formatter // ============================ diff --git a/src/core/tools/memory-search.ts b/src/core/tools/memory-search.ts index dc9d2c2..76145c0 100644 --- a/src/core/tools/memory-search.ts +++ b/src/core/tools/memory-search.ts @@ -31,6 +31,8 @@ export interface MemorySearchResultItem { type: string; priority: number; scene_name: string; + session_key: string; + session_id: string; score: number; created_at: string; updated_at: string; @@ -45,6 +47,8 @@ export interface MemorySearchResult { } const TAG = "[memory-tdai][tdai_memory_search]"; +const FILTERED_SEARCH_INITIAL_CANDIDATES = 50; +const FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES = 500; // ============================ // RRF (Reciprocal Rank Fusion) @@ -90,6 +94,7 @@ export async function executeMemorySearch(params: { limit: number; type?: string; scene?: string; + sessionKeyPrefixes?: string[]; vectorStore?: IMemoryStore; embeddingService?: EmbeddingService; logger?: Logger; @@ -99,14 +104,17 @@ export async function executeMemorySearch(params: { limit, type: typeFilter, scene: sceneFilter, + sessionKeyPrefixes, vectorStore, embeddingService, logger, } = params; + const normalizedSessionPrefixes = normalizeSessionPrefixes(sessionKeyPrefixes); logger?.debug?.( `${TAG} CALLED: query="${query.slice(0, 100)}", limit=${limit}, ` + `typeFilter=${typeFilter ?? "(none)"}, sceneFilter=${sceneFilter ?? "(none)"}, ` + + `sessionPrefixFilter=${normalizedSessionPrefixes.join("|") || "(none)"}, ` + `vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` + `embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`, ); @@ -138,12 +146,96 @@ export async function executeMemorySearch(params: { }; } - // ── Over-retrieve for later filtering and RRF merging ── - const candidateK = limit * 3; + let candidateK = normalizedSessionPrefixes.length > 0 + ? Math.max(limit * 6, FILTERED_SEARCH_INITIAL_CANDIDATES) + : limit * 3; + const maxCandidateK = normalizedSessionPrefixes.length > 0 + ? await scopedSearchMaxCandidates({ + count: () => vectorStore.countL1(), + candidateK, + logger, + }) + : candidateK; + + while (true) { + const search = await collectMemoryCandidates({ + query, + candidateK, + hasFts, + hasEmbedding, + vectorStore, + embeddingService, + logger, + }); + + if (search.results.length === 0) { + logger?.debug?.(`${TAG} Both search paths returned 0 results`); + return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" }; + } + + const filtered = filterMemoryResults(search.results, { + typeFilter, + sceneFilter, + sessionPrefixes: normalizedSessionPrefixes, + logger, + }); + const trimmed = filtered.slice(0, limit); + + if ( + trimmed.length >= limit || + normalizedSessionPrefixes.length === 0 || + !search.mayHaveMore || + candidateK >= maxCandidateK + ) { + logger?.debug?.( + `${TAG} RESULT (strategy=${search.strategy}, candidateK=${candidateK}): returning ${trimmed.length} memories ` + + `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`, + ); + + return { + results: trimmed, + total: trimmed.length, + strategy: search.strategy, + }; + } + + candidateK = Math.min(candidateK * 2, maxCandidateK); + logger?.debug?.(`${TAG} Expanding scoped search window to candidateK=${candidateK}`); + } +} + +async function scopedSearchMaxCandidates(params: { + count: () => number | Promise; + candidateK: number; + logger?: Logger; +}): Promise { + const { count, candidateK, logger } = params; + try { + const total = await count(); + if (Number.isFinite(total) && total > 0) { + return Math.max(candidateK, Math.floor(total)); + } + } catch (err) { + logger?.warn?.( + `${TAG} Scoped search could not count records; falling back to ${FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES} candidates: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + return Math.max(candidateK, FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES); +} + +async function collectMemoryCandidates(params: { + query: string; + candidateK: number; + hasFts: boolean; + hasEmbedding: boolean; + vectorStore: IMemoryStore; + embeddingService?: EmbeddingService; + logger?: Logger; +}): Promise<{ results: MemorySearchResultItem[]; strategy: string; mayHaveMore: boolean }> { + const { query, candidateK, hasFts, hasEmbedding, vectorStore, embeddingService, logger } = params; - // ── Run available search strategies in parallel ── const [ftsItems, vecItems] = await Promise.all([ - // FTS5 keyword search (async (): Promise => { if (!hasFts) return []; try { @@ -155,16 +247,7 @@ export async function executeMemorySearch(params: { logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`); const ftsResults = await vectorStore.searchL1Fts(ftsQuery, candidateK); logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`); - return ftsResults.map((r) => ({ - id: r.record_id, - content: r.content, - type: r.type, - priority: r.priority, - scene_name: r.scene_name, - score: r.score, - created_at: r.timestamp_start, - updated_at: r.timestamp_end, - })); + return ftsResults.map(memoryResultItemFromStore); } catch (err) { logger?.warn?.( `${TAG} [hybrid-fts] FTS5 search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, @@ -172,8 +255,6 @@ export async function executeMemorySearch(params: { return []; } })(), - - // Vector embedding search (async (): Promise => { if (!hasEmbedding) return []; try { @@ -184,16 +265,7 @@ export async function executeMemorySearch(params: { ); const vecResults: L1SearchResult[] = await vectorStore.searchL1Vector(queryEmbedding, candidateK, query); logger?.debug?.(`${TAG} [hybrid-vec] Vector search returned ${vecResults.length} candidates`); - return vecResults.map((r) => ({ - id: r.record_id, - content: r.content, - type: r.type, - priority: r.priority, - scene_name: r.scene_name, - score: r.score, - created_at: r.timestamp_start, - updated_at: r.timestamp_end, - })); + return vecResults.map(memoryResultItemFromStore); } catch (err) { logger?.warn?.( `${TAG} [hybrid-vec] Embedding search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, @@ -203,61 +275,80 @@ export async function executeMemorySearch(params: { })(), ]); - // ── Determine effective strategy ── const ftsOk = ftsItems.length > 0; const vecOk = vecItems.length > 0; - let strategy: string; - - if (ftsOk && vecOk) { - strategy = "hybrid"; - } else if (vecOk) { - strategy = "embedding"; - } else if (ftsOk) { - strategy = "fts"; - } else { - logger?.debug?.(`${TAG} Both search paths returned 0 results`); - return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" }; - } + const strategy = ftsOk && vecOk ? "hybrid" : vecOk ? "embedding" : ftsOk ? "fts" : "none"; + const results = strategy === "hybrid" + ? rrfMergeL1(ftsItems, vecItems) + : ftsOk ? ftsItems : vecItems; - // ── Merge results ── - let results: MemorySearchResultItem[]; if (strategy === "hybrid") { - results = rrfMergeL1(ftsItems, vecItems); logger?.debug?.( `${TAG} [hybrid] RRF merged: fts=${ftsItems.length}, vec=${vecItems.length} → ${results.length} unique`, ); - } else { - // Single-source: use whichever list has results (already sorted by score) - results = ftsOk ? ftsItems : vecItems; } - // ── Apply secondary filters (type, scene) ── + return { + results, + strategy, + mayHaveMore: (hasFts && ftsItems.length >= candidateK) || (hasEmbedding && vecItems.length >= candidateK), + }; +} + +function memoryResultItemFromStore(r: L1SearchResult): MemorySearchResultItem { + return { + id: r.record_id, + content: r.content, + type: r.type, + priority: r.priority, + scene_name: r.scene_name, + session_key: r.session_key, + session_id: r.session_id, + score: r.score, + created_at: r.timestamp_start, + updated_at: r.timestamp_end, + }; +} + +function filterMemoryResults( + results: MemorySearchResultItem[], + filters: { + typeFilter?: string; + sceneFilter?: string; + sessionPrefixes: string[]; + logger?: Logger; + }, +): MemorySearchResultItem[] { + const { typeFilter, sceneFilter, sessionPrefixes, logger } = filters; const preFilterCount = results.length; + let filtered = results; if (typeFilter) { - results = results.filter((r) => r.type === typeFilter); - logger?.debug?.(`${TAG} After type filter "${typeFilter}": ${results.length}/${preFilterCount}`); + filtered = filtered.filter((r) => r.type === typeFilter); + logger?.debug?.(`${TAG} After type filter "${typeFilter}": ${filtered.length}/${preFilterCount}`); } if (sceneFilter) { const normalizedScene = sceneFilter.toLowerCase(); - results = results.filter((r) => + filtered = filtered.filter((r) => r.scene_name.toLowerCase().includes(normalizedScene), ); - logger?.debug?.(`${TAG} After scene filter "${sceneFilter}": ${results.length}/${preFilterCount}`); + logger?.debug?.(`${TAG} After scene filter "${sceneFilter}": ${filtered.length}/${preFilterCount}`); } + if (sessionPrefixes.length > 0) { + const beforeSessionFilter = filtered.length; + filtered = filtered.filter((r) => + sessionPrefixes.some((prefix) => r.session_key.startsWith(prefix)), + ); + logger?.debug?.(`${TAG} After session-prefix filter: ${filtered.length}/${beforeSessionFilter}`); + } + return filtered; +} - // ── Trim to requested limit ── - const trimmed = results.slice(0, limit); - - logger?.debug?.( - `${TAG} RESULT (strategy=${strategy}): returning ${trimmed.length} memories ` + - `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`, - ); - - return { - results: trimmed, - total: trimmed.length, - strategy, - }; +function normalizeSessionPrefixes(prefixes: string[] | undefined): string[] { + if (!Array.isArray(prefixes)) return []; + return prefixes + .map((prefix) => typeof prefix === "string" ? prefix.trim() : "") + .filter(Boolean) + .slice(0, 20); } // ============================ diff --git a/src/core/tools/search-prefix-filter.test.ts b/src/core/tools/search-prefix-filter.test.ts new file mode 100644 index 0000000..04fcd4d --- /dev/null +++ b/src/core/tools/search-prefix-filter.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; +import { executeConversationSearch } from "./conversation-search.js"; +import { executeMemorySearch } from "./memory-search.js"; + +const logger = { + info: () => {}, + warn: () => {}, + error: () => {}, +}; + +describe("session-prefix search filters", () => { + it("filters L1 memory search results by session-key prefix", async () => { + const rows = [ + ...Array.from({ length: 700 }, (_, i) => l1Result(`other-${i}`, "codex:def456:session-b")), + l1Result("a", "codex:abc123:session-a"), + l1Result("c", "codex-import:abc123:session-c"), + ]; + const vectorStore = { + isFtsAvailable: () => true, + countL1: () => rows.length, + searchL1Fts: (_query: string, limit: number) => rows.slice(0, limit), + }; + + const result = await executeMemorySearch({ + query: "project note", + limit: 2, + sessionKeyPrefixes: ["codex:abc123:", "codex-import:abc123:"], + vectorStore: vectorStore as any, + logger, + }); + + expect(result.results.map((item) => item.id)).toEqual(["a", "c"]); + }); + + it("filters L0 conversation search results by session-key prefix", async () => { + const rows = [ + ...Array.from({ length: 700 }, (_, i) => l0Result(`other-${i}`, "codex:def456:session-b")), + l0Result("a", "codex:abc123:session-a"), + l0Result("c", "codex-import:abc123:session-c"), + ]; + const vectorStore = { + isFtsAvailable: () => true, + countL0: () => rows.length, + searchL0Fts: (_query: string, limit: number) => rows.slice(0, limit), + }; + + const result = await executeConversationSearch({ + query: "previous command", + limit: 2, + sessionKeyPrefixes: ["codex:abc123:", "codex-import:abc123:"], + vectorStore: vectorStore as any, + logger, + }); + + expect(result.results.map((item) => item.id)).toEqual(["a", "c"]); + }); +}); + +function l1Result(id: string, sessionKey: string) { + return { + record_id: id, + content: `memory ${id}`, + type: "episodic", + priority: 2, + scene_name: "test", + score: 1, + timestamp_str: "", + timestamp_start: "", + timestamp_end: "", + session_key: sessionKey, + session_id: id, + metadata_json: "{}", + }; +} + +function l0Result(id: string, sessionKey: string) { + return { + record_id: id, + session_key: sessionKey, + role: "assistant", + message_text: `conversation ${id}`, + score: 1, + recorded_at: "", + }; +} diff --git a/src/core/types.ts b/src/core/types.ts index 8585b50..5444129 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -231,6 +231,7 @@ export interface MemorySearchParams { limit?: number; type?: string; scene?: string; + sessionKeyPrefixes?: string[]; } /** Search parameters for L0 conversation search. */ @@ -238,4 +239,5 @@ export interface ConversationSearchParams { query: string; limit?: number; sessionKey?: string; + sessionKeyPrefixes?: string[]; } diff --git a/src/gateway/auth.test.ts b/src/gateway/auth.test.ts new file mode 100644 index 0000000..3372671 --- /dev/null +++ b/src/gateway/auth.test.ts @@ -0,0 +1,331 @@ +import fs from "node:fs"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { TdaiGateway } from "./server.js"; + +async function request(params: { + port: number; + path: string; + method?: string; + headers?: Record; + body?: unknown; +}): Promise<{ status: number; body: string; wwwAuth?: string }> { + return new Promise((resolve, reject) => { + const body = params.body === undefined ? "" : JSON.stringify(params.body); + const req = http.request( + { + host: "127.0.0.1", + port: params.port, + path: params.path, + method: params.method ?? "GET", + headers: { + ...(body ? { "Content-Type": "application/json", "Content-Length": Buffer.byteLength(body).toString() } : {}), + ...(params.headers ?? {}), + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (chunk) => chunks.push(chunk)); + res.on("end", () => resolve({ + status: res.statusCode ?? 0, + body: Buffer.concat(chunks).toString("utf-8"), + wwwAuth: res.headers["www-authenticate"] as string | undefined, + })); + }, + ); + req.on("error", reject); + if (body) req.write(body); + req.end(); + }); +} + +describe("Gateway bearer auth", () => { + const port = 18451; + const token = "test-token-abc-123"; + let gateway: TdaiGateway; + let tmpDir: string; + + beforeAll(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-gateway-auth-")); + vi.stubEnv("TDAI_DATA_DIR", tmpDir); + vi.stubEnv("TDAI_GATEWAY_TOKEN", token); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + gateway = new TdaiGateway({ server: { port, host: "127.0.0.1" } } as never); + await gateway.start(); + }); + + beforeEach(() => { + vi.stubEnv("TDAI_GATEWAY_TOKEN", token); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + }); + + afterAll(async () => { + await gateway.stop(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("requires the bearer token when a token is configured", async () => { + const missing = await request({ port, path: "/health" }); + expect(missing.status).toBe(401); + expect(missing.wwwAuth).toMatch(/^Bearer\s+realm=/); + + const wrong = await request({ port, path: "/health", headers: { Authorization: "Bearer wrong" } }); + expect(wrong.status).toBe(401); + + const ok = await request({ port, path: "/health", headers: { Authorization: `Bearer ${token}` } }); + expect(ok.status).toBe(200); + }); + + it("requires bearer auth across all POST endpoints", async () => { + const endpoints = [ + "/recall", + "/capture", + "/search/memories", + "/search/conversations", + "/session/end", + "/seed", + ]; + + for (const path of endpoints) { + const missing = await request({ port, path, method: "POST", body: {} }); + expect(missing.status, `${path} missing token`).toBe(401); + + const wrong = await request({ + port, + path, + method: "POST", + headers: { Authorization: "Bearer wrong" }, + body: {}, + }); + expect(wrong.status, `${path} wrong token`).toBe(401); + + const authorized = await request({ + port, + path, + method: "POST", + headers: { Authorization: `Bearer ${token}` }, + body: {}, + }); + expect(authorized.status, `${path} valid token`).not.toBe(401); + } + }); + + it("accepts RFC 6750 bearer scheme case variants", async () => { + for (const scheme of ["Bearer", "bearer", "BEARER", "BeArEr"]) { + const result = await request({ + port, + path: "/health", + headers: { Authorization: `${scheme} ${token}` }, + }); + expect(result.status, scheme).toBe(200); + } + }); + + it("rejects malformed authorization headers", async () => { + const headers = [ + "Basic dGVzdA==", + "", + "Bearer", + "Bearer wrong", + `Bearer ${token} trailing`, + `prefix Bearer ${token}`, + `${token}`, + ]; + + for (const authorization of headers) { + const result = await request({ + port, + path: "/health", + headers: authorization ? { Authorization: authorization } : {}, + }); + expect(result.status, authorization || "(empty)").toBe(401); + expect(result.wwwAuth).toMatch(/^Bearer\s+realm=/); + } + }); + + it("rejects non-loopback CORS origins before route handling", async () => { + const result = await request({ + port, + path: "/seed", + method: "OPTIONS", + headers: { Origin: "https://example.invalid" }, + }); + expect(result.status).toBe(403); + }); +}); + +describe("Gateway loopback compatibility without a token", () => { + const port = 18452; + let gateway: TdaiGateway; + let tmpDir: string; + + beforeAll(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-gateway-auth-none-")); + vi.stubEnv("TDAI_DATA_DIR", tmpDir); + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + gateway = new TdaiGateway({ server: { port, host: "127.0.0.1" } } as never); + await gateway.start(); + }); + + beforeEach(() => { + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + }); + + afterAll(async () => { + await gateway.stop(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("preserves tokenless loopback health checks when no token is configured", async () => { + const result = await request({ port, path: "/health" }); + expect(result.status).toBe(200); + }); + + it("requires a token for tokenless loopback POST routes by default", async () => { + const result = await request({ + port, + path: "/search/memories", + method: "POST", + body: {}, + }); + expect(result.status).toBe(401); + expect(result.wwwAuth).toMatch(/^Bearer\s+realm=/); + }); +}); + +describe("Gateway explicit tokenless loopback development mode", () => { + const port = 18456; + let gateway: TdaiGateway; + let tmpDir: string; + + beforeAll(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-gateway-auth-disabled-")); + vi.stubEnv("TDAI_DATA_DIR", tmpDir); + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + vi.stubEnv("TDAI_GATEWAY_AUTH_DISABLED", "true"); + gateway = new TdaiGateway({ server: { port, host: "127.0.0.1" } } as never); + await gateway.start(); + }); + + beforeEach(() => { + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + vi.stubEnv("TDAI_GATEWAY_AUTH_DISABLED", "true"); + }); + + afterAll(async () => { + await gateway.stop(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("allows POST routes only when explicitly disabled on loopback", async () => { + const result = await request({ + port, + path: "/search/memories", + method: "POST", + body: {}, + }); + expect(result.status).toBe(400); + expect(result.body).toContain("Missing required field: query"); + }); +}); + +describe("Gateway token file safety", () => { + const port = 18453; + let gateway: TdaiGateway; + let tmpDir: string; + + beforeAll(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-gateway-auth-missing-")); + vi.stubEnv("TDAI_DATA_DIR", tmpDir); + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", path.join(tmpDir, "missing-token")); + gateway = new TdaiGateway({ server: { port, host: "127.0.0.1" } } as never); + await gateway.start(); + }); + + beforeEach(() => { + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", path.join(tmpDir, "missing-token")); + }); + + afterAll(async () => { + await gateway.stop(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("does not silently downgrade to tokenless mode when TDAI_TOKEN_PATH is configured but unreadable", async () => { + const result = await request({ port, path: "/health" }); + expect(result.status).toBe(401); + }); +}); + +describe("Gateway empty token file safety", () => { + const port = 18455; + let gateway: TdaiGateway; + let tmpDir: string; + let tokenPath: string; + + beforeAll(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-gateway-auth-empty-")); + tokenPath = path.join(tmpDir, "empty-token"); + fs.writeFileSync(tokenPath, "", { mode: 0o600 }); + vi.stubEnv("TDAI_DATA_DIR", tmpDir); + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", tokenPath); + gateway = new TdaiGateway({ server: { port, host: "127.0.0.1" } } as never); + await gateway.start(); + }); + + beforeEach(() => { + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", tokenPath); + }); + + afterAll(async () => { + await gateway.stop(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("does not silently downgrade to tokenless mode when TDAI_TOKEN_PATH is empty", async () => { + const result = await request({ port, path: "/health" }); + expect(result.status).toBe(401); + }); +}); + +describe("Gateway non-loopback tokenless safety", () => { + const port = 18454; + let gateway: TdaiGateway; + let tmpDir: string; + + beforeAll(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-gateway-auth-remote-")); + vi.stubEnv("TDAI_DATA_DIR", tmpDir); + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + vi.stubEnv("TDAI_GATEWAY_AUTH_DISABLED", "true"); + gateway = new TdaiGateway({ server: { port, host: "0.0.0.0" } } as never); + await gateway.start(); + }); + + beforeEach(() => { + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + vi.stubEnv("TDAI_GATEWAY_AUTH_DISABLED", "true"); + }); + + afterAll(async () => { + await gateway.stop(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("rejects tokenless non-loopback access even when auth-disabled is set", async () => { + const result = await request({ port, path: "/health" }); + expect(result.status).toBe(401); + }); +}); diff --git a/src/gateway/cli.ts b/src/gateway/cli.ts new file mode 100644 index 0000000..b8e98e8 --- /dev/null +++ b/src/gateway/cli.ts @@ -0,0 +1,107 @@ +#!/usr/bin/env node +/** + * Standalone Gateway daemon entry for host adapters. + * + * This bin keeps host plugins small: Codex/Claude-style plugins can spawn + * `tdai-memory-gateway` from the installed npm package instead of importing + * package dependencies from the copied plugin directory. + */ + +import { readFileSync, statSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { TdaiGateway } from "./server.js"; + +const LOOPBACK_HOSTS = new Set(["127.0.0.1", "localhost", "::1", "[::1]", "::ffff:127.0.0.1"]); + +function assertSafeHost(): void { + const host = process.env.TDAI_GATEWAY_HOST?.trim(); + if (!host || LOOPBACK_HOSTS.has(host)) return; + if (process.env.TDAI_GATEWAY_ALLOW_REMOTE === "1" || process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK === "true") { + return; + } + process.stderr.write( + `tdai-memory-gateway: refusing non-loopback TDAI_GATEWAY_HOST=${host}. ` + + "Set TDAI_GATEWAY_ALLOW_REMOTE=1 to opt in.\n", + ); + process.exit(2); +} + +function loadTokenFromFile(): void { + const tokenPath = expandHome(process.env.TDAI_TOKEN_PATH); + if (!tokenPath) return; + try { + const stat = statSync(tokenPath); + if (process.platform !== "win32" && (stat.mode & 0o077) !== 0) { + process.stderr.write(`tdai-memory-gateway: token file permissions are too loose: ${tokenPath}\n`); + process.exit(2); + } + if (process.platform !== "win32" && typeof process.getuid === "function" && stat.uid !== process.getuid()) { + process.stderr.write(`tdai-memory-gateway: token file owner mismatch: ${tokenPath}\n`); + process.exit(2); + } + const token = readFileSync(tokenPath, "utf-8").trim(); + if (!token) { + process.stderr.write(`tdai-memory-gateway: token file is empty: ${tokenPath}\n`); + process.exit(2); + } + // This mutates only Node's in-process env object, not the execve env block. + process.env.TDAI_GATEWAY_TOKEN = token; + } catch (err) { + process.stderr.write(`tdai-memory-gateway: failed to read TDAI_TOKEN_PATH=${tokenPath}: ${String(err)}\n`); + process.exit(2); + } +} + +function expandHome(value: string | undefined): string { + if (!value) return ""; + if (value === "~") return os.homedir(); + if (value.startsWith("~/")) return path.join(os.homedir(), value.slice(2)); + return value; +} + +async function main(): Promise { + assertSafeHost(); + loadTokenFromFile(); + + const gateway = new TdaiGateway(); + await gateway.start(); + + let shuttingDown = false; + const shutdown = async (reason: string): Promise => { + if (shuttingDown) return; + shuttingDown = true; + try { + await Promise.race([ + gateway.stop(), + new Promise((resolve) => setTimeout(resolve, 5_000)), + ]); + } catch { + // Best effort shutdown. + } + process.exit(reason === "error" ? 1 : 0); + }; + + process.on("SIGTERM", () => void shutdown("SIGTERM")); + process.on("SIGINT", () => void shutdown("SIGINT")); + + const parentPid = Number(process.env.TDAI_CODEX_PARENT_PID || process.env.TDAI_CC_PID || 0); + if (Number.isFinite(parentPid) && parentPid > 0) { + const timer = setInterval(() => { + try { + process.kill(parentPid, 0); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ESRCH") { + clearInterval(timer); + void shutdown("parent-exit"); + } + } + }, 15_000); + timer.unref(); + } +} + +main().catch((err) => { + process.stderr.write(`tdai-memory-gateway failed: ${String(err)}\n`); + process.exit(1); +}); diff --git a/src/gateway/server.ts b/src/gateway/server.ts index bd7d0a0..35f5c25 100644 --- a/src/gateway/server.ts +++ b/src/gateway/server.ts @@ -2,18 +2,21 @@ * TDAI Gateway — HTTP server for the Hermes sidecar. * * Exposes TDAI Core capabilities as HTTP endpoints: + * GET / — Service metadata for browser/local preview probes * GET /health — Health check * POST /recall — Memory recall (prefetch) * POST /capture — Conversation capture (sync_turn) * POST /search/memories — L1 memory search * POST /search/conversations — L0 conversation search * POST /session/end — Session end + flush - * POST /seed — Batch seed historical conversations (L0 → L1) + * POST /seed — Batch seed historical conversations (L0 → L1, optionally L2/L3) * * Built with Node.js native `http` module — no Express/Fastify dependency. * Designed to run as a managed sidecar alongside Hermes. */ +import crypto from "node:crypto"; +import fs from "node:fs"; import http from "node:http"; import { URL } from "node:url"; import { TdaiCore } from "../core/tdai-core.js"; @@ -23,6 +26,7 @@ import type { GatewayConfig } from "./config.js"; import { initDataDirectories } from "../utils/pipeline-factory.js"; import { SessionFilter } from "../utils/session-filter.js"; import type { + RootResponse, HealthResponse, RecallRequest, RecallResponse, @@ -173,10 +177,7 @@ export class TdaiGateway { const method = req.method?.toUpperCase() ?? "GET"; const pathname = url.pathname; - // CORS headers (for development) - res.setHeader("Access-Control-Allow-Origin", "*"); - res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS"); - res.setHeader("Access-Control-Allow-Headers", "Content-Type"); + if (!this.applyCors(req, res)) return; if (method === "OPTIONS") { res.writeHead(204); @@ -184,8 +185,12 @@ export class TdaiGateway { return; } + if (!this.authorizeRequest(req, res, method)) return; + try { switch (`${method} ${pathname}`) { + case "GET /": + return this.handleRoot(res); case "GET /health": return this.handleHealth(res); case "POST /recall": @@ -214,6 +219,73 @@ export class TdaiGateway { // Route handlers // ============================ + private applyCors(req: http.IncomingMessage, res: http.ServerResponse): boolean { + res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS"); + res.setHeader("Access-Control-Allow-Headers", "Content-Type, Authorization"); + + const origin = String(req.headers.origin ?? ""); + if (!origin) return true; + if (!isAllowedCorsOrigin(origin)) { + sendError(res, 403, "CORS origin not allowed"); + return false; + } + + res.setHeader("Access-Control-Allow-Origin", origin); + res.setHeader("Vary", "Origin"); + return true; + } + + private authorizeRequest(req: http.IncomingMessage, res: http.ServerResponse, method: string): boolean { + const token = expectedGatewayToken(); + + if (!token) { + if (!isLoopbackHost(this.config.server.host)) { + sendError(res, 401, "Unauthorized: Gateway token is required for non-loopback routes"); + return false; + } + + if (method === "GET") { + return true; + } + + if (process.env.TDAI_GATEWAY_AUTH_DISABLED === "true") { + return true; + } + + res.setHeader("WWW-Authenticate", 'Bearer realm="tdai-gateway"'); + sendError(res, 401, "Unauthorized: Gateway token is required for POST routes; set TDAI_GATEWAY_AUTH_DISABLED=true only for trusted loopback development"); + return false; + } + + const authorization = String(req.headers.authorization ?? ""); + const match = authorization.match(/^Bearer\s+(\S+)\s*$/i); + if (!match || !safeTokenEqual(match[1], token)) { + res.setHeader("WWW-Authenticate", 'Bearer realm="tdai-gateway"'); + sendError(res, 401, "Unauthorized"); + return false; + } + return true; + } + + private handleRoot(res: http.ServerResponse): void { + const response: RootResponse = { + service: "TencentDB Agent Memory Gateway", + kind: "api", + version: VERSION, + message: "This local service is an API sidecar, not a web UI. Use GET /health for readiness.", + endpoints: [ + { method: "GET", path: "/health", description: "Gateway readiness and store status" }, + { method: "POST", path: "/recall", description: "Memory recall for a session query" }, + { method: "POST", path: "/capture", description: "Conversation turn capture" }, + { method: "POST", path: "/search/memories", description: "Structured memory search" }, + { method: "POST", path: "/search/conversations", description: "Raw conversation search" }, + { method: "POST", path: "/session/end", description: "Session flush" }, + { method: "POST", path: "/seed", description: "Batch seed historical conversations" }, + ], + }; + sendJson(res, 200, response); + } + private handleHealth(res: http.ServerResponse): void { const response: HealthResponse = { status: this.core.getVectorStore() ? "ok" : "degraded", @@ -267,6 +339,7 @@ export class TdaiGateway { ], sessionKey: body.session_key, sessionId: body.session_id, + startedAt: typeof body.started_at === "number" ? body.started_at : undefined, }); const elapsed = Date.now() - startMs; @@ -292,6 +365,7 @@ export class TdaiGateway { limit: body.limit, type: body.type, scene: body.scene, + sessionKeyPrefixes: body.session_key_prefixes, }); const response: MemorySearchResponse = { @@ -314,6 +388,7 @@ export class TdaiGateway { query: body.query, limit: body.limit, sessionKey: body.session_key, + sessionKeyPrefixes: body.session_key_prefixes, }); const response: ConversationSearchResponse = { @@ -366,7 +441,8 @@ export class TdaiGateway { this.logger.info( `Seed request: ${input.sessions.length} session(s), ` + - `${input.totalRounds} round(s), ${input.totalMessages} message(s)`, + `${input.totalRounds} round(s), ${input.totalMessages} message(s), ` + + `waitFullPipeline=${body.wait_for_full_pipeline === true}`, ); // Resolve output directory: use gateway's data dir with a timestamped subfolder @@ -392,6 +468,15 @@ export class TdaiGateway { }, }; if (body.config_override) { + const blockedOverridePaths = findBlockedConfigOverridePaths(body.config_override); + if (blockedOverridePaths.length > 0) { + sendJson(res, 400, { + error: "config_override contains blocked credential-bearing or network-routing keys", + blocked_paths: blockedOverridePaths, + }); + return; + } + for (const key of Object.keys(body.config_override)) { const baseVal = pluginConfig[key]; const overVal = body.config_override[key]; @@ -409,6 +494,10 @@ export class TdaiGateway { outputDir, openclawConfig: {}, pluginConfig, + waitForFullPipeline: body.wait_for_full_pipeline === true, + fullPipelineFlushTimeoutMs: typeof body.full_pipeline_timeout_ms === "number" + ? body.full_pipeline_timeout_ms + : undefined, logger: this.logger as import("../utils/pipeline-factory.js").PipelineLogger, onProgress: (progress: SeedProgress) => { this.logger.debug?.( @@ -428,6 +517,7 @@ export class TdaiGateway { rounds_processed: summary.roundsProcessed, messages_processed: summary.messagesProcessed, l0_recorded: summary.l0RecordedCount, + full_pipeline_flushed: summary.fullPipelineFlushed, duration_ms: summary.durationMs, output_dir: summary.outputDir, }; @@ -435,6 +525,79 @@ export class TdaiGateway { } } +function expectedGatewayToken(): string { + const direct = process.env.TDAI_GATEWAY_TOKEN?.trim(); + if (direct) return direct; + + const tokenPath = process.env.TDAI_TOKEN_PATH?.trim(); + if (!tokenPath) return ""; + try { + const stat = fs.statSync(tokenPath); + if (process.platform !== "win32" && (stat.mode & 0o077) !== 0) return "\0"; + if (process.platform !== "win32" && typeof process.getuid === "function" && stat.uid !== process.getuid()) { + return "\0"; + } + return fs.readFileSync(tokenPath, "utf-8").trim() || "\0"; + } catch { + return "\0"; + } +} + +function isAllowedCorsOrigin(origin: string): boolean { + const explicit = process.env.TDAI_GATEWAY_CORS_ORIGINS?.trim(); + if (explicit) { + const allowed = explicit.split(",").map((item) => item.trim()).filter(Boolean); + if (allowed.includes("*")) return true; + return allowed.includes(origin); + } + + try { + const url = new URL(origin); + return isLoopbackHost(url.hostname); + } catch { + return false; + } +} + +function isLoopbackHost(host: string): boolean { + const normalized = String(host || "").trim().toLowerCase(); + return normalized === "localhost" || + normalized === "127.0.0.1" || + normalized === "::1" || + normalized === "[::1]"; +} + +function findBlockedConfigOverridePaths(value: unknown, prefix = ""): string[] { + if (!value || typeof value !== "object" || Array.isArray(value)) return []; + const blocked: string[] = []; + for (const [key, child] of Object.entries(value as Record)) { + const current = prefix ? `${prefix}.${key}` : key; + if (isBlockedConfigOverridePath(current)) { + blocked.push(current); + continue; + } + blocked.push(...findBlockedConfigOverridePaths(child, current)); + } + return blocked; +} + +function isBlockedConfigOverridePath(path: string): boolean { + const parts = path.toLowerCase().split("."); + const leaf = parts.at(-1) || ""; + if (parts[0] === "tcvdb") return true; + if (parts[0] === "embedding") return true; + if (parts[0] === "llm" && ["apikey", "baseurl", "enabled"].includes(leaf)) return true; + if (parts[0] === "offload" && ["backendurl", "backendapikey", "mode"].includes(leaf)) return true; + return /(?:apikey|secret|token|password|authorization|credential|baseurl|backendurl|proxyurl)$/.test(leaf); +} + +function safeTokenEqual(actual: string, expected: string): boolean { + const actualBuffer = Buffer.from(actual); + const expectedBuffer = Buffer.from(expected); + if (actualBuffer.length !== expectedBuffer.length) return false; + return crypto.timingSafeEqual(actualBuffer, expectedBuffer); +} + // ============================ // CLI entry point // ============================ diff --git a/src/gateway/types.ts b/src/gateway/types.ts index 50b2ff4..13fd50f 100644 --- a/src/gateway/types.ts +++ b/src/gateway/types.ts @@ -11,6 +11,22 @@ export interface GatewayErrorResponse { code?: string; } +// ============================ +// / +// ============================ + +export interface RootResponse { + service: "TencentDB Agent Memory Gateway"; + kind: "api"; + version: string; + message: string; + endpoints: { + method: "GET" | "POST"; + path: string; + description: string; + }[]; +} + // ============================ // /health // ============================ @@ -50,6 +66,8 @@ export interface CaptureRequest { assistant_content: string; session_key: string; session_id?: string; + /** Epoch ms when the captured turn began. Hosts that reconstruct turns out-of-process should set this. */ + started_at?: number; user_id?: string; messages?: unknown[]; } @@ -68,6 +86,8 @@ export interface MemorySearchRequest { limit?: number; type?: string; scene?: string; + /** Optional session-key prefixes for host/project scoped search. */ + session_key_prefixes?: string[]; } export interface MemorySearchResponse { @@ -84,6 +104,8 @@ export interface ConversationSearchRequest { query: string; limit?: number; session_key?: string; + /** Optional session-key prefixes for host/project scoped search. */ + session_key_prefixes?: string[]; } export interface ConversationSearchResponse { @@ -129,6 +151,10 @@ export interface SeedRequest { strict_round_role?: boolean; /** Auto-fill missing timestamps (default: true). */ auto_fill_timestamps?: boolean; + /** Wait for final L1→L2→L3 processing before returning (default: false). */ + wait_for_full_pipeline?: boolean; + /** Max wait time for final L1→L2→L3 processing. */ + full_pipeline_timeout_ms?: number; /** Plugin config overrides (deep-merged on top of gateway memory config). */ config_override?: Record; } @@ -138,6 +164,7 @@ export interface SeedResponse { rounds_processed: number; messages_processed: number; l0_recorded: number; + full_pipeline_flushed?: boolean; duration_ms: number; output_dir: string; } diff --git a/src/offload/backend-client.ts b/src/offload/backend-client.ts index 30dc622..9daf096 100644 --- a/src/offload/backend-client.ts +++ b/src/offload/backend-client.ts @@ -296,6 +296,10 @@ export class BackendClient { const parsed = new URL(url); const isHttps = parsed.protocol === "https:"; const transport = isHttps ? https : http; + const allowInsecureTls = process.env.TDAI_OFFLOAD_INSECURE_TLS === "true"; + if (isHttps && allowInsecureTls) { + this.logger.warn("[context-offload] TDAI_OFFLOAD_INSECURE_TLS=true disables backend TLS certificate verification"); + } return new Promise((resolve, reject) => { const timer = setTimeout(() => { @@ -309,7 +313,7 @@ export class BackendClient { path: parsed.pathname + parsed.search, method: "POST", headers: reqHeaders, - ...(isHttps ? { rejectUnauthorized: false } : {}), + ...(isHttps && allowInsecureTls ? { rejectUnauthorized: false } : {}), }, (res) => { let data = ""; diff --git a/src/utils/pipeline-manager.ts b/src/utils/pipeline-manager.ts index b56234c..d8d2d75 100644 --- a/src/utils/pipeline-manager.ts +++ b/src/utils/pipeline-manager.ts @@ -146,6 +146,40 @@ export interface PipelineConfig { }; } +export interface PipelineQueueSizes { + l1: number; + l2: number; + l3: number; + l1Pending: boolean; + l2Pending: boolean; + l3Pending: boolean; + l1Idle: boolean; + l2Idle: boolean; + l3Idle: boolean; +} + +export interface PipelineFlushOptions { + /** Human-readable reason for diagnostics. */ + reason?: string; + /** Maximum time to wait before rejecting. Omit or set to 0 for no timeout. */ + timeoutMs?: number; + /** Poll interval for the final stability check. */ + pollIntervalMs?: number; + /** Number of consecutive idle polls required before returning. */ + stableRounds?: number; + /** + * Whether L2 should arm the follow-up max-interval timer after this flush. + * Seed/import callers should set this false because they are about to tear + * down the pipeline and do not need a periodic L2 timer. + */ + armFollowUpL2Timers?: boolean; +} + +export interface PipelineFlushResult { + durationMs: number; + queueSizes: PipelineQueueSizes; +} + /** Result returned by the L1 runner. */ export interface L1RunnerResult { /** Number of messages successfully processed */ @@ -216,6 +250,7 @@ export class MemoryPipelineManager { // L3 dedup flag private l3Pending = false; private l3Running = false; + private suppressL2MaxInterval = false; // Per-session state private readonly sessionStates = new Map(); @@ -502,6 +537,48 @@ export class MemoryPipelineManager { this.logger?.debug?.(`${TAG} [${sessionKey}] flushSession: complete`); } + /** + * Flush immediate pipeline work without destroying the scheduler. + * + * This is stronger than {@link flushSession}: it drains pending L1 work, + * flushes currently scheduled L2 timers, waits for L2, then waits for the + * L3 persona runner triggered by L2. It intentionally ignores future + * max-interval L2 timers, which are periodic maintenance rather than work + * created by the current seed/import batch. + */ + async flushPendingWork(options: PipelineFlushOptions = {}): Promise { + const start = Date.now(); + const reason = options.reason ?? "manual"; + + this.logger?.info(`${TAG} Flushing pending work (${reason})...`); + + let timeoutId: ReturnType | undefined; + const flushPromise = this._doFlush(options); + + if (options.timeoutMs && options.timeoutMs > 0) { + await Promise.race([ + flushPromise, + new Promise((_, reject) => { + timeoutId = setTimeout( + () => reject(new Error(`flush timeout after ${options.timeoutMs}ms`)), + options.timeoutMs, + ); + }), + ]).finally(() => { + if (timeoutId !== undefined) clearTimeout(timeoutId); + }); + } else { + await flushPromise; + } + + const result: PipelineFlushResult = { + durationMs: Date.now() - start, + queueSizes: this.getQueueSizes(), + }; + this.logger?.info(`${TAG} Pending work flushed (${reason}) in ${result.durationMs}ms`); + return result; + } + /** * Maximum time (ms) to wait for pipeline flush during destroy. * Must be shorter than the gateway_stop hook timeout (3 s) to leave @@ -560,37 +637,77 @@ export class MemoryPipelineManager { * Internal: attempt to flush all pending pipeline work (L1 → L2 → L3). * Extracted from destroy() so it can be wrapped with a timeout. */ - private async _doFlush(): Promise { - // Step 1: Flush all L1 idle timers — only enqueue if there are buffered messages - for (const [sessionKey, timers] of this.sessionTimers) { - if (timers.l1Idle.pending) { - timers.l1Idle.cancel(); // don't fire the idle callback directly - const buffer = this.messageBuffers.get(sessionKey); - if (buffer && buffer.length > 0) { - this.logger?.debug?.(`${TAG} [${sessionKey}] Flush: enqueuing L1 for ${buffer.length} buffered messages`); - this.enqueueL1(sessionKey, "flush"); + private async _doFlush(options: PipelineFlushOptions = {}): Promise { + const previousSuppressL2MaxInterval = this.suppressL2MaxInterval; + if (options.armFollowUpL2Timers === false) { + this.suppressL2MaxInterval = true; + } + + try { + // Step 1: Flush all L1 idle timers — only enqueue if there are buffered messages + for (const [sessionKey, timers] of this.sessionTimers) { + if (timers.l1Idle.pending) { + timers.l1Idle.cancel(); // don't fire the idle callback directly + const buffer = this.messageBuffers.get(sessionKey); + if (buffer && buffer.length > 0) { + this.logger?.debug?.(`${TAG} [${sessionKey}] Flush: enqueuing L1 for ${buffer.length} buffered messages`); + this.enqueueL1(sessionKey, "flush"); + } } } - } - // Step 2: Wait for L1 queue to drain - this.logger?.debug?.(`${TAG} Waiting for L1 queue to drain (size=${this.l1Queue.size})`); - await this.l1Queue.onIdle(); + // Step 2: Wait for L1 queue to drain + this.logger?.debug?.(`${TAG} Waiting for L1 queue to drain (size=${this.l1Queue.size})`); + await this.l1Queue.onIdle(); - // Step 3: Flush all L2 schedule timers - for (const [sessionKey, timers] of this.sessionTimers) { - if (timers.l2Schedule.pending) { - this.logger?.debug?.(`${TAG} [${sessionKey}] Flush: triggering L2 schedule timer`); - timers.l2Schedule.flush(); + // Step 3: Flush all L2 schedule timers + for (const [sessionKey, timers] of this.sessionTimers) { + if (timers.l2Schedule.pending) { + this.logger?.debug?.(`${TAG} [${sessionKey}] Flush: triggering L2 schedule timer`); + timers.l2Schedule.flush(); + } } + + // Step 4: Wait for all remaining queues to drain + this.logger?.debug?.(`${TAG} Waiting for queues to drain (l2=${this.l2Queue.size}, l3=${this.l3Queue.size})`); + await this.l2Queue.onIdle(); + await this.l3Queue.onIdle(); + + // L3 is enqueued by L2 completion. Depending on microtask ordering, an + // early l3Queue.onIdle() observer can miss a just-enqueued follow-up run. + // Require a short stable idle window that also checks the L3 dedupe flags. + await this.waitForImmediateQueuesIdle({ + pollIntervalMs: options.pollIntervalMs, + stableRounds: options.stableRounds, + }); + } finally { + this.suppressL2MaxInterval = previousSuppressL2MaxInterval; } + } + + private async waitForImmediateQueuesIdle(options: Pick = {}): Promise { + const pollIntervalMs = options.pollIntervalMs ?? 50; + const stableRounds = options.stableRounds ?? 2; + let consecutiveIdle = 0; + + while (true) { + const queues = this.getQueueSizes(); + const idle = + queues.l1Idle && + queues.l2Idle && + queues.l3Idle && + !this.l3Running && + !this.l3Pending; + + if (idle) { + consecutiveIdle += 1; + if (consecutiveIdle >= stableRounds) return; + } else { + consecutiveIdle = 0; + } - // Step 4: Wait for all remaining queues to drain - this.logger?.debug?.(`${TAG} Waiting for queues to drain (l2=${this.l2Queue.size}, l3=${this.l3Queue.size})`); - await Promise.all([ - this.l2Queue.onIdle(), - this.l3Queue.onIdle(), - ]); + await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)); + } } // ============================ @@ -915,8 +1032,11 @@ export class MemoryPipelineManager { this.logger?.debug?.(`${TAG} [${sessionKey}] L2 complete`); - // Arm the maxInterval timer for the next cycle - this.armL2MaxInterval(sessionKey); + // Arm the maxInterval timer for the next cycle unless this L2 was forced + // by a one-shot seed/import flush that is about to tear the pipeline down. + if (!this.suppressL2MaxInterval) { + this.armL2MaxInterval(sessionKey); + } // Trigger L3 this.triggerL3(); @@ -1139,11 +1259,7 @@ export class MemoryPipelineManager { } /** Queue sizes and running state for monitoring. */ - getQueueSizes(): { - l1: number; l2: number; l3: number; - l1Pending: boolean; l2Pending: boolean; l3Pending: boolean; - l1Idle: boolean; l2Idle: boolean; l3Idle: boolean; - } { + getQueueSizes(): PipelineQueueSizes { return { l1: this.l1Queue.size, l2: this.l2Queue.size, diff --git a/src/utils/sanitize.ts b/src/utils/sanitize.ts index 80ee636..3e7b55f 100644 --- a/src/utils/sanitize.ts +++ b/src/utils/sanitize.ts @@ -17,6 +17,18 @@ export function sanitizeText(text: string): string { cleaned = cleaned.replace(/[\s\S]*?<\/user-persona>/g, ""); cleaned = cleaned.replace(/[\s\S]*?<\/relevant-scenes>/g, ""); cleaned = cleaned.replace(/[\s\S]*?<\/scene-navigation>/g, ""); + cleaned = cleaned.replace(//g, ""); + + // Remove Codex adapter injected context before L0/L1 persistence. Codex does + // not expose an OpenClaw-style before_message_write rewrite hook, so the + // Gateway-side sanitizer is the final guard against recall feedback loops. + cleaned = cleaned.replace(//g, ""); + cleaned = cleaned.replace(//g, ""); + cleaned = cleaned.replace(//g, ""); + cleaned = cleaned.replace(//g, ""); + cleaned = cleaned.replace(//g, ""); + cleaned = cleaned.replace(//g, ""); + cleaned = cleaned.replace(//g, ""); // Remove offload-injected task context blocks (MMD mermaid diagrams) cleaned = cleaned.replace(/[\s\S]*?<\/current_task_context>/g, ""); diff --git a/tsdown.config.ts b/tsdown.config.ts index 16b0073..89b0e06 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -11,7 +11,7 @@ function collectExternalDependencies(): string[] { } export default defineConfig({ - entry: ["./index.ts"], + entry: ["./index.ts", "./src/gateway/cli.ts"], outDir: "./dist", format: "esm", platform: "node", diff --git a/vitest.config.ts b/vitest.config.ts index 1f9ce29..3f5446c 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,7 +4,7 @@ export default defineConfig({ test: { environment: "node", pool: "forks", - include: ["src/**/*.test.ts", "__tests__/**/*.test.ts"], + include: ["src/**/*.test.ts", "__tests__/**/*.test.ts", "codex-plugin/**/*.test.mjs"], exclude: ["dist/**", "node_modules/**", "**/*.e2e.test.ts"], testTimeout: 120_000, hookTimeout: 120_000, From a5590109793a3a588c64369b4e3abd6854169ce3 Mon Sep 17 00:00:00 2001 From: Siyao Zheng Date: Wed, 20 May 2026 12:06:27 +0800 Subject: [PATCH 2/5] fix(codex): harden import pipeline for review Signed-off-by: Siyao Zheng --- codex-plugin/README.md | 45 +- codex-plugin/adapter-profile.json | 62 +++ codex-plugin/scripts/codex-security.test.mjs | 17 + codex-plugin/scripts/doctor.mjs | 266 +++++++++++ codex-plugin/scripts/import-codex-history.mjs | 53 ++- codex-plugin/scripts/lib.mjs | 49 ++ codex-plugin/scripts/query.mjs | 12 +- codex-plugin/tdai-gateway.example.json | 3 +- index.ts | 2 +- openclaw.plugin.json | 1 + src/adapters/standalone/llm-runner.ts | 15 +- src/cli/README.md | 6 +- src/cli/commands/seed.ts | 16 + src/config.ts | 3 + src/core/hooks/auto-capture.ts | 19 +- src/core/prompts/l1-extraction.ts | 11 +- src/core/record/l1-dedup.ts | 72 ++- src/core/record/l1-extractor.ts | 70 ++- src/core/seed/input.test.ts | 34 ++ src/core/seed/input.ts | 12 + src/core/seed/seed-runtime.ts | 439 +++++++++++++++--- src/core/seed/types.ts | 10 + src/gateway/server.ts | 15 +- src/gateway/types.ts | 11 + src/utils/pipeline-factory.ts | 19 +- src/utils/pipeline-manager.test.ts | 151 ++++++ src/utils/pipeline-manager.ts | 114 ++++- src/utils/serial-queue.test.ts | 26 ++ src/utils/serial-queue.ts | 83 ++-- 29 files changed, 1465 insertions(+), 171 deletions(-) create mode 100644 codex-plugin/adapter-profile.json create mode 100644 codex-plugin/scripts/doctor.mjs create mode 100644 src/core/seed/input.test.ts create mode 100644 src/utils/pipeline-manager.test.ts create mode 100644 src/utils/serial-queue.test.ts diff --git a/codex-plugin/README.md b/codex-plugin/README.md index c87f69a..1f094be 100644 --- a/codex-plugin/README.md +++ b/codex-plugin/README.md @@ -95,9 +95,11 @@ The plugin manifest is: codex-plugin/.codex-plugin/plugin.json ``` -It declares the Codex skill, bundled hook config, and bundled MCP server config: +It declares the Codex skill, bundled hook config, and bundled MCP server config. +The adapter also ships a machine-readable reuse contract: ```text +codex-plugin/adapter-profile.json codex-plugin/hooks/hooks.codex.json codex-plugin/.mcp.json ``` @@ -110,6 +112,40 @@ environment variables. The bundled MCP config exposes memory search and offload lookup tools; the manual `codex mcp add` command below is a fallback for local development or older Codex builds. +## Reuse Contract + +The adapter is intended to be installable from a copied plugin directory, a +Codex plugin cache, a package release, or a forked source checkout without +editing script files. The stable contract is: + +- `adapter-profile.json` describes the adapter ID, host, entrypoints, + runtime requirements, environment variables, and extension points. +- Hook and MCP configs refer to the plugin root through Codex-provided root + variables instead of machine-specific absolute paths. +- Per-user state lives under `TDAI_CODEX_DATA_DIR` or the default + `~/.memory-tencentdb/codex-memory-tdai`; copied adapters do not share state + unless that directory is explicitly shared. +- Gateway autostart uses the package binary by default, so a copied adapter can + run without importing dependencies from the plugin directory. +- Source-tree development is still supported by setting `TDAI_CODEX_TDAI_ROOT` + to a local checkout. +- Fork, release, and tarball validation can override + `TDAI_CODEX_GATEWAY_PACKAGE` without changing the adapter scripts. + +Run the doctor before publishing, copying, or handing the adapter to another +Codex environment: + +```bash +node codex-plugin/scripts/doctor.mjs +node codex-plugin/scripts/doctor.mjs --start --require-healthy --strict +node codex-plugin/scripts/query.mjs doctor --json +``` + +The doctor checks that manifest entrypoints exist, hook/MCP configs are +portable, adapter state is writable with private adapter-owned subdirectories, +the Gateway URL is loopback unless explicitly allowed, and the Gateway can be +launched from either a source checkout or package binary. + ## Setup From the TencentDB-Agent-Memory repository root: @@ -264,7 +300,12 @@ the generated `/seed` payload before writing. By default, a real import requests `wait_for_full_pipeline`, so Gateway `/seed` records L0, waits for L1, flushes L2 scene extraction, and waits for L3 persona generation before returning. Use `--no-full-pipeline` when the faster L0/L1-only -seed behavior is preferred. +seed behavior is preferred. For large trusted local imports, `--l1-concurrency` +or `TDAI_CODEX_IMPORT_L1_CONCURRENCY` can raise bounded L1 extraction +parallelism without changing the live host default. The importer also sends +`l2_batch_size` by default, which lets Gateway coalesce many short historical +Codex sessions into larger L2 scene-extraction batches while keeping live +runtime L2 scheduling unchanged. ## Short-Term Context Offload diff --git a/codex-plugin/adapter-profile.json b/codex-plugin/adapter-profile.json new file mode 100644 index 0000000..3b83ee6 --- /dev/null +++ b/codex-plugin/adapter-profile.json @@ -0,0 +1,62 @@ +{ + "schemaVersion": 1, + "adapterId": "memory-tencentdb-codex", + "displayName": "TencentDB Agent Memory Codex Adapter", + "host": "codex", + "packageName": "@tencentdb-agent-memory/memory-tencentdb", + "entrypoints": { + "pluginManifest": ".codex-plugin/plugin.json", + "hooks": "hooks/hooks.codex.json", + "mcp": ".mcp.json", + "skill": "skills/tdai-memory/SKILL.md", + "doctor": "scripts/doctor.mjs", + "gateway": "scripts/gateway.mjs", + "query": "scripts/query.mjs" + }, + "requiredRuntime": { + "node": ">=22.16.0", + "gatewayDefaultUrl": "http://127.0.0.1:8420" + }, + "installContract": { + "rootEnv": [ + "PLUGIN_ROOT", + "CLAUDE_PLUGIN_ROOT" + ], + "dataDirEnv": [ + "TDAI_CODEX_DATA_DIR", + "TDAI_DATA_DIR" + ], + "gatewayRootEnv": [ + "TDAI_CODEX_TDAI_ROOT", + "TDAI_INSTALL_DIR" + ], + "gatewayPackageEnv": "TDAI_CODEX_GATEWAY_PACKAGE", + "gatewayUrlEnv": "TDAI_CODEX_GATEWAY_URL", + "tokenEnv": [ + "TDAI_CODEX_GATEWAY_TOKEN", + "TDAI_GATEWAY_TOKEN", + "TDAI_TOKEN_PATH" + ] + }, + "reuseContract": { + "noRepositoryAbsolutePathsRequired": true, + "safeDefaultGatewayScope": "loopback-only", + "authDefault": "private bearer token generated under the adapter data directory", + "stateIsolation": "per-user data lives under TDAI_CODEX_DATA_DIR or ~/.memory-tencentdb/codex-memory-tdai", + "sourceCheckoutOptional": "package-bin autostart is the default; TDAI_CODEX_TDAI_ROOT is only for source-tree development", + "extensionPoints": [ + "override TDAI_CODEX_GATEWAY_PACKAGE to validate a fork, version, or tarball", + "override TDAI_CODEX_TDAI_ROOT to run the Gateway from a local source checkout", + "override TDAI_CODEX_DATA_DIR to isolate state for another user, test, or product", + "extend hooks/hooks.codex.json with host-specific hooks while keeping shared scripts intact", + "extend .mcp.json with additional MCP tools while preserving tdai-memory tools" + ] + }, + "diagnosticChecks": [ + "manifest files exist relative to the plugin root", + "hook and MCP configs do not require machine-specific absolute paths", + "data directory is writable and private when created", + "Gateway URL is loopback unless explicitly overridden", + "Gateway can be launched from package-bin or an explicit source checkout" + ] +} diff --git a/codex-plugin/scripts/codex-security.test.mjs b/codex-plugin/scripts/codex-security.test.mjs index cdeb94d..7259b68 100644 --- a/codex-plugin/scripts/codex-security.test.mjs +++ b/codex-plugin/scripts/codex-security.test.mjs @@ -17,6 +17,7 @@ import { sanitizeMemoryText, sessionKeyFromPayload, } from "./lib.mjs"; +import { buildAdapterDoctorReport } from "./doctor.mjs"; import { lookupCodexOffload, recordCodexToolOffload, @@ -336,6 +337,22 @@ ${privateKeyBlock} await expect(httpPost("/capture", { user_content: "secret" })).resolves.toBeNull(); expect(fetchMock).not.toHaveBeenCalled(); }); + + it("ships a portable adapter profile and doctor report for reuse", async () => { + process.env.TDAI_CODEX_AUTOSTART = "false"; + process.env.TDAI_CODEX_GATEWAY_URL = "http://127.0.0.1:9"; + + const report = await buildAdapterDoctorReport({ + pluginRoot: path.resolve("codex-plugin"), + }); + + expect(report.adapter.id).toBe("memory-tencentdb-codex"); + expect(report.checks.find((check) => check.name === "entrypoint:pluginManifest")?.ok).toBe(true); + expect(report.checks.find((check) => check.name === "entrypoint:doctor")?.ok).toBe(true); + expect(report.checks.find((check) => check.name === "portable:hooks.codex.json")?.ok).toBe(true); + expect(report.checks.find((check) => check.name === "portable:.mcp.json")?.ok).toBe(true); + expect(report.ok).toBe(true); + }); }); function offloadParams(cwd, sessionId, toolUseId) { diff --git a/codex-plugin/scripts/doctor.mjs b/codex-plugin/scripts/doctor.mjs new file mode 100644 index 0000000..71aa892 --- /dev/null +++ b/codex-plugin/scripts/doctor.mjs @@ -0,0 +1,266 @@ +#!/usr/bin/env node +import fs from "node:fs/promises"; +import fsSync from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { + configuredGatewayTokenPath, + ensureGateway, + expandHome, + gatewayUrl, + healthCheck, + pluginRoot, + resolveTdaiRoot, + tdaiDataDir, +} from "./lib.mjs"; + +const DEFAULT_PROFILE = "adapter-profile.json"; + +export async function buildAdapterDoctorReport(options = {}) { + const root = options.pluginRoot ? path.resolve(options.pluginRoot) : pluginRoot(); + const profilePath = path.resolve(root, options.profilePath || DEFAULT_PROFILE); + const profile = await readJson(profilePath); + const checks = []; + + addCheck(checks, "profile", Boolean(profile?.adapterId), { + adapterId: profile?.adapterId || null, + profilePath, + }); + + const entrypointChecks = await checkEntrypoints(root, profile); + checks.push(...entrypointChecks); + + const hookConfigPath = path.join(root, profile.entrypoints?.hooks || "hooks/hooks.codex.json"); + const mcpConfigPath = path.join(root, profile.entrypoints?.mcp || ".mcp.json"); + const portability = await checkPortableConfigs([hookConfigPath, mcpConfigPath], root); + checks.push(...portability); + + const dataDir = tdaiDataDir(); + const dataDirCheck = await checkDataDir(dataDir); + checks.push(dataDirCheck); + + const url = gatewayUrl(); + const loopback = isLoopbackGatewayUrl(url); + addCheck(checks, "gateway_loopback", loopback || process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK === "true", { + gatewayUrl: url, + allowNonLoopback: process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK === "true", + }); + + const tdaiRoot = resolveTdaiRoot(); + const sourceRootUsable = tdaiRoot ? await hasSourceGateway(tdaiRoot) : false; + const packageSpec = process.env.TDAI_CODEX_GATEWAY_PACKAGE || profile.packageName || "@tencentdb-agent-memory/memory-tencentdb"; + addCheck(checks, "gateway_launch_source_or_package", Boolean(sourceRootUsable || packageSpec), { + mode: sourceRootUsable ? "source-tree" : "package-bin", + tdaiRoot, + packageSpec, + }); + + const tokenPath = configuredGatewayTokenPath(); + addCheck(checks, "token_path_configured", Boolean(tokenPath), { + tokenPath, + tokenPathPrivateParent: await privateParentStatus(tokenPath), + }); + + const healthyBeforeStart = await healthCheck(); + let healthyAfterStart = healthyBeforeStart; + if (!healthyBeforeStart && options.start === true) { + healthyAfterStart = await ensureGateway(); + } + addCheck(checks, options.start ? "gateway_health_after_start" : "gateway_health", options.requireHealthy ? healthyAfterStart : true, { + healthy: healthyAfterStart, + attemptedStart: options.start === true, + }); + + const ok = checks.every((check) => check.ok); + return { + ok, + adapter: { + id: profile?.adapterId || "unknown", + displayName: profile?.displayName || "unknown", + host: profile?.host || "codex", + profilePath, + pluginRoot: root, + }, + runtime: { + node: process.version, + platform: `${os.platform()}-${os.arch()}`, + gatewayUrl: url, + dataDir, + tdaiRoot, + gatewayPackage: packageSpec, + }, + checks, + next: ok + ? "Codex adapter contract looks portable. Installers can reuse this plugin root and override env vars without editing scripts." + : "Fix failed checks before publishing or handing this adapter to another Codex environment.", + }; +} + +async function checkEntrypoints(root, profile) { + const checks = []; + const entrypoints = profile?.entrypoints || {}; + for (const [name, relPath] of Object.entries(entrypoints)) { + const filePath = path.resolve(root, relPath); + addCheck(checks, `entrypoint:${name}`, fsSync.existsSync(filePath), { + path: filePath, + relativePath: relPath, + }); + } + return checks; +} + +async function checkPortableConfigs(configPaths, root) { + const checks = []; + for (const configPath of configPaths) { + let text = ""; + try { + text = await fs.readFile(configPath, "utf-8"); + } catch (err) { + addCheck(checks, `portable:${path.basename(configPath)}`, false, { + path: configPath, + error: err instanceof Error ? err.message : String(err), + }); + continue; + } + + const absolutePaths = findSuspiciousAbsolutePaths(text) + .filter((value) => !value.startsWith(root)); + addCheck(checks, `portable:${path.basename(configPath)}`, absolutePaths.length === 0, { + path: configPath, + suspiciousAbsolutePaths: absolutePaths, + usesPluginRootVariable: text.includes("${PLUGIN_ROOT}") || text.includes("${CLAUDE_PLUGIN_ROOT}"), + }); + } + return checks; +} + +async function checkDataDir(dataDir) { + const adapterDir = path.join(dataDir, "codex-adapter"); + const details = { + dataDir, + adapterDir, + created: false, + writable: false, + dataDirMode: null, + adapterDirMode: null, + }; + try { + await fs.mkdir(dataDir, { recursive: true, mode: 0o700 }); + await fs.mkdir(adapterDir, { recursive: true, mode: 0o700 }); + details.created = true; + const probe = path.join(adapterDir, ".doctor-probe"); + await fs.writeFile(probe, "ok\n", { mode: 0o600 }); + await fs.rm(probe, { force: true }); + details.writable = true; + details.dataDirMode = modeString(dataDir); + details.adapterDirMode = modeString(adapterDir); + } catch (err) { + details.error = err instanceof Error ? err.message : String(err); + } + const adapterDirPrivate = details.adapterDirMode === "700"; + return { + name: "data_dir_writable", + ok: details.writable && adapterDirPrivate, + details, + }; +} + +async function hasSourceGateway(root) { + if (!root) return false; + const resolved = path.resolve(expandHome(root)); + return fsSync.existsSync(path.join(resolved, "package.json")) && + fsSync.existsSync(path.join(resolved, "src", "gateway", "server.ts")); +} + +function findSuspiciousAbsolutePaths(text) { + const values = new Set(); + for (const match of text.matchAll(/"(\/(?:Users|home|private|tmp|var|opt)\/[^"]+)"/g)) { + values.add(match[1]); + } + return Array.from(values); +} + +function isLoopbackGatewayUrl(value) { + try { + const url = new URL(value); + return ["127.0.0.1", "localhost", "::1", "[::1]"].includes(url.hostname); + } catch { + return false; + } +} + +async function privateParentStatus(filePath) { + const parent = path.dirname(filePath); + if (!fsSync.existsSync(parent)) return { exists: false, mode: null }; + return { exists: true, mode: modeString(parent) }; +} + +function modeString(filePath) { + try { + return (fsSync.statSync(filePath).mode & 0o777).toString(8); + } catch { + return null; + } +} + +function addCheck(checks, name, ok, details = {}) { + checks.push({ name, ok: Boolean(ok), details }); +} + +async function readJson(filePath) { + const text = await fs.readFile(filePath, "utf-8"); + return JSON.parse(text); +} + +function parseArgs(args) { + const opts = { + start: false, + requireHealthy: false, + strict: false, + pretty: true, + }; + + for (const arg of args) { + if (arg === "--start") opts.start = true; + else if (arg === "--require-healthy") opts.requireHealthy = true; + else if (arg === "--strict") opts.strict = true; + else if (arg === "--json") opts.pretty = false; + else if (arg === "--help" || arg === "-h") opts.help = true; + else throw new Error(`Unknown option: ${arg}`); + } + return opts; +} + +function usage(code = 0) { + const message = `Usage: node scripts/doctor.mjs [--start] [--require-healthy] [--strict] [--json] + +Checks whether the Codex adapter can be installed, inherited, or reused from the +current plugin root without machine-specific edits. + +Options: + --start Attempt to start the Gateway before reporting health. + --require-healthy Treat Gateway health as a required check. + --strict Exit non-zero when any required check fails. + --json Print compact JSON. +`; + (code === 0 ? console.log : console.error)(message); + process.exit(code); +} + +export async function doctorCli(args = process.argv.slice(2)) { + const opts = parseArgs(args); + if (opts.help) usage(0); + const report = await buildAdapterDoctorReport(opts); + console.log(JSON.stringify(report, null, opts.pretty ? 2 : 0)); + if (opts.strict && !report.ok) process.exit(1); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { + try { + await doctorCli(); + } catch (err) { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(2); + } +} diff --git a/codex-plugin/scripts/import-codex-history.mjs b/codex-plugin/scripts/import-codex-history.mjs index 220997c..99e4a62 100644 --- a/codex-plugin/scripts/import-codex-history.mjs +++ b/codex-plugin/scripts/import-codex-history.mjs @@ -81,8 +81,12 @@ export async function importCodexHistoryCli(args = process.argv.slice(2)) { data: seedData, strict_round_role: true, auto_fill_timestamps: false, + wait_for_l1: opts.waitForL1, + l1_concurrency: opts.l1Concurrency, + l2_batch_size: opts.l2BatchSize, wait_for_full_pipeline: opts.fullPipeline, - full_pipeline_timeout_ms: opts.fullPipelineTimeoutMs + full_pipeline_timeout_ms: opts.fullPipelineTimeoutMs, + import_into_current_store: opts.importIntoCurrentStore }, Number(process.env.TDAI_CODEX_SEED_TIMEOUT_MS || DEFAULT_SEED_TIMEOUT_MS)); console.log(JSON.stringify({ @@ -125,6 +129,7 @@ async function parseCodexRollout(entry, opts) { let sessionTimestamp = 0; let source = ""; const messages = []; + let messageIndex = 0; for (const line of text.split(/\r?\n/)) { if (!line.trim()) continue; @@ -150,11 +155,14 @@ async function parseCodexRollout(entry, opts) { const content = sanitizeMemoryText(contentToText(payload.content)); if (shouldSkipMessage(payload.role, content)) continue; + const timestamp = timestampMs(row.timestamp) || sessionTimestamp || Date.now(); messages.push({ + id: stableMessageId(entry.file, sessionId, messageIndex, payload.role, timestamp, content), role: payload.role, content, - timestamp: timestampMs(row.timestamp) || sessionTimestamp || Date.now() + timestamp }); + messageIndex++; } if (opts.since && sessionTimestamp && sessionTimestamp < opts.since) { @@ -255,6 +263,10 @@ function summarize(files, sessions, skipped, opts) { archivedDir: opts.includeArchived ? opts.archivedDir : null, includeArchived: opts.includeArchived, waitForFullPipeline: opts.fullPipeline, + waitForL1: opts.waitForL1, + l1Concurrency: opts.l1Concurrency, + l2BatchSize: opts.l2BatchSize, + importIntoCurrentStore: opts.importIntoCurrentStore, fullPipelineTimeoutMs: opts.fullPipelineTimeoutMs, cwd: opts.cwd || null, since: opts.since ? new Date(opts.since).toISOString() : null, @@ -273,6 +285,10 @@ function parseArgs(args) { archivedDir: path.resolve(expandHome(process.env.CODEX_ARCHIVED_SESSIONS_DIR || DEFAULT_ARCHIVED_DIR)), includeArchived: true, fullPipeline: true, + importIntoCurrentStore: true, + waitForL1: true, + l1Concurrency: positiveInteger(process.env.TDAI_CODEX_IMPORT_L1_CONCURRENCY, 8), + l2BatchSize: positiveInteger(process.env.TDAI_CODEX_IMPORT_L2_BATCH_SIZE, 32, 128), fullPipelineTimeoutMs: positiveNumber(process.env.TDAI_CODEX_FULL_PIPELINE_TIMEOUT_MS, DEFAULT_FULL_PIPELINE_TIMEOUT_MS), dryRun: false, yes: false, @@ -288,8 +304,15 @@ function parseArgs(args) { else if (arg === "--dry-run") opts.dryRun = true; else if (arg === "--yes" || arg === "-y") opts.yes = true; else if (arg === "--no-archived") opts.includeArchived = false; - else if (arg === "--no-full-pipeline") opts.fullPipeline = false; + else if (arg === "--no-full-pipeline") { + opts.fullPipeline = false; + opts.waitForL1 = false; + } + else if (arg === "--no-wait-for-l1") opts.waitForL1 = false; + else if (arg === "--snapshot-seed") opts.importIntoCurrentStore = false; else if (arg === "--full-pipeline-timeout-ms") opts.fullPipelineTimeoutMs = positiveNumber(next(args, ++i, arg), 0); + else if (arg === "--l1-concurrency") opts.l1Concurrency = positiveInteger(next(args, ++i, arg), 1); + else if (arg === "--l2-batch-size") opts.l2BatchSize = positiveInteger(next(args, ++i, arg), 1, 128); else if (arg === "--sessions-dir") opts.sessionsDir = path.resolve(expandHome(next(args, ++i, arg))); else if (arg === "--archived-dir") opts.archivedDir = path.resolve(expandHome(next(args, ++i, arg))); else if (arg === "--cwd") opts.cwd = next(args, ++i, arg); @@ -321,6 +344,11 @@ function positiveNumber(value, fallback) { return Number.isFinite(n) && n > 0 ? n : fallback; } +function positiveInteger(value, fallback, max = 32) { + const n = Number(value); + return Number.isFinite(n) && n > 0 ? Math.min(max, Math.max(1, Math.floor(n))) : fallback; +} + function timestampMs(value) { if (typeof value === "number") return value < 10_000_000_000 ? value * 1000 : value; if (typeof value === "string") { @@ -341,6 +369,19 @@ function safeKey(value) { return String(value).replace(/[^a-zA-Z0-9_.:-]/g, "_").slice(0, 120); } +function stableMessageId(file, sessionId, index, role, timestamp, content) { + const digest = sha1([ + "codex-import-message", + path.resolve(file), + sessionId, + index, + role, + timestamp, + content + ].join("\0")).slice(0, 20); + return `codex_import_${digest}`; +} + function usage(code = 0) { const message = `Usage: node scripts/import-codex-history.mjs [options] @@ -353,7 +394,11 @@ Options: --sessions-dir Active Codex sessions directory. Default: ${DEFAULT_SESSIONS_DIR} --archived-dir Archived Codex sessions directory. Default: ${DEFAULT_ARCHIVED_DIR} --no-archived Do not include archived Codex JSONL files. - --no-full-pipeline Only seed through the Gateway's default L0/L1 path. + --no-full-pipeline Only write L0 records; skip the final L1/L2/L3 flush. + --no-wait-for-l1 Do not wait for per-session L1 batches; intended for L0-only imports. + --l1-concurrency Concurrent L1 extraction tasks for this import. Default: 8. + --l2-batch-size L1 records per bulk L2 scene batch. Default: 32. + --snapshot-seed Write to an isolated seed-* directory instead of the current memory store. --full-pipeline-timeout-ms Max wait for the final L1/L2/L3 flush. Default: ${DEFAULT_FULL_PIPELINE_TIMEOUT_MS} --cwd Import only sessions whose session_meta.cwd matches this path. diff --git a/codex-plugin/scripts/lib.mjs b/codex-plugin/scripts/lib.mjs index 2ffcb5f..751a01a 100644 --- a/codex-plugin/scripts/lib.mjs +++ b/codex-plugin/scripts/lib.mjs @@ -2,6 +2,8 @@ import crypto from "node:crypto"; import { execFile, spawn } from "node:child_process"; import fs from "node:fs/promises"; import fsSync from "node:fs"; +import http from "node:http"; +import https from "node:https"; import { fileURLToPath } from "node:url"; import os from "node:os"; import path from "node:path"; @@ -993,6 +995,12 @@ export async function httpPost(route, body, timeoutMs = DEFAULT_RECALL_TIMEOUT_M if (!isAllowedGatewayEndpoint()) return null; if (await isGatewayCircuitOpen()) return null; try { + if (timeoutMs > 300_000 || route === "/seed") { + const json = await httpPostLong(route, body, timeoutMs); + await recordGatewaySuccess(); + return json; + } + const headers = { "Content-Type": "application/json", ...await gatewayAuthHeaders() @@ -1018,6 +1026,47 @@ export async function httpPost(route, body, timeoutMs = DEFAULT_RECALL_TIMEOUT_M } } +async function httpPostLong(route, body, timeoutMs) { + const url = new URL(`${gatewayUrl()}${route}`); + const payload = JSON.stringify(body); + const headers = { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(payload), + ...await gatewayAuthHeaders() + }; + const client = url.protocol === "https:" ? https : http; + + return await new Promise((resolve, reject) => { + const req = client.request(url, { + method: "POST", + headers, + timeout: timeoutMs, + }, (res) => { + const chunks = []; + res.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + res.on("end", () => { + const text = Buffer.concat(chunks).toString("utf-8"); + if ((res.statusCode ?? 0) < 200 || (res.statusCode ?? 0) >= 300) { + reject(new Error(`Gateway ${route} returned ${res.statusCode}: ${text}`)); + return; + } + try { + resolve(text ? JSON.parse(text) : null); + } catch (err) { + reject(new Error(`Gateway ${route} returned invalid JSON: ${err instanceof Error ? err.message : String(err)}`)); + } + }); + }); + + req.on("timeout", () => { + req.destroy(new Error(`Gateway ${route} timed out after ${timeoutMs}ms`)); + }); + req.on("error", reject); + req.write(payload); + req.end(); + }); +} + function isAllowedGatewayEndpoint() { let url; try { diff --git a/codex-plugin/scripts/query.mjs b/codex-plugin/scripts/query.mjs index 03d731b..c61ff81 100644 --- a/codex-plugin/scripts/query.mjs +++ b/codex-plugin/scripts/query.mjs @@ -74,11 +74,13 @@ if (command === "status") { } const fullPipelineTimeoutMs = positiveNumber(process.env.TDAI_CODEX_FULL_PIPELINE_TIMEOUT_MS, 900000); const seedTimeoutMs = positiveNumber(process.env.TDAI_CODEX_SEED_TIMEOUT_MS, 960000); + const l1Concurrency = positiveInteger(process.env.TDAI_CODEX_SEED_L1_CONCURRENCY, 1); const dataPath = path.resolve(expandHome(file)); const data = JSON.parse(await readFile(dataPath, "utf-8")); const result = await httpPost("/seed", { data, session_key: sessionKeyFromPayload(payload), + l1_concurrency: l1Concurrency, wait_for_full_pipeline: process.env.TDAI_CODEX_SEED_FULL_PIPELINE !== "false", full_pipeline_timeout_ms: fullPipelineTimeoutMs }, seedTimeoutMs); @@ -86,6 +88,9 @@ if (command === "status") { } else if (command === "import-codex-history") { const { importCodexHistoryCli } = await import("./import-codex-history.mjs"); await importCodexHistoryCli(args); +} else if (command === "doctor") { + const { doctorCli } = await import("./doctor.mjs"); + await doctorCli(args); } else if (command === "offload") { const { offloadCli } = await import("./offload-store.mjs"); await offloadCli(args, { @@ -97,7 +102,7 @@ if (command === "status") { } function usage(code = 0) { - console.error("Usage: node scripts/query.mjs [status|memory |conversation |remember |flush|seed |import-codex-history [options]|offload ]"); + console.error("Usage: node scripts/query.mjs [status|memory |conversation |remember |flush|seed |import-codex-history [options]|doctor [options]|offload ]"); process.exit(code); } @@ -112,3 +117,8 @@ function positiveNumber(value, fallback) { const n = Number(value); return Number.isFinite(n) && n > 0 ? n : fallback; } + +function positiveInteger(value, fallback) { + const n = Number(value); + return Number.isFinite(n) && n > 0 ? Math.min(32, Math.max(1, Math.floor(n))) : fallback; +} diff --git a/codex-plugin/tdai-gateway.example.json b/codex-plugin/tdai-gateway.example.json index 89e7cf9..ee0cfbe 100644 --- a/codex-plugin/tdai-gateway.example.json +++ b/codex-plugin/tdai-gateway.example.json @@ -27,7 +27,8 @@ "pipeline": { "everyNConversations": 3, "enableWarmup": true, - "l1IdleTimeoutSeconds": 300 + "l1IdleTimeoutSeconds": 300, + "l1Concurrency": 1 }, "extraction": { "enabled": true, diff --git a/index.ts b/index.ts index b7f67ed..90e54a0 100644 --- a/index.ts +++ b/index.ts @@ -156,7 +156,7 @@ export default function register(api: OpenClawPluginApi) { `capture=${cfg.capture.enabled}, ` + `recall=${cfg.recall.enabled}(maxResults=${cfg.recall.maxResults}), ` + `extraction=${cfg.extraction.enabled}(dedup=${cfg.extraction.enableDedup}, maxMem=${cfg.extraction.maxMemoriesPerSession}), ` + - `pipeline=(everyN=${cfg.pipeline.everyNConversations}, warmup=${cfg.pipeline.enableWarmup}, l1Idle=${cfg.pipeline.l1IdleTimeoutSeconds}s, l2DelayAfterL1=${cfg.pipeline.l2DelayAfterL1Seconds}s, l2Min=${cfg.pipeline.l2MinIntervalSeconds}s, l2Max=${cfg.pipeline.l2MaxIntervalSeconds}s, activeWindow=${cfg.pipeline.sessionActiveWindowHours}h), ` + + `pipeline=(everyN=${cfg.pipeline.everyNConversations}, warmup=${cfg.pipeline.enableWarmup}, l1Idle=${cfg.pipeline.l1IdleTimeoutSeconds}s, l1Concurrency=${cfg.pipeline.l1Concurrency}, l2DelayAfterL1=${cfg.pipeline.l2DelayAfterL1Seconds}s, l2Min=${cfg.pipeline.l2MinIntervalSeconds}s, l2Max=${cfg.pipeline.l2MaxIntervalSeconds}s, activeWindow=${cfg.pipeline.sessionActiveWindowHours}h), ` + `persona(triggerEvery=${cfg.persona.triggerEveryN}, backupCount=${cfg.persona.backupCount}, sceneBackupCount=${cfg.persona.sceneBackupCount}), ` + `memoryCleanup(enabled=${cfg.memoryCleanup.enabled}, retentionDays=${cfg.memoryCleanup.retentionDays ?? "(disabled)"}, cleanTime=${cfg.memoryCleanup.cleanTime}), ` + `offload(enabled=${cfg.offload.enabled}, backendUrl=${cfg.offload.backendUrl ?? "(none)"}, mildRatio=${cfg.offload.mildOffloadRatio}, aggressiveRatio=${cfg.offload.aggressiveCompressRatio}, retentionDays=${cfg.offload.offloadRetentionDays})`, diff --git a/openclaw.plugin.json b/openclaw.plugin.json index 514b5fd..eb6b8e9 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -57,6 +57,7 @@ "everyNConversations": { "type": "number", "default": 5, "description": "每 N 轮对话触发 L1 批处理" }, "enableWarmup": { "type": "boolean", "default": true, "description": "Warm-up 模式：新 session 从 1 轮触发开始，每次 L1 后翻倍（1→2→4→...→everyN），加速早期记忆提取" }, "l1IdleTimeoutSeconds": { "type": "number", "default": 600, "description": "L1 空闲超时（秒）：用户停止对话后多久触发 L1 批处理" }, + "l1Concurrency": { "type": "number", "default": 1, "description": "L1 抽取任务并发数；默认 1 保持实时宿主顺序，历史 seed/import 可提高此值" }, "l2DelayAfterL1Seconds": { "type": "number", "default": 90, "description": "L1 完成后延迟多久触发 L2（秒）" }, "l2MinIntervalSeconds": { "type": "number", "default": 900, "description": "同一 session 两次 L2 抽取的最小间隔（秒）" }, "l2MaxIntervalSeconds": { "type": "number", "default": 3600, "description": "同一活跃 session 的 L2 最大轮询间隔（秒）" }, diff --git a/src/adapters/standalone/llm-runner.ts b/src/adapters/standalone/llm-runner.ts index 372d83f..39ae6e7 100644 --- a/src/adapters/standalone/llm-runner.ts +++ b/src/adapters/standalone/llm-runner.ts @@ -226,12 +226,6 @@ function createSandboxedTools(workspaceDir: string, logger?: Logger) { }; } -/** Read-only tool subset — used when enableTools=false to avoid empty tools rejection. */ -function createReadOnlyTools(workspaceDir: string, logger?: Logger) { - const all = createSandboxedTools(workspaceDir, logger); - return { read_file: all.read_file }; -} - // ============================ // StandaloneLLMRunner // ============================ @@ -274,18 +268,19 @@ export class StandaloneLLMRunner implements LLMRunner { compatibility: "compatible", }); - // Select tools based on mode + // Text-only tasks (L1 extraction, dedup) must not receive tools. Even a + // read-only tool lets models drift into "I'll inspect a file first" instead + // of returning the strict JSON the pipeline expects. const tools = this.enableTools ? createSandboxedTools(workspaceDir, this.logger) - : createReadOnlyTools(workspaceDir, this.logger); + : undefined; try { const result = await generateText({ model: provider.chat(this.model), system: params.systemPrompt, prompt: params.prompt, - tools, - stopWhen: stepCountIs(this.enableTools ? MAX_TOOL_ITERATIONS : 1), + ...(tools ? { tools, stopWhen: stepCountIs(MAX_TOOL_ITERATIONS) } : {}), maxOutputTokens: maxTokens, abortSignal: AbortSignal.timeout(timeoutMs), }); diff --git a/src/cli/README.md b/src/cli/README.md index 1252692..1d20145 100644 --- a/src/cli/README.md +++ b/src/cli/README.md @@ -26,6 +26,8 @@ openclaw memory-tdai seed --input [options] | `--session-key ` | — | 回退 session key（当输入数据缺少时使用） | | `--config ` | — | 配置覆盖文件（JSON，与 openclaw.json 插件配置深度合并） | | `--strict-round-role` | — | 严格校验每轮对话必须包含 user 和 assistant 消息 | +| `--no-wait-for-l1` | — | 不在每批 L1 边界暂停；通常只用于大规模导入 | +| `--l1-concurrency ` | — | 本次 seed 的 L1 抽取并发数（默认使用配置值） | | `--wait-for-full-pipeline` | — | 返回前等待最终 L1→L2→L3 flush 完成 | | `--full-pipeline-timeout-ms ` | — | 最终 L1→L2→L3 flush 的最长等待时间（默认 900000） | | `--yes` | — | 跳过交互确认（如时间戳自动填充确认） | @@ -121,6 +123,7 @@ openclaw memory-tdai seed --input data.json --config seed-config.json --strict-r "everyNConversations": 3, "enableWarmup": false, "l1IdleTimeoutSeconds": 2, + "l1Concurrency": 4, "l2DelayAfterL1Seconds": 1, "l2MinIntervalSeconds": 1, "l2MaxIntervalSeconds": 10 @@ -139,7 +142,8 @@ openclaw memory-tdai seed --input data.json --config seed-config.json --strict-r "pipeline": { "everyNConversations": 3, "enableWarmup": false, - "l1IdleTimeoutSeconds": 2 + "l1IdleTimeoutSeconds": 2, + "l1Concurrency": 4 } } ``` diff --git a/src/cli/commands/seed.ts b/src/cli/commands/seed.ts index ee68944..19fbe37 100644 --- a/src/cli/commands/seed.ts +++ b/src/cli/commands/seed.ts @@ -31,6 +31,9 @@ export function registerSeedCommand(parent: Command, ctx: SeedCliContext): void .option("--session-key ", "Fallback session key when input lacks one") .option("--config ", "Path to memory-tdai config override file (JSON, deep-merged on top of current plugin config)") .option("--strict-round-role", "Require each round to have both user and assistant messages", false) + .option("--no-wait-for-l1", "Do not pause at per-batch L1 boundaries; useful with --wait-for-full-pipeline for large imports") + .option("--l1-concurrency ", "Bounded concurrent L1 extraction tasks for this seed run") + .option("--l2-batch-size ", "Coalesce pending L2 records into batches during final full-pipeline flush") .option("--wait-for-full-pipeline", "Wait for final L1→L2→L3 processing before returning", false) .option("--full-pipeline-timeout-ms ", "Max wait time for final L1→L2→L3 processing", "900000") .option("--yes", "Skip interactive confirmations (e.g. timestamp auto-fill)", false) @@ -50,6 +53,13 @@ Examples: yes: rawOpts.yes === true, configFile: rawOpts.config as string | undefined, waitForFullPipeline: rawOpts.waitForFullPipeline === true, + waitForL1: rawOpts.waitForL1 !== false, + l1Concurrency: rawOpts.l1Concurrency === undefined + ? undefined + : Number(rawOpts.l1Concurrency) || undefined, + l2BatchSize: rawOpts.l2BatchSize === undefined + ? undefined + : Number(rawOpts.l2BatchSize) || undefined, fullPipelineTimeoutMs: Number(rawOpts.fullPipelineTimeoutMs) || undefined, }; @@ -70,6 +80,9 @@ async function runSeedCommand(opts: SeedCommandOptions, ctx: SeedCliContext): Pr logger.info(`${TAG} sessionKey: ${opts.sessionKey ?? "(from input)"}`); logger.info(`${TAG} config: ${opts.configFile ?? "(default)"}`); logger.info(`${TAG} strict: ${opts.strictRoundRole}`); + logger.info(`${TAG} waitL1: ${opts.waitForL1 !== false}`); + logger.info(`${TAG} l1Conc: ${opts.l1Concurrency ?? "(config)"}`); + logger.info(`${TAG} l2Batch: ${opts.l2BatchSize ?? "(disabled)"}`); logger.info(`${TAG} full: ${opts.waitForFullPipeline === true}`); logger.info(`${TAG} yes: ${opts.yes}`); @@ -154,6 +167,9 @@ async function runSeedCommand(opts: SeedCommandOptions, ctx: SeedCliContext): Pr openclawConfig: ctx.config, pluginConfig: mergedPluginConfig, inputFile: opts.input, + waitForL1: opts.waitForL1 !== false, + l1Concurrency: opts.l1Concurrency, + l2BatchSize: opts.l2BatchSize, waitForFullPipeline: opts.waitForFullPipeline === true, fullPipelineFlushTimeoutMs: opts.fullPipelineTimeoutMs, logger, diff --git a/src/config.ts b/src/config.ts index 30a68e3..cffa719 100644 --- a/src/config.ts +++ b/src/config.ts @@ -64,6 +64,8 @@ export interface PipelineTriggerConfig { enableWarmup: boolean; /** L1 idle timeout: trigger L1 after this many seconds of inactivity (default: 600) */ l1IdleTimeoutSeconds: number; + /** L1 task concurrency. Default 1 preserves live-host ordering; seed/import can raise it. */ + l1Concurrency: number; /** L2 delay after L1: wait this many seconds after L1 completes before triggering L2 (default: 90) */ l2DelayAfterL1Seconds: number; /** L2 min interval: minimum seconds between L2 runs per session (default: 900 = 15 min) */ @@ -478,6 +480,7 @@ export function parseConfig(raw: Record | undefined): MemoryTda everyNConversations: num(pipelineGroup, "everyNConversations") ?? 5, enableWarmup: bool(pipelineGroup, "enableWarmup") ?? true, l1IdleTimeoutSeconds: num(pipelineGroup, "l1IdleTimeoutSeconds") ?? 600, + l1Concurrency: Math.min(32, Math.max(1, Math.floor(num(pipelineGroup, "l1Concurrency") ?? 1))), l2DelayAfterL1Seconds: num(pipelineGroup, "l2DelayAfterL1Seconds") ?? 90, l2MinIntervalSeconds: num(pipelineGroup, "l2MinIntervalSeconds") ?? 900, l2MaxIntervalSeconds: num(pipelineGroup, "l2MaxIntervalSeconds") ?? 3600, diff --git a/src/core/hooks/auto-capture.ts b/src/core/hooks/auto-capture.ts index 18eaaee..8edaf95 100644 --- a/src/core/hooks/auto-capture.ts +++ b/src/core/hooks/auto-capture.ts @@ -43,7 +43,16 @@ export interface AutoCaptureResult { * Generate a unique L0 record ID for vector indexing. * Includes an index to distinguish multiple messages within the same round. */ -function generateL0RecordId(sessionKey: string, index: number): string { +function generateL0RecordId(sessionKey: string, msg: ConversationMessage, index: number): string { + if (msg.id && msg.id.trim()) { + const digest = crypto + .createHash("sha1") + .update(["l0", sessionKey, msg.id].join("\0")) + .digest("hex") + .slice(0, 32); + return `l0_${digest}`; + } + return `l0_${sessionKey}_${Date.now()}_${index}_${crypto.randomBytes(3).toString("hex")}`; } @@ -183,7 +192,7 @@ export async function performAutoCapture(params: { const msg = filteredMessages[i]; try { const l0Record: L0Record = { - id: generateL0RecordId(sessionKey, i), + id: generateL0RecordId(sessionKey, msg, i), sessionKey, sessionId: sessionId || "", role: msg.role, @@ -304,7 +313,7 @@ export async function performAutoCapture(params: { // ============================ const tNotifyStart = performance.now(); // Pass empty array: L1 Runner reads from VectorStore DB (or L0 JSONL fallback), not from in-memory buffers. - if (scheduler) { + if (scheduler && filteredMessages.length > 0) { await scheduler.notifyConversation(sessionKey, []); logger?.debug?.(`${TAG} Scheduler notified of conversation round (sessionKey=${sessionKey})`); @@ -327,6 +336,10 @@ export async function performAutoCapture(params: { }; } + if (scheduler && filteredMessages.length === 0) { + logger?.debug?.(`${TAG} Scheduler notification skipped: no captured L0 messages (sessionKey=${sessionKey})`); + } + const totalMs = performance.now() - tCaptureStart; const vecDetail = supportsBgEmbed ? `metadata-only, embed=background, msgs=${filteredMessages.length}` diff --git a/src/core/prompts/l1-extraction.ts b/src/core/prompts/l1-extraction.ts index 6378598..3522291 100644 --- a/src/core/prompts/l1-extraction.ts +++ b/src/core/prompts/l1-extraction.ts @@ -42,12 +42,14 @@ export const EXTRACT_MEMORIES_SYSTEM_PROMPT = `你是专业的"情境切分与 2. 客观事件记忆 (type: "episodic") - 定义：客观发生的动作、决定、计划或达成结果。绝不包含纯主观感受。 + - 对 AI 编程、研究、文档、爬虫、数据、服务器、插件等工作流，以下内容即使只属于某个项目，也应提取为 episodic：明确交付物、验收标准、关键路径、真实文件路径/仓库路径、数据产物、运行环境、已确认的 blocker、root cause、修复决定、复现结论、用户纠正过的技术假设、已完成/未完成状态。 - 提取句式："用户（[姓名]）在 [最好是精确绝对时间] 于 [地点] [做了某事（可以包含起因、经过、结果）]"。 - 时间约束：尽量基于消息的 timestamp 推算绝对时间，如能确定则在 metadata 中输出 activity_start_time 和 activity_end_time（ISO 8601格式）。无法确定时可省略。 - 打分 (priority)：80-100（重要事件/计划）；60-70（一般完整活动）；<60（琐碎事项，直接丢弃）。 3. 全局指令记忆 (type: "instruction") - 定义：用户对 AI 提出的长期行为规则、格式偏好、语气控制。 + - 如果用户在项目工作流中定义了可复用的执行规则（例如必须直接修改源文件、交付物是最终数据而不是代码、必须实时监控 ETA、不要在 HPC 登录节点跑重活、先验证路径/依赖/代理可达性），即使它来自某个历史任务，也应提取为 instruction，除非文本明确限定为"只限这一次"。 - 提取句式："用户要求/希望 AI 以后回答时..." - 触发词：以后都、从现在开始、记住、必须。 - 打分 (priority)：-1（极其严格的全局死命令）；90-100（核心行为规则）；70-80（重要要求）；<70（临时要求，直接丢弃）。 @@ -60,11 +62,14 @@ export const EXTRACT_MEMORIES_SYSTEM_PROMPT = `你是专业的"情境切分与 - 重复的内容；AI助手自身的行为或输出 - 不属于以上3类的信息 - 纯主观感受（不带客观事件的情绪表达） +- 不要因为内容是项目级、工作流级、agent 执行级而丢弃；只要未来继续该项目、复现 bug、查找产物或理解用户偏好时有帮助，就应保留。 --- ### 任务三：输出格式规范（JSON） -返回且仅返回一个合法的 JSON 数组。数组的每一项是一个情境，包含该情境的消息范围和抽取到的记忆： +返回且仅返回一个合法的 JSON 数组。第一个字符必须是 "["，最后一个字符必须是 "]"。 +禁止输出分析过程、标题、Markdown、自然语言解释、工具调用意图或任何 JSON 外文本。 +数组的每一项是一个情境，包含该情境的消息范围和抽取到的记忆： [ { @@ -125,7 +130,9 @@ export function formatExtractionPrompt(params: { .map((m) => `[${m.id}] [${m.role}] [${new Date(m.timestamp).toISOString()}]: ${m.content}`) .join("\n\n"); - return `【上一个情境】：${previousSceneName} + return `只返回 JSON 数组。不要写分析、标题、说明或 Markdown。 + +【上一个情境】：${previousSceneName} 【背景对话】（仅供理解上下文推断关系/时间，严禁从中提取记忆）： ${bgText} diff --git a/src/core/record/l1-dedup.ts b/src/core/record/l1-dedup.ts index 5d833e1..c2f1ab9 100644 --- a/src/core/record/l1-dedup.ts +++ b/src/core/record/l1-dedup.ts @@ -326,19 +326,9 @@ function parseBatchResult( cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, ""); } - // Extract JSON array - const arrayMatch = cleaned.match(/\[[\s\S]*\]/); - if (!arrayMatch) { - logger?.warn?.(`${TAG} No JSON array found in conflict detection response`); - return fallbackStoreAll(memories); - } - - // Sanitize control characters inside JSON string literals that LLM may produce - const sanitized = sanitizeJsonForParse(arrayMatch[0]); - const parsed = JSON.parse(sanitized) as unknown[]; - - if (!Array.isArray(parsed)) { - logger?.warn?.(`${TAG} Conflict detection response is not an array`); + const parsed = parseFirstJsonArray(cleaned); + if (!parsed) { + logger?.warn?.(`${TAG} No valid JSON array found in conflict detection response`); return fallbackStoreAll(memories); } @@ -367,7 +357,7 @@ function parseBatchResult( action: validActions.includes(action) ? (action as DedupDecision["action"]) : "store", target_ids: Array.isArray(d.target_ids) ? d.target_ids.map(String) : [], merged_content: typeof d.merged_content === "string" ? d.merged_content : undefined, - merged_type: VALID_TYPES.includes(d.merged_type as MemoryType) ? (d.merged_type as MemoryType) : undefined, + merged_type: typeof d.merged_type === "string" ? normalizeType(d.merged_type) ?? undefined : undefined, merged_priority: typeof d.merged_priority === "number" ? d.merged_priority : undefined, merged_timestamps: Array.isArray(d.merged_timestamps) ? d.merged_timestamps.map(String) : undefined, }); @@ -393,6 +383,60 @@ function parseBatchResult( } } +function parseFirstJsonArray(text: string): unknown[] | null { + for (let start = 0; start < text.length; start++) { + if (text[start] !== "[") continue; + + let depth = 0; + let inString = false; + let escaped = false; + + for (let end = start; end < text.length; end++) { + const ch = text[end]; + + if (inString) { + if (escaped) { + escaped = false; + } else if (ch === "\\") { + escaped = true; + } else if (ch === "\"") { + inString = false; + } + continue; + } + + if (ch === "\"") { + inString = true; + } else if (ch === "[") { + depth += 1; + } else if (ch === "]") { + depth -= 1; + if (depth === 0) { + const candidate = sanitizeJsonForParse(text.slice(start, end + 1)); + try { + const parsed = JSON.parse(candidate) as unknown; + if (Array.isArray(parsed)) return parsed; + } catch { + break; + } + } + } + } + } + + return null; +} + +function normalizeType(raw: string): MemoryType | null { + const lower = raw.toLowerCase().trim(); + const compact = lower.replace(/[^a-z]/g, ""); + if (VALID_TYPES.includes(lower as MemoryType)) return lower as MemoryType; + if (compact === "episode") return "episodic"; + if (compact === "instruct" || compact === "insruction") return "instruction"; + if (compact === "preference") return "persona"; + return null; +} + /** * Fallback: store all memories when parsing fails. */ diff --git a/src/core/record/l1-extractor.ts b/src/core/record/l1-extractor.ts index ad0a8f1..ab87810 100644 --- a/src/core/record/l1-extractor.ts +++ b/src/core/record/l1-extractor.ts @@ -365,10 +365,12 @@ function parseExtractionResult(raw: string, logger?: Logger): SceneSegment[] { cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, ""); } - // Try to extract JSON array - const arrayMatch = cleaned.match(/\[[\s\S]*\]/); - if (!arrayMatch) { - logger?.warn?.(`${TAG} No JSON array found in extraction response`); + // Try to extract the first valid JSON array. Avoid a greedy regex here: + // narrative failures can contain message IDs like "[l0_xxx]" before the + // real array, and matching from the first "[" corrupts parsing. + const parsed = parseFirstJsonArray(cleaned); + if (!parsed) { + logger?.warn?.(`${TAG} No valid JSON array found in extraction response`); // [l1-debug] NO_JSON — dump the full raw so we can see what the LLM actually said const rawPreview = raw.slice(0, 2048); logger?.warn?.( @@ -377,15 +379,6 @@ function parseExtractionResult(raw: string, logger?: Logger): SceneSegment[] { return []; } - // Sanitize control characters inside JSON string literals that LLM may produce - const sanitized = sanitizeJsonForParse(arrayMatch[0]); - const parsed = JSON.parse(sanitized) as unknown[]; - - if (!Array.isArray(parsed)) { - logger?.warn?.(`${TAG} Extraction response is not an array`); - return []; - } - const scenes: SceneSegment[] = []; for (const item of parsed) { if (!item || typeof item !== "object") continue; @@ -415,6 +408,50 @@ function parseExtractionResult(raw: string, logger?: Logger): SceneSegment[] { } } +function parseFirstJsonArray(text: string): unknown[] | null { + for (let start = 0; start < text.length; start++) { + if (text[start] !== "[") continue; + + let depth = 0; + let inString = false; + let escaped = false; + + for (let end = start; end < text.length; end++) { + const ch = text[end]; + + if (inString) { + if (escaped) { + escaped = false; + } else if (ch === "\\") { + escaped = true; + } else if (ch === "\"") { + inString = false; + } + continue; + } + + if (ch === "\"") { + inString = true; + } else if (ch === "[") { + depth += 1; + } else if (ch === "]") { + depth -= 1; + if (depth === 0) { + const candidate = sanitizeJsonForParse(text.slice(start, end + 1)); + try { + const parsed = JSON.parse(candidate) as unknown; + if (Array.isArray(parsed)) return parsed; + } catch { + break; + } + } + } + } + } + + return null; +} + // ============================ // Write helpers // ============================ @@ -524,12 +561,13 @@ const VALID_TYPES: MemoryType[] = ["persona", "episodic", "instruction"]; function normalizeType(raw: string): MemoryType | null { const lower = raw.toLowerCase().trim(); + const compact = lower.replace(/[^a-z]/g, ""); if (VALID_TYPES.includes(lower as MemoryType)) { return lower as MemoryType; } // Handle legacy type names - if (lower === "episode") return "episodic"; - if (lower === "instruct") return "instruction"; - if (lower === "preference") return "persona"; // fold preference into persona + if (compact === "episode") return "episodic"; + if (compact === "instruct" || compact === "insruction") return "instruction"; + if (compact === "preference") return "persona"; // fold preference into persona return null; } diff --git a/src/core/seed/input.test.ts b/src/core/seed/input.test.ts new file mode 100644 index 0000000..5d946af --- /dev/null +++ b/src/core/seed/input.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { validateAndNormalizeRaw, SeedValidationError } from "./input.js"; + +describe("seed input normalization", () => { + it("preserves stable message IDs for idempotent imports", () => { + const input = validateAndNormalizeRaw({ + sessions: [{ + sessionKey: "codex-import:test", + sessionId: "session-1", + conversations: [[ + { id: "msg-user-1", role: "user", content: "hello", timestamp: 1 }, + { id: "msg-assistant-1", role: "assistant", content: "hi", timestamp: 2 }, + ]], + }], + }, { strictRoundRole: true, autoFillTimestamps: false }); + + expect(input.sessions[0]!.rounds[0]!.messages.map((msg) => msg.id)).toEqual([ + "msg-user-1", + "msg-assistant-1", + ]); + }); + + it("rejects empty message IDs when provided", () => { + expect(() => validateAndNormalizeRaw({ + sessions: [{ + sessionKey: "codex-import:test", + conversations: [[ + { id: "", role: "user", content: "hello", timestamp: 1 }, + { role: "assistant", content: "hi", timestamp: 2 }, + ]], + }], + }, { strictRoundRole: true, autoFillTimestamps: false })).toThrow(SeedValidationError); + }); +}); diff --git a/src/core/seed/input.ts b/src/core/seed/input.ts index 8eb05b6..bae3c3e 100644 --- a/src/core/seed/input.ts +++ b/src/core/seed/input.ts @@ -356,6 +356,17 @@ function validateSessions( }); } + if (msg.id !== undefined && (typeof msg.id !== "string" || msg.id.trim() === "")) { + errors.push({ + stage: "message", + sourceIndex: si, + sessionKey: session.sessionKey, + roundIndex: ri, + messageIndex: mi, + message: '"id" must be a non-empty string when provided.', + }); + } + if (!msg.content || typeof msg.content !== "string" || msg.content.trim() === "") { errors.push({ stage: "message", @@ -465,6 +476,7 @@ function normalizeSessions( if (!Array.isArray(rawRound)) continue; const messages: NormalizedMessage[] = rawRound.map((msg) => ({ + id: msg.id, role: msg.role, content: msg.content, // Normalize timestamp: ISO string → epoch ms, number → pass-through, missing → 0 (filled later) diff --git a/src/core/seed/seed-runtime.ts b/src/core/seed/seed-runtime.ts index 34ec09a..8f5b7ca 100644 --- a/src/core/seed/seed-runtime.ts +++ b/src/core/seed/seed-runtime.ts @@ -6,13 +6,13 @@ * module focused on seed-specific concerns: * - Synchronous per-round L0 capture with progress reporting * - waitForL1Idle polling at batch boundaries - * - Optional final full-pipeline flush for callers that need L2/L3 artifacts + * - Optional L1 waiting and final full-pipeline flush for callers that need + * extracted artifacts immediately * - Ctrl+C graceful shutdown * * By default, seed preserves the historical CLI behavior and waits for L1 at - * batch boundaries. Callers such as the Codex history importer can opt into a - * final L1→L2→L3 flush before shutdown when they need higher-level artifacts - * to be immediately available for recall injection. + * batch boundaries. Bulk import callers can disable L1 waiting when they only + * need L0 records to become searchable immediately. */ import path from "node:path"; @@ -21,12 +21,19 @@ import type { MemoryTdaiConfig } from "../../config.js"; import { performAutoCapture } from "../hooks/auto-capture.js"; import { createPipeline, createL2Runner, createL3Runner } from "../../utils/pipeline-factory.js"; import type { PipelineInstance, PipelineLogger } from "../../utils/pipeline-factory.js"; +import { CheckpointManager } from "../../utils/checkpoint.js"; import { readManifest, writeManifest } from "../../utils/manifest.js"; import { StandaloneLLMRunnerFactory } from "../../adapters/standalone/llm-runner.js"; import type { MemoryPipelineManager } from "../../utils/pipeline-manager.js"; import type { LLMRunner } from "../types.js"; +import { queryMemoryRecords, readMemoryRecords } from "../record/l1-reader.js"; +import type { MemoryRecord } from "../record/l1-reader.js"; +import { SceneExtractor } from "../scene/scene-extractor.js"; +import { pullProfilesToLocal, syncLocalProfilesToStore } from "../profile/profile-sync.js"; +import type { IMemoryStore } from "../store/types.js"; import type { NormalizedInput, + NormalizedSession, SeedProgress, SeedSummary, } from "./types.js"; @@ -46,10 +53,18 @@ export interface SeedRuntimeOptions { pluginConfig?: Record; /** Original input file path (for manifest traceability). */ inputFile?: string; + /** Wait for L1 extraction to drain after each batch/session. */ + waitForL1?: boolean; + /** Bounded L1 extraction concurrency for this seed run. */ + l1Concurrency?: number; + /** Coalesce pending L2 records into batches during final full-pipeline flush. */ + l2BatchSize?: number; /** Wait for a final L1→L2→L3 flush before returning. */ waitForFullPipeline?: boolean; /** Max time for the final L1→L2→L3 flush. */ fullPipelineFlushTimeoutMs?: number; + /** Whether the seed pipeline owns and should close store resources. */ + ownsStoreResources?: boolean; /** Logger instance. */ logger: PipelineLogger; /** Progress callback (called after each round). */ @@ -64,15 +79,24 @@ export interface SeedRuntimeOptions { * Create a seed pipeline using the shared factory, with L2/L3 runners * wired via shared factory functions (same logic as index.ts live runtime). */ -async function createSeedPipeline(opts: SeedRuntimeOptions): Promise<{ pipeline: PipelineInstance; cfg: MemoryTdaiConfig }> { +async function createSeedPipeline(opts: SeedRuntimeOptions): Promise<{ pipeline: PipelineInstance; cfg: MemoryTdaiConfig; l2l3LlmRunner?: LLMRunner }> { const { outputDir, openclawConfig, pluginConfig, logger } = opts; // Parse config — all values come from pluginConfig (or parseConfig defaults) const cfg = parseConfig(pluginConfig); + if (opts.l1Concurrency !== undefined) { + cfg.pipeline.l1Concurrency = Math.min(32, Math.max(1, Math.floor(opts.l1Concurrency))); + } + if (opts.waitForFullPipeline) { + // Seed/import should not let L2/L3 consume LLM capacity while L0/L1 is + // still ingesting. The final flush explicitly triggers pending L2 timers. + cfg.pipeline.l2DelayAfterL1Seconds = Math.max(cfg.pipeline.l2DelayAfterL1Seconds, 24 * 60 * 60); + } logger.info( `${TAG} Creating seed pipeline: outputDir=${outputDir}, ` + `everyN=${cfg.pipeline.everyNConversations}, l1Idle=${cfg.pipeline.l1IdleTimeoutSeconds}s, ` + + `l1Concurrency=${cfg.pipeline.l1Concurrency}, ` + `l2Delay=${cfg.pipeline.l2DelayAfterL1Seconds}s, l2Min=${cfg.pipeline.l2MinIntervalSeconds}s, l2Max=${cfg.pipeline.l2MaxIntervalSeconds}s`, ); @@ -105,6 +129,7 @@ async function createSeedPipeline(opts: SeedRuntimeOptions): Promise<{ pipeline: openclawConfig, logger, l1LlmRunner, + ownsStoreResources: opts.ownsStoreResources, }); // Wire L2 runner via shared factory (same logic as index.ts live runtime) @@ -127,7 +152,7 @@ async function createSeedPipeline(opts: SeedRuntimeOptions): Promise<{ pipeline: llmRunner: l2l3LlmRunner, })); - return { pipeline, cfg }; + return { pipeline, cfg, l2l3LlmRunner }; } // ============================ @@ -146,6 +171,7 @@ async function waitForL1Idle( pollIntervalMs?: number; stableRounds?: number; maxWaitMs?: number; + failOnTimeout?: boolean; } = {}, ): Promise { const pollInterval = opts.pollIntervalMs ?? 1_000; @@ -158,27 +184,18 @@ async function waitForL1Idle( while (true) { const elapsed = Date.now() - startTime; if (elapsed > maxWait) { - logger.warn(`${TAG} [waitL1] Max wait time reached (${(maxWait / 1000).toFixed(0)}s), proceeding`); + const message = `Max wait time reached (${(maxWait / 1000).toFixed(0)}s)`; + if (opts.failOnTimeout) { + throw new Error(`${TAG} [waitL1] ${message}`); + } + logger.warn(`${TAG} [waitL1] ${message}, proceeding`); break; } const queues = scheduler.getQueueSizes(); - - // Check per-session: buffered messages + conversation count - let totalBuffered = 0; - let totalConversationCount = 0; - for (const key of sessionKeys) { - totalBuffered += scheduler.getBufferedMessageCount(key); - const state = scheduler.getSessionState(key); - if (state) { - totalConversationCount += state.conversation_count; - } - } - - const isIdle = - queues.l1Idle && - totalBuffered === 0 && - totalConversationCount === 0; + const pendingSessionKeys = sessionKeys.filter((key) => scheduler.hasPendingL1Work(key)); + const pendingSessionCount = pendingSessionKeys.length; + const isIdle = pendingSessionCount === 0; if (isIdle) { consecutiveIdle++; @@ -187,10 +204,16 @@ async function waitForL1Idle( return; } } else { + if (queues.l1Idle && pendingSessionKeys.length > 0) { + logger.warn( + `${TAG} [waitL1] L1 queue is idle but ${pendingSessionKeys.length} session(s) still report pending work; flushing target sessions`, + ); + await Promise.all(pendingSessionKeys.map((key) => scheduler.flushSession(key))); + } consecutiveIdle = 0; logger.debug?.( `${TAG} [waitL1] Waiting: l1Queue=${queues.l1}, l1Pending=${queues.l1Pending}, l1Idle=${queues.l1Idle}, ` + - `buffered=${totalBuffered}, convCount=${totalConversationCount}`, + `pendingSessions=${pendingSessionCount}/${sessionKeys.length}`, ); } @@ -198,6 +221,214 @@ async function waitForL1Idle( } } +// ============================ +// Bulk L2/L3 flush for historical imports +// ============================ + +interface PendingL2SessionGroup { + sessionKey: string; + records: Array<{ + content: string; + created_at: string; + id: string; + updatedAt: string; + }>; +} + +function supportsProfileSyncWrite(store?: IMemoryStore): boolean { + return !!(store?.syncProfiles || store?.deleteProfiles); +} + +function chunkPendingL2Groups(groups: PendingL2SessionGroup[], batchSize: number): PendingL2SessionGroup[][] { + const chunks: PendingL2SessionGroup[][] = []; + let current: PendingL2SessionGroup[] = []; + let currentRecords = 0; + + for (const group of groups) { + const groupSize = Math.max(1, group.records.length); + if (current.length > 0 && currentRecords + groupSize > batchSize) { + chunks.push(current); + current = []; + currentRecords = 0; + } + current.push(group); + currentRecords += groupSize; + } + + if (current.length > 0) chunks.push(current); + return chunks; +} + +async function collectPendingL2Groups( + pipeline: PipelineInstance, + outputDir: string, + logger: PipelineLogger, +): Promise<{ pendingSessionKeys: string[]; groups: PendingL2SessionGroup[]; noRecordSessionKeys: string[] }> { + const pendingSessionKeys = pipeline.scheduler.getPendingL2SessionKeys(); + const groups: PendingL2SessionGroup[] = []; + const noRecordSessionKeys: string[] = []; + + for (const sessionKey of pendingSessionKeys) { + const state = pipeline.scheduler.getSessionState(sessionKey); + const cursor = state?.last_extraction_updated_time || undefined; + let sessionRecords: MemoryRecord[] = []; + + if (pipeline.vectorStore && !pipeline.vectorStore.isDegraded()) { + sessionRecords = await queryMemoryRecords(pipeline.vectorStore, { + sessionKey, + updatedAfter: cursor, + }, logger); + } else { + sessionRecords = await readMemoryRecords(sessionKey, outputDir, logger); + if (cursor) { + sessionRecords = sessionRecords.filter((r) => (r.updatedAt || r.createdAt || "") > cursor); + } + } + + if (sessionRecords.length === 0) { + noRecordSessionKeys.push(sessionKey); + continue; + } + + groups.push({ + sessionKey, + records: sessionRecords + .map((r) => ({ + content: r.content, + created_at: r.createdAt, + id: r.id, + updatedAt: r.updatedAt, + })) + .sort((a, b) => a.updatedAt.localeCompare(b.updatedAt)), + }); + } + + groups.sort((a, b) => { + const aFirst = a.records[0]?.updatedAt ?? ""; + const bFirst = b.records[0]?.updatedAt ?? ""; + return aFirst.localeCompare(bFirst); + }); + + return { pendingSessionKeys, groups, noRecordSessionKeys }; +} + +async function flushSeedFullPipelineInBatches( + pipeline: PipelineInstance, + cfg: MemoryTdaiConfig, + opts: SeedRuntimeOptions, + openclawConfig: unknown, + llmRunner: LLMRunner | undefined, +): Promise { + const batchSize = Math.max(1, Math.floor(opts.l2BatchSize ?? 1)); + const { logger, outputDir } = opts; + const { pendingSessionKeys, groups, noRecordSessionKeys } = await collectPendingL2Groups(pipeline, outputDir, logger); + const recordCount = groups.reduce((sum, group) => sum + group.records.length, 0); + + logger.info( + `${TAG} Bulk L2 flush: pendingSessions=${pendingSessionKeys.length}, ` + + `sessionsWithRecords=${groups.length}, records=${recordCount}, batchSize=${batchSize}`, + ); + + if (noRecordSessionKeys.length > 0) { + await pipeline.scheduler.markL2FlushedForSessions(noRecordSessionKeys); + logger.info(`${TAG} Bulk L2 flush: marked ${noRecordSessionKeys.length} session(s) with no new L1 records as flushed`); + } + + if (recordCount > 0 && !openclawConfig && !llmRunner) { + throw new Error(`${TAG} Bulk L2 flush requires OpenClaw config or a standalone LLM runner`); + } + + let profileBaseline = new Map(); + if (pipeline.vectorStore && !pipeline.vectorStore.isDegraded() && supportsProfileSyncWrite(pipeline.vectorStore)) { + profileBaseline = await pullProfilesToLocal(outputDir, pipeline.vectorStore, logger); + } + + const chunks = chunkPendingL2Groups(groups, batchSize); + for (let i = 0; i < chunks.length; i++) { + const chunk = chunks[i]!; + const records = chunk.flatMap((group) => group.records); + const sessionKeys = chunk.map((group) => group.sessionKey); + + logger.info( + `${TAG} Bulk L2 batch ${i + 1}/${chunks.length}: ` + + `sessions=${sessionKeys.length}, records=${records.length}`, + ); + + const extractor = new SceneExtractor({ + dataDir: outputDir, + config: openclawConfig, + model: cfg.persona.model, + maxScenes: cfg.persona.maxScenes, + sceneBackupCount: cfg.persona.sceneBackupCount, + logger, + llmRunner, + }); + + const checkpoint = new CheckpointManager(outputDir, logger); + const preState = await checkpoint.read(); + const extractResult = await extractor.extract(records.map((r) => ({ + content: r.content, + created_at: r.created_at, + id: r.id, + }))); + + if (!extractResult.success) { + throw new Error(`${TAG} Bulk L2 batch ${i + 1}/${chunks.length} failed: ${extractResult.error ?? "unknown error"}`); + } + + const postState = await checkpoint.read(); + if ( + postState.scenes_processed < preState.scenes_processed || + postState.total_processed < preState.total_processed + ) { + logger.warn( + `${TAG} Bulk L2 checkpoint regression detected; repairing counters ` + + `(scenes ${preState.scenes_processed}→${postState.scenes_processed}, ` + + `total ${preState.total_processed}→${postState.total_processed})`, + ); + await checkpoint.write({ + ...postState, + scenes_processed: Math.max(postState.scenes_processed, preState.scenes_processed), + total_processed: Math.max(postState.total_processed, preState.total_processed), + memories_since_last_persona: Math.max(postState.memories_since_last_persona, preState.memories_since_last_persona), + }); + } + + if (pipeline.vectorStore && supportsProfileSyncWrite(pipeline.vectorStore)) { + await syncLocalProfilesToStore(outputDir, pipeline.vectorStore, profileBaseline, logger); + } + + await checkpoint.incrementScenesProcessed(); + + const latestCursorBySession = new Map(); + for (const group of chunk) { + const latest = group.records.reduce((cursor, record) => ( + record.updatedAt > cursor ? record.updatedAt : cursor + ), ""); + if (latest) latestCursorBySession.set(group.sessionKey, latest); + } + await pipeline.scheduler.markL2FlushedForSessions(sessionKeys, latestCursorBySession); + } + + if (recordCount > 0) { + const checkpoint = new CheckpointManager(outputDir, logger); + await checkpoint.setPersonaUpdateRequest("seed full-pipeline bulk L2 flush completed"); + logger.info(`${TAG} Bulk L2 flush complete; running final L3 persona pass`); + } else { + logger.info(`${TAG} Bulk L2 flush found no L1 records requiring scene extraction; running final L3 check`); + } + + const l3Runner = createL3Runner({ + pluginDataDir: outputDir, + cfg, + openclawConfig, + vectorStore: pipeline.vectorStore, + logger, + llmRunner, + }); + await l3Runner(); +} + // ============================ // Main execution function // ============================ @@ -205,9 +436,8 @@ async function waitForL1Idle( /** * Execute the seed pipeline: feed normalized input through L0 → L1. * - * L2/L3 runners are wired. Their completion is awaited only when - * `waitForFullPipeline` is true; otherwise seed preserves the faster historical - * behavior and returns after L1 drains. + * L2/L3 runners are wired. L1 completion is awaited by default, but callers can + * disable L1 waiting for large L0-only historical imports. * * This is the core runtime called by `src/cli/commands/seed.ts` after * all input validation and user confirmation are complete. @@ -243,8 +473,22 @@ export async function executeSeed( const seed = await createSeedPipeline(opts); pipeline = seed.pipeline; const seedCfg = seed.cfg; - - pipeline.scheduler.start({}); + const waitForL1 = opts.waitForL1 !== false; + const l1Concurrency = seedCfg.pipeline.l1Concurrency; + const l1WaitMaxMs = opts.waitForFullPipeline + ? Math.max( + opts.fullPipelineFlushTimeoutMs ?? 0, + 300_000, + (seedCfg.llm.timeoutMs ?? 180_000) + 120_000, + ) + : Math.max(300_000, (seedCfg.llm.timeoutMs ?? 180_000) + 120_000); + const failOnL1Timeout = opts.waitForFullPipeline === true; + + const checkpoint = new CheckpointManager(opts.outputDir, logger); + const restoredCheckpoint = await checkpoint.read(); + const restoredPipelineStates = checkpoint.getAllPipelineStates(restoredCheckpoint); + pipeline.scheduler.start(restoredPipelineStates); + logger.info(`${TAG} Pipeline restored ${Object.keys(restoredPipelineStates).length} checkpoint session state(s)`); logger.info(`${TAG} Pipeline started, processing ${input.sessions.length} session(s), ${input.totalRounds} round(s)`); // Seed-specific: use 0 so the cold-start guard in captureAtomically() @@ -253,17 +497,19 @@ export async function executeSeed( // but seed intentionally feeds all historical data. const captureStartTimestamp = 0; - // Process each session → each round - // Key invariant: after every everyNConversations rounds we must wait for L1 - // to finish before feeding more rounds. Without this pause the for-loop - // would dump all rounds into L0 back-to-back and L1 would only run once - // with the full batch (defeating the "every N" batching semantics). + // Process each session → each round. + // + // Key invariant: within a single session, after every + // everyNConversations rounds we must wait for that session's L1 to finish + // before feeding more rounds. Without the per-session pause, one L1 run + // could read an oversized L0 batch and advance the cursor past messages + // that were never eligible for extraction. For bulk imports we can still + // parallelize across sessions, because each session keeps its own cursor. const everyN = seedCfg.pipeline.everyNConversations; - for (const session of input.sessions) { - if (interrupted) break; - + const processSession = async (session: NormalizedSession): Promise => { logger.info(`${TAG} Session: key="${session.sessionKey}" id="${session.sessionId}" rounds=${session.rounds.length}`); + let l0RecordedSinceLastWait = 0; for (let ri = 0; ri < session.rounds.length; ri++) { if (interrupted) break; @@ -275,6 +521,7 @@ export async function executeSeed( // Field must be named "timestamp" (not "ts") because l0-recorder's // extractUserAssistantMessages reads m.timestamp for incremental filtering. const messages = round.messages.map((m) => ({ + id: m.id, role: m.role, content: m.content, timestamp: m.timestamp, @@ -295,6 +542,7 @@ export async function executeSeed( }); totalL0Recorded += result.l0RecordedCount; + l0RecordedSinceLastWait += result.l0RecordedCount; } catch (err) { logger.error( `${TAG} L0 capture failed for session="${session.sessionKey}" round=${ri}: ` + @@ -314,7 +562,16 @@ export async function executeSeed( // feeding the next batch. This keeps L1 batches aligned with the // everyNConversations boundary instead of letting all rounds pile up. const roundInSession = ri + 1; // 1-based - if (roundInSession % everyN === 0 && !interrupted) { + if (waitForL1 && roundInSession % everyN === 0 && !interrupted) { + const hasL1Work = l0RecordedSinceLastWait > 0 || pipeline.scheduler.hasPendingL1Work(session.sessionKey); + if (!hasL1Work) { + logger.debug?.( + `${TAG} Skipping L1 wait after round ${roundInSession}/${session.rounds.length} ` + + `for session="${session.sessionKey}" because no new L0 was captured`, + ); + continue; + } + onProgress?.({ currentRound: roundsProcessed, totalRounds: input.totalRounds, @@ -327,18 +584,34 @@ export async function executeSeed( `for session="${session.sessionKey}" — waiting for L1 to drain`, ); + await pipeline.scheduler.flushSession(session.sessionKey); + l0RecordedSinceLastWait = 0; + await waitForL1Idle( pipeline.scheduler, [session.sessionKey], logger, - { pollIntervalMs: 500, stableRounds: 2, maxWaitMs: 120_000 }, + { + pollIntervalMs: 500, + stableRounds: 2, + maxWaitMs: l1WaitMaxMs, + failOnTimeout: failOnL1Timeout, + }, ); } } - // After all rounds for this session, wait for any residual L1 work - // (handles the tail when total rounds is not a multiple of everyN) - if (!interrupted) { + // After all rounds for this session, flush any residual L1 work (handles + // the tail when total rounds is not a multiple of everyN). Polling alone + // is not enough here: one-round historical sessions may never cross the + // threshold and their idle timer can be minutes away. + if (waitForL1 && !interrupted) { + const hasTailL1Work = l0RecordedSinceLastWait > 0 || pipeline.scheduler.hasPendingL1Work(session.sessionKey); + if (!hasTailL1Work) { + logger.debug?.(`${TAG} Skipping final L1 wait for session="${session.sessionKey}" because no new L0 was captured`); + return; + } + onProgress?.({ currentRound: roundsProcessed, totalRounds: input.totalRounds, @@ -346,27 +619,65 @@ export async function executeSeed( stage: "l1_waiting", }); + await pipeline.scheduler.flushSession(session.sessionKey); + l0RecordedSinceLastWait = 0; + await waitForL1Idle( pipeline.scheduler, [session.sessionKey], logger, - { pollIntervalMs: 1_000, stableRounds: 3, maxWaitMs: 300_000 }, + { + pollIntervalMs: 1_000, + stableRounds: 3, + maxWaitMs: l1WaitMaxMs, + failOnTimeout: failOnL1Timeout, + }, ); logger.info(`${TAG} L1 idle for session="${session.sessionKey}"`); } + }; + + if (waitForL1 && l1Concurrency > 1) { + let nextSessionIndex = 0; + const workerCount = Math.min(l1Concurrency, input.sessions.length); + await Promise.all(Array.from({ length: workerCount }, async () => { + while (!interrupted) { + const session = input.sessions[nextSessionIndex++]; + if (!session) break; + await processSession(session); + } + })); + } else { + for (const session of input.sessions) { + if (interrupted) break; + await processSession(session); + } } // Final wait for all sessions - if (!interrupted) { - const allKeys = input.sessions.map((s) => s.sessionKey); - logger.info(`${TAG} Final L1 idle wait for all sessions...`); - await waitForL1Idle( - pipeline.scheduler, - allKeys, - logger, - { pollIntervalMs: 1_000, stableRounds: 3, maxWaitMs: 300_000 }, - ); + if (waitForL1 && !interrupted) { + const pendingKeys = input.sessions + .map((s) => s.sessionKey) + .filter((key) => pipeline.scheduler.hasPendingL1Work(key)); + if (pendingKeys.length > 0) { + logger.info(`${TAG} Final L1 idle wait for ${pendingKeys.length} pending session(s)...`); + await waitForL1Idle( + pipeline.scheduler, + pendingKeys, + logger, + { + pollIntervalMs: 1_000, + stableRounds: 3, + maxWaitMs: Math.max(600_000, l1WaitMaxMs), + failOnTimeout: failOnL1Timeout, + }, + ); + } else { + logger.debug?.(`${TAG} Final L1 idle wait skipped: no pending sessions`); + } + } else if (!waitForL1) { + logger.info(`${TAG} L1 waiting disabled; returning after L0 capture`); } if (!interrupted && opts.waitForFullPipeline) { @@ -378,13 +689,23 @@ export async function executeSeed( }); logger.info(`${TAG} Final full pipeline flush requested (L1→L2→L3)...`); - await pipeline.scheduler.flushPendingWork({ - reason: "seed", - timeoutMs: opts.fullPipelineFlushTimeoutMs ?? 900_000, - pollIntervalMs: 100, - stableRounds: 3, - armFollowUpL2Timers: false, - }); + if ((opts.l2BatchSize ?? 1) > 1) { + await flushSeedFullPipelineInBatches( + pipeline, + seedCfg, + opts, + opts.openclawConfig, + seed.l2l3LlmRunner, + ); + } else { + await pipeline.scheduler.flushPendingWork({ + reason: "seed", + timeoutMs: opts.fullPipelineFlushTimeoutMs ?? 900_000, + pollIntervalMs: 100, + stableRounds: 3, + armFollowUpL2Timers: false, + }); + } fullPipelineFlushed = true; } } finally { diff --git a/src/core/seed/types.ts b/src/core/seed/types.ts index eb09477..5f98462 100644 --- a/src/core/seed/types.ts +++ b/src/core/seed/types.ts @@ -13,6 +13,8 @@ /** A single message in a conversation round. */ export interface RawMessage { + /** Optional stable message ID. Useful for idempotent historical imports. */ + id?: string; role: string; content: string; /** @@ -43,6 +45,8 @@ export type FormatB = RawSession[]; // ============================ export interface NormalizedMessage { + /** Stable message ID when provided by the input. */ + id?: string; role: string; content: string; /** Epoch ms — always present after normalization (filled if originally missing). */ @@ -113,6 +117,12 @@ export interface SeedCommandOptions { configFile?: string; /** Wait for final L1→L2→L3 processing before returning. */ waitForFullPipeline?: boolean; + /** Wait for L1 at per-batch boundaries before feeding more rounds. */ + waitForL1?: boolean; + /** Bounded L1 extraction concurrency for this seed run. */ + l1Concurrency?: number; + /** Coalesce pending L2 records into batches during final full-pipeline flush. */ + l2BatchSize?: number; /** Max wait time for final L1→L2→L3 processing. */ fullPipelineTimeoutMs?: number; } diff --git a/src/gateway/server.ts b/src/gateway/server.ts index 35f5c25..4fd8dbc 100644 --- a/src/gateway/server.ts +++ b/src/gateway/server.ts @@ -442,16 +442,23 @@ export class TdaiGateway { this.logger.info( `Seed request: ${input.sessions.length} session(s), ` + `${input.totalRounds} round(s), ${input.totalMessages} message(s), ` + + `waitL1=${body.wait_for_l1 !== false}, ` + + `l1Concurrency=${body.l1_concurrency ?? "(config)"}, ` + + `l2BatchSize=${body.l2_batch_size ?? "(disabled)"}, ` + + `currentStore=${body.import_into_current_store === true}, ` + `waitFullPipeline=${body.wait_for_full_pipeline === true}`, ); - // Resolve output directory: use gateway's data dir with a timestamped subfolder + // Resolve output directory. Normal seed runs are isolated for inspection; + // trusted local importers can explicitly target the live store. const now = new Date(); const pad = (n: number) => String(n).padStart(2, "0"); const ts = `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}-` + `${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`; - const outputDir = `${this.config.data.baseDir}/seed-${ts}`; + const outputDir = body.import_into_current_store === true + ? this.config.data.baseDir + : `${this.config.data.baseDir}/seed-${ts}`; // Merge config overrides if provided // Start with the base memory config + inject llm config from gateway settings @@ -494,10 +501,14 @@ export class TdaiGateway { outputDir, openclawConfig: {}, pluginConfig, + waitForL1: body.wait_for_l1 !== false, + l1Concurrency: body.l1_concurrency, + l2BatchSize: body.l2_batch_size, waitForFullPipeline: body.wait_for_full_pipeline === true, fullPipelineFlushTimeoutMs: typeof body.full_pipeline_timeout_ms === "number" ? body.full_pipeline_timeout_ms : undefined, + ownsStoreResources: body.import_into_current_store !== true, logger: this.logger as import("../utils/pipeline-factory.js").PipelineLogger, onProgress: (progress: SeedProgress) => { this.logger.debug?.( diff --git a/src/gateway/types.ts b/src/gateway/types.ts index 13fd50f..ad31699 100644 --- a/src/gateway/types.ts +++ b/src/gateway/types.ts @@ -151,10 +151,21 @@ export interface SeedRequest { strict_round_role?: boolean; /** Auto-fill missing timestamps (default: true). */ auto_fill_timestamps?: boolean; + /** Wait for L1 extraction to drain before returning (default: true). */ + wait_for_l1?: boolean; + /** Bounded L1 extraction concurrency for this seed run. */ + l1_concurrency?: number; + /** Coalesce pending L2 records into batches during final full-pipeline flush. */ + l2_batch_size?: number; /** Wait for final L1→L2→L3 processing before returning (default: false). */ wait_for_full_pipeline?: boolean; /** Max wait time for final L1→L2→L3 processing. */ full_pipeline_timeout_ms?: number; + /** + * Write seed output into the currently running memory store instead of an + * isolated timestamped seed directory. Intended for trusted local importers. + */ + import_into_current_store?: boolean; /** Plugin config overrides (deep-merged on top of gateway memory config). */ config_override?: Record; } diff --git a/src/utils/pipeline-factory.ts b/src/utils/pipeline-factory.ts index 4595753..7f5aab1 100644 --- a/src/utils/pipeline-factory.ts +++ b/src/utils/pipeline-factory.ts @@ -73,6 +73,11 @@ export interface PipelineFactoryOptions { l1LlmRunner?: import("../core/types.js").LLMRunner; /** Host-neutral LLM runner for L2/L3 (tool-call enabled, enableTools=true). */ l2l3LlmRunner?: import("../core/types.js").LLMRunner; + /** + * Whether this pipeline owns the shared store resources for `pluginDataDir`. + * Set false for sidecar/seed pipelines that target an already-live store. + */ + ownsStoreResources?: boolean; } // ============================ @@ -664,7 +669,10 @@ export function createPipelineManager( { everyNConversations: cfg.pipeline.everyNConversations, enableWarmup: cfg.pipeline.enableWarmup, - l1: { idleTimeoutSeconds: cfg.pipeline.l1IdleTimeoutSeconds }, + l1: { + idleTimeoutSeconds: cfg.pipeline.l1IdleTimeoutSeconds, + concurrency: cfg.pipeline.l1Concurrency, + }, l2: { delayAfterL1Seconds: cfg.pipeline.l2DelayAfterL1Seconds, minIntervalSeconds: cfg.pipeline.l2MinIntervalSeconds, @@ -691,6 +699,7 @@ export function createPipelineManager( */ export async function createPipeline(opts: PipelineFactoryOptions): Promise { const { pluginDataDir, cfg, openclawConfig, logger, sessionFilter, l1LlmRunner } = opts; + const ownsStoreResources = opts.ownsStoreResources !== false; // Ensure data directories exist initDataDirectories(pluginDataDir); @@ -720,11 +729,11 @@ export async function createPipeline(opts: PipelineFactoryOptions): Promise { logger.info(`${TAG} Destroying pipeline...`); await scheduler.destroy(); - if (vectorStore) { + if (ownsStoreResources && vectorStore) { logger.info(`${TAG} Closing VectorStore`); vectorStore.close(); } - if (embeddingService?.close) { + if (ownsStoreResources && embeddingService?.close) { try { logger.info(`${TAG} Closing EmbeddingService`); await embeddingService.close(); @@ -732,7 +741,9 @@ export async function createPipeline(opts: PipelineFactoryOptions): Promise new Promise((resolve) => setTimeout(resolve, ms)); + +const config: PipelineConfig = { + everyNConversations: 5, + enableWarmup: false, + l1: { idleTimeoutSeconds: 3600 }, + l2: { + delayAfterL1Seconds: 3600, + minIntervalSeconds: 0, + maxIntervalSeconds: 3600, + sessionActiveWindowHours: 24, + }, +}; + +const logger = { + info: () => undefined, + warn: () => undefined, + error: () => undefined, +}; + +describe("MemoryPipelineManager", () => { + it("flushSession drains below-threshold buffered L1 work", async () => { + const scheduler = new MemoryPipelineManager(config, logger); + const seen: CapturedMessage[][] = []; + + scheduler.setL1Runner(async ({ msg }) => { + seen.push(msg); + return { processedCount: msg.length }; + }); + + await scheduler.notifyConversation("seed-session", [{ + role: "user", + content: "The final deliverable is the full crawled dataset.", + timestamp: "2026-05-18T00:00:00.000Z", + }]); + + expect(seen).toHaveLength(0); + expect(scheduler.getSessionState("seed-session")?.conversation_count).toBe(1); + expect(scheduler.getBufferedMessageCount("seed-session")).toBe(1); + + await scheduler.flushSession("seed-session"); + + expect(seen).toHaveLength(1); + expect(seen[0]).toHaveLength(1); + expect(scheduler.getSessionState("seed-session")?.conversation_count).toBe(0); + expect(scheduler.getSessionState("seed-session")?.l2_pending_l1_count).toBe(1); + expect(scheduler.getBufferedMessageCount("seed-session")).toBe(0); + + await scheduler.destroy(); + }); + + it("flushSession drains DB-backed work when the in-memory buffer is empty", async () => { + const scheduler = new MemoryPipelineManager(config, logger); + let runs = 0; + + scheduler.setL1Runner(async ({ msg }) => { + runs += 1; + return { processedCount: msg.length }; + }); + + await scheduler.notifyConversation("seed-db-session", []); + + expect(scheduler.getSessionState("seed-db-session")?.conversation_count).toBe(1); + expect(scheduler.getBufferedMessageCount("seed-db-session")).toBe(0); + + await scheduler.flushSession("seed-db-session"); + + expect(runs).toBe(1); + expect(scheduler.getSessionState("seed-db-session")?.conversation_count).toBe(0); + + await scheduler.destroy(); + }); + + it("can run L1 work for multiple sessions concurrently", async () => { + const scheduler = new MemoryPipelineManager({ + ...config, + l1: { idleTimeoutSeconds: 3600, concurrency: 2 }, + }, logger); + let running = 0; + let maxRunning = 0; + + scheduler.setL1Runner(async ({ msg }) => { + running += 1; + maxRunning = Math.max(maxRunning, running); + await delay(10); + running -= 1; + return { processedCount: msg.length }; + }); + + await scheduler.notifyConversation("seed-session-a", [{ + role: "user", + content: "Session A needs to be extracted.", + timestamp: "2026-05-18T00:00:00.000Z", + }]); + await scheduler.notifyConversation("seed-session-b", [{ + role: "user", + content: "Session B needs to be extracted.", + timestamp: "2026-05-18T00:00:01.000Z", + }]); + + await Promise.all([ + scheduler.flushSession("seed-session-a"), + scheduler.flushSession("seed-session-b"), + ]); + + expect(maxRunning).toBe(2); + expect(scheduler.getBufferedMessageCount("seed-session-a")).toBe(0); + expect(scheduler.getBufferedMessageCount("seed-session-b")).toBe(0); + + await scheduler.destroy(); + }); + + it("flushSession waits for the target session instead of global L1 idle", async () => { + const scheduler = new MemoryPipelineManager({ + ...config, + l1: { idleTimeoutSeconds: 3600, concurrency: 2 }, + }, logger); + + scheduler.setL1Runner(async ({ sessionKey }) => { + if (sessionKey === "seed-session-a") { + await delay(80); + } + return { processedCount: 1 }; + }); + + await scheduler.notifyConversation("seed-session-a", [{ + role: "user", + content: "Session A is deliberately slow.", + timestamp: "2026-05-18T00:00:00.000Z", + }]); + const slowFlush = scheduler.flushSession("seed-session-a"); + await delay(5); + + await scheduler.notifyConversation("seed-session-b", [{ + role: "user", + content: "Session B should not wait for session A after its own L1 completes.", + timestamp: "2026-05-18T00:00:01.000Z", + }]); + + const start = Date.now(); + await scheduler.flushSession("seed-session-b"); + expect(Date.now() - start).toBeLessThan(70); + + await slowFlush; + await scheduler.destroy(); + }); +}); diff --git a/src/utils/pipeline-manager.ts b/src/utils/pipeline-manager.ts index d8d2d75..4464bff 100644 --- a/src/utils/pipeline-manager.ts +++ b/src/utils/pipeline-manager.ts @@ -123,6 +123,8 @@ export interface PipelineConfig { l1: { /** Idle timeout before triggering L1 (seconds, default: 60) */ idleTimeoutSeconds: number; + /** Number of L1 extraction tasks allowed to run concurrently (default: 1). */ + concurrency?: number; }; l2: { @@ -221,6 +223,8 @@ interface SessionTimerState { l2Schedule: ManagedTimer; /** Whether an L1 task is already queued or running for this session. */ l1Queued: boolean; + /** Promise for the currently queued/running L1 task for this session. */ + l1Promise?: Promise; /** Whether an L2 task is already queued or running for this session. */ l2Queued: boolean; /** Consecutive L1 failure count for retry limiting. Reset on success or new conversation. */ @@ -243,7 +247,7 @@ export class MemoryPipelineManager { private readonly L1_MAX_RETRIES = 5; // Queues (named for diagnostics) - private readonly l1Queue = new SerialQueue("L1"); + private readonly l1Queue: SerialQueue; private readonly l2Queue = new SerialQueue("L2"); private readonly l3Queue = new SerialQueue("L3"); @@ -296,11 +300,13 @@ export class MemoryPipelineManager { this.sessionActiveWindowMs = config.l2.sessionActiveWindowHours * 60 * 60 * 1000; this.logger = logger; this.sessionFilter = sessionFilter ?? new SessionFilter(); + this.l1Queue = new SerialQueue("L1", config.l1.concurrency ?? 1); this.logger?.debug?.( `${TAG} Initialized: everyNConversations=${config.everyNConversations}, ` + `warmup=${config.enableWarmup ? "enabled" : "disabled"}, ` + `l1IdleTimeout=${config.l1.idleTimeoutSeconds}s, ` + + `l1Concurrency=${this.l1Queue.concurrency}, ` + `l2DelayAfterL1=${config.l2.delayAfterL1Seconds}s, ` + `l2MinInterval=${config.l2.minIntervalSeconds}s, ` + `l2MaxInterval=${config.l2.maxIntervalSeconds}s, ` + @@ -493,10 +499,10 @@ export class MemoryPipelineManager { * fires for this key). * 2. If the session's message buffer still holds work, enqueue an * immediate L1 run for this session (``triggerReason="flush"``). - * 3. Await the shared ``l1Queue`` so the caller observes L1 - * completion before returning. We do not selectively wait - * because L1 is already a single-consumer SerialQueue — waiting - * for ``onIdle`` is the cheapest correct signal. + * 3. Await this session's queued/running L1 task so the caller observes + * target-session completion before returning. This intentionally does + * not wait for the whole L1 queue, because unrelated seed/import workers + * may be processing other sessions concurrently. * * What it deliberately does NOT do: * - Touch other sessions' timers / buffers / pipeline state. @@ -513,26 +519,27 @@ export class MemoryPipelineManager { const timers = this.sessionTimers.get(sessionKey); const buffer = this.messageBuffers.get(sessionKey); + const state = this.sessionStates.get(sessionKey); // Step 1: cancel the idle timer so it won't fire after we return. if (timers?.l1Idle.pending) { timers.l1Idle.cancel(); } - // Step 2: flush pending buffered messages through L1 if any. - if (buffer && buffer.length > 0) { + // Step 2: flush pending L1 work if any. In the Gateway/seed path the L1 + // runner reads L0 from the store, so the in-memory buffer can be empty + // while conversation_count still represents unextracted DB-backed work. + if ((buffer && buffer.length > 0) || (state && state.conversation_count > 0)) { this.logger?.debug?.( - `${TAG} [${sessionKey}] flushSession: enqueuing L1 for ${buffer.length} buffered message(s)`, + `${TAG} [${sessionKey}] flushSession: enqueuing L1 for ` + + `${buffer?.length ?? 0} buffered message(s), conversations=${state?.conversation_count ?? 0}`, ); this.enqueueL1(sessionKey, "flush"); } - // Step 3: wait for L1 to drain. L1 is a single-consumer SerialQueue - // so this is the cheapest correct signal; it will not starve other - // sessions because any cross-session interleaving L1 work was either - // already queued or will be queued concurrently by their own capture - // paths. - await this.l1Queue.onIdle(); + // Step 3: wait for this session's L1 task only. Waiting for the shared + // queue to become globally idle would serialize unrelated seed workers. + await timers?.l1Promise; this.logger?.debug?.(`${TAG} [${sessionKey}] flushSession: complete`); } @@ -644,15 +651,22 @@ export class MemoryPipelineManager { } try { - // Step 1: Flush all L1 idle timers — only enqueue if there are buffered messages + // Step 1: Flush all immediate L1 work. Do not key this solely off the + // idle timer: recovery/seed paths can have DB-backed conversation_count + // without a pending timer, and final flush must still advance it. for (const [sessionKey, timers] of this.sessionTimers) { if (timers.l1Idle.pending) { timers.l1Idle.cancel(); // don't fire the idle callback directly - const buffer = this.messageBuffers.get(sessionKey); - if (buffer && buffer.length > 0) { - this.logger?.debug?.(`${TAG} [${sessionKey}] Flush: enqueuing L1 for ${buffer.length} buffered messages`); - this.enqueueL1(sessionKey, "flush"); - } + } + + const buffer = this.messageBuffers.get(sessionKey); + const state = this.sessionStates.get(sessionKey); + if ((buffer && buffer.length > 0) || (state && state.conversation_count > 0)) { + this.logger?.debug?.( + `${TAG} [${sessionKey}] Flush: enqueuing L1 for ` + + `${buffer?.length ?? 0} buffered messages, conversations=${state?.conversation_count ?? 0}`, + ); + this.enqueueL1(sessionKey, "flush"); } } @@ -762,7 +776,7 @@ export class MemoryPipelineManager { }); } - this.l1Queue.add(async () => { + const taskPromise = this.l1Queue.add(async () => { await this.runL1(sessionKey); }).catch((err) => { this.logger?.error( @@ -770,7 +784,9 @@ export class MemoryPipelineManager { ); }).finally(() => { timers.l1Queued = false; + timers.l1Promise = undefined; }); + timers.l1Promise = taskPromise; } /** @@ -1248,11 +1264,67 @@ export class MemoryPipelineManager { return this.messageBuffers.get(sessionKey)?.length ?? 0; } + /** Whether a specific session still has L1 work queued, running, or buffered. */ + hasPendingL1Work(sessionKey: string): boolean { + const timers = this.sessionTimers.get(sessionKey); + const state = this.sessionStates.get(sessionKey); + const buffered = this.messageBuffers.get(sessionKey)?.length ?? 0; + const conversations = state?.conversation_count ?? 0; + + return Boolean(timers?.l1Queued) || buffered > 0 || conversations > 0; + } + /** Get all session keys being tracked. */ getSessionKeys(): string[] { return Array.from(this.sessionStates.keys()); } + /** Get session keys with L1 output that still needs L2 scene extraction. */ + getPendingL2SessionKeys(): string[] { + const keys: string[] = []; + for (const [sessionKey, state] of this.sessionStates) { + if ((state.l2_pending_l1_count ?? 0) > 0) keys.push(sessionKey); + } + return keys; + } + + /** + * Mark L2 complete for a set of sessions after an external/bulk L2 flush. + * + * Seed imports can coalesce many small historical Codex sessions into larger + * L2 batches to avoid thousands of tiny scene-extraction calls. This method + * updates the scheduler-owned state and cancels stale L2 timers so the later + * shutdown flush does not replay already-coalesced work. + */ + async markL2FlushedForSessions( + sessionKeys: string[], + latestCursorBySession: Map = new Map(), + ): Promise { + if (sessionKeys.length === 0) return; + + const nowIso = new Date().toISOString(); + for (const sessionKey of sessionKeys) { + const state = this.sessionStates.get(sessionKey); + if (!state) continue; + + state.l2_pending_l1_count = 0; + state.last_extraction_time = nowIso; + state.l2_last_extraction_time = nowIso; + + const latestCursor = latestCursorBySession.get(sessionKey); + if (latestCursor) { + state.last_extraction_updated_time = latestCursor; + } + + const timers = this.sessionTimers.get(sessionKey); + if (timers?.l2Schedule.pending) { + timers.l2Schedule.cancel(); + } + } + + await this.persistStates(); + } + /** Whether the pipeline has been destroyed. */ get isDestroyed(): boolean { return this.destroyed; diff --git a/src/utils/serial-queue.test.ts b/src/utils/serial-queue.test.ts new file mode 100644 index 0000000..805e196 --- /dev/null +++ b/src/utils/serial-queue.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { SerialQueue } from "./serial-queue.js"; + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +describe("SerialQueue", () => { + it("runs tasks with bounded concurrency", async () => { + const queue = new SerialQueue("test", 2); + let running = 0; + let maxRunning = 0; + + const tasks = Array.from({ length: 5 }, (_, index) => queue.add(async () => { + running += 1; + maxRunning = Math.max(maxRunning, running); + await delay(10); + running -= 1; + return index; + })); + + const results = await Promise.all(tasks); + + expect(results).toEqual([0, 1, 2, 3, 4]); + expect(maxRunning).toBe(2); + expect(queue.idle).toBe(true); + }); +}); diff --git a/src/utils/serial-queue.ts b/src/utils/serial-queue.ts index 63d179b..fa76408 100644 --- a/src/utils/serial-queue.ts +++ b/src/utils/serial-queue.ts @@ -1,9 +1,9 @@ /** - * SerialQueue: a lightweight task queue with concurrency=1. + * SerialQueue: a lightweight FIFO task queue. * - * Equivalent to `new PQueue({ concurrency: 1 })` but with zero external + * Equivalent to `new PQueue({ concurrency })` but with zero external * dependencies. Supports: - * - Serial execution (FIFO) + * - FIFO execution with bounded concurrency * - `add(fn)` to enqueue a task (returns the task's result promise) * - `onIdle()` to wait until all queued tasks have completed * - `pause()` / `start()` to suspend/resume execution @@ -22,17 +22,19 @@ interface QueueEntry { export class SerialQueue { /** Human-readable name for logging / diagnostics. */ public readonly name: string; + public readonly concurrency: number; private queue: QueueEntry[] = []; - private running = false; + private runningCount = 0; private paused = false; private idleResolvers: Array<() => void> = []; /** Optional debug logger — receives diagnostic messages for enqueue/dequeue/complete. */ private debugFn?: (msg: string) => void; - constructor(name = "unnamed") { + constructor(name = "unnamed", concurrency = 1) { this.name = name; + this.concurrency = Math.max(1, Math.floor(concurrency)); } /** Set a debug logger for queue diagnostics. */ @@ -47,12 +49,12 @@ export class SerialQueue { /** Whether a task is currently executing. */ get pending(): boolean { - return this.running; + return this.runningCount > 0; } /** Whether the queue is idle (no queued tasks and nothing running). */ get idle(): boolean { - return this.queue.length === 0 && !this.running; + return this.queue.length === 0 && this.runningCount === 0; } /** Add a task to the queue. Returns the task's result promise. */ @@ -63,7 +65,10 @@ export class SerialQueue { resolve: resolve as (value: unknown) => void, reject, }); - this.debugFn?.(`[queue:${this.name}] enqueued, pending=${this.queue.length}, running=${this.running}`); + this.debugFn?.( + `[queue:${this.name}] enqueued, pending=${this.queue.length}, ` + + `running=${this.runningCount}/${this.concurrency}`, + ); this.drain(); }); } @@ -81,7 +86,7 @@ export class SerialQueue { /** Returns a promise that resolves when all queued tasks have completed. */ onIdle(): Promise { - if (this.queue.length === 0 && !this.running) { + if (this.queue.length === 0 && this.runningCount === 0) { return Promise.resolve(); } return new Promise((resolve) => { @@ -98,28 +103,46 @@ export class SerialQueue { } private drain(): void { - if (this.running || this.paused || this.queue.length === 0) return; - - const entry = this.queue.shift()!; - this.running = true; - - this.debugFn?.(`[queue:${this.name}] dequeued, starting execution (remaining=${this.queue.length})`); - - entry - .task() - .then((result) => entry.resolve(result)) - .catch((err) => entry.reject(err)) - .finally(() => { - this.running = false; - this.debugFn?.(`[queue:${this.name}] task completed (remaining=${this.queue.length})`); - if (this.queue.length === 0) { - // Notify idle waiters - const resolvers = this.idleResolvers; - this.idleResolvers = []; - for (const resolve of resolvers) resolve(); + while (!this.paused && this.runningCount < this.concurrency && this.queue.length > 0) { + const entry = this.queue.shift()!; + this.runningCount++; + + this.debugFn?.( + `[queue:${this.name}] dequeued, starting execution ` + + `(remaining=${this.queue.length}, running=${this.runningCount}/${this.concurrency})`, + ); + + void (async () => { + let result: unknown; + let error: unknown; + let failed = false; + try { + result = await entry.task(); + } catch (err) { + error = err; + failed = true; + } finally { + this.runningCount--; + this.debugFn?.( + `[queue:${this.name}] task completed ` + + `(remaining=${this.queue.length}, running=${this.runningCount}/${this.concurrency})`, + ); + if (this.queue.length === 0 && this.runningCount === 0) { + // Notify idle waiters + const resolvers = this.idleResolvers; + this.idleResolvers = []; + for (const resolve of resolvers) resolve(); + } else { + this.drain(); + } + } + + if (failed) { + entry.reject(error); } else { - this.drain(); + entry.resolve(result); } - }); + })(); + } } } From c9961da15c9cee292e514ceca4218e126a23b6d8 Mon Sep 17 00:00:00 2001 From: Siyao Zheng Date: Wed, 20 May 2026 12:35:40 +0800 Subject: [PATCH 3/5] fix(codex): bound scoped search candidates Signed-off-by: Siyao Zheng --- src/core/store/sqlite.ts | 121 ++++++++++++++++++-- src/core/store/tcvdb.ts | 112 +++++++++++++++--- src/core/store/types.ts | 16 ++- src/core/tools/conversation-search.ts | 116 +++++++------------ src/core/tools/memory-search.ts | 115 +++++++------------ src/core/tools/search-prefix-filter.test.ts | 36 ++++-- 6 files changed, 332 insertions(+), 184 deletions(-) diff --git a/src/core/store/sqlite.ts b/src/core/store/sqlite.ts index 6252419..f57f35c 100644 --- a/src/core/store/sqlite.ts +++ b/src/core/store/sqlite.ts @@ -32,6 +32,7 @@ import type { L1FtsResult, L0SearchResult, L0FtsResult, + SearchScopeOptions, } from "./types.js"; // ============================ @@ -302,6 +303,59 @@ export function bm25RankToScore(rank: number): number { return 1 / (1 + rank); } +function normalizeSearchScope(scope?: SearchScopeOptions): { sessionKey?: string; sessionKeyPrefixes: string[] } { + const sessionKey = typeof scope?.sessionKey === "string" ? scope.sessionKey.trim() : ""; + const sessionKeyPrefixes = Array.isArray(scope?.sessionKeyPrefixes) + ? scope.sessionKeyPrefixes + .map((prefix) => typeof prefix === "string" ? prefix.trim() : "") + .filter(Boolean) + .slice(0, 20) + : []; + const normalized: { sessionKey?: string; sessionKeyPrefixes: string[] } = { sessionKeyPrefixes }; + if (sessionKey) normalized.sessionKey = sessionKey; + return normalized; +} + +function hasSearchScope(scope?: SearchScopeOptions): boolean { + const normalized = normalizeSearchScope(scope); + return !!normalized.sessionKey || normalized.sessionKeyPrefixes.length > 0; +} + +function matchesSearchScope(sessionKey: string, scope?: SearchScopeOptions): boolean { + const normalized = normalizeSearchScope(scope); + if (normalized.sessionKey && sessionKey !== normalized.sessionKey) return false; + if ( + normalized.sessionKeyPrefixes.length > 0 && + !normalized.sessionKeyPrefixes.some((prefix) => sessionKey.startsWith(prefix)) + ) { + return false; + } + return true; +} + +function escapeSqliteLike(value: string): string { + return value.replace(/[\\%_]/g, (ch) => `\\${ch}`); +} + +function buildSqliteSessionScopeFilter(scope?: SearchScopeOptions): { clause: string; params: string[] } | undefined { + const normalized = normalizeSearchScope(scope); + const clauses: string[] = []; + const params: string[] = []; + + if (normalized.sessionKey) { + clauses.push("session_key = ?"); + params.push(normalized.sessionKey); + } + + if (normalized.sessionKeyPrefixes.length > 0) { + clauses.push(`(${normalized.sessionKeyPrefixes.map(() => "session_key LIKE ? ESCAPE '\\'").join(" OR ")})`); + params.push(...normalized.sessionKeyPrefixes.map((prefix) => `${escapeSqliteLike(prefix)}%`)); + } + + if (clauses.length === 0) return undefined; + return { clause: clauses.join(" AND "), params }; +} + /** FTS5 search result for L1 records. */ export interface FtsSearchResult { record_id: string; @@ -935,6 +989,9 @@ export class VectorStore implements IMemoryStore { */ private static readonly ZERO_VEC_BUFFER = 10; + /** Bounded extra vector candidates when a metadata scope must be applied. */ + private static readonly SCOPED_VECTOR_EXTRA_CANDIDATES = 200; + /** Default result limit for FTS5 keyword searches. */ private static readonly FTS_DEFAULT_LIMIT = 20; @@ -1109,7 +1166,12 @@ export class VectorStore implements IMemoryStore { * **Fault-tolerant**: returns an empty array on any error (e.g. dimension * mismatch, corrupted DB) so callers can fall back to keyword search. */ - searchL1Vector(queryEmbedding: Float32Array, topK = 5): VectorSearchResult[] { + searchL1Vector( + queryEmbedding: Float32Array, + topK = 5, + _queryText?: string, + scope?: SearchScopeOptions, + ): VectorSearchResult[] { if (this.degraded || !this.vecTablesReady) { if (this.degraded) this.logger?.warn(`${TAG} [L1-search] SKIPPED (degraded mode)`); return []; @@ -1122,8 +1184,8 @@ export class VectorStore implements IMemoryStore { // in KNN results. A small buffer of 10 is sufficient for remnants. // NOTE: "AND distance IS NOT NULL" is NOT usable because vec0 does not // support that constraint — it causes an empty result set. - const ZERO_VEC_BUFFER = 10; - const retrieveCount = topK + ZERO_VEC_BUFFER; + const scopeExtra = hasSearchScope(scope) ? VectorStore.SCOPED_VECTOR_EXTRA_CANDIDATES : 0; + const retrieveCount = topK + VectorStore.ZERO_VEC_BUFFER + scopeExtra; this.logger?.debug?.( `${TAG} [L1-search] START topK=${topK}, retrieveCount=${retrieveCount}, ` + @@ -1171,6 +1233,9 @@ export class VectorStore implements IMemoryStore { this.logger?.warn(`${TAG} [L1-search] record_id=${record_id} has vector but NO metadata (orphan)`); continue; } + if (!matchesSearchScope(meta.session_key, scope)) { + continue; + } const score = 1.0 - distance; this.logger?.debug?.( @@ -1192,6 +1257,7 @@ export class VectorStore implements IMemoryStore { session_id: meta.session_id, metadata_json: meta.metadata_json, }); + if (results.length >= topK) break; } // Trim back to the caller's requested topK (we over-fetched above). @@ -1543,7 +1609,12 @@ export class VectorStore implements IMemoryStore { * * **Fault-tolerant**: returns an empty array on any error. */ - searchL0Vector(queryEmbedding: Float32Array, topK = 5): L0VectorSearchResult[] { + searchL0Vector( + queryEmbedding: Float32Array, + topK = 5, + _queryText?: string, + scope?: SearchScopeOptions, + ): L0VectorSearchResult[] { if (this.degraded || !this.vecTablesReady) { if (this.degraded) this.logger?.warn(`${TAG} [L0-search] SKIPPED (degraded mode)`); return []; @@ -1556,7 +1627,8 @@ export class VectorStore implements IMemoryStore { // in KNN results. // NOTE: "AND distance IS NOT NULL" is NOT usable because vec0 does not // support that constraint — it causes an empty result set. - const retrieveCount = topK + VectorStore.ZERO_VEC_BUFFER; + const scopeExtra = hasSearchScope(scope) ? VectorStore.SCOPED_VECTOR_EXTRA_CANDIDATES : 0; + const retrieveCount = topK + VectorStore.ZERO_VEC_BUFFER + scopeExtra; this.logger?.debug?.( `${TAG} [L0-search] START topK=${topK}, retrieveCount=${retrieveCount}, ` + @@ -1600,6 +1672,9 @@ export class VectorStore implements IMemoryStore { this.logger?.warn(`${TAG} [L0-search] record_id=${record_id} has vector but NO metadata (orphan)`); continue; } + if (!matchesSearchScope(meta.session_key, scope)) { + continue; + } const score = 1.0 - distance; this.logger?.debug?.( @@ -1617,6 +1692,7 @@ export class VectorStore implements IMemoryStore { recorded_at: meta.recorded_at, timestamp: meta.timestamp ?? 0, }); + if (results.length >= topK) break; } // Trim back to the caller's requested topK (we over-fetched above). @@ -2026,10 +2102,22 @@ export class VectorStore implements IMemoryStore { * * **Fault-tolerant**: returns an empty array on any error. */ - searchL1Fts(ftsQuery: string, limit = 20): FtsSearchResult[] { + searchL1Fts(ftsQuery: string, limit = 20, scope?: SearchScopeOptions): FtsSearchResult[] { if (this.degraded || !this.ftsAvailable) return []; try { - const rows = this.stmtL1FtsSearch.all(ftsQuery, limit) as Array<{ + const scopeFilter = buildSqliteSessionScopeFilter(scope); + const rows = (scopeFilter + ? this.db.prepare(` + SELECT record_id, content_original AS content, type, priority, scene_name, + session_key, session_id, timestamp_str, timestamp_start, timestamp_end, + metadata_json, + bm25(l1_fts) AS rank + FROM l1_fts + WHERE l1_fts MATCH ? AND ${scopeFilter.clause} + ORDER BY rank ASC + LIMIT ? + `).all(ftsQuery, ...scopeFilter.params, limit) + : this.stmtL1FtsSearch.all(ftsQuery, limit)) as Array<{ record_id: string; content: string; type: string; @@ -2075,10 +2163,25 @@ export class VectorStore implements IMemoryStore { * * **Fault-tolerant**: returns an empty array on any error. */ - searchL0Fts(ftsQuery: string, limit = VectorStore.FTS_DEFAULT_LIMIT): L0FtsSearchResult[] { + searchL0Fts( + ftsQuery: string, + limit = VectorStore.FTS_DEFAULT_LIMIT, + scope?: SearchScopeOptions, + ): L0FtsSearchResult[] { if (this.degraded || !this.ftsAvailable) return []; try { - const rows = this.stmtL0FtsSearch.all(ftsQuery, limit) as Array<{ + const scopeFilter = buildSqliteSessionScopeFilter(scope); + const rows = (scopeFilter + ? this.db.prepare(` + SELECT record_id, message_text_original AS message_text, session_key, + session_id, role, recorded_at, timestamp, + bm25(l0_fts) AS rank + FROM l0_fts + WHERE l0_fts MATCH ? AND ${scopeFilter.clause} + ORDER BY rank ASC + LIMIT ? + `).all(ftsQuery, ...scopeFilter.params, limit) + : this.stmtL0FtsSearch.all(ftsQuery, limit)) as Array<{ record_id: string; message_text: string; session_key: string; diff --git a/src/core/store/tcvdb.ts b/src/core/store/tcvdb.ts index 35fe6a9..07195ea 100644 --- a/src/core/store/tcvdb.ts +++ b/src/core/store/tcvdb.ts @@ -27,6 +27,7 @@ import type { L0SessionGroup, ProfileRecord, ProfileSyncRecord, + SearchScopeOptions, StoreLogger, } from "./types.js"; import { TcvdbClient, TcvdbApiError } from "./tcvdb-client.js"; @@ -112,6 +113,68 @@ function extractAgentId(sessionKey: string): string { return ""; } +function normalizeSearchScope(scope?: SearchScopeOptions): { sessionKey?: string; sessionKeyPrefixes: string[] } { + const sessionKey = typeof scope?.sessionKey === "string" ? scope.sessionKey.trim() : ""; + const sessionKeyPrefixes = Array.isArray(scope?.sessionKeyPrefixes) + ? scope.sessionKeyPrefixes + .map((prefix) => typeof prefix === "string" ? prefix.trim() : "") + .filter(Boolean) + .slice(0, 20) + : []; + const normalized: { sessionKey?: string; sessionKeyPrefixes: string[] } = { sessionKeyPrefixes }; + if (sessionKey) normalized.sessionKey = sessionKey; + return normalized; +} + +function hasSearchScope(scope?: SearchScopeOptions): boolean { + const normalized = normalizeSearchScope(scope); + return !!normalized.sessionKey || normalized.sessionKeyPrefixes.length > 0; +} + +function matchesSearchScope(sessionKey: string, scope?: SearchScopeOptions): boolean { + const normalized = normalizeSearchScope(scope); + if (normalized.sessionKey && sessionKey !== normalized.sessionKey) return false; + if ( + normalized.sessionKeyPrefixes.length > 0 && + !normalized.sessionKeyPrefixes.some((prefix) => sessionKey.startsWith(prefix)) + ) { + return false; + } + return true; +} + +function escapeTcvdbFilterString(value: string): string { + return value.replace(/\\/g, "\\\\").replace(/"/g, "\\\""); +} + +function buildTcvdbSessionScopeFilter(scope?: SearchScopeOptions): string | undefined { + const normalized = normalizeSearchScope(scope); + const clauses: string[] = []; + + if (normalized.sessionKey) { + clauses.push(`session_key = "${escapeTcvdbFilterString(normalized.sessionKey)}"`); + } + + if (normalized.sessionKeyPrefixes.length > 0) { + const prefixClauses = normalized.sessionKeyPrefixes.map( + (prefix) => `session_key like "${escapeTcvdbFilterString(prefix)}%"`, + ); + clauses.push(`(${prefixClauses.join(" or ")})`); + } + + return clauses.length > 0 ? clauses.join(" and ") : undefined; +} + +function filterL1ResultsByScope(results: L1SearchResult[], scope?: SearchScopeOptions): L1SearchResult[] { + if (!hasSearchScope(scope)) return results; + return results.filter((result) => matchesSearchScope(result.session_key, scope)); +} + +function filterL0ResultsByScope(results: L0SearchResult[], scope?: SearchScopeOptions): L0SearchResult[] { + if (!hasSearchScope(scope)) return results; + return results.filter((result) => matchesSearchScope(result.session_key, scope)); +} + // ============================ // TcvdbMemoryStore // ============================ @@ -615,21 +678,26 @@ export class TcvdbMemoryStore implements IMemoryStore { // ── L1 Search Operations ───────────────────────────────── - async searchL1Vector(_queryEmbedding: Float32Array, topK?: number, queryText?: string): Promise { + async searchL1Vector( + _queryEmbedding: Float32Array, + topK?: number, + queryText?: string, + scope?: SearchScopeOptions, + ): Promise { // TCVDB uses server-side embedding — delegate to hybrid search with text if (queryText) { - return this.searchL1HybridAsync({ queryText, topK }); + return this.searchL1HybridAsync({ queryText, topK, scope }); } // No queryText and TCVDB can't use client embeddings directly via embeddingItems // Return empty — callers should pass queryText for TCVDB return []; } - async searchL1Fts(ftsQuery: string, limit?: number): Promise { + async searchL1Fts(ftsQuery: string, limit?: number, scope?: SearchScopeOptions): Promise { // TCVDB has no pure FTS — use hybrid search with sparse-only path // The ftsQuery is raw text, use it as queryText for hybrid if (!ftsQuery) return []; - const results = await this.searchL1HybridAsync({ queryText: ftsQuery, topK: limit }); + const results = await this.searchL1HybridAsync({ queryText: ftsQuery, topK: limit, scope }); // L1SearchResult and L1FtsResult have identical shapes return results; } @@ -639,10 +707,11 @@ export class TcvdbMemoryStore implements IMemoryStore { queryEmbedding?: Float32Array; sparseVector?: SparseVector; topK?: number; + scope?: SearchScopeOptions; }): Promise { const queryText = params.query; if (!queryText) return []; - return this.searchL1HybridAsync({ queryText, topK: params.topK }); + return this.searchL1HybridAsync({ queryText, topK: params.topK, scope: params.scope }); } /** @@ -652,8 +721,9 @@ export class TcvdbMemoryStore implements IMemoryStore { async searchL1HybridAsync(params: { queryText: string; topK?: number; + scope?: SearchScopeOptions; }): Promise { - const { queryText, topK = 10 } = params; + const { queryText, topK = 10, scope } = params; if (!queryText) return []; try { @@ -665,6 +735,8 @@ export class TcvdbMemoryStore implements IMemoryStore { limit: topK, outputFields: L1_OUTPUT_FIELDS, }; + const filterExpr = buildTcvdbSessionScopeFilter(scope); + if (filterExpr) searchParams.filter = filterExpr; // ann: use embedding field name "text" for server-side embedding // (per SDK: AnnSearch(field_name="text", data='query string')) @@ -693,7 +765,7 @@ export class TcvdbMemoryStore implements IMemoryStore { searchParams.rerank = { method: "rrf", k: 60 }; const resp = await this.client.hybridSearch(this.l1Collection, searchParams); - return this._parseL1SearchResults(resp.documents); + return filterL1ResultsByScope(this._parseL1SearchResults(resp.documents), scope); } else { // Dense-only fallback (BM25 unavailable) — use /document/search with embeddingItems const denseSearch: Record = { @@ -702,8 +774,9 @@ export class TcvdbMemoryStore implements IMemoryStore { retrieveVector: false, outputFields: L1_OUTPUT_FIELDS, }; + if (filterExpr) denseSearch.filter = filterExpr; const resp = await this.client.search(this.l1Collection, denseSearch); - return this._parseL1SearchResults(resp.documents); + return filterL1ResultsByScope(this._parseL1SearchResults(resp.documents), scope); } } catch (err) { this.logger?.warn(`${TAG} [L1-hybridSearch] FAILED: ${err instanceof Error ? err.message : String(err)}`); @@ -929,18 +1002,23 @@ export class TcvdbMemoryStore implements IMemoryStore { // ── L0 Search Operations ───────────────────────────────── - async searchL0Vector(_queryEmbedding: Float32Array, topK?: number, queryText?: string): Promise { + async searchL0Vector( + _queryEmbedding: Float32Array, + topK?: number, + queryText?: string, + scope?: SearchScopeOptions, + ): Promise { // TCVDB uses server-side embedding — delegate to hybrid search with text if (queryText) { - return this.searchL0HybridAsync({ queryText, topK }); + return this.searchL0HybridAsync({ queryText, topK, scope }); } return []; } - async searchL0Fts(ftsQuery: string, limit?: number): Promise { + async searchL0Fts(ftsQuery: string, limit?: number, scope?: SearchScopeOptions): Promise { if (!ftsQuery) return []; // Use hybrid search; L0SearchResult and L0FtsResult have identical shapes - return this.searchL0HybridAsync({ queryText: ftsQuery, topK: limit }); + return this.searchL0HybridAsync({ queryText: ftsQuery, topK: limit, scope }); } /** @@ -949,8 +1027,9 @@ export class TcvdbMemoryStore implements IMemoryStore { async searchL0HybridAsync(params: { queryText: string; topK?: number; + scope?: SearchScopeOptions; }): Promise { - const { queryText, topK = 10 } = params; + const { queryText, topK = 10, scope } = params; if (!queryText) return []; try { @@ -961,6 +1040,8 @@ export class TcvdbMemoryStore implements IMemoryStore { limit: topK, outputFields: L0_OUTPUT_FIELDS, }; + const filterExpr = buildTcvdbSessionScopeFilter(scope); + if (filterExpr) searchParams.filter = filterExpr; // ann: use embedding field name "message_text" for L0 server-side embedding const ann = [{ @@ -986,7 +1067,7 @@ export class TcvdbMemoryStore implements IMemoryStore { searchParams.match = match; searchParams.rerank = { method: "rrf", k: 60 }; const resp = await this.client.hybridSearch(this.l0Collection, searchParams); - return this._parseL0SearchResults(resp.documents); + return filterL0ResultsByScope(this._parseL0SearchResults(resp.documents), scope); } else { const denseSearch: Record = { embeddingItems: [queryText], @@ -994,8 +1075,9 @@ export class TcvdbMemoryStore implements IMemoryStore { retrieveVector: false, outputFields: L0_OUTPUT_FIELDS, }; + if (filterExpr) denseSearch.filter = filterExpr; const resp = await this.client.search(this.l0Collection, denseSearch); - return this._parseL0SearchResults(resp.documents); + return filterL0ResultsByScope(this._parseL0SearchResults(resp.documents), scope); } } catch (err) { this.logger?.warn(`${TAG} [L0-hybridSearch] FAILED: ${err instanceof Error ? err.message : String(err)}`); diff --git a/src/core/store/types.ts b/src/core/store/types.ts index cfcb50a..8b0ad4e 100644 --- a/src/core/store/types.ts +++ b/src/core/store/types.ts @@ -236,6 +236,13 @@ export interface ProfileSyncRecord extends ProfileRecord { */ export type MaybePromise = T | Promise; +export interface SearchScopeOptions { + /** Exact session key constraint. */ + sessionKey?: string; + /** Prefix constraints used by Codex project-scoped sessions. */ + sessionKeyPrefixes?: string[]; +} + export interface IMemoryStore { // ── Capabilities ─────────────────────────────────────────── @@ -270,13 +277,14 @@ export interface IMemoryStore { // ── L1 Search ──────────────────────────────────────────── - searchL1Vector(queryEmbedding: Float32Array, topK?: number, queryText?: string): MaybePromise; - searchL1Fts(ftsQuery: string, limit?: number): MaybePromise; + searchL1Vector(queryEmbedding: Float32Array, topK?: number, queryText?: string, scope?: SearchScopeOptions): MaybePromise; + searchL1Fts(ftsQuery: string, limit?: number, scope?: SearchScopeOptions): MaybePromise; searchL1Hybrid?(params: { query?: string; queryEmbedding?: Float32Array; sparseVector?: Array<[number, number]>; topK?: number; + scope?: SearchScopeOptions; }): MaybePromise; // ── L0 Write ───────────────────────────────────────────── @@ -296,8 +304,8 @@ export interface IMemoryStore { // ── L0 Search ──────────────────────────────────────────── - searchL0Vector(queryEmbedding: Float32Array, topK?: number, queryText?: string): MaybePromise; - searchL0Fts(ftsQuery: string, limit?: number): MaybePromise; + searchL0Vector(queryEmbedding: Float32Array, topK?: number, queryText?: string, scope?: SearchScopeOptions): MaybePromise; + searchL0Fts(ftsQuery: string, limit?: number, scope?: SearchScopeOptions): MaybePromise; pullProfiles?(): Promise; syncProfiles?(records: ProfileSyncRecord[]): Promise; diff --git a/src/core/tools/conversation-search.ts b/src/core/tools/conversation-search.ts index 423a702..d4acd17 100644 --- a/src/core/tools/conversation-search.ts +++ b/src/core/tools/conversation-search.ts @@ -10,7 +10,7 @@ * The tool is registered via `api.registerTool()` in index.ts. */ -import type { IMemoryStore, L0SearchResult } from "../store/types.js"; +import type { IMemoryStore, L0SearchResult, SearchScopeOptions } from "../store/types.js"; import { buildFtsQuery } from "../store/sqlite.js"; import type { EmbeddingService } from "../store/embedding.js"; @@ -47,7 +47,6 @@ export interface ConversationSearchResult { const TAG = "[memory-tdai][tdai_conversation_search]"; const FILTERED_SEARCH_INITIAL_CANDIDATES = 50; -const FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES = 500; // ============================ // RRF (Reciprocal Rank Fusion) @@ -144,81 +143,49 @@ export async function executeConversationSearch(params: { } const hasSessionScope = !!sessionFilter || normalizedSessionPrefixes.length > 0; - let candidateK = hasSessionScope + const searchScope: SearchScopeOptions | undefined = hasSessionScope + ? { + ...(sessionFilter ? { sessionKey: sessionFilter } : {}), + sessionKeyPrefixes: normalizedSessionPrefixes, + } + : undefined; + const candidateK = hasSessionScope ? Math.max(limit * 6, FILTERED_SEARCH_INITIAL_CANDIDATES) : limit * 3; - const maxCandidateK = hasSessionScope - ? await scopedSearchMaxCandidates({ - count: () => vectorStore.countL0(), - candidateK, - logger, - }) - : candidateK; - - while (true) { - const search = await collectConversationCandidates({ - query, - candidateK, - hasFts, - hasEmbedding, - vectorStore, - embeddingService, - logger, - }); - - if (search.results.length === 0) { - logger?.debug?.(`${TAG} Both search paths returned 0 results`); - return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" }; - } - const filtered = filterConversationResults(search.results, { - sessionFilter, - sessionPrefixes: normalizedSessionPrefixes, - logger, - }); - const trimmed = filtered.slice(0, limit); - - if ( - trimmed.length >= limit || - !hasSessionScope || - !search.mayHaveMore || - candidateK >= maxCandidateK - ) { - logger?.debug?.( - `${TAG} RESULT (strategy=${search.strategy}, candidateK=${candidateK}): returning ${trimmed.length} messages ` + - `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`, - ); - - return { - results: trimmed, - total: trimmed.length, - strategy: search.strategy, - }; - } + const search = await collectConversationCandidates({ + query, + candidateK, + hasFts, + hasEmbedding, + vectorStore, + embeddingService, + searchScope, + logger, + }); - candidateK = Math.min(candidateK * 2, maxCandidateK); - logger?.debug?.(`${TAG} Expanding scoped search window to candidateK=${candidateK}`); + if (search.results.length === 0) { + logger?.debug?.(`${TAG} Both search paths returned 0 results`); + return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" }; } -} -async function scopedSearchMaxCandidates(params: { - count: () => number | Promise; - candidateK: number; - logger?: Logger; -}): Promise { - const { count, candidateK, logger } = params; - try { - const total = await count(); - if (Number.isFinite(total) && total > 0) { - return Math.max(candidateK, Math.floor(total)); - } - } catch (err) { - logger?.warn?.( - `${TAG} Scoped search could not count records; falling back to ${FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES} candidates: ` + - `${err instanceof Error ? err.message : String(err)}`, - ); - } - return Math.max(candidateK, FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES); + const filtered = filterConversationResults(search.results, { + sessionFilter, + sessionPrefixes: normalizedSessionPrefixes, + logger, + }); + const trimmed = filtered.slice(0, limit); + + logger?.debug?.( + `${TAG} RESULT (strategy=${search.strategy}, candidateK=${candidateK}): returning ${trimmed.length} messages ` + + `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`, + ); + + return { + results: trimmed, + total: trimmed.length, + strategy: search.strategy, + }; } async function collectConversationCandidates(params: { @@ -228,9 +195,10 @@ async function collectConversationCandidates(params: { hasEmbedding: boolean; vectorStore: IMemoryStore; embeddingService?: EmbeddingService; + searchScope?: SearchScopeOptions; logger?: Logger; }): Promise<{ results: ConversationSearchResultItem[]; strategy: string; mayHaveMore: boolean }> { - const { query, candidateK, hasFts, hasEmbedding, vectorStore, embeddingService, logger } = params; + const { query, candidateK, hasFts, hasEmbedding, vectorStore, embeddingService, searchScope, logger } = params; const [ftsItems, vecItems] = await Promise.all([ (async (): Promise => { @@ -242,7 +210,7 @@ async function collectConversationCandidates(params: { return []; } logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`); - const ftsResults = await vectorStore.searchL0Fts(ftsQuery, candidateK); + const ftsResults = await vectorStore.searchL0Fts(ftsQuery, candidateK, searchScope); logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`); return ftsResults.map(conversationResultItemFromStore); } catch (err) { @@ -260,7 +228,7 @@ async function collectConversationCandidates(params: { logger?.debug?.( `${TAG} [hybrid-vec] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`, ); - const vecResults: L0SearchResult[] = await vectorStore.searchL0Vector(queryEmbedding, candidateK, query); + const vecResults: L0SearchResult[] = await vectorStore.searchL0Vector(queryEmbedding, candidateK, query, searchScope); logger?.debug?.(`${TAG} [hybrid-vec] Vector search returned ${vecResults.length} candidates`); return vecResults.map(conversationResultItemFromStore); } catch (err) { diff --git a/src/core/tools/memory-search.ts b/src/core/tools/memory-search.ts index 76145c0..3487bdc 100644 --- a/src/core/tools/memory-search.ts +++ b/src/core/tools/memory-search.ts @@ -10,7 +10,7 @@ * The tool is registered via `api.registerTool()` in index.ts. */ -import type { IMemoryStore, L1SearchResult } from "../store/types.js"; +import type { IMemoryStore, L1SearchResult, SearchScopeOptions } from "../store/types.js"; import { buildFtsQuery } from "../store/sqlite.js"; import type { EmbeddingService } from "../store/embedding.js"; @@ -48,7 +48,6 @@ export interface MemorySearchResult { const TAG = "[memory-tdai][tdai_memory_search]"; const FILTERED_SEARCH_INITIAL_CANDIDATES = 50; -const FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES = 500; // ============================ // RRF (Reciprocal Rank Fusion) @@ -146,82 +145,47 @@ export async function executeMemorySearch(params: { }; } - let candidateK = normalizedSessionPrefixes.length > 0 + const searchScope: SearchScopeOptions | undefined = normalizedSessionPrefixes.length > 0 + ? { sessionKeyPrefixes: normalizedSessionPrefixes } + : undefined; + const candidateK = normalizedSessionPrefixes.length > 0 ? Math.max(limit * 6, FILTERED_SEARCH_INITIAL_CANDIDATES) : limit * 3; - const maxCandidateK = normalizedSessionPrefixes.length > 0 - ? await scopedSearchMaxCandidates({ - count: () => vectorStore.countL1(), - candidateK, - logger, - }) - : candidateK; - - while (true) { - const search = await collectMemoryCandidates({ - query, - candidateK, - hasFts, - hasEmbedding, - vectorStore, - embeddingService, - logger, - }); - - if (search.results.length === 0) { - logger?.debug?.(`${TAG} Both search paths returned 0 results`); - return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" }; - } - const filtered = filterMemoryResults(search.results, { - typeFilter, - sceneFilter, - sessionPrefixes: normalizedSessionPrefixes, - logger, - }); - const trimmed = filtered.slice(0, limit); - - if ( - trimmed.length >= limit || - normalizedSessionPrefixes.length === 0 || - !search.mayHaveMore || - candidateK >= maxCandidateK - ) { - logger?.debug?.( - `${TAG} RESULT (strategy=${search.strategy}, candidateK=${candidateK}): returning ${trimmed.length} memories ` + - `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`, - ); - - return { - results: trimmed, - total: trimmed.length, - strategy: search.strategy, - }; - } + const search = await collectMemoryCandidates({ + query, + candidateK, + hasFts, + hasEmbedding, + vectorStore, + embeddingService, + searchScope, + logger, + }); - candidateK = Math.min(candidateK * 2, maxCandidateK); - logger?.debug?.(`${TAG} Expanding scoped search window to candidateK=${candidateK}`); + if (search.results.length === 0) { + logger?.debug?.(`${TAG} Both search paths returned 0 results`); + return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" }; } -} -async function scopedSearchMaxCandidates(params: { - count: () => number | Promise; - candidateK: number; - logger?: Logger; -}): Promise { - const { count, candidateK, logger } = params; - try { - const total = await count(); - if (Number.isFinite(total) && total > 0) { - return Math.max(candidateK, Math.floor(total)); - } - } catch (err) { - logger?.warn?.( - `${TAG} Scoped search could not count records; falling back to ${FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES} candidates: ` + - `${err instanceof Error ? err.message : String(err)}`, - ); - } - return Math.max(candidateK, FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES); + const filtered = filterMemoryResults(search.results, { + typeFilter, + sceneFilter, + sessionPrefixes: normalizedSessionPrefixes, + logger, + }); + const trimmed = filtered.slice(0, limit); + + logger?.debug?.( + `${TAG} RESULT (strategy=${search.strategy}, candidateK=${candidateK}): returning ${trimmed.length} memories ` + + `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`, + ); + + return { + results: trimmed, + total: trimmed.length, + strategy: search.strategy, + }; } async function collectMemoryCandidates(params: { @@ -231,9 +195,10 @@ async function collectMemoryCandidates(params: { hasEmbedding: boolean; vectorStore: IMemoryStore; embeddingService?: EmbeddingService; + searchScope?: SearchScopeOptions; logger?: Logger; }): Promise<{ results: MemorySearchResultItem[]; strategy: string; mayHaveMore: boolean }> { - const { query, candidateK, hasFts, hasEmbedding, vectorStore, embeddingService, logger } = params; + const { query, candidateK, hasFts, hasEmbedding, vectorStore, embeddingService, searchScope, logger } = params; const [ftsItems, vecItems] = await Promise.all([ (async (): Promise => { @@ -245,7 +210,7 @@ async function collectMemoryCandidates(params: { return []; } logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`); - const ftsResults = await vectorStore.searchL1Fts(ftsQuery, candidateK); + const ftsResults = await vectorStore.searchL1Fts(ftsQuery, candidateK, searchScope); logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`); return ftsResults.map(memoryResultItemFromStore); } catch (err) { @@ -263,7 +228,7 @@ async function collectMemoryCandidates(params: { logger?.debug?.( `${TAG} [hybrid-vec] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`, ); - const vecResults: L1SearchResult[] = await vectorStore.searchL1Vector(queryEmbedding, candidateK, query); + const vecResults: L1SearchResult[] = await vectorStore.searchL1Vector(queryEmbedding, candidateK, query, searchScope); logger?.debug?.(`${TAG} [hybrid-vec] Vector search returned ${vecResults.length} candidates`); return vecResults.map(memoryResultItemFromStore); } catch (err) { diff --git a/src/core/tools/search-prefix-filter.test.ts b/src/core/tools/search-prefix-filter.test.ts index 04fcd4d..5cfbc37 100644 --- a/src/core/tools/search-prefix-filter.test.ts +++ b/src/core/tools/search-prefix-filter.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { executeConversationSearch } from "./conversation-search.js"; import { executeMemorySearch } from "./memory-search.js"; @@ -15,21 +15,32 @@ describe("session-prefix search filters", () => { l1Result("a", "codex:abc123:session-a"), l1Result("c", "codex-import:abc123:session-c"), ]; + const prefixes = ["codex:abc123:", "codex-import:abc123:"]; + const countL1 = vi.fn(() => rows.length); + const searchL1Fts = vi.fn((_query: string, limit: number, scope?: { sessionKeyPrefixes?: string[] }) => + rows + .filter((row) => scope?.sessionKeyPrefixes?.some((prefix) => row.session_key.startsWith(prefix))) + .slice(0, limit), + ); const vectorStore = { isFtsAvailable: () => true, - countL1: () => rows.length, - searchL1Fts: (_query: string, limit: number) => rows.slice(0, limit), + countL1, + searchL1Fts, }; const result = await executeMemorySearch({ query: "project note", limit: 2, - sessionKeyPrefixes: ["codex:abc123:", "codex-import:abc123:"], + sessionKeyPrefixes: prefixes, vectorStore: vectorStore as any, logger, }); expect(result.results.map((item) => item.id)).toEqual(["a", "c"]); + expect(countL1).not.toHaveBeenCalled(); + expect(searchL1Fts).toHaveBeenCalledTimes(1); + expect(searchL1Fts.mock.calls[0][1]).toBeLessThan(rows.length); + expect(searchL1Fts.mock.calls[0][2]).toEqual({ sessionKeyPrefixes: prefixes }); }); it("filters L0 conversation search results by session-key prefix", async () => { @@ -38,21 +49,32 @@ describe("session-prefix search filters", () => { l0Result("a", "codex:abc123:session-a"), l0Result("c", "codex-import:abc123:session-c"), ]; + const prefixes = ["codex:abc123:", "codex-import:abc123:"]; + const countL0 = vi.fn(() => rows.length); + const searchL0Fts = vi.fn((_query: string, limit: number, scope?: { sessionKeyPrefixes?: string[] }) => + rows + .filter((row) => scope?.sessionKeyPrefixes?.some((prefix) => row.session_key.startsWith(prefix))) + .slice(0, limit), + ); const vectorStore = { isFtsAvailable: () => true, - countL0: () => rows.length, - searchL0Fts: (_query: string, limit: number) => rows.slice(0, limit), + countL0, + searchL0Fts, }; const result = await executeConversationSearch({ query: "previous command", limit: 2, - sessionKeyPrefixes: ["codex:abc123:", "codex-import:abc123:"], + sessionKeyPrefixes: prefixes, vectorStore: vectorStore as any, logger, }); expect(result.results.map((item) => item.id)).toEqual(["a", "c"]); + expect(countL0).not.toHaveBeenCalled(); + expect(searchL0Fts).toHaveBeenCalledTimes(1); + expect(searchL0Fts.mock.calls[0][1]).toBeLessThan(rows.length); + expect(searchL0Fts.mock.calls[0][2]).toEqual({ sessionKeyPrefixes: prefixes }); }); }); From 8b62ade9414547c2a20d2fe4a887683759c273f5 Mon Sep 17 00:00:00 2001 From: Siyao Zheng Date: Wed, 20 May 2026 14:46:35 +0800 Subject: [PATCH 4/5] fix(codex): capture Codex App prompts from transcript Signed-off-by: Siyao Zheng --- codex-plugin/scripts/codex-security.test.mjs | 99 ++++++++++++++++++++ codex-plugin/scripts/lib.mjs | 92 ++++++++++++++++-- 2 files changed, 184 insertions(+), 7 deletions(-) diff --git a/codex-plugin/scripts/codex-security.test.mjs b/codex-plugin/scripts/codex-security.test.mjs index 7259b68..9adbca0 100644 --- a/codex-plugin/scripts/codex-security.test.mjs +++ b/codex-plugin/scripts/codex-security.test.mjs @@ -14,6 +14,7 @@ import { loadSessionState, recallForPrompt, readGatewayAuthToken, + promptFromPayload, sanitizeMemoryText, sessionKeyFromPayload, } from "./lib.mjs"; @@ -148,6 +149,104 @@ ${privateKeyBlock} expect(cleaned).toContain("DB_PASSWORD=[REDACTED]"); }); + it("extracts Codex App prompts from user message content arrays", () => { + const payload = { + message: { + type: "message", + role: "user", + content: [ + { type: "input_text", text: "capture this Codex App prompt" }, + ], + }, + }; + + expect(promptFromPayload(payload)).toBe("capture this Codex App prompt"); + }); + + it("does not treat assistant messages as user prompts", () => { + const payload = { + message: { + type: "message", + role: "assistant", + content: [ + { type: "output_text", text: "assistant output should not be captured" }, + ], + }, + prompt: "", + }; + + expect(promptFromPayload(payload)).toBe(""); + }); + + it("falls back to the latest real user message in the Codex transcript", () => { + const transcriptPath = path.join(tmpDir, "rollout.jsonl"); + fs.writeFileSync(transcriptPath, [ + JSON.stringify({ + timestamp: "2026-05-20T05:00:00.000Z", + type: "response_item", + payload: { + type: "message", + role: "user", + content: [{ type: "input_text", text: "earlier real prompt" }], + }, + }), + JSON.stringify({ + timestamp: "2026-05-20T05:01:00.000Z", + type: "response_item", + payload: { + type: "message", + role: "user", + content: [{ type: "input_text", text: "\nsynthetic interruption" }], + }, + }), + JSON.stringify({ + timestamp: "2026-05-20T05:02:00.000Z", + type: "response_item", + payload: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "assistant response" }], + }, + }), + JSON.stringify({ + timestamp: "2026-05-20T05:03:00.000Z", + type: "response_item", + payload: { + type: "message", + role: "user", + content: [{ type: "input_text", text: "latest real Codex App prompt" }], + }, + }), + ].join("\n") + "\n"); + + expect(promptFromPayload({ transcript_path: transcriptPath })).toBe("latest real Codex App prompt"); + }); + + it("stores transcript fallback text when beginning a turn", async () => { + const transcriptPath = path.join(tmpDir, "begin-turn-rollout.jsonl"); + fs.writeFileSync(transcriptPath, JSON.stringify({ + timestamp: "2026-05-20T05:10:00.000Z", + type: "response_item", + payload: { + type: "message", + role: "user", + content: [{ type: "input_text", text: "write this prompt into memory" }], + }, + }) + "\n"); + + const payload = { + cwd: process.cwd(), + session_id: "transcript-fallback", + transcript_path: transcriptPath, + }; + const sessionKey = sessionKeyFromPayload(payload); + + await beginTurn(payload); + const state = await loadSessionState(sessionKey); + + expect(state.currentTurn.userPrompt).toBe("write this prompt into memory"); + }); + it("redacts full Authorization and Proxy-Authorization header values", () => { const cleaned = sanitizeMemoryText([ "Authorization: Basic dXNlcjpwYXNz", diff --git a/codex-plugin/scripts/lib.mjs b/codex-plugin/scripts/lib.mjs index 751a01a..a4d8be9 100644 --- a/codex-plugin/scripts/lib.mjs +++ b/codex-plugin/scripts/lib.mjs @@ -147,13 +147,91 @@ export function sessionIdFromPayload(payload) { } export function promptFromPayload(payload) { - return String( - payload.prompt ?? - payload.user_prompt ?? - payload.userPrompt ?? - payload.input ?? - "" - ); + const directPrompt = firstNonEmptyText([ + payload.prompt, + payload.user_prompt, + payload.userPrompt, + payload.message, + payload.input, + payload.payload + ]); + return directPrompt || transcriptPromptFromPayload(payload); +} + +function firstNonEmptyText(values) { + for (const value of values) { + const text = textFromCodexContent(value).trim(); + if (text) return text; + } + return ""; +} + +function textFromCodexContent(value) { + if (value == null) return ""; + if (typeof value === "string") return value; + if (typeof value === "number" || typeof value === "boolean") return String(value); + if (Array.isArray(value)) { + return value.map(textFromCodexContent).filter(Boolean).join("\n"); + } + if (typeof value === "object") { + if (value.type === "message") { + return value.role === "user" ? textFromCodexContent(value.content) : ""; + } + if (typeof value.text === "string") return value.text; + if (typeof value.content === "string" || Array.isArray(value.content)) { + return textFromCodexContent(value.content); + } + } + return ""; +} + +function transcriptPromptFromPayload(payload) { + const transcriptPath = payload.transcript_path || payload.transcriptPath; + if (!transcriptPath || typeof transcriptPath !== "string") return ""; + return latestUserPromptFromTranscript(transcriptPath); +} + +function latestUserPromptFromTranscript(transcriptPath) { + try { + const resolved = path.resolve(expandHome(transcriptPath)); + const stat = fsSync.statSync(resolved); + if (!stat.isFile() || stat.size === 0) return ""; + const maxBytes = 1_000_000; + const start = Math.max(0, stat.size - maxBytes); + const fd = fsSync.openSync(resolved, "r"); + try { + const buffer = Buffer.alloc(stat.size - start); + fsSync.readSync(fd, buffer, 0, buffer.length, start); + const chunk = buffer.toString("utf-8"); + const lines = chunk.slice(start === 0 ? 0 : chunk.indexOf("\n") + 1).trimEnd().split("\n"); + for (let i = lines.length - 1; i >= 0; i--) { + const text = userPromptFromTranscriptLine(lines[i]); + if (text && !isSyntheticUserPrompt(text)) return text; + } + } finally { + fsSync.closeSync(fd); + } + } catch { + return ""; + } + return ""; +} + +function userPromptFromTranscriptLine(line) { + if (!line?.trim()) return ""; + try { + const entry = JSON.parse(line); + const payload = entry?.payload; + if (!payload || payload.type !== "message" || payload.role !== "user") return ""; + return textFromCodexContent(payload.content).trim(); + } catch { + return ""; + } +} + +function isSyntheticUserPrompt(text) { + const trimmed = text.trim(); + return trimmed.startsWith("") || trimmed.startsWith(" Date: Wed, 20 May 2026 19:55:25 +0800 Subject: [PATCH 5/5] fix(codex): stabilize gateway, importer, and hook paths Signed-off-by: Siyao Zheng --- codex-plugin/.mcp.json | 2 +- codex-plugin/README.md | 5 +- codex-plugin/adapter-profile.json | 7 + codex-plugin/scripts/cli-smoke.test.mjs | 81 +++++++++++ codex-plugin/scripts/codex-security.test.mjs | 35 +++++ codex-plugin/scripts/doctor.mjs | 9 +- codex-plugin/scripts/hooks.smoke.test.mjs | 64 +++++++++ codex-plugin/scripts/import-codex-history.mjs | 88 +++++++----- codex-plugin/scripts/lib.mjs | 76 ++++++---- codex-plugin/scripts/loopback.mjs | 33 +++++ codex-plugin/scripts/offload-store.mjs | 30 +++- codex-plugin/scripts/query.mjs | 8 +- codex-plugin/scripts/seed-constants.mjs | 10 ++ src/config.ts | 4 +- src/core/seed/constants.test.ts | 14 ++ src/core/seed/constants.ts | 3 + src/core/seed/seed-runtime.ts | 6 +- src/core/store/sqlite.ts | 8 +- src/core/tools/search-prefix-filter.test.ts | 56 +++++++- src/gateway/auth.test.ts | 76 ++++++++++ src/gateway/cli.ts | 5 +- src/gateway/loopback.test.ts | 32 +++++ src/gateway/loopback.ts | 5 + src/gateway/server.ts | 134 ++++++++++++++---- src/utils/manifest.ts | 1 + src/utils/pipeline-manager.test.ts | 21 +++ src/utils/pipeline-manager.ts | 9 +- src/utils/sanitize.test.ts | 15 ++ src/utils/sanitize.ts | 2 +- 29 files changed, 719 insertions(+), 120 deletions(-) create mode 100644 codex-plugin/scripts/cli-smoke.test.mjs create mode 100644 codex-plugin/scripts/hooks.smoke.test.mjs create mode 100644 codex-plugin/scripts/loopback.mjs create mode 100644 codex-plugin/scripts/seed-constants.mjs create mode 100644 src/core/seed/constants.test.ts create mode 100644 src/core/seed/constants.ts create mode 100644 src/gateway/loopback.test.ts create mode 100644 src/gateway/loopback.ts create mode 100644 src/utils/sanitize.test.ts diff --git a/codex-plugin/.mcp.json b/codex-plugin/.mcp.json index d4ec3a9..963f09a 100644 --- a/codex-plugin/.mcp.json +++ b/codex-plugin/.mcp.json @@ -3,7 +3,7 @@ "tdai-memory": { "command": "node", "args": [ - "${CLAUDE_PLUGIN_ROOT}/scripts/mcp-server.mjs" + "${PLUGIN_ROOT}/scripts/mcp-server.mjs" ], "env": { "TDAI_CODEX_AUTOSTART": "true" diff --git a/codex-plugin/README.md b/codex-plugin/README.md index 1f094be..1b9bdcd 100644 --- a/codex-plugin/README.md +++ b/codex-plugin/README.md @@ -162,7 +162,8 @@ export TDAI_CODEX_DATA_DIR="$HOME/.memory-tencentdb/codex-memory-tdai" export TDAI_CODEX_GATEWAY_URL="http://127.0.0.1:8420" export TDAI_CODEX_AUTOSTART=true export TDAI_CODEX_FLUSH_EVERY_N_TURNS=5 -export TDAI_CODEX_TOOL_OFFLOAD=true +# Tool-output offload is enabled by default; uncomment to disable it. +# export TDAI_CODEX_TOOL_OFFLOAD=false ``` When the adapter autostarts the Gateway it keeps the service on loopback by @@ -282,7 +283,7 @@ By default it reads: ```text ~/.codex/sessions/**/*.jsonl -~/.codex/archived_sessions/*.jsonl +~/.codex/archived_sessions/**/*.jsonl ``` The importer is opt-in and runs as a dry run unless `--yes` is provided: diff --git a/codex-plugin/adapter-profile.json b/codex-plugin/adapter-profile.json index 3b83ee6..172b673 100644 --- a/codex-plugin/adapter-profile.json +++ b/codex-plugin/adapter-profile.json @@ -36,6 +36,13 @@ "TDAI_CODEX_GATEWAY_TOKEN", "TDAI_GATEWAY_TOKEN", "TDAI_TOKEN_PATH" + ], + "behaviorEnv": [ + "TDAI_CODEX_TOOL_OFFLOAD", + "TDAI_CODEX_AUTOSTART", + "TDAI_CODEX_CIRCUIT_BREAKER", + "TDAI_CODEX_ALLOW_NON_LOOPBACK", + "TDAI_CODEX_DEBUG" ] }, "reuseContract": { diff --git a/codex-plugin/scripts/cli-smoke.test.mjs b/codex-plugin/scripts/cli-smoke.test.mjs new file mode 100644 index 0000000..c47431b --- /dev/null +++ b/codex-plugin/scripts/cli-smoke.test.mjs @@ -0,0 +1,81 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterAll, describe, expect, it } from "vitest"; + +const scriptsDir = path.dirname(fileURLToPath(import.meta.url)); +const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-codex-cli-")); + +describe("Codex adapter CLI entry scripts", () => { + afterAll(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("imports Codex JSONL history in dry-run mode", async () => { + const sessionsDir = path.join(tmpDir, "sessions"); + fs.mkdirSync(sessionsDir, { recursive: true }); + const cwd = path.join(tmpDir, "project"); + fs.mkdirSync(cwd, { recursive: true }); + const sessionPath = path.join(sessionsDir, "sample.jsonl"); + fs.writeFileSync(sessionPath, [ + JSON.stringify({ type: "session_meta", timestamp: "2026-05-20T00:00:00.000Z", payload: { id: "smoke", cwd, source: "codex-cli" } }), + JSON.stringify({ type: "response_item", timestamp: "2026-05-20T00:00:01.000Z", payload: { type: "message", role: "user", content: [{ text: "What did we decide?" }] } }), + JSON.stringify({ type: "response_item", timestamp: "2026-05-20T00:00:02.000Z", payload: { type: "message", role: "assistant", content: [{ text: "We decided to keep the adapter portable." }] } }), + "", + ].join("\n")); + + const result = await runScript("import-codex-history.mjs", [ + "--sessions-dir", sessionsDir, + "--no-archived", + "--dry-run", + "--cwd", cwd, + "--limit", "1", + ]); + expect(result.stderr).toBe(""); + expect(result.code).toBe(0); + expect(JSON.parse(result.stdout)).toEqual(expect.objectContaining({ + sessionsPrepared: 1, + roundsPrepared: 1, + messagesPrepared: 2, + skipped: expect.objectContaining({ parseError: 0 }), + })); + }); + + it("prints query status JSON without autostarting Gateway", async () => { + const result = await runScript("query.mjs", ["status"], { + CLAUDE_PROJECT_DIR: tmpDir, + TDAI_CODEX_AUTOSTART: "false", + TDAI_CODEX_GATEWAY_URL: "http://127.0.0.1:9", + TDAI_CODEX_DATA_DIR: tmpDir, + }); + expect(result.stderr).toBe(""); + expect(result.code).toBe(0); + expect(JSON.parse(result.stdout)).toEqual(expect.objectContaining({ + healthy: false, + gatewayUrl: "http://127.0.0.1:9", + sessionKey: expect.stringContaining("codex:"), + })); + }); +}); + +function runScript(script, args = [], env = {}) { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [path.join(scriptsDir, script), ...args], { + env: { + ...process.env, + TDAI_DATA_DIR: tmpDir, + TDAI_CODEX_AUTOSTART: "false", + ...env, + }, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { stdout += chunk; }); + child.stderr.on("data", (chunk) => { stderr += chunk; }); + child.on("error", reject); + child.on("close", (code) => resolve({ code, stdout, stderr })); + }); +} diff --git a/codex-plugin/scripts/codex-security.test.mjs b/codex-plugin/scripts/codex-security.test.mjs index 9adbca0..179e3c9 100644 --- a/codex-plugin/scripts/codex-security.test.mjs +++ b/codex-plugin/scripts/codex-security.test.mjs @@ -113,6 +113,14 @@ ${privateKeyBlock} expect(cleaned).toContain("[REDACTED_PRIVATE_KEY]"); }); + it("redacts local Gateway token diagnostics", () => { + const token = "a".repeat(43); + const cleaned = sanitizeMemoryText(`gateway token: ${token}`); + + expect(cleaned).not.toContain(token); + expect(cleaned).toContain("[REDACTED"); + }); + it("redacts JSON-style credential fields", () => { const cleaned = sanitizeMemoryText(JSON.stringify({ apiKey: "plain-secret-123", @@ -312,6 +320,22 @@ ${privateKeyBlock} expect(all.matches.map((entry) => entry.tool_call_id).sort()).toEqual(["tool-a", "tool-b"]); }); + it("escapes Mermaid labels for offloaded tool results", async () => { + const cwd = path.join(tmpDir, "project-mermaid"); + fs.mkdirSync(cwd); + const result = await recordCodexToolOffload({ + ...offloadParams(cwd, "session-mermaid", "tool-mermaid"), + toolName: "tool\"] --> EVIL[\"x", + inputSummary: "payload [brackets]", + }); + + const canvas = fs.readFileSync(result.paths.canvasPath, "utf-8"); + expect(canvas).not.toContain("