From 38edc2271b4c49f655faba181b760b8040b9ffe9 Mon Sep 17 00:00:00 2001 From: zho Date: Tue, 19 May 2026 01:55:30 +0800 Subject: [PATCH 1/5] addressed coverage and docs issues --- .github/workflows/ci.yml | 5 + .markdown-link-check.json | 13 ++ CHANGELOG.md | 5 +- README.md | 25 +++- docs/CI_CD.md | 44 ++++++ docs/CONFIGURATION.md | 68 ++++++++++ docs/CONTRIBUTING.md | 46 +++++++ docs/FAQ.md | 29 ++++ docs/MIGRATION.md | 129 ++++++++++++++++++ docs/README.md | 16 +++ docs/RELEASING.md | 18 +++ docs/SECURITY.md | 37 +++++ docs/TOOLS.md | 209 +++++++++++++++++++++++++++++ examples/README.md | 5 +- examples/guided-query-demo.ts | 55 ++++++++ examples/library-embedding-demo.ts | 49 +++++++ examples/suggest-flow-demo.ts | 59 ++++++++ vitest.config.ts | 11 ++ 18 files changed, 817 insertions(+), 6 deletions(-) create mode 100644 .markdown-link-check.json create mode 100644 docs/CI_CD.md create mode 100644 docs/CONFIGURATION.md create mode 100644 docs/CONTRIBUTING.md create mode 100644 docs/FAQ.md create mode 100644 docs/MIGRATION.md create mode 100644 docs/README.md create mode 100644 docs/RELEASING.md create mode 100644 docs/SECURITY.md create mode 100644 docs/TOOLS.md create mode 100644 examples/guided-query-demo.ts create mode 100644 examples/library-embedding-demo.ts create mode 100644 examples/suggest-flow-demo.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33681d7..7bed065 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,3 +84,8 @@ jobs: - name: Package dry run run: npm pack --dry-run + + - name: Verify internal markdown links + run: | + npx --yes markdown-link-check@3 -c .markdown-link-check.json README.md CHANGELOG.md + find docs -type f -name '*.md' -print0 | xargs -0 -I{} npx --yes markdown-link-check@3 -c .markdown-link-check.json "{}" diff --git a/.markdown-link-check.json b/.markdown-link-check.json new file mode 100644 index 0000000..6eabdd3 --- /dev/null +++ b/.markdown-link-check.json @@ -0,0 +1,13 @@ +{ + "ignorePatterns": [ + { + "pattern": "^https?://" + }, + { + "pattern": "^mailto:" + } + ], + "retryOn429": true, + "retryCount": 2, + "aliveStatusCodes": [200, 206, 403] +} diff --git a/CHANGELOG.md b/CHANGELOG.md index a56f1de..cc535a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,9 +21,10 @@ Tagged releases are published to npm from GitHub Actions when a **GitHub Release - `--version` CLI flag prints the package version and exits. - `list_namespaces` response now includes `expires_at_iso` so clients see the cache expiry as an ISO-8601 timestamp without converting `cache_ttl_seconds`. - `examples/README.md` describing the library embedding sample. -- GitHub Actions **CI** matrix across **ubuntu-latest**, **windows-latest**, and **macos-latest**, each with **Node.js** **20.x** and **22.x**: typecheck, lint, Prettier, build, `test:coverage`, **CycloneDX** SBOM artifact upload (per job), **Codecov** upload (**Ubuntu** + Node **20.x** only), plus a separate **quality** job (`npm audit`, `npm pack --dry-run`). -- `npm run test:coverage` with Vitest coverage thresholds (see `vitest.config.ts`). +- GitHub Actions **CI** matrix across **ubuntu-latest**, **windows-latest**, and **macos-latest**, each with **Node.js** **20.x** and **22.x**: typecheck, lint, Prettier, build, `test:coverage`, **CycloneDX** SBOM artifact upload (per job), **Codecov** upload (**Ubuntu** + Node **20.x** only), plus a separate **quality** job (`npm audit`, `npm pack --dry-run`, **markdown-link-check** on README/CHANGELOG/docs). +- Vitest **global** coverage thresholds in `vitest.config.ts` (lines 73%, statements 72%, branches 58%, functions 76% — measured baseline minus slack); `npm run test:coverage` exits non-zero when any bucket regresses. - `@vitest/coverage-v8` devDependency for coverage reports (`lcov`, `json-summary`, HTML). +- `docs/` reference set (TOOLS, CONFIGURATION, SECURITY, CONTRIBUTING, CI_CD, FAQ, MIGRATION, RELEASING) and worked examples `examples/suggest-flow-demo.ts`, `examples/guided-query-demo.ts`, `examples/library-embedding-demo.ts`. ### Changed diff --git a/README.md b/README.md index f4ca377..83cd019 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,9 @@ A Model Context Protocol (MCP) server that provides semantic search over Pinecon | [docs/FAQ.md](docs/FAQ.md) | Common questions | | [docs/MIGRATION.md](docs/MIGRATION.md) | Deprecations & breaking changes | | [docs/CI_CD.md](docs/CI_CD.md) | GitHub Actions, SBOM, Docker, releases | -| [RELEASING.md](RELEASING.md) | Pointer to the full release guide in `docs/` | -| [CONTRIBUTING.md](CONTRIBUTING.md) | How to contribute | -| [SECURITY.md](SECURITY.md) | Vulnerability reporting | +| [docs/RELEASING.md](docs/RELEASING.md) | npm publish via GitHub Releases | +| [docs/CONTRIBUTING.md](docs/CONTRIBUTING.md) | How to contribute | +| [docs/SECURITY.md](docs/SECURITY.md) | Vulnerability reporting | ## Error responses @@ -35,6 +35,8 @@ When a tool fails, the MCP tool result sets **`isError: true`**. The `text` cont Success payloads are unchanged and do **not** wrap `ToolError`. Clients that still expect `{ "status": "error", "message": "..." }` must migrate to the shape above. +For successful `query` / `guided_query` payloads, **rerank/hybrid fidelity** is described in [docs/TOOLS.md](docs/TOOLS.md) (row-level `reranked`, current lack of a top-level `degraded` envelope). + ## Features - **Hybrid Search**: Combines dense and sparse embeddings for superior recall @@ -102,6 +104,12 @@ Run `pinecone-read-only-mcp --help` for CLI equivalents (`--cache-ttl-seconds`, The server uses **process-global** memory for the suggest-flow gate (`suggest_query_params` context), namespaces cache, URL generator registry, and active configuration. **Stdio MCP (one client per Node process)** matches this model. If you embed `setupServer` behind a multi-tenant HTTP transport, isolate those structures per session yourself or treat the suggest-flow guard as best-effort only. +### Library embedding (`setupServer`) + +Treat **`setupServer()` as one logical server per Node process**: it mutates shared module singletons (suggest-flow map, namespaces cache, URL registry, config context, shared `PineconeClient` slot). A second `setupServer()` without a coordinated teardown can leave stale or mixed state for in-flight requests — **spawn a separate process** per isolated instance until an explicit lifecycle API is documented in the changelog. + +Recommended pattern: `resolveConfig` → `setPineconeClient(new PineconeClient(...))` → `await setupServer(config)` → connect one MCP transport. See [examples/library-embedding-demo.ts](examples/library-embedding-demo.ts) and [docs/TOOLS.md](docs/TOOLS.md#suggest-flow-gate). + ### Custom URL generators Namespaces other than `mailing` and `slack-Cpplang` (or different URL rules for any namespace) can use programmatic registration — no fork required. @@ -130,6 +138,17 @@ registerUrlGenerator('product-docs', myDocs); A fuller embedding sample lives in [examples/custom-url-generator.ts](examples/custom-url-generator.ts). +### Examples + +| File | Description | +| ---- | ----------- | +| [examples/suggest-flow-demo.ts](examples/suggest-flow-demo.ts) | Manual **suggest_query_params → query** flow with namespace consistency notes. | +| [examples/guided-query-demo.ts](examples/guided-query-demo.ts) | **guided_query** orchestration and how to read `decision_trace`. | +| [examples/library-embedding-demo.ts](examples/library-embedding-demo.ts) | Programmatic **setupServer** wiring without the CLI binary. | +| [examples/custom-url-generator.ts](examples/custom-url-generator.ts) | Custom **URL generator** registration for `generate_urls` / row enrichment. | + +Run with `npx tsx examples/.ts` from a checkout (requires valid Pinecone env for live paths). + ### Claude Desktop Configuration Add to your `claude_desktop_config.json`: diff --git a/docs/CI_CD.md b/docs/CI_CD.md new file mode 100644 index 0000000..a3c5413 --- /dev/null +++ b/docs/CI_CD.md @@ -0,0 +1,44 @@ +# CI/CD + +## Workflow overview + +| Workflow | File | Trigger | +| -------- | ---- | ------- | +| **CI** | [`.github/workflows/ci.yml`](../.github/workflows/ci.yml) | PR/push to `main`, `workflow_call` | +| **CodeQL** | [`.github/workflows/codeql.yml`](../.github/workflows/codeql.yml) | PR/push to `main`, weekly schedule | +| **Publish** | [`.github/workflows/publish.yml`](../.github/workflows/publish.yml) | GitHub Release `published` | + +--- + +## CI matrix (`ci.yml`) + +**Job `build-and-test`** + +- **OS:** `ubuntu-latest`, `windows-latest`, `macos-latest` +- **Node:** `20.x`, `22.x` (see workflow comments — Node 18 is unsupported) +- **Steps:** checkout → `npm ci` → `npm run typecheck` → `npm run lint` → `npm run format:check` → `npm run build` → smoke CLI `--help` → `npm run test:coverage` → CycloneDX SBOM (`@cyclonedx/cyclonedx-npm`) → upload SBOM artifact +- **Codecov:** Ubuntu + Node 20 only (`codecov-action`, non-blocking on upload failure flag) + +**Job `quality`** + +- Ubuntu + Node 20: `npm audit --audit-level=moderate` (continue-on-error) → `npm pack --dry-run` → **markdown-link-check** on documentation paths (see workflow). + +--- + +## CodeQL (`codeql.yml`) + +Static analysis for **JavaScript** (`matrix.language: javascript`). Runs GitHub’s CodeQL init/autobuild/analyze on each PR/push to `main` and on a weekly cron. + +--- + +## SBOM + +Each matrix cell uploads `sbom.cdx.json` (CycloneDX) as a workflow artifact for supply-chain review. + +--- + +## Releases & npm + +Publishing is **not** done on every tag push alone: the **Publish** workflow runs when a GitHub **Release** is published. It reuses CI via `workflow_call`, then `npm publish --provenance --access public` with `NPM_TOKEN`. + +Details: [RELEASING.md](./RELEASING.md). diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md new file mode 100644 index 0000000..dc15d2d --- /dev/null +++ b/docs/CONFIGURATION.md @@ -0,0 +1,68 @@ +# Configuration + +Configuration is built from **CLI flags** (when using the binary), **environment variables**, and **defaults**. Library callers use `resolveConfig(overrides)` with optional `ConfigOverrides`. + +## Precedence + +**CLI / `ConfigOverrides` > environment variables > built-in defaults.** + +`resolveConfig` in `src/config.ts` applies this order for each field. + +--- + +## `ServerConfig` (resolved shape) + +| Field | Source | Default / notes | +| ----- | ------ | --------------- | +| `apiKey` | `apiKey` / `PINECONE_API_KEY` | **Required** (non-empty after trim) | +| `indexName` | `indexName` / `PINECONE_INDEX_NAME` | `rag-hybrid` | +| `sparseIndexName` | `sparseIndexName` / `PINECONE_SPARSE_INDEX_NAME` | `{indexName}-sparse` | +| `rerankModel` | `rerankModel` / `PINECONE_RERANK_MODEL` | `bge-reranker-v2-m3` | +| `defaultTopK` | `defaultTopK` / `PINECONE_TOP_K` | `10` (positive int) | +| `logLevel` | `logLevel` / `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | `INFO` (`DEBUG`–`ERROR`) | +| `logFormat` | `logFormat` / `PINECONE_READ_ONLY_MCP_LOG_FORMAT` | `text` or `json` | +| `cacheTtlMs` | `cacheTtlSeconds` / `PINECONE_CACHE_TTL_SECONDS` | `1800` seconds → ms | +| `requestTimeoutMs` | `requestTimeoutMs` / `PINECONE_REQUEST_TIMEOUT_MS` | `15000` | +| `disableSuggestFlow` | `disableSuggestFlow` / `PINECONE_DISABLE_SUGGEST_FLOW` | `false` (bool parsing: true/1/yes/on) | +| `checkIndexes` | `checkIndexes` / `PINECONE_CHECK_INDEXES` | `false` | + +**Throws** if `apiKey` is missing after trim. + +--- + +## CLI flags (`parseCli` / `src/cli.ts`) + +| Flag | Maps to | +| ---- | ------- | +| `--api-key` | `apiKey` | +| `--index-name` | `indexName` | +| `--sparse-index-name` | `sparseIndexName` | +| `--rerank-model` | `rerankModel` | +| `--top-k` | `defaultTopK` | +| `--log-level` | `logLevel` | +| `--log-format` | `logFormat` | +| `--cache-ttl-seconds` | `cacheTtlSeconds` | +| `--request-timeout-ms` | `requestTimeoutMs` | +| `--disable-suggest-flow` | `disableSuggestFlow: true` | +| `--check-indexes` | `checkIndexes: true` | +| `--help` / `-h` | Print help and exit | +| `--version` / `-v` | Print version and exit | + +--- + +## Library embedding + +1. Build `ServerConfig` with `resolveConfig({ apiKey: '...', ... })` or pass explicit overrides. +2. Construct `PineconeClient` and `setPineconeClient(client)` before `setupServer(config)` (mirrors `src/index.ts`). +3. `await setupServer(config)` then connect an MCP transport. + +See [README deployment model](../README.md#deployment-model) and [examples/library-embedding-demo.ts](../examples/library-embedding-demo.ts). + +--- + +## Logging + +- **Levels:** `DEBUG`, `INFO`, `WARN`, `ERROR`. +- **Formats:** `text` (human lines to stderr) or `json` (one JSON object per line). + +Secrets are redacted (see [SECURITY.md](./SECURITY.md)). diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md new file mode 100644 index 0000000..1adb345 --- /dev/null +++ b/docs/CONTRIBUTING.md @@ -0,0 +1,46 @@ +# Contributing + +## Prerequisites + +- **Node.js ≥ 20.12** (see `engines` in `package.json` — Vitest 4 / coverage require it). +- npm (lockfile is `package-lock.json`). + +## Setup + +```bash +git clone https://github.com/CppDigest/pinecone-read-only-mcp-typescript.git +cd pinecone-read-only-mcp-typescript +npm ci +``` + +## Commands + +| Script | Purpose | +| ------ | ------- | +| `npm run build` | Clean `dist/` and `tsc` compile | +| `npm run typecheck` | `tsc --noEmit` | +| `npm run lint` | ESLint on `src/` | +| `npm run lint:fix` | ESLint with `--fix` | +| `npm run format` | Prettier write (`src/**/*.ts`, config JSON) | +| `npm run format:check` | Prettier check | +| `npm test` | Vitest once | +| `npm run test:coverage` | Vitest + coverage thresholds (`vitest.config.ts`) | +| `npm run ci` | Full local gate (typecheck, lint, format, build, coverage) | + +## Coding conventions + +- **TypeScript strict** options enabled (`strict`, `noUncheckedIndexedAccess`, etc.). +- Prefer explicit types on exported APIs; use Zod at MCP tool boundaries. +- **No `process.env` reads** in feature code outside `resolveConfig` / CLI — thread `ServerConfig`. +- Tool errors: return `jsonErrorResponse` with `ToolError` shapes from `tool-error.ts`. +- Tests live beside sources as `*.test.ts`; use Vitest. + +## Pull requests + +- Run `npm run ci` before pushing. +- Keep changes focused; update `CHANGELOG.md` `[Unreleased]` for user-visible behavior. +- Documentation changes should keep [README](../README.md) links and run markdown link check locally if you touch many relative links. + +## Documentation + +Authoritative reference lives under [`docs/`](./README.md). When adding tools or config knobs, update `docs/TOOLS.md` and `docs/CONFIGURATION.md` in the same PR when possible. diff --git a/docs/FAQ.md b/docs/FAQ.md new file mode 100644 index 0000000..fc35f4f --- /dev/null +++ b/docs/FAQ.md @@ -0,0 +1,29 @@ +# FAQ + +## Why does `query` return `FLOW_GATE`? + +Call **`suggest_query_params`** for that namespace first (within the cache TTL). Or use **`guided_query`**, which performs suggestion internally. + +## Does `keyword_search` need `suggest_query_params`? + +**No.** Only `query`, `count`, and `query_documents` are gated. + +## What happened to `query_fast` / `query_detailed`? + +They are unified into **`query`** with `preset`: `fast`, `detailed`, or `full`. See [MIGRATION.md](./MIGRATION.md). + +## How do I disable the suggest gate for testing? + +Set **`PINECONE_DISABLE_SUGGEST_FLOW=true`** or pass **`--disable-suggest-flow`**. Prefer fixing the client flow in production. + +## Where are benchmarks? + +There is a `npm run benchmark` script (`benchmarks/latency.ts`); published README claims remain qualitative until benchmark results are checked in. + +## Which field is the document id? + +Use **`document_id`** on rows. **`paper_number`** is a deprecated alias scheduled for removal in the next major release. + +## Node version errors with Vitest + +Use **Node ≥ 20.12** — Vitest 4’s bundler and `@vitest/coverage-v8` require newer `node:util` / inspector APIs. diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md new file mode 100644 index 0000000..f95465b --- /dev/null +++ b/docs/MIGRATION.md @@ -0,0 +1,129 @@ +# Migration guide + +This guide is for **library and MCP client authors** upgrading from earlier **0.1.x** lines. The `[Unreleased]` section of [`CHANGELOG.md`](../CHANGELOG.md) is the authoritative list of changes; this document shows **how** to migrate. + +## Migrating to v0.2.0 (upcoming) + +### Namespace trimming and suggest-flow + +**Rationale:** The in-process suggest-flow gate keys state by `namespace` string. If `suggest_query_params` is called with `" docs "` and `query` with `"docs"`, the gate may not match. + +**Migration:** + +1. Normalize namespaces once (e.g. `namespace.trim()`). +2. Pass the **same** string to `suggest_query_params`, `query`, `count`, and `query_documents`. + +```ts +const ns = userInput.trim(); +// call suggest_query_params({ namespace: ns, ... }) +// then query({ namespace: ns, ... }) +``` + +--- + +### 1. `ToolError` replaces `{ status: 'error', message }` + +**Rationale:** Typed, machine-readable errors for MCP clients and `switch` on `code`. + +**Before (conceptual):** + +```json +{ + "status": "error", + "message": "Query text cannot be empty" +} +``` + +**After (`ToolError`, `isError: true` body):** + +```json +{ + "code": "VALIDATION", + "message": "Query text cannot be empty", + "recoverable": true, + "field": "query_text" +} +``` + +**`code` values (discriminated union):** + +| `code` | `recoverable` | Notes | +| ------ | --------------- | ----- | +| `FLOW_GATE` | `true` | Suggestion: call `suggest_query_params` for the namespace first | +| `VALIDATION` | `true` | **`field` required** — input or `metadata_filter` dot-path | +| `PINECONE_ERROR` | `true` or `false` | Upstream / network / Pinecone failures | +| `TIMEOUT` | `true` | Outbound deadline exceeded | + +**Migration steps:** + +1. Parse JSON with `toolErrorSchema` / `ToolError` from `@will-cppa/pinecone-read-only-mcp` if using Zod. +2. Replace checks for `status === 'error'` with `isError === true` and `code` branching. +3. Map `VALIDATION` UX to `field` for inline form errors. + +--- + +### 2. `QueryResponse` / `KeywordSearchResponse` — error fields removed + +**Rationale:** Success and error paths are separated: success DTOs never carry `status: 'error'` embedded fields. + +**Before:** Some clients read `status: 'error'` **inside** a success-shaped response. + +**After:** On failure, the MCP layer sets `isError: true` and returns `ToolError` JSON only. Successful `query` responses are always `QueryResponse` with `status: 'success'` and optional `results`, etc. + +**Migration steps:** + +1. Treat HTTP/MCP errors only via `isError` + `ToolError`. +2. Remove dead branches that looked for `QueryResponse.status === 'error'`. + +--- + +### 3. `recommended_tool` string values + +**Rationale:** Align routing hints with the unified `query` tool vocabulary. + +| Old (legacy) | New | +| ------------ | --- | +| `query_fast` | `fast` | +| `query_detailed` | `detailed` | +| `count` | `count` (unchanged) | +| _(n/a)_ | `full` (explicit preset) | + +**Migration steps:** + +1. Update `switch (recommended_tool)` / routing tables to the new literals. +2. When forwarding to `query`, map to `preset` (next section). + +--- + +### 4. Unified `query` tool (replaces `query_fast` / `query_detailed`) + +**Rationale:** One hybrid tool with a `preset` knob instead of duplicate registrations. + +| Legacy tool call | New `query` call | +| ---------------- | ---------------- | +| `query_fast({ ...params })` | `query({ ...params, preset: 'fast' })` | +| `query_detailed({ ...params })` | `query({ ...params, preset: 'detailed' })` | +| Custom / explicit rerank+fields | `query({ ...params, preset: 'full', use_reranking?, fields? })` | + +**Example:** + +```ts +// was: tool "query_fast" with { query_text, namespace, top_k, metadata_filter } +await callTool('query', { + query_text, + namespace, + top_k, + metadata_filter, + preset: 'fast', +}); +``` + +--- + +## Summary checklist + +- [ ] Normalize and reuse namespace strings across suggest + gated tools. +- [ ] Adopt `ToolError` parsing for all tool failures. +- [ ] Remove reliance on in-body `status: 'error'` for query responses. +- [ ] Update `recommended_tool` handling to `count` \| `fast` \| `detailed` \| `full`. +- [ ] Map legacy fast/detailed tool calls to `query` + `preset`. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..1e46489 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,16 @@ +# Documentation index + +Guides for operators, MCP client authors, and library embedders. + +| Guide | Description | +| ----- | ----------- | +| [TOOLS.md](./TOOLS.md) | All MCP tools, parameters, success/error shapes, suggest-flow | +| [CONFIGURATION.md](./CONFIGURATION.md) | Environment variables, CLI flags, `resolveConfig`, precedence | +| [SECURITY.md](./SECURITY.md) | API keys, log redaction, Docker hardening, reporting issues | +| [CONTRIBUTING.md](./CONTRIBUTING.md) | Local dev, tests, lint/format, PR expectations | +| [CI_CD.md](./CI_CD.md) | GitHub Actions, CodeQL, SBOM, Codecov, releases | +| [FAQ.md](./FAQ.md) | Common questions | +| [MIGRATION.md](./MIGRATION.md) | Breaking changes and upgrade paths | +| [RELEASING.md](./RELEASING.md) | npm publish via GitHub Releases | + +The main [README.md](../README.md) remains the high-level overview; deep reference lives here. diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 0000000..faaec85 --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,18 @@ +# Releasing + +Packages are published to npm as **`@will-cppa/pinecone-read-only-mcp`**. + +## Mechanism + +1. Merge changes to `main` with an updated [`CHANGELOG.md`](../CHANGELOG.md). +2. Create a **GitHub Release** (publish event) with a version tag aligned with semver. +3. The [Publish workflow](../.github/workflows/publish.yml) runs the full CI suite (`workflow_call` to `ci.yml`), then executes `npm publish --provenance --access public` on Ubuntu with `NODE_AUTH_TOKEN`. + +## Requirements + +- **`NPM_TOKEN`** secret configured on the repository. +- **`prepublishOnly`** runs `npm run ci` — local `npm publish` must pass the same gates. + +## Version source + +`SERVER_VERSION` is read from `package.json` at runtime so MCP `serverInfo` matches the published package. diff --git a/docs/SECURITY.md b/docs/SECURITY.md new file mode 100644 index 0000000..180727a --- /dev/null +++ b/docs/SECURITY.md @@ -0,0 +1,37 @@ +# Security + +## API keys + +- **Never** commit real Pinecone API keys. Use environment variables (`PINECONE_API_KEY`) or secret managers in CI. +- The CLI and `resolveConfig` read keys only from argv/env/overrides — logs must not echo raw keys. + +## Log redaction + +`src/logger.ts` implements `redactApiKey` and recursive redaction for structured log data: + +- UUID-shaped tokens (`xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`) → `***` +- Substrings after `apiKey` / `api_key` / similar patterns → masked +- `Authorization: Bearer …` tokens → masked + +Logs go to **stderr**; use `PINECONE_READ_ONLY_MCP_LOG_FORMAT=json` for pipelines and ensure downstream sinks treat stderr as sensitive. + +## Docker image + +The multi-stage [`Dockerfile`](../Dockerfile): + +1. **Build stage** (`node:20-bookworm-slim`): `npm ci`, `npm run build`. +2. **Runtime stage**: `npm ci --omit=dev`, copies `dist/` only. +3. Creates a non-root user **`mcpuser`** (uid `10001`) and runs `node dist/index.js` as that user (`USER mcpuser`). + +Do not run the production image as root unless you have a compensating security model. + +## Supply chain + +- CI runs `npm audit --audit-level=moderate` (see [CI_CD.md](./CI_CD.md)). +- SBOM: CycloneDX JSON is generated per CI matrix job. + +## Reporting vulnerabilities + +Open a **private** security advisory or issue per repository policy on [GitHub](https://github.com/CppDigest/pinecone-read-only-mcp-typescript/security). Do not post exploit details in public issues before a fix is available. + +Include: affected version, reproduction steps, and impact assessment. diff --git a/docs/TOOLS.md b/docs/TOOLS.md new file mode 100644 index 0000000..0785e3d --- /dev/null +++ b/docs/TOOLS.md @@ -0,0 +1,209 @@ +# MCP tools reference + +The server registers **nine** tools. Unless noted, failures return MCP `isError: true` with JSON matching `ToolError` (see [MIGRATION.md](./MIGRATION.md) and [README error table](../README.md#error-responses)). + +## Suggest-flow gate + +Tools **`query`**, **`count`**, and **`query_documents`** require a prior successful **`suggest_query_params`** call for the **same namespace string** within the cache TTL (see `PINECONE_CACHE_TTL_SECONDS`). The gate is in-process memory (`requireSuggested`). + +**Namespace consistency:** use the **exact same** `namespace` value (including trimming — avoid leading/trailing spaces in one call and not the other) for `suggest_query_params` and downstream gated tools. Mismatches yield `FLOW_GATE` with a suggestion to call `suggest_query_params` first. + +**Bypass:** set `PINECONE_DISABLE_SUGGEST_FLOW=true` or CLI `--disable-suggest-flow` (not recommended for production). + +--- + +## 1. `list_namespaces` + +**Purpose:** Discover namespaces, metadata field names, and record counts. Results are cached (~30 minutes; see response `expires_at_iso`). + +| | | +| --- | --- | +| **Input** | _(empty object)_ | +| **Success** | `{ status: 'success', cache_hit, cache_ttl_seconds, expires_at_iso, count, namespaces: [{ name, record_count, metadata_fields }] }` | +| **Errors** | `PINECONE_ERROR`, `TIMEOUT`, etc. | + +**Example (conceptual MCP params):** + +```json +{} +``` + +--- + +## 2. `namespace_router` + +**Purpose:** Rank candidate namespaces from natural-language intent (optional step before `suggest_query_params`). + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `user_query` | string | yes | User question / intent | +| `top_n` | int | no (default 3) | Max suggestions, 1–5 | + +**Success:** `{ status: 'success', cache_hit, user_query, suggestions, recommended_namespace }`. + +**Example:** + +```json +{ "user_query": "Where is the allocator documented?", "top_n": 3 } +``` + +--- + +## 3. `suggest_query_params` + +**Purpose:** Mandatory gate before `query` / `count` / `query_documents`. Returns field hints and `recommended_tool`. + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `namespace` | string | yes | Target namespace (must exist in cached `list_namespaces`) | +| `user_query` | string | yes | Natural-language task | + +**Success:** `{ status: 'success', cache_hit, ...suggestQueryParams fields including suggested_fields, recommended_tool, use_count_tool, explanation, namespace_found }`. + +**Example:** + +```json +{ + "namespace": "mailing", + "user_query": "Summarize discussions about coroutines from last month" +} +``` + +--- + +## 4. `count` + +**Purpose:** Semantic count of **unique documents** (dedupe by `document_number` / `url` / `doc_id`). Requires suggest-flow. + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `namespace` | string | yes | Namespace | +| `query_text` | string | yes | Query text (use broad text like `"document"` for metadata-only counts) | +| `metadata_filter` | object | no | Pinecone metadata filter | + +**Success:** `{ status: 'success', count, truncated, namespace, metadata_filter? }`. + +--- + +## 5. `query` + +**Purpose:** Hybrid dense+sparse retrieval with optional reranking. Requires suggest-flow. + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `query_text` | string | yes | Search text | +| `namespace` | string | yes | Namespace | +| `top_k` | int | no (default 10) | 1–100 | +| `preset` | `"fast"` \| `"detailed"` \| `"full"` | no (default `"full"`) | `fast`: no rerank + light fields; `detailed` / `full`: reranking (see Zod in source) | +| `use_reranking` | boolean | no | When preset allows reranking | +| `metadata_filter` | object | no | Metadata filter | +| `fields` | string[] | no | Pinecone fields to return | + +**Success (`QueryResponse`):** `{ status: 'success', mode?: 'query' \| 'query_fast' \| 'query_detailed', query, namespace, metadata_filter?, result_count, results[], fields? }`. + +Each row: `document_id`, `paper_number` (deprecated alias), `title`, `author`, `url`, `content`, `score`, `reranked`, optional `metadata`. + +**Example:** + +```json +{ + "query_text": "exception safety guarantees", + "namespace": "mailing", + "preset": "detailed", + "top_k": 8 +} +``` + +### Rerank fallback and row-level fidelity + +When reranking is requested but the rerank API fails, the server still returns **`status: 'success'`** with rows where `reranked: false`. Treat **`reranked: false`** as lower confidence when reranking was expected (`preset` detailed/full). Structured stderr logs include the failure; there is **no** separate top-level `degraded` flag in the current JSON envelope—client UX should combine `preset`, `use_reranking`, and per-row `reranked` (see project issue backlog for envelope-level degradation). + +--- + +## 6. `keyword_search` + +**Purpose:** Lexical / sparse-index search only (no hybrid merge, no rerank). **Does not** require `suggest_query_params`. + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `query_text` | string | yes | Keyword-style query | +| `namespace` | string | yes | Namespace | +| `top_k` | int | no | 1–100 | +| `metadata_filter` | object | no | Filter | +| `fields` | string[] | no | Returned fields | + +**Success:** Similar row shape to `query` (`KeywordSearchResponse`). + +--- + +## 7. `query_documents` + +**Purpose:** Fetch chunks, rerank, **reassemble** whole documents (merge chunk text). Requires suggest-flow. + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `query_text` | string | yes | Query | +| `namespace` | string | yes | Namespace | +| `top_k` | int | no | Documents to return (see constants, default 5, max 20) | +| `metadata_filter` | object | no | Filter | +| `max_chunks_per_document` | int | no | Cap merged chunks per doc (default 200, max 500) | + +**Success:** `{ status: 'success', query, namespace, metadata_filter?, result_count, documents: [{ document_id, merged_content, metadata, chunk_count, best_score }] }`. + +--- + +## 8. `guided_query` + +**Purpose:** Single-call orchestration: namespace routing + internal `suggest_query_params` + `count` or `query`. **Does not** require the client to call `suggest_query_params` first (it calls `markSuggested` internally). + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `user_query` | string | yes | User intent | +| `namespace` | string | no | Pin to explicit namespace | +| `metadata_filter` | object | no | Filter | +| `top_k` | int | no | For query paths | +| `preferred_tool` | `auto` \| `count` \| `fast` \| `detailed` \| `full` | no | Override automated tool choice | +| `enrich_urls` | boolean | no (default true) | Run URL generator when metadata lacks `url` | + +**Success:** `{ status: 'success', decision_trace, result }` where `result` is either a count payload or a `QueryResponse`-shaped query payload. + +**`decision_trace` fields (non-exhaustive):** `cache_hit`, `input_namespace`, `routed_namespace`, `selected_namespace`, `ranked_namespaces`, `suggested_fields`, `suggested_tool`, `selected_tool`, `explanation`, `enrich_urls`. + +**Example:** + +```json +{ + "user_query": "How many messages mention modules TS?", + "preferred_tool": "auto" +} +``` + +--- + +## 9. `generate_urls` + +**Purpose:** Synthesize URLs from metadata via per-namespace generators. + +| Field | Type | Required | Description | +| ----- | ---- | -------- | ----------- | +| `namespace` | string | yes | Namespace | +| `records` | object[] | yes | Up to 500 records (metadata object or `{ metadata: {...} }`) | + +**Success:** `{ status: 'success', namespace, count, results: [{ index, url, method, reason, metadata }] }`. + +--- + +## Tool ordering cheat sheet + +```text +Typical manual flow: + list_namespaces → (optional) namespace_router → suggest_query_params → query | count | query_documents + +Keyword-only: + list_namespaces → keyword_search # no suggest gate + +Single-shot: + guided_query +``` + +Canonical Zod schemas live beside each handler under `src/server/tools/*.ts`. diff --git a/examples/README.md b/examples/README.md index 9fe3606..c595b3f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -3,5 +3,8 @@ | File | Description | |------|-------------| | [custom-url-generator.ts](./custom-url-generator.ts) | Embed the MCP server as a library, register a **custom URL generator** for a namespace, and wire `PineconeClient` + `setupServer()`. | +| [suggest-flow-demo.ts](./suggest-flow-demo.ts) | Document the **suggest_query_params → query** gate sequence and trimmed namespace usage. | +| [guided-query-demo.ts](./guided-query-demo.ts) | Document **guided_query** and the **`decision_trace`** payload. | +| [library-embedding-demo.ts](./library-embedding-demo.ts) | Minimal **library embedding** (`resolveConfig`, `setPineconeClient`, `setupServer`). | -Run with `npx tsx examples/custom-url-generator.ts` after `npm install` (requires valid Pinecone credentials in env). +Run with `npx tsx examples/.ts` after `npm install` (live Pinecone calls need `PINECONE_API_KEY` and related env). diff --git a/examples/guided-query-demo.ts b/examples/guided-query-demo.ts new file mode 100644 index 0000000..f95e3f2 --- /dev/null +++ b/examples/guided-query-demo.ts @@ -0,0 +1,55 @@ +/** + * Worked example: `guided_query` single-call orchestration. + * + * Unlike the manual flow, `guided_query` bundles: + * - optional implicit `namespace_router` ranking + * - internal `suggest_query_params` + `markSuggested` + * - execution of `count` or hybrid `query` with `fast` / `detailed` / `full` + * + * The success payload includes **`decision_trace`**: cache hit, routed vs + * selected namespace, suggested fields/tools, and the final `selected_tool`. + * Use the trace in UIs or logs to explain why a path was chosen. + * + * **Reranking fidelity:** when reranking was expected, inspect each row's + * `reranked` boolean; `false` can indicate rerank was skipped or failed while + * still returning HTTP/MCP success (see docs/TOOLS.md). + */ + +import { + PineconeClient, + resolveConfig, + setPineconeClient, + setupServer, +} from '@will-cppa/pinecone-read-only-mcp'; + +async function main(): Promise { + const apiKey = process.env['PINECONE_API_KEY']?.trim(); + if (!apiKey) { + console.log( + '[guided-query-demo] Set PINECONE_API_KEY to run live. ' + + 'Call guided_query with user_query; read decision_trace + result in the JSON response.' + ); + return; + } + + const config = resolveConfig({ apiKey }); + setPineconeClient( + new PineconeClient({ + apiKey: config.apiKey, + indexName: config.indexName, + sparseIndexName: config.sparseIndexName, + rerankModel: config.rerankModel, + defaultTopK: config.defaultTopK, + requestTimeoutMs: config.requestTimeoutMs, + }) + ); + + const server = await setupServer(config); + void server; + console.log('Server ready — call guided_query({ user_query, preferred_tool?: "auto" }).'); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/examples/library-embedding-demo.ts b/examples/library-embedding-demo.ts new file mode 100644 index 0000000..97f679a --- /dev/null +++ b/examples/library-embedding-demo.ts @@ -0,0 +1,49 @@ +/** + * Library embedding: build the MCP server from a Node script (not the CLI). + * + * Pattern (mirrors `src/index.ts`): + * 1. `resolveConfig({ apiKey, ... })` — never rely on ambient env alone in libraries. + * 2. `new PineconeClient({ ... })` + `setPineconeClient(client)`. + * 3. `await setupServer(config)` then `server.connect(transport)`. + * + * **Single process:** `setupServer` registers tools against process-global + * singletons (suggest-flow state, namespaces cache, URL registry, active config). + * Do **not** call `setupServer` twice in one process for isolated tenants unless + * you accept shared state — prefer **one server per Node process** or external + * process isolation. (A future release may add an explicit teardown API; see + * CHANGELOG when available.) + */ + +import { + PineconeClient, + resolveConfig, + setPineconeClient, + setupServer, +} from '@will-cppa/pinecone-read-only-mcp'; + +async function main(): Promise { + const apiKey = process.env['PINECONE_API_KEY']?.trim() ?? 'demo-key-for-types'; + const config = resolveConfig({ apiKey }); + + setPineconeClient( + new PineconeClient({ + apiKey: config.apiKey, + indexName: config.indexName, + sparseIndexName: config.sparseIndexName, + rerankModel: config.rerankModel, + defaultTopK: config.defaultTopK, + requestTimeoutMs: config.requestTimeoutMs, + }) + ); + + const server = await setupServer(config); + // const transport = new StdioServerTransport(); + // await server.connect(transport); + void server; + console.log('Embedded server constructed — connect your MCP transport (stdio, HTTP, etc.).'); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/examples/suggest-flow-demo.ts b/examples/suggest-flow-demo.ts new file mode 100644 index 0000000..24ce76d --- /dev/null +++ b/examples/suggest-flow-demo.ts @@ -0,0 +1,59 @@ +/** + * Worked example: suggest-then-query (manual multi-step flow). + * + * Stage 1 — discovery: call `list_namespaces` (not shown) so the model knows + * valid namespaces and metadata fields. + * + * Stage 2 — gate: call `suggest_query_params` with a **trimmed** namespace and + * the user query. This records in-process state (`markSuggested`) so the gate + * opens for that namespace until the cache TTL expires. + * + * Stage 3 — retrieval: call `query` with the **same** namespace string, passing + * `preset` aligned with `recommended_tool` (`fast` | `detailed` | `full`) and + * optional `fields` from `suggested_fields`. + * + * This file is runnable without Pinecone only in **documentation mode**; set + * `PINECONE_API_KEY` and wire an MCP transport to execute real tool calls. + */ + +import { + PineconeClient, + resolveConfig, + setPineconeClient, + setupServer, +} from '@will-cppa/pinecone-read-only-mcp'; + +async function main(): Promise { + const apiKey = process.env['PINECONE_API_KEY']?.trim(); + if (!apiKey) { + console.log( + '[suggest-flow-demo] Set PINECONE_API_KEY to run against Pinecone. ' + + 'Flow: list_namespaces → suggest_query_params → query (same trimmed namespace).' + ); + return; + } + + const config = resolveConfig({ apiKey }); + setPineconeClient( + new PineconeClient({ + apiKey: config.apiKey, + indexName: config.indexName, + sparseIndexName: config.sparseIndexName, + rerankModel: config.rerankModel, + defaultTopK: config.defaultTopK, + requestTimeoutMs: config.requestTimeoutMs, + }) + ); + + const server = await setupServer(config); + // With an MCP client connected to `server`, invoke tools in order: + // 1) suggest_query_params({ namespace: "mailing".trim(), user_query: "..." }) + // 2) query({ query_text, namespace: "mailing", preset: "detailed", ... }) + void server; + console.log('Server ready — connect a transport and issue suggest_query_params then query.'); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/vitest.config.ts b/vitest.config.ts index fbae058..95efacf 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,5 +1,10 @@ import { defineConfig } from 'vitest/config'; +/** + * Global coverage gates (v8). Thresholds sit ~2–3% below the last measured + * `npm run test:coverage` totals on main so normal fluctuation does not fail CI, + * while still catching meaningful regressions. Re-measure after large refactors. + */ export default defineConfig({ test: { globals: true, @@ -8,6 +13,12 @@ export default defineConfig({ provider: 'v8', reporter: ['text', 'json', 'html', 'lcov'], exclude: ['node_modules/', 'dist/', '**/*.test.ts'], + thresholds: { + lines: 73, + statements: 72, + branches: 58, + functions: 76, + }, }, }, }); From 327b5ad17174f9469eab05b5c3a8622e42ac6e73 Mon Sep 17 00:00:00 2001 From: zho Date: Tue, 19 May 2026 02:30:36 +0800 Subject: [PATCH 2/5] added baseline.json --- benchmarks/baseline.json | 53 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 benchmarks/baseline.json diff --git a/benchmarks/baseline.json b/benchmarks/baseline.json new file mode 100644 index 0000000..edc238b --- /dev/null +++ b/benchmarks/baseline.json @@ -0,0 +1,53 @@ +{ + "generated_at": "2026-05-13T21:21:47.950Z", + "node": "v22.22.0", + "warmup_iterations": 10, + "measured_iterations": 200, + "results": [ + { + "name": "query_no_rerank", + "p50": 0.0583, + "p95": 0.0942, + "p99": 0.8578, + "min": 0.0243, + "max": 1.581, + "iterations": 200 + }, + { + "name": "query_with_rerank", + "p50": 0.0261, + "p95": 0.0484, + "p99": 0.1141, + "min": 0.0245, + "max": 0.2182, + "iterations": 200 + }, + { + "name": "guided_query_end_to_end", + "p50": 0.0166, + "p95": 0.02, + "p99": 0.0324, + "min": 0.0161, + "max": 0.2947, + "iterations": 200 + }, + { + "name": "list_namespaces_cache_miss", + "p50": 0.0005, + "p95": 0.0009, + "p99": 0.0014, + "min": 0.0004, + "max": 0.0596, + "iterations": 200 + }, + { + "name": "list_namespaces_cache_hit", + "p50": 0.0004, + "p95": 0.0005, + "p99": 0.0009, + "min": 0.0003, + "max": 0.1018, + "iterations": 200 + } + ] +} From d6f01a555c361d5c353811c583c90054e4fc4f43 Mon Sep 17 00:00:00 2001 From: zho Date: Tue, 19 May 2026 03:43:58 +0800 Subject: [PATCH 3/5] addressed ai reviews --- docs/CONTRIBUTING.md | 2 +- examples/library-embedding-demo.ts | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 1adb345..f337700 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -8,7 +8,7 @@ ## Setup ```bash -git clone https://github.com/CppDigest/pinecone-read-only-mcp-typescript.git +git clone https://github.com/cppalliance/pinecone-read-only-mcp-typescript.git cd pinecone-read-only-mcp-typescript npm ci ``` diff --git a/examples/library-embedding-demo.ts b/examples/library-embedding-demo.ts index 97f679a..cfc777a 100644 --- a/examples/library-embedding-demo.ts +++ b/examples/library-embedding-demo.ts @@ -22,7 +22,13 @@ import { } from '@will-cppa/pinecone-read-only-mcp'; async function main(): Promise { - const apiKey = process.env['PINECONE_API_KEY']?.trim() ?? 'demo-key-for-types'; + const apiKey = process.env['PINECONE_API_KEY']?.trim(); + if (!apiKey) { + console.log( + 'Set PINECONE_API_KEY to run this example. Skipping live setup in doc-only mode.' + ); + return; + } const config = resolveConfig({ apiKey }); setPineconeClient( From adcdccd65a35d75eafb06c51bac781ef931483b4 Mon Sep 17 00:00:00 2001 From: zho Date: Tue, 19 May 2026 13:30:19 +0800 Subject: [PATCH 4/5] rename CppDigest as cppalliance --- CHANGELOG.md | 8 ++-- README.md | 100 ++++++++++++++++++++++++----------------------- docs/SECURITY.md | 2 +- package.json | 6 +-- 4 files changed, 59 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc535a2..2869663 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,7 +90,7 @@ details. Newer shipped changes are recorded in this changelog by version. - Environment variable support - Full documentation and examples -[Unreleased]: https://github.com/CppDigest/pinecone-read-only-mcp-typescript/compare/v0.1.6...HEAD -[0.1.6]: https://github.com/CppDigest/pinecone-read-only-mcp-typescript/compare/v0.1.1...v0.1.6 -[0.1.1]: https://github.com/CppDigest/pinecone-read-only-mcp-typescript/compare/v0.1.0...v0.1.1 -[0.1.0]: https://github.com/CppDigest/pinecone-read-only-mcp-typescript/releases/tag/v0.1.0 +[Unreleased]: https://github.com/cppalliance/pinecone-read-only-mcp-typescript/compare/v0.1.6...HEAD +[0.1.6]: https://github.com/cppalliance/pinecone-read-only-mcp-typescript/compare/v0.1.1...v0.1.6 +[0.1.1]: https://github.com/cppalliance/pinecone-read-only-mcp-typescript/compare/v0.1.0...v0.1.1 +[0.1.0]: https://github.com/cppalliance/pinecone-read-only-mcp-typescript/releases/tag/v0.1.0 diff --git a/README.md b/README.md index 83cd019..c5d846c 100644 --- a/README.md +++ b/README.md @@ -3,35 +3,35 @@ [![npm version](https://img.shields.io/npm/v/@will-cppa/pinecone-read-only-mcp.svg)](https://www.npmjs.com/package/@will-cppa/pinecone-read-only-mcp) [![Node.js Version](https://img.shields.io/node/v/@will-cppa/pinecone-read-only-mcp.svg)](https://nodejs.org) [![License: BSL-1.0](https://img.shields.io/badge/License-BSL--1.0-blue.svg)](https://opensource.org/licenses/BSL-1.0) -[![CI](https://github.com/CppDigest/pinecone-read-only-mcp-typescript/workflows/CI/badge.svg)](https://github.com/CppDigest/pinecone-read-only-mcp-typescript/actions) +[![CI](https://github.com/cppalliance/pinecone-read-only-mcp-typescript/workflows/CI/badge.svg)](https://github.com/cppalliance/pinecone-read-only-mcp-typescript/actions) A Model Context Protocol (MCP) server that provides semantic search over Pinecone vector databases using hybrid search (dense + sparse) with reranking. ## Documentation -| Doc | Description | -|-----|---------------| -| [docs/README.md](docs/README.md) | Index of all guides | -| [docs/TOOLS.md](docs/TOOLS.md) | Tool catalog & flows | -| [docs/CONFIGURATION.md](docs/CONFIGURATION.md) | Env vars, CLI flags, library config | -| [docs/FAQ.md](docs/FAQ.md) | Common questions | -| [docs/MIGRATION.md](docs/MIGRATION.md) | Deprecations & breaking changes | -| [docs/CI_CD.md](docs/CI_CD.md) | GitHub Actions, SBOM, Docker, releases | -| [docs/RELEASING.md](docs/RELEASING.md) | npm publish via GitHub Releases | -| [docs/CONTRIBUTING.md](docs/CONTRIBUTING.md) | How to contribute | -| [docs/SECURITY.md](docs/SECURITY.md) | Vulnerability reporting | +| Doc | Description | +| ---------------------------------------------- | -------------------------------------- | +| [docs/README.md](docs/README.md) | Index of all guides | +| [docs/TOOLS.md](docs/TOOLS.md) | Tool catalog & flows | +| [docs/CONFIGURATION.md](docs/CONFIGURATION.md) | Env vars, CLI flags, library config | +| [docs/FAQ.md](docs/FAQ.md) | Common questions | +| [docs/MIGRATION.md](docs/MIGRATION.md) | Deprecations & breaking changes | +| [docs/CI_CD.md](docs/CI_CD.md) | GitHub Actions, SBOM, Docker, releases | +| [docs/RELEASING.md](docs/RELEASING.md) | npm publish via GitHub Releases | +| [docs/CONTRIBUTING.md](docs/CONTRIBUTING.md) | How to contribute | +| [docs/SECURITY.md](docs/SECURITY.md) | Vulnerability reporting | ## Error responses When a tool fails, the MCP tool result sets **`isError: true`**. The `text` content is JSON matching **`ToolError`** (parse with `toolErrorSchema` from `@will-cppa/pinecone-read-only-mcp`). -| Field | Description | -| ----- | ----------- | -| `code` | `FLOW_GATE` — `suggest_query_params` was not run for this namespace (or context expired). `VALIDATION` — bad input or metadata filter. `PINECONE_ERROR` — SDK / network / server failure. `TIMEOUT` — outbound Pinecone call exceeded `--request-timeout-ms`. | -| `message` | Human-readable detail (`DEBUG` log level may surface raw SDK messages in the message for `PINECONE_ERROR` / `TIMEOUT`). | -| `recoverable` | Whether the client can plausibly fix the issue and retry (`true` for flow gate, validation, timeouts; typically `false` for generic Pinecone errors). | -| `suggestion` | Optional hint. **`FLOW_GATE`** always includes: `Call suggest_query_params for namespace '' first`. **`TIMEOUT`** suggests retrying or increasing the request timeout. | -| `field` | **Required when `code` is `VALIDATION`:** the input parameter name (e.g. `query_text`, `namespace`) or a dot-path into `metadata_filter` (e.g. `author.$in`). | +| Field | Description | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `code` | `FLOW_GATE` — `suggest_query_params` was not run for this namespace (or context expired). `VALIDATION` — bad input or metadata filter. `PINECONE_ERROR` — SDK / network / server failure. `TIMEOUT` — outbound Pinecone call exceeded `--request-timeout-ms`. | +| `message` | Human-readable detail (`DEBUG` log level may surface raw SDK messages in the message for `PINECONE_ERROR` / `TIMEOUT`). | +| `recoverable` | Whether the client can plausibly fix the issue and retry (`true` for flow gate, validation, timeouts; typically `false` for generic Pinecone errors). | +| `suggestion` | Optional hint. **`FLOW_GATE`** always includes: `Call suggest_query_params for namespace '' first`. **`TIMEOUT`** suggests retrying or increasing the request timeout. | +| `field` | **Required when `code` is `VALIDATION`:** the input parameter name (e.g. `query_text`, `namespace`) or a dot-path into `metadata_filter` (e.g. `author.$in`). | Success payloads are unchanged and do **not** wrap `ToolError`. Clients that still expect `{ "status": "error", "message": "..." }` must migrate to the shape above. @@ -78,7 +78,7 @@ npm install -g @will-cppa/pinecone-read-only-mcp ### From Source ```bash -git clone https://github.com/CppDigest/pinecone-read-only-mcp-typescript.git +git clone https://github.com/cppallance/pinecone-read-only-mcp-typescript.git cd pinecone-read-only-mcp-typescript npm install npm run build @@ -90,13 +90,13 @@ You need a **Pinecone API key** and (by default) a **dense** index plus matching Quick reference: -| Variable | Required | Default | -| -------- | -------- | ------- | -| `PINECONE_API_KEY` | Yes (for live Pinecone) | — | -| `PINECONE_INDEX_NAME` | No | `rag-hybrid` | -| `PINECONE_SPARSE_INDEX_NAME` | No | `{index}-sparse` | -| `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | No | `INFO` (`DEBUG`–`ERROR`) | -| `PINECONE_READ_ONLY_MCP_LOG_FORMAT` | No | `text` (`json` for log pipelines) | +| Variable | Required | Default | +| ----------------------------------- | ----------------------- | --------------------------------- | +| `PINECONE_API_KEY` | Yes (for live Pinecone) | — | +| `PINECONE_INDEX_NAME` | No | `rag-hybrid` | +| `PINECONE_SPARSE_INDEX_NAME` | No | `{index}-sparse` | +| `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | No | `INFO` (`DEBUG`–`ERROR`) | +| `PINECONE_READ_ONLY_MCP_LOG_FORMAT` | No | `text` (`json` for log pipelines) | Run `pinecone-read-only-mcp --help` for CLI equivalents (`--cache-ttl-seconds`, `--request-timeout-ms`, `--disable-suggest-flow`, etc.). @@ -140,12 +140,12 @@ A fuller embedding sample lives in [examples/custom-url-generator.ts](examples/c ### Examples -| File | Description | -| ---- | ----------- | -| [examples/suggest-flow-demo.ts](examples/suggest-flow-demo.ts) | Manual **suggest_query_params → query** flow with namespace consistency notes. | -| [examples/guided-query-demo.ts](examples/guided-query-demo.ts) | **guided_query** orchestration and how to read `decision_trace`. | -| [examples/library-embedding-demo.ts](examples/library-embedding-demo.ts) | Programmatic **setupServer** wiring without the CLI binary. | -| [examples/custom-url-generator.ts](examples/custom-url-generator.ts) | Custom **URL generator** registration for `generate_urls` / row enrichment. | +| File | Description | +| ------------------------------------------------------------------------ | ------------------------------------------------------------------------------ | +| [examples/suggest-flow-demo.ts](examples/suggest-flow-demo.ts) | Manual **suggest_query_params → query** flow with namespace consistency notes. | +| [examples/guided-query-demo.ts](examples/guided-query-demo.ts) | **guided_query** orchestration and how to read `decision_trace`. | +| [examples/library-embedding-demo.ts](examples/library-embedding-demo.ts) | Programmatic **setupServer** wiring without the CLI binary. | +| [examples/custom-url-generator.ts](examples/custom-url-generator.ts) | Custom **URL generator** registration for `generate_urls` / row enrichment. | Run with `npx tsx examples/.ts` from a checkout (requires valid Pinecone env for live paths). @@ -335,9 +335,9 @@ Suggests which **fields** to request and which path to use (`count`, or hybrid q **Parameters:** -| Parameter | Type | Required | Description | -| ------------ | ------ | -------- | ----------------------------------------------------------------------------------------------- | -| `namespace` | string | Yes | Namespace to query (must match a name from `list_namespaces`) | +| Parameter | Type | Required | Description | +| ------------ | ------ | -------- | -------------------------------------------------------------------------------------------------- | +| `namespace` | string | Yes | Namespace to query (must match a name from `list_namespaces`) | | `user_query` | string | Yes | User’s question or intent (e.g. "list papers by John Doe with titles", "how many papers by Wong?") | **Returns:** `suggested_fields` (only fields that exist in that namespace), `use_count_tool`, `recommended_tool`, `explanation`, and `namespace_found`. @@ -375,7 +375,7 @@ It returns both the final result and a `decision_trace` for transparency. | `namespace` | string | No | - | Optional explicit namespace | | `metadata_filter` | object | No | - | Optional metadata filter | | `top_k` | integer | No | `10` | Query result size for query paths (1-100) | -| `preferred_tool` | enum | No | `auto` | One of `auto`, `count`, `fast`, `detailed`, `full` | +| `preferred_tool` | enum | No | `auto` | One of `auto`, `count`, `fast`, `detailed`, `full` | | `enrich_urls` | boolean | No | `true` | Auto-generate URLs for `mailing` and `slack-Cpplang` when `metadata.url` is missing | **Returns:** JSON containing `decision_trace` and `result`. @@ -416,7 +416,7 @@ Returns the **unique document count** matching a metadata filter and semantic qu | ----------------- | ------ | -------- | -------------------------------------------------------------------------------------------- | | `namespace` | string | Yes | Namespace to count in (use `list_namespaces` to discover) | | `query_text` | string | Yes | Search query; use a broad term (e.g. `"paper"`, `"document"`) when counting by metadata only | -| `metadata_filter` | object | No | Same operators as `query` (e.g. `{"author": {"$in": ["John Doe"]}}` for wg21-papers) | +| `metadata_filter` | object | No | Same operators as `query` (e.g. `{"author": {"$in": ["John Doe"]}}` for wg21-papers) | **Returns:** JSON with `count` (unique documents, up to 10,000), and `truncated: true` if there are at least 10,000 matches. @@ -438,13 +438,13 @@ Performs **keyword (lexical/sparse-only)** search over the dedicated sparse inde **Parameters:** -| Parameter | Type | Required | Default | Description | -| ----------------- | -------- | -------- | ------- | --------------------------------------------------------------------------- | -| `query_text` | string | Yes | - | Search query text (keyword/lexical match) | -| `namespace` | string | Yes | - | Namespace to search (use `list_namespaces` to discover) | -| `top_k` | integer | No | `10` | Number of results to return (1-100) | -| `metadata_filter` | object | No | - | Optional metadata filter (same operators as `query`) | -| `fields` | string[] | No | - | Optional field names to return; omit for all fields | +| Parameter | Type | Required | Default | Description | +| ----------------- | -------- | -------- | ------- | ------------------------------------------------------- | +| `query_text` | string | Yes | - | Search query text (keyword/lexical match) | +| `namespace` | string | Yes | - | Namespace to search (use `list_namespaces` to discover) | +| `top_k` | integer | No | `10` | Number of results to return (1-100) | +| `metadata_filter` | object | No | - | Optional metadata filter (same operators as `query`) | +| `fields` | string[] | No | - | Optional field names to return; omit for all fields | **Returns:** JSON with `status`, `query`, `namespace`, `index` (sparse index name), `result_count`, and `results` (ids, metadata, scores). Result rows match the `query` tool shape (e.g. `paper_number`, `title`, `author`, `url`, `content`, `score`, `reranked: false`). @@ -586,7 +586,7 @@ Metadata filters allow you to narrow down search results based on document prope ### Setup Development Environment ```bash -git clone https://github.com/CppDigest/pinecone-read-only-mcp-typescript.git +git clone https://github.com/cppalliance/pinecone-read-only-mcp-typescript.git cd pinecone-read-only-mcp-typescript npm install ``` @@ -617,9 +617,11 @@ The script prints a table of p50, p95, and p99 latencies in milliseconds and wri 1. **Connectivity and keyword search (script):** Run the search test script (includes a keyword search step against the sparse index): + ```bash PINECONE_API_KEY=your-key npm run test:search ``` + If the sparse index (`rag-hybrid-sparse` by default) does not exist or has no data, the keyword search step is skipped with a warning. 2. **Via MCP client:** @@ -681,7 +683,7 @@ npm run dev -- --api-key YOUR_API_KEY ## Comparison with Python Version -This TypeScript implementation grew out of the [Python version](https://github.com/CppDigest/pinecone-read-only-mcp) and now exposes a strict superset of its tool surface, including: +This TypeScript implementation grew out of the [Python version](https://github.com/cppalliance/pinecone-read-only-mcp) and now exposes a strict superset of its tool surface, including: - `guided_query` (single-call orchestrator with decision trace) - `query_documents` (full-document reassembly from chunks) @@ -739,14 +741,14 @@ This project uses: ## Related Projects -- [Python version](https://github.com/CppDigest/pinecone-read-only-mcp) - Original Python implementation +- [Python version](https://github.com/cppalliance/pinecone-read-only-mcp) - Original Python implementation - [Pinecone MCP](https://github.com/pinecone-io/pinecone-mcp) - Full-featured Pinecone MCP with write capabilities ## Support For issues and questions: -- GitHub Issues: [https://github.com/CppDigest/pinecone-read-only-mcp-typescript/issues](https://github.com/CppDigest/pinecone-read-only-mcp-typescript/issues) +- GitHub Issues: [https://github.com/cppalliance/pinecone-read-only-mcp-typescript/issues](https://github.com/cppalliance/pinecone-read-only-mcp-typescript/issues) - Email: will@cppalliance.org ## Changelog diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 180727a..fff6e94 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -32,6 +32,6 @@ Do not run the production image as root unless you have a compensating security ## Reporting vulnerabilities -Open a **private** security advisory or issue per repository policy on [GitHub](https://github.com/CppDigest/pinecone-read-only-mcp-typescript/security). Do not post exploit details in public issues before a fix is available. +Open a **private** security advisory or issue per repository policy on [GitHub](https://github.com/cppalliance/pinecone-read-only-mcp-typescript/security). Do not post exploit details in public issues before a fix is available. Include: affected version, reproduction steps, and impact assessment. diff --git a/package.json b/package.json index d8bdd3e..24ea6b4 100644 --- a/package.json +++ b/package.json @@ -35,12 +35,12 @@ "license": "BSL-1.0", "repository": { "type": "git", - "url": "git+https://github.com/CppDigest/pinecone-read-only-mcp-typescript.git" + "url": "git+https://github.com/cppalliance/pinecone-read-only-mcp-typescript.git" }, "bugs": { - "url": "https://github.com/CppDigest/pinecone-read-only-mcp-typescript/issues" + "url": "https://github.com/cppalliance/pinecone-read-only-mcp-typescript/issues" }, - "homepage": "https://github.com/CppDigest/pinecone-read-only-mcp-typescript#readme", + "homepage": "https://github.com/cppalliance/pinecone-read-only-mcp-typescript#readme", "engines": { "node": ">=20.12.0" }, From 7cd894536d6a95ab1ce29f77b91c37ed29109ca2 Mon Sep 17 00:00:00 2001 From: zho Date: Tue, 19 May 2026 13:38:45 +0800 Subject: [PATCH 5/5] addressed first human review results --- .github/workflows/ci.yml | 4 +--- CHANGELOG.md | 2 +- docs/CI_CD.md | 2 +- docs/CONTRIBUTING.md | 2 +- package.json | 1 + scripts/docs-link-check.mjs | 35 +++++++++++++++++++++++++++++++++++ 6 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 scripts/docs-link-check.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7bed065..d6ae439 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,4 @@ jobs: run: npm pack --dry-run - name: Verify internal markdown links - run: | - npx --yes markdown-link-check@3 -c .markdown-link-check.json README.md CHANGELOG.md - find docs -type f -name '*.md' -print0 | xargs -0 -I{} npx --yes markdown-link-check@3 -c .markdown-link-check.json "{}" + run: npm run docs:link-check diff --git a/CHANGELOG.md b/CHANGELOG.md index 2869663..089df37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,7 @@ Tagged releases are published to npm from GitHub Actions when a **GitHub Release - `.env.example` log-level options corrected to the four levels actually supported (`DEBUG`, `INFO`, `WARN`, `ERROR`); the stale `WARNING`/`CRITICAL` values are gone. - README Slack URL example now matches the generator output (`https://app.slack.com/client/{team_id}/{channel_id}/p{messageId}`). - README "Comparison with Python Version" no longer claims an identical API; the new TypeScript-only tools (`guided_query`, `query_documents`, `keyword_search`, `namespace_router`, `suggest_query_params`, `count`, `generate_urls`) are listed explicitly. -- `npm run ci` now runs `test:coverage` so merges are gated on coverage thresholds. +- CI **quality** job: `npm run docs:link-check` runs `markdown-link-check` in a single `npx` invocation over `README.md`, `CHANGELOG.md`, and all `docs/**/*.md` (via `scripts/docs-link-check.mjs`) instead of one `npx` per file under `docs/`. - **Breaking (runtime / tooling):** `engines.node` is now **>=20.12.0**. Vitest **4** (bundled **rolldown**) imports `util.styleText` from `node:util` (added in Node **20.12**), and **`@vitest/coverage-v8`** uses `node:inspector/promises` (Node **≥19**). CI tests only **20.x** and **22.x**. - Dependabot groups related **vitest**, **typescript-eslint**, and **eslint/prettier** updates. diff --git a/docs/CI_CD.md b/docs/CI_CD.md index a3c5413..1243b18 100644 --- a/docs/CI_CD.md +++ b/docs/CI_CD.md @@ -21,7 +21,7 @@ **Job `quality`** -- Ubuntu + Node 20: `npm audit --audit-level=moderate` (continue-on-error) → `npm pack --dry-run` → **markdown-link-check** on documentation paths (see workflow). +- Ubuntu + Node 20: `npm audit --audit-level=moderate` (continue-on-error) → `npm pack --dry-run` → **`npm run docs:link-check`** (single `npx markdown-link-check` over `README.md`, `CHANGELOG.md`, and all `docs/**/*.md`). --- diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index f337700..fe9863f 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -39,7 +39,7 @@ npm ci - Run `npm run ci` before pushing. - Keep changes focused; update `CHANGELOG.md` `[Unreleased]` for user-visible behavior. -- Documentation changes should keep [README](../README.md) links and run markdown link check locally if you touch many relative links. +- Documentation changes should keep [README](../README.md) links; run `npm run docs:link-check` locally if you touch many relative links. ## Documentation diff --git a/package.json b/package.json index 24ea6b4..adb61f0 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,7 @@ "lint:fix": "eslint src/ --fix", "format": "prettier --write \"src/**/*.ts\" \"*.json\" \".prettierrc\"", "format:check": "prettier --check \"src/**/*.ts\" \"*.json\" \".prettierrc\"", + "docs:link-check": "node scripts/docs-link-check.mjs", "typecheck": "tsc --noEmit", "ci": "npm run typecheck && npm run lint && npm run format:check && npm run build && npm run test:coverage", "release:check": "npm run ci && npm pack --dry-run --ignore-scripts", diff --git a/scripts/docs-link-check.mjs b/scripts/docs-link-check.mjs new file mode 100644 index 0000000..7f93db0 --- /dev/null +++ b/scripts/docs-link-check.mjs @@ -0,0 +1,35 @@ +/** + * Run markdown-link-check once for README, CHANGELOG, and every *.md under docs/. + * Avoids per-file `npx` invocations (slow / flaky under registry hiccups). + */ + +import { spawnSync } from 'node:child_process'; +import { readdirSync, statSync } from 'node:fs'; +import { join } from 'node:path'; + +/** @param {string} dir @returns {string[]} */ +function walkMarkdownFiles(dir) { + const out = []; + try { + if (!statSync(dir, { throwIfNoEntry: false })?.isDirectory()) return out; + for (const ent of readdirSync(dir, { withFileTypes: true })) { + const p = join(dir, ent.name); + if (ent.isDirectory()) out.push(...walkMarkdownFiles(p)); + else if (ent.isFile() && ent.name.endsWith('.md')) out.push(p); + } + } catch { + // missing or unreadable dir + } + return out; +} + +const paths = ['README.md', 'CHANGELOG.md', ...walkMarkdownFiles('docs')]; + +const shell = process.platform === 'win32'; +const r = spawnSync( + 'npx', + ['--yes', 'markdown-link-check@3', '-c', '.markdown-link-check.json', ...paths], + { stdio: 'inherit', shell } +); + +process.exit(r.status === null ? 1 : r.status);