From 671c9452a071b287f026aec3746385dee70b9321 Mon Sep 17 00:00:00 2001 From: Ankit Kotnala Date: Sat, 23 May 2026 16:01:23 +0530 Subject: [PATCH 1/6] Adding local XMem setup --- .gitignore | 8 +- README.md | 42 +-- docker-compose.local.yml | 66 +++++ package.json | 22 ++ packages/create-xmem/bin/create-xmem.js | 126 ++++++++ packages/create-xmem/package.json | 21 ++ scripts/check-npm-publish.js | 68 +++++ scripts/configure-xmem-env.ps1 | 174 +++++++++++ scripts/context-export.ps1 | 20 ++ scripts/context-import.ps1 | 20 ++ scripts/context-sync.ps1 | 20 ++ scripts/context.py | 368 ++++++++++++++++++++++++ scripts/doctor.ps1 | 155 ++++++++++ scripts/install.ps1 | 273 ++++++++++++++++++ scripts/patch-extension-local.ps1 | 161 +++++++++++ scripts/start.ps1 | 230 +++++++++++++++ scripts/verify.ps1 | 140 +++++++++ scripts/xmem.js | 160 +++++++++++ src/agents/base.py | 13 +- src/api/app.py | 84 +++++- src/api/routes/memory.py | 30 +- src/api/schemas.py | 38 ++- src/config/settings.py | 8 + templates/xmem.env.local | 92 ++++++ 24 files changed, 2295 insertions(+), 44 deletions(-) create mode 100644 docker-compose.local.yml create mode 100644 package.json create mode 100644 packages/create-xmem/bin/create-xmem.js create mode 100644 packages/create-xmem/package.json create mode 100644 scripts/check-npm-publish.js create mode 100644 scripts/configure-xmem-env.ps1 create mode 100644 scripts/context-export.ps1 create mode 100644 scripts/context-import.ps1 create mode 100644 scripts/context-sync.ps1 create mode 100644 scripts/context.py create mode 100644 scripts/doctor.ps1 create mode 100644 scripts/install.ps1 create mode 100644 scripts/patch-extension-local.ps1 create mode 100644 scripts/start.ps1 create mode 100644 scripts/verify.ps1 create mode 100644 scripts/xmem.js create mode 100644 templates/xmem.env.local diff --git a/.gitignore b/.gitignore index b028d5d..02e560a 100644 --- a/.gitignore +++ b/.gitignore @@ -52,9 +52,6 @@ logs/ # Dev-only folders frontend/ -scripts/ -!scripts/ -!scripts/benchmark_v2_ingest.py tests/ !tests/ !tests/**/*.py @@ -96,3 +93,8 @@ src/api/routes/search.py # Misc error.json MIGRATION_PLAN.md +node_modules/ +repos/ +exports/ +tmp/ +.xmem/ diff --git a/README.md b/README.md index e841508..efba9bf 100644 --- a/README.md +++ b/README.md @@ -309,36 +309,46 @@ The industry standard benchmark for long-term conversational memory. Tests wheth ## Quickstart -### 1. Start the XMem Server +### Local XMem ```bash -git clone https://github.com/XortexLabs/xmem.git +npx create-xmem@latest cd xmem +npm run dev +``` -# Install (requires Python 3.11+) -pip install -e . +This creates a local XMem workspace, installs the backend, starts local storage, builds the Chrome extension, and launches the API at `http://localhost:8000`. -# Configure environment -cp .env.example .env # Add your API keys +After setup, load the extension from: -# Start -uvicorn src.api.app:create_app --factory --host 0.0.0.0 --port 8000 +```text +repos/xmem-extension/dist ``` -### 2. Install the Chrome Extension +Chrome path: `chrome://extensions` -> enable Developer mode -> Load unpacked. + +### Local Commands ```bash -git clone https://github.com/XortexAI/xmem-extension.git -npm install && npm run build +npm run setup +npm run start +npm run verify +npm run doctor ``` -Load `dist/` in Chrome via `chrome://extensions` → "Load unpacked". Point it to your server URL. +If `.env` contains a real cloud LLM key, XMem uses that provider and keeps embeddings local with FastEmbed. If no cloud key is configured, XMem falls back to local Ollama and pulls the required local models during setup. +### Context Portability -https://github.com/user-attachments/assets/605985c3-ef27-4096-a28c-b0b4cc6f8b8d +```bash +npm run context:export +npm run context:import -- --file ./exports/xmem-context.json +npm run context:sync -- --file ./exports/xmem-context.json --server https://api.xmem.in --api-key +``` +`context:export` writes a local context bundle that can be imported later or synced to an XMem server. -### 3. Index a Repository (Optional) +### Index a Repository ```bash python -m src.scanner.runner \ @@ -352,8 +362,8 @@ python -m src.scanner.runner \ > For a fully local setup with no cloud dependencies: > ```ini > FALLBACK_ORDER='["ollama"]' -> EMBEDDING_PROVIDER=fastembed -> VECTOR_STORE_PROVIDER=chroma +> EMBEDDING_PROVIDER=ollama +> VECTOR_STORE_PROVIDER=pgvector > ``` > Then install local extras: `pip install -e ".[local]"` diff --git a/docker-compose.local.yml b/docker-compose.local.yml new file mode 100644 index 0000000..6be171d --- /dev/null +++ b/docker-compose.local.yml @@ -0,0 +1,66 @@ +name: xmem + +services: + postgres: + image: pgvector/pgvector:pg16 + container_name: xmem-postgres + environment: + POSTGRES_DB: xmem + POSTGRES_USER: xmem + POSTGRES_PASSWORD: xmem + ports: + - "15432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U xmem -d xmem"] + interval: 10s + timeout: 5s + retries: 10 + + mongo: + image: mongo:7 + container_name: xmem-mongo + ports: + - "27018:27017" + volumes: + - mongo_data:/data/db + healthcheck: + test: + [ + "CMD-SHELL", + "mongosh --quiet --eval \"db.adminCommand('ping').ok\" || exit 1" + ] + interval: 10s + timeout: 5s + retries: 12 + + neo4j: + image: neo4j:5-community + container_name: xmem-neo4j + environment: + NEO4J_AUTH: neo4j/local-password + NEO4J_server_memory_heap_initial__size: 512m + NEO4J_server_memory_heap_max__size: 2G + NEO4J_server_memory_pagecache_size: 1G + ports: + - "17474:7474" + - "17687:7687" + volumes: + - neo4j_data:/data + - neo4j_logs:/logs + healthcheck: + test: + [ + "CMD-SHELL", + "cypher-shell -u neo4j -p local-password 'RETURN 1' || exit 1" + ] + interval: 10s + timeout: 5s + retries: 12 + +volumes: + postgres_data: + mongo_data: + neo4j_data: + neo4j_logs: diff --git a/package.json b/package.json new file mode 100644 index 0000000..b30b4a5 --- /dev/null +++ b/package.json @@ -0,0 +1,22 @@ +{ + "name": "xmem", + "version": "0.1.0", + "private": true, + "description": "XMem local-first memory layer.", + "scripts": { + "setup": "node ./scripts/xmem.js setup", + "dev": "node ./scripts/xmem.js dev", + "start": "node ./scripts/xmem.js start", + "verify": "node ./scripts/xmem.js verify", + "doctor": "node ./scripts/xmem.js doctor", + "context:export": "node ./scripts/xmem.js context:export", + "context:import": "node ./scripts/xmem.js context:import", + "context:sync": "node ./scripts/xmem.js context:sync", + "check:npm": "node ./scripts/check-npm-publish.js", + "pack:create": "npm pack ./packages/create-xmem --dry-run", + "publish:create": "npm publish ./packages/create-xmem --access public" + }, + "engines": { + "node": ">=20" + } +} diff --git a/packages/create-xmem/bin/create-xmem.js b/packages/create-xmem/bin/create-xmem.js new file mode 100644 index 0000000..fc98de8 --- /dev/null +++ b/packages/create-xmem/bin/create-xmem.js @@ -0,0 +1,126 @@ +#!/usr/bin/env node + +const fs = require("node:fs"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); + +const DEFAULT_REPO = "https://github.com/XortexAI/XMem.git"; +const DEFAULT_BRANCH = "main"; + +function usage(exitCode = 0) { + console.log(`Create a local XMem workspace + +Usage: + npx create-xmem@latest + npx create-xmem@latest my-xmem + +Options: + --repo XMem git repository URL + --branch XMem branch to use + --help Show this message + +After creation: + cd xmem + npm run dev +`); + process.exit(exitCode); +} + +function parseArgs(argv) { + const options = { + target: "xmem", + repo: process.env.XMEM_REPO || DEFAULT_REPO, + branch: process.env.XMEM_BRANCH || DEFAULT_BRANCH, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + + if (arg === "--help" || arg === "-h") { + usage(0); + } + + if (arg === "--repo") { + options.repo = argv[index + 1]; + index += 1; + continue; + } + + if (arg === "--branch") { + options.branch = argv[index + 1]; + index += 1; + continue; + } + + if (arg.startsWith("-")) { + console.error(`[create-xmem] Unknown option: ${arg}`); + usage(1); + } + + options.target = arg; + } + + if (!options.repo || !options.branch) { + console.error("[create-xmem] --repo and --branch require values."); + usage(1); + } + + return options; +} + +function runGit(args, cwd) { + const result = spawnSync("git", args, { + cwd, + stdio: "inherit", + shell: false, + }); + + if (result.error) { + console.error(`[create-xmem] Git is required: ${result.error.message}`); + console.error("[create-xmem] Install Git, reopen your terminal, and run the command again."); + process.exit(1); + } + + if (result.status !== 0) { + process.exit(result.status || 1); + } +} + +function assertCleanTarget(targetPath) { + if (!fs.existsSync(targetPath)) { + return; + } + + const entries = fs.readdirSync(targetPath); + if (entries.length > 0) { + console.error(`[create-xmem] Target folder is not empty: ${targetPath}`); + console.error("[create-xmem] Choose a new folder name or empty the existing folder."); + process.exit(1); + } +} + +function removeGitMetadata(targetPath) { + fs.rmSync(path.join(targetPath, ".git"), { + recursive: true, + force: true, + }); +} + +const options = parseArgs(process.argv.slice(2)); +const targetPath = path.resolve(process.cwd(), options.target); + +assertCleanTarget(targetPath); + +console.log(`[create-xmem] Creating XMem workspace in ${targetPath}`); +runGit(["clone", "--depth", "1", "--branch", options.branch, options.repo, targetPath], process.cwd()); +removeGitMetadata(targetPath); + +console.log(""); +console.log("[create-xmem] Created local XMem workspace."); +console.log(""); +console.log("Next:"); +console.log(` cd ${path.relative(process.cwd(), targetPath) || "."}`); +console.log(" npm run dev"); +console.log(""); +console.log("Chrome extension after setup:"); +console.log(" Load unpacked: repos/xmem-extension/dist"); diff --git a/packages/create-xmem/package.json b/packages/create-xmem/package.json new file mode 100644 index 0000000..7b1e189 --- /dev/null +++ b/packages/create-xmem/package.json @@ -0,0 +1,21 @@ +{ + "name": "create-xmem", + "version": "0.1.1", + "description": "Create a local XMem workspace.", + "bin": { + "create-xmem": "bin/create-xmem.js" + }, + "files": [ + "bin" + ], + "keywords": [ + "xmem", + "xortexai", + "memory", + "local-ai" + ], + "license": "MIT", + "engines": { + "node": ">=20" + } +} diff --git a/scripts/check-npm-publish.js b/scripts/check-npm-publish.js new file mode 100644 index 0000000..ce16a4c --- /dev/null +++ b/scripts/check-npm-publish.js @@ -0,0 +1,68 @@ +#!/usr/bin/env node + +const { spawnSync } = require("node:child_process"); + +function runNpm(args) { + return spawnSync("npm", args, { + encoding: "utf8", + shell: process.platform === "win32", + }); +} + +function printOutput(result) { + if (result.error) { + console.error(result.error.message); + } + if ((result.stdout || "").trim()) { + console.log(result.stdout.trim()); + } + if ((result.stderr || "").trim()) { + console.error(result.stderr.trim()); + } +} + +const whoami = runNpm(["whoami"]); +if (whoami.status !== 0) { + console.error("[xmem] npm is not logged in. Run: npm login"); + printOutput(whoami); + process.exit(1); +} + +console.log(`[xmem] npm user: ${whoami.stdout.trim()}`); + +const profile = runNpm(["profile", "get", "--json"]); +if (profile.status !== 0) { + const combinedOutput = `${profile.stdout}\n${profile.stderr}`; + if (combinedOutput.includes("E403")) { + console.log("[xmem] npm profile is not readable with this token."); + console.log("[xmem] That is okay for granular publish tokens; continuing package checks."); + } else { + console.error("[xmem] Could not read npm profile."); + printOutput(profile); + process.exit(1); + } +} else { + const profileJson = JSON.parse(profile.stdout); + console.log(`[xmem] npm email verified: ${profileJson.email_verified}`); + console.log(`[xmem] npm 2FA enabled: ${profileJson.tfa}`); + if (!profileJson.tfa) { + console.log("[xmem] Enable npm 2FA or use a granular publish token before publishing."); + } +} + +const packageView = runNpm(["view", "create-xmem", "name", "version", "--json"]); +if (packageView.status === 0) { + console.log("[xmem] create-xmem already exists on npm:"); + printOutput(packageView); + process.exit(0); +} + +const combinedOutput = `${packageView.stdout}\n${packageView.stderr}`; +if (combinedOutput.includes("E404")) { + console.log("[xmem] create-xmem is available on npm."); + process.exit(0); +} + +console.error("[xmem] Could not check create-xmem package availability."); +printOutput(packageView); +process.exit(packageView.status || 1); diff --git a/scripts/configure-xmem-env.ps1 b/scripts/configure-xmem-env.ps1 new file mode 100644 index 0000000..236d635 --- /dev/null +++ b/scripts/configure-xmem-env.ps1 @@ -0,0 +1,174 @@ +param( + [string]$EnvPath = "", + [switch]$Quiet +) + +$ErrorActionPreference = "Stop" + +function Write-Step { + param([string]$Message) + if (-not $Quiet) { + Write-Host "[xmem] $Message" + } +} + +function Get-DotEnvValue { + param( + [string]$Path, + [string]$Name + ) + + if (-not (Test-Path $Path)) { + return "" + } + + $pattern = "^\s*$([regex]::Escape($Name))\s*=\s*(.*)\s*$" + foreach ($line in Get-Content -Path $Path) { + if ($line -match $pattern) { + $value = $Matches[1].Trim() + if ( + ($value.StartsWith('"') -and $value.EndsWith('"')) -or + ($value.StartsWith("'") -and $value.EndsWith("'")) + ) { + $value = $value.Substring(1, $value.Length - 2) + } + return $value.Trim() + } + } + + return "" +} + +function Set-DotEnvValue { + param( + [string]$Path, + [string]$Name, + [string]$Value + ) + + $lines = @() + if (Test-Path $Path) { + $lines = @(Get-Content -Path $Path) + } + + $pattern = "^\s*$([regex]::Escape($Name))\s*=" + $updated = $false + $next = foreach ($line in $lines) { + if ($line -match $pattern) { + $updated = $true + "$Name=$Value" + } else { + $line + } + } + + if (-not $updated) { + $next += "$Name=$Value" + } + + Set-Content -Path $Path -Value $next +} + +function Test-SecretValue { + param([string]$Value) + + if (-not $Value) { + return $false + } + + $trimmed = $Value.Trim() + if (-not $trimmed) { + return $false + } + + $placeholderPatterns = @( + "^your[_-]", + "your_.*_key", + "example", + "sample", + "placeholder", + "change[-_]?me", + "^dummy([-_].*)?$", + "^fake([-_].*)?$", + "^test([-_].*)?$" + ) + + foreach ($pattern in $placeholderPatterns) { + if ($trimmed -match $pattern) { + return $false + } + } + + return $true +} + +function Get-ConfiguredValue { + param( + [string]$Path, + [string]$Name + ) + + $envValue = [Environment]::GetEnvironmentVariable($Name) + if (Test-SecretValue $envValue) { + return $envValue + } + + $fileValue = Get-DotEnvValue -Path $Path -Name $Name + if (Test-SecretValue $fileValue) { + return $fileValue + } + + return "" +} + +if (-not $EnvPath) { + $Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path + $EnvPath = Join-Path $Root ".env" +} + +if (-not (Test-Path $EnvPath)) { + throw "XMem .env not found at $EnvPath" +} + +$providers = @() +if (Get-ConfiguredValue -Path $EnvPath -Name "OPENROUTER_API_KEY") { + $providers += "openrouter" +} +if (Get-ConfiguredValue -Path $EnvPath -Name "GEMINI_API_KEY") { + $providers += "gemini" +} +if (Get-ConfiguredValue -Path $EnvPath -Name "CLAUDE_API_KEY") { + $providers += "claude" +} +if (Get-ConfiguredValue -Path $EnvPath -Name "OPENAI_API_KEY") { + $providers += "openai" +} + +$awsAccessKey = Get-ConfiguredValue -Path $EnvPath -Name "AWS_ACCESS_KEY_ID" +$awsSecretKey = Get-ConfiguredValue -Path $EnvPath -Name "AWS_SECRET_ACCESS_KEY" +if ($awsAccessKey -and $awsSecretKey) { + $providers += "bedrock" +} + +if ($providers.Count -gt 0) { + $providerJson = "[" + (($providers | ForEach-Object { '"' + $_ + '"' }) -join ",") + "]" + Set-DotEnvValue -Path $EnvPath -Name "FALLBACK_ORDER" -Value "'$providerJson'" + + # Keep embeddings local and non-Ollama when a cloud LLM key is available. + Set-DotEnvValue -Path $EnvPath -Name "EMBEDDING_PROVIDER" -Value "fastembed" + Set-DotEnvValue -Path $EnvPath -Name "FASTEMBED_MODEL" -Value "BAAI/bge-small-en-v1.5" + Set-DotEnvValue -Path $EnvPath -Name "EMBEDDING_MODEL" -Value "BAAI/bge-small-en-v1.5" + Set-DotEnvValue -Path $EnvPath -Name "PINECONE_DIMENSION" -Value "384" + + Write-Step "Detected cloud LLM provider(s): $($providers -join ', ')" + Write-Step "Configured XMem to avoid Ollama for LLM and embedding calls." +} else { + Set-DotEnvValue -Path $EnvPath -Name "FALLBACK_ORDER" -Value "'[`"ollama`"]'" + Set-DotEnvValue -Path $EnvPath -Name "EMBEDDING_PROVIDER" -Value "ollama" + Set-DotEnvValue -Path $EnvPath -Name "OLLAMA_EMBEDDING_MODEL" -Value "nomic-embed-text" + Set-DotEnvValue -Path $EnvPath -Name "EMBEDDING_MODEL" -Value "nomic-embed-text" + Set-DotEnvValue -Path $EnvPath -Name "PINECONE_DIMENSION" -Value "768" + + Write-Step "No cloud LLM provider keys detected." + Write-Step "Configured XMem to use local Ollama for LLM and embedding calls." +} diff --git a/scripts/context-export.ps1 b/scripts/context-export.ps1 new file mode 100644 index 0000000..46037c2 --- /dev/null +++ b/scripts/context-export.ps1 @@ -0,0 +1,20 @@ +param( + [string]$ReposDir = "" +) + +$ErrorActionPreference = "Stop" + +$Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +if (-not $ReposDir) { + $ReposDir = Join-Path $Root "repos" +} + +$pythonExe = Join-Path $Root ".venv\Scripts\python.exe" +if (-not (Test-Path $pythonExe)) { + throw "XMem virtualenv not found. Run npm run setup first." +} + +& $pythonExe (Join-Path $Root "scripts\context.py") export @args +if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE +} diff --git a/scripts/context-import.ps1 b/scripts/context-import.ps1 new file mode 100644 index 0000000..6879f88 --- /dev/null +++ b/scripts/context-import.ps1 @@ -0,0 +1,20 @@ +param( + [string]$ReposDir = "" +) + +$ErrorActionPreference = "Stop" + +$Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +if (-not $ReposDir) { + $ReposDir = Join-Path $Root "repos" +} + +$pythonExe = Join-Path $Root ".venv\Scripts\python.exe" +if (-not (Test-Path $pythonExe)) { + throw "XMem virtualenv not found. Run npm run setup first." +} + +& $pythonExe (Join-Path $Root "scripts\context.py") import @args +if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE +} diff --git a/scripts/context-sync.ps1 b/scripts/context-sync.ps1 new file mode 100644 index 0000000..0a7d741 --- /dev/null +++ b/scripts/context-sync.ps1 @@ -0,0 +1,20 @@ +param( + [string]$ReposDir = "" +) + +$ErrorActionPreference = "Stop" + +$Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +if (-not $ReposDir) { + $ReposDir = Join-Path $Root "repos" +} + +$pythonExe = Join-Path $Root ".venv\Scripts\python.exe" +if (-not (Test-Path $pythonExe)) { + throw "XMem virtualenv not found. Run npm run setup first." +} + +& $pythonExe (Join-Path $Root "scripts\context.py") sync @args +if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE +} diff --git a/scripts/context.py b/scripts/context.py new file mode 100644 index 0000000..d95e276 --- /dev/null +++ b/scripts/context.py @@ -0,0 +1,368 @@ +from __future__ import annotations + +import argparse +import json +import re +import sys +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +FORMAT = "xmem-context-v1" + + +def normalize_user_id(value: str) -> str: + text = str(value or "").strip() + text = re.sub(r"[^A-Za-z0-9_.@-]+", "_", text) + text = re.sub(r"_+", "_", text).strip("_") + return text[:256] + + +def read_env(path: Path) -> dict[str, str]: + values: dict[str, str] = {} + if not path.exists(): + raise SystemExit(f"XMem .env not found at {path}. Run npm run setup first.") + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + value = value.strip().strip('"').strip("'") + values[key.strip()] = value + return values + + +def json_default(value: Any) -> str: + if isinstance(value, datetime): + return value.astimezone(timezone.utc).isoformat() + return str(value) + + +def project_paths() -> tuple[Path, Path, Path]: + root = Path(__file__).resolve().parents[1] + xmem_dir = root + return root, xmem_dir, xmem_dir / ".env" + + +def load_bundle(path: Path) -> dict[str, Any]: + bundle = json.loads(path.read_text(encoding="utf-8")) + if bundle.get("format") != FORMAT: + raise SystemExit(f"Unsupported context bundle format: {bundle.get('format')!r}") + return bundle + + +def connect_postgres(env: dict[str, str]): + import psycopg + + return psycopg.connect(env.get("PGVECTOR_URL") or "postgresql://xmem:xmem@localhost:15432/xmem") + + +def user_filter_values(user_id: str | None) -> list[str]: + if not user_id: + return [] + values = [user_id.strip(), normalize_user_id(user_id)] + return sorted({value for value in values if value}) + + +def export_pgvector(env: dict[str, str], user_id: str | None) -> list[dict[str, Any]]: + table = env.get("PGVECTOR_TABLE") or "xmem_vectors" + filters = user_filter_values(user_id) + params: list[Any] = [] + where = "" + if filters: + where = "WHERE metadata->>'user_id' = ANY(%s)" + params.append(filters) + + with connect_postgres(env) as conn: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT namespace, id, content, embedding::text AS embedding, + metadata, created_at, updated_at + FROM {table} + {where} + ORDER BY created_at, namespace, id + """, + params, + ) + rows = cur.fetchall() + + return [ + { + "namespace": row[0], + "id": row[1], + "content": row[2], + "embedding": row[3], + "metadata": row[4] or {}, + "created_at": row[5], + "updated_at": row[6], + } + for row in rows + ] + + +def export_neo4j_events(env: dict[str, str], user_id: str | None) -> list[dict[str, Any]]: + try: + from neo4j import GraphDatabase + except ImportError: + return [] + + filters = user_filter_values(user_id) + where = "WHERE size($users) = 0 OR u.user_id IN $users" + query = f""" + MATCH (u:User)-[r:HAS_EVENT]->(d:Date) + {where} + RETURN u.user_id AS user_id, d.date AS date, properties(r) AS properties + ORDER BY user_id, date, properties(r).event_name + """ + + driver = GraphDatabase.driver( + env.get("NEO4J_URI") or "bolt://localhost:17687", + auth=(env.get("NEO4J_USERNAME") or "neo4j", env.get("NEO4J_PASSWORD") or "local-password"), + ) + try: + with driver.session() as session: + rel_types = [record["relationshipType"] for record in session.run("CALL db.relationshipTypes()")] + if "HAS_EVENT" not in rel_types: + return [] + records = session.run(query, users=filters) + return [ + { + "user_id": record["user_id"], + "date": record["date"], + "properties": dict(record["properties"] or {}), + } + for record in records + ] + finally: + driver.close() + + +def export_context(args: argparse.Namespace) -> None: + root, _, env_path = project_paths() + env = read_env(env_path) + vectors = export_pgvector(env, args.user_id) + events = export_neo4j_events(env, args.user_id) + users = sorted( + { + str(row.get("metadata", {}).get("user_id") or "") + for row in vectors + if row.get("metadata", {}).get("user_id") + } + | {str(event.get("user_id") or "") for event in events if event.get("user_id")} + ) + + out = Path(args.out) if args.out else root / "exports" / f"xmem-context-{datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')}.json" + out.parent.mkdir(parents=True, exist_ok=True) + bundle = { + "format": FORMAT, + "exported_at": datetime.now(timezone.utc), + "source": { + "workspace": str(root), + "vector_store": "pgvector", + "graph_store": "neo4j", + }, + "filter": {"user_id": args.user_id or None}, + "users": users, + "stores": { + "pgvector": { + "table": env.get("PGVECTOR_TABLE") or "xmem_vectors", + "rows": vectors, + }, + "neo4j": { + "temporal_events": events, + }, + }, + } + out.write_text(json.dumps(bundle, indent=2, default=json_default), encoding="utf-8") + print(f"[xmem] Exported {len(vectors)} vector memories and {len(events)} temporal events.") + print(f"[xmem] Context bundle: {out}") + + +def import_pgvector(env: dict[str, str], rows: list[dict[str, Any]], user_id: str | None) -> int: + if not rows: + return 0 + from psycopg.types.json import Jsonb + + table = env.get("PGVECTOR_TABLE") or "xmem_vectors" + with connect_postgres(env) as conn: + conn.autocommit = True + with conn.cursor() as cur: + for row in rows: + metadata = dict(row.get("metadata") or {}) + if user_id: + metadata["user_id"] = normalize_user_id(user_id) + cur.execute( + f""" + INSERT INTO {table}(namespace, id, content, embedding, metadata, created_at, updated_at) + VALUES (%s, %s, %s, %s::vector, %s, COALESCE(%s::timestamptz, now()), COALESCE(%s::timestamptz, now())) + ON CONFLICT(namespace, id) DO UPDATE SET + content = excluded.content, + embedding = excluded.embedding, + metadata = excluded.metadata, + updated_at = now() + """, + ( + row["namespace"], + row["id"], + row["content"], + row["embedding"], + Jsonb(metadata), + row.get("created_at"), + row.get("updated_at"), + ), + ) + return len(rows) + + +def import_neo4j_events(env: dict[str, str], events: list[dict[str, Any]], user_id: str | None) -> int: + if not events: + return 0 + from neo4j import GraphDatabase + + driver = GraphDatabase.driver( + env.get("NEO4J_URI") or "bolt://localhost:17687", + auth=(env.get("NEO4J_USERNAME") or "neo4j", env.get("NEO4J_PASSWORD") or "local-password"), + ) + query = """ + MERGE (u:User {user_id: $user_id}) + MERGE (d:Date {date: $date}) + MERGE (u)-[r:HAS_EVENT {event_name: $event_name}]->(d) + SET r += $properties + """ + try: + with driver.session() as session: + for event in events: + props = dict(event.get("properties") or {}) + target_user = normalize_user_id(user_id) if user_id else event.get("user_id") + session.run( + query, + user_id=target_user, + date=event.get("date"), + event_name=props.get("event_name") or "", + properties=props, + ) + finally: + driver.close() + return len(events) + + +def import_context(args: argparse.Namespace) -> None: + _, _, env_path = project_paths() + env = read_env(env_path) + bundle = load_bundle(Path(args.file)) + rows = bundle.get("stores", {}).get("pgvector", {}).get("rows", []) + events = bundle.get("stores", {}).get("neo4j", {}).get("temporal_events", []) + row_count = import_pgvector(env, rows, args.user_id) + event_count = import_neo4j_events(env, events, args.user_id) + print(f"[xmem] Imported {row_count} vector memories and {event_count} temporal events.") + + +def api_post_json(url: str, api_key: str, payload: dict[str, Any], timeout: int) -> dict[str, Any]: + request = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise SystemExit(f"Remote sync failed: HTTP {exc.code}\n{body}") from exc + + +def sync_context(args: argparse.Namespace) -> None: + bundle = load_bundle(Path(args.file)) + rows = bundle.get("stores", {}).get("pgvector", {}).get("rows", []) + if not rows: + print("[xmem] No vector memories found in bundle.") + return + + server = args.server.rstrip("/") + api_key = args.api_key + if not api_key: + raise SystemExit("Missing --api-key for remote sync.") + + items: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + for row in rows: + metadata = row.get("metadata") or {} + target_user = normalize_user_id(args.user_id) if args.user_id else metadata.get("user_id") or "xmem-imported-user" + content = str(row.get("content") or "").strip() + if not content: + continue + key = (target_user, content) + if key in seen: + continue + seen.add(key) + items.append( + { + "user_query": content, + "agent_response": "Imported from an XMem local context bundle.", + "user_id": target_user, + "effort_level": "low", + } + ) + + if args.dry_run: + print(f"[xmem] Dry run: would sync {len(items)} memories to {server}.") + return + + synced = 0 + for index in range(0, len(items), args.batch_size): + batch = items[index : index + args.batch_size] + response = api_post_json( + f"{server}/v1/memory/batch-ingest", + api_key, + {"items": batch}, + args.timeout, + ) + if response.get("status") != "ok": + raise SystemExit(f"Remote sync failed: {response}") + synced += len(batch) + print(f"[xmem] Synced {synced}/{len(items)} memories") + + print(f"[xmem] Remote sync complete: {synced} memories sent to {server}.") + + +def main() -> None: + parser = argparse.ArgumentParser(description="XMem local context export/import/sync") + subparsers = parser.add_subparsers(dest="command", required=True) + + export_parser = subparsers.add_parser("export", help="Export local XMem context to a JSON bundle") + export_parser.add_argument("--user-id", default="", help="Optional user id/name filter") + export_parser.add_argument("--out", default="", help="Output JSON path") + export_parser.set_defaults(func=export_context) + + import_parser = subparsers.add_parser("import", help="Import a JSON context bundle into local XMem storage") + import_parser.add_argument("--file", required=True, help="Context bundle JSON path") + import_parser.add_argument("--user-id", default="", help="Optional target user id override") + import_parser.set_defaults(func=import_context) + + sync_parser = subparsers.add_parser("sync", help="Send a context bundle to a remote XMem API") + sync_parser.add_argument("--file", required=True, help="Context bundle JSON path") + sync_parser.add_argument("--server", required=True, help="Remote XMem server URL") + sync_parser.add_argument("--api-key", required=True, help="Remote XMem API key") + sync_parser.add_argument("--user-id", default="", help="Optional target user id override") + sync_parser.add_argument("--batch-size", type=int, default=20, help="Batch size for remote ingest") + sync_parser.add_argument("--timeout", type=int, default=900, help="HTTP timeout seconds per batch") + sync_parser.add_argument("--dry-run", action="store_true", help="Show what would be synced without sending") + sync_parser.set_defaults(func=sync_context) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + sys.exit(130) diff --git a/scripts/doctor.ps1 b/scripts/doctor.ps1 new file mode 100644 index 0000000..c8805f9 --- /dev/null +++ b/scripts/doctor.ps1 @@ -0,0 +1,155 @@ +param( + [string]$BaseUrl = "http://localhost:8000", + [string]$ReposDir = "" +) + +$ErrorActionPreference = "Stop" + +function Write-Check { + param( + [string]$Name, + [bool]$Ok, + [string]$Message, + [string]$Fix = "" + ) + + $label = if ($Ok) { "OK" } else { "FIX" } + $color = if ($Ok) { "Green" } else { "Yellow" } + Write-Host "[$label] $Name - $Message" -ForegroundColor $color + if (-not $Ok -and $Fix) { + Write-Host " $Fix" + } +} + +function Test-CommandExists { + param([string]$Name) + return [bool](Get-Command $Name -ErrorAction SilentlyContinue) +} + +function Get-DotEnvValue { + param( + [string]$Path, + [string]$Name, + [string]$Default = "" + ) + + if (-not (Test-Path $Path)) { + return $Default + } + + $pattern = "^\s*$([regex]::Escape($Name))\s*=\s*(.*)\s*$" + foreach ($line in Get-Content -Path $Path) { + if ($line -match $pattern) { + $value = $Matches[1].Trim() + if ( + ($value.StartsWith('"') -and $value.EndsWith('"')) -or + ($value.StartsWith("'") -and $value.EndsWith("'")) + ) { + $value = $value.Substring(1, $value.Length - 2) + } + return $value + } + } + + return $Default +} + +function Test-NativeOk { + param([scriptblock]$Command) + + $oldErrorActionPreference = $ErrorActionPreference + $oldNativePreference = $null + $hasNativePreference = Get-Variable -Name PSNativeCommandUseErrorActionPreference -ErrorAction SilentlyContinue + + try { + $ErrorActionPreference = "Continue" + if ($hasNativePreference) { + $oldNativePreference = $PSNativeCommandUseErrorActionPreference + $PSNativeCommandUseErrorActionPreference = $false + } + $null = & $Command 2>&1 + return ($LASTEXITCODE -eq 0) + } finally { + $ErrorActionPreference = $oldErrorActionPreference + if ($hasNativePreference) { + $PSNativeCommandUseErrorActionPreference = $oldNativePreference + } + } +} + +$Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +if (-not $ReposDir) { + $ReposDir = Join-Path $Root "repos" +} + +$xmemDir = $Root +$extensionDir = Join-Path $ReposDir "xmem-extension" +$envPath = Join-Path $xmemDir ".env" +$failures = 0 + +Write-Host "[xmem] Doctor report" +Write-Host "" + +foreach ($cmd in @("git", "python", "node", "npm")) { + $ok = Test-CommandExists $cmd + if (-not $ok) { $failures++ } + Write-Check $cmd $ok "command lookup" "Install $cmd and reopen this terminal." +} + +$dockerCommand = Test-CommandExists "docker" +$dockerRunning = $dockerCommand -and (Test-NativeOk { docker info }) +if (-not $dockerCommand -or -not $dockerRunning) { $failures++ } +Write-Check "Docker" $dockerRunning "local database runtime" "Start Docker Desktop, then rerun npm run dev." + +$xmemExists = Test-Path (Join-Path $xmemDir "pyproject.toml") +if (-not $xmemExists) { $failures++ } +Write-Check "XMem repo" $xmemExists $xmemDir "Run this from the XMem repository root." + +$extensionExists = Test-Path $extensionDir +if (-not $extensionExists) { $failures++ } +Write-Check "Extension repo" $extensionExists $extensionDir "Run npm run setup." + +$envExists = Test-Path $envPath +if (-not $envExists) { $failures++ } +Write-Check "XMem .env" $envExists $envPath "Run npm run setup to create it from templates/xmem.env.local." + +if ($envExists) { + $usesOllama = [bool]((Get-Content -Raw -Path $envPath) -match "(?m)^\s*FALLBACK_ORDER\s*=.*ollama") + if ($usesOllama) { + $ollamaCommand = Test-CommandExists "ollama" + $ollamaRunning = $ollamaCommand -and (Test-NativeOk { ollama list }) + if (-not $ollamaCommand -or -not $ollamaRunning) { $failures++ } + Write-Check "Ollama" $ollamaRunning "required because no cloud LLM key is configured" "Start Ollama, or add a cloud LLM key to .env." + + if ($ollamaRunning) { + $chatModel = Get-DotEnvValue -Path $envPath -Name "OLLAMA_MODEL" -Default "qwen2.5:1.5b" + $embeddingModel = Get-DotEnvValue -Path $envPath -Name "OLLAMA_EMBEDDING_MODEL" -Default "nomic-embed-text" + $installed = (& ollama list 2>$null | Select-Object -Skip 1) -join "`n" + foreach ($model in @($chatModel, $embeddingModel)) { + $escaped = [regex]::Escape($model) + $ok = ($installed -match "(?m)^$escaped(\s|:latest\s)") + if (-not $ok) { $failures++ } + Write-Check "Ollama model $model" $ok "local model availability" "Run: ollama pull $model" + } + } + } else { + Write-Check "LLM routing" $true "cloud key detected; Ollama is not required" + } +} + +try { + $health = Invoke-RestMethod -Uri "$BaseUrl/health" -Method GET -TimeoutSec 5 + $ready = if ($health.data) { [bool]$health.data.pipelines_ready } else { [bool]$health.pipelines_ready } + if (-not $ready) { $failures++ } + Write-Check "XMem API" $ready "$BaseUrl/health" "Start it with npm run dev and wait for pipelines_ready=true." +} catch { + $failures++ + Write-Check "XMem API" $false "$BaseUrl is not reachable" "Start it with npm run dev." +} + +Write-Host "" +if ($failures -eq 0) { + Write-Host "[xmem] Everything looks ready." -ForegroundColor Green +} else { + Write-Host "[xmem] Found $failures setup item(s) to fix." -ForegroundColor Yellow +} diff --git a/scripts/install.ps1 b/scripts/install.ps1 new file mode 100644 index 0000000..2dca0e4 --- /dev/null +++ b/scripts/install.ps1 @@ -0,0 +1,273 @@ +param( + [string]$ReposDir = "", + [switch]$IncludeMcp, + [switch]$IncludeSdk, + [switch]$SkipModelPull, + [switch]$SkipPythonInstall, + [switch]$SkipNodeInstall, + [switch]$SkipDocker +) + +$ErrorActionPreference = "Stop" + +function Write-Step { + param([string]$Message) + Write-Host "[xmem] $Message" +} + +function Test-Command { + param([string]$Name) + return [bool](Get-Command $Name -ErrorAction SilentlyContinue) +} + +function Invoke-Native { + param([scriptblock]$Command) + & $Command + if ($LASTEXITCODE -ne 0) { + throw "Command failed with exit code $LASTEXITCODE" + } +} + +function Test-DockerRunning { + if (-not (Test-Command "docker")) { + Write-Host "[xmem] Docker was not found." -ForegroundColor Yellow + Write-Host "[xmem] Install Docker Desktop or rerun npm run setup -- -SkipDocker to skip local database startup." + return $false + } + + $oldErrorActionPreference = $ErrorActionPreference + $oldNativePreference = $null + $hasNativePreference = Get-Variable -Name PSNativeCommandUseErrorActionPreference -ErrorAction SilentlyContinue + + try { + $ErrorActionPreference = "Continue" + if ($hasNativePreference) { + $oldNativePreference = $PSNativeCommandUseErrorActionPreference + $PSNativeCommandUseErrorActionPreference = $false + } + + $null = & docker info 2>&1 + $dockerExitCode = $LASTEXITCODE + } finally { + $ErrorActionPreference = $oldErrorActionPreference + if ($hasNativePreference) { + $PSNativeCommandUseErrorActionPreference = $oldNativePreference + } + } + + if ($dockerExitCode -ne 0) { + Write-Host "[xmem] Docker Desktop is installed but not running." -ForegroundColor Yellow + Write-Host "[xmem] Start Docker Desktop, wait until it says Docker is running, then rerun this script." + Write-Host "[xmem] Temporary escape hatch: rerun npm run setup -- -SkipDocker to continue cloning/building without local databases." + return $false + } + + return $true +} + +function Test-OllamaRunning { + if (-not (Test-Command "ollama")) { + Write-Host "[xmem] Ollama was not found." -ForegroundColor Yellow + Write-Host "[xmem] Install Ollama, or add a cloud LLM key to .env and rerun." + return $false + } + + $oldErrorActionPreference = $ErrorActionPreference + $oldNativePreference = $null + $hasNativePreference = Get-Variable -Name PSNativeCommandUseErrorActionPreference -ErrorAction SilentlyContinue + + try { + $ErrorActionPreference = "Continue" + if ($hasNativePreference) { + $oldNativePreference = $PSNativeCommandUseErrorActionPreference + $PSNativeCommandUseErrorActionPreference = $false + } + + $null = & ollama list 2>&1 + $ollamaExitCode = $LASTEXITCODE + } finally { + $ErrorActionPreference = $oldErrorActionPreference + if ($hasNativePreference) { + $PSNativeCommandUseErrorActionPreference = $oldNativePreference + } + } + + if ($ollamaExitCode -ne 0) { + Write-Host "[xmem] Ollama is installed but not running." -ForegroundColor Yellow + Write-Host "[xmem] Start Ollama, or add a cloud LLM key to .env and rerun." + return $false + } + + return $true +} + +function Test-XMemUsesOllama { + param([string]$EnvPath) + if (-not (Test-Path $EnvPath)) { + return $true + } + return [bool]((Get-Content -Raw -Path $EnvPath) -match "(?m)^\s*FALLBACK_ORDER\s*=.*ollama") +} + +function Wait-ContainerHealthy { + param( + [string[]]$ContainerNames, + [int]$TimeoutSeconds = 180 + ) + + $deadline = (Get-Date).AddSeconds($TimeoutSeconds) + $pending = @($ContainerNames) + + while ((Get-Date) -lt $deadline) { + $stillPending = @() + foreach ($name in $pending) { + $status = (& docker inspect --format "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}" $name 2>$null) + if ($LASTEXITCODE -ne 0) { + $stillPending += $name + continue + } + + $status = ($status | Select-Object -First 1).Trim() + if ($status -in @("healthy", "running")) { + continue + } + + if ($status -eq "unhealthy") { + throw "Container $name is unhealthy. Run npm run doctor or inspect it with: docker logs $name" + } + + $stillPending += $name + } + + if ($stillPending.Count -eq 0) { + return + } + + Write-Host "[xmem] Waiting for local database containers: $($stillPending -join ', ')" + $pending = $stillPending + Start-Sleep -Seconds 5 + } + + throw "Timed out waiting for local database containers: $($pending -join ', '). Run npm run doctor for details." +} + +function Sync-Repo { + param( + [string]$Name, + [string]$Url, + [string]$Branch + ) + + $target = Join-Path $ReposDir $Name + if (Test-Path $target) { + if (-not (Test-Path (Join-Path $target ".git"))) { + throw "$target exists but is not a git checkout." + } + Write-Step "Updating $Name" + Invoke-Native { git -C $target fetch origin } + Invoke-Native { git -C $target checkout $Branch } + Invoke-Native { git -C $target pull --ff-only origin $Branch } + } else { + Write-Step "Cloning $Name" + Invoke-Native { git clone --branch $Branch $Url $target } + } +} + +$Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +if (-not $ReposDir) { + $ReposDir = Join-Path $Root "repos" +} + +New-Item -ItemType Directory -Force -Path $ReposDir | Out-Null + +foreach ($cmd in @("git", "python", "node", "npm")) { + if (-not (Test-Command $cmd)) { + throw "$cmd is required. Install it, then run this script again." + } +} + +Sync-Repo "xmem-extension" "https://github.com/XortexAI/xmem-extension.git" "main" + +if ($IncludeMcp) { + Sync-Repo "xmem-mcp" "https://github.com/XortexAI/xmem-mcp.git" "main" +} + +if ($IncludeSdk) { + Sync-Repo "xmem-sdk" "https://github.com/XortexAI/xmem-sdk.git" "master" +} + +$XmemDir = $Root +$ExtensionDir = Join-Path $ReposDir "xmem-extension" + +$envTemplate = Join-Path $Root "templates\xmem.env.local" +$envTarget = Join-Path $XmemDir ".env" +if (-not (Test-Path $envTarget)) { + Copy-Item $envTemplate $envTarget + Write-Step "Created .env from local template" +} else { + Write-Step ".env already exists; leaving it unchanged" +} + +Invoke-Native { powershell -ExecutionPolicy Bypass -File (Join-Path $Root "scripts\configure-xmem-env.ps1") -EnvPath $envTarget } +$usesOllama = Test-XMemUsesOllama -EnvPath $envTarget +$dockerSkipped = $false +$ollamaSkipped = $false + +if (-not $SkipModelPull) { + if ($usesOllama) { + if (Test-OllamaRunning) { + Write-Step "Pulling Ollama chat model" + Invoke-Native { ollama pull qwen2.5:1.5b } + Write-Step "Pulling Ollama embedding model" + Invoke-Native { ollama pull nomic-embed-text } + } else { + $ollamaSkipped = $true + } + } else { + Write-Step "Cloud LLM provider key detected; skipping Ollama model pulls" + } +} + +if (-not $SkipDocker) { + if (Test-DockerRunning) { + Write-Step "Starting local Docker services" + Invoke-Native { docker compose -f (Join-Path $Root "docker-compose.local.yml") up -d --remove-orphans } + Wait-ContainerHealthy -ContainerNames @("xmem-postgres", "xmem-mongo", "xmem-neo4j") + } else { + $dockerSkipped = $true + } +} + +if (-not $SkipPythonInstall) { + $venvPython = Join-Path $XmemDir ".venv\Scripts\python.exe" + if (-not (Test-Path $venvPython)) { + Write-Step "Creating XMem virtualenv" + Invoke-Native { python -m venv (Join-Path $XmemDir ".venv") } + } + Write-Step "Installing XMem local dependencies" + Invoke-Native { & $venvPython -m pip install --upgrade pip } + Invoke-Native { & $venvPython -m pip install -e "$XmemDir[local,dev]" } +} + +Write-Step "Patching extension for local API" +Invoke-Native { powershell -ExecutionPolicy Bypass -File (Join-Path $Root "scripts\patch-extension-local.ps1") -ExtensionDir $ExtensionDir } + +if (-not $SkipNodeInstall) { + Write-Step "Installing and building Chrome extension" + Invoke-Native { npm --prefix $ExtensionDir install } + Invoke-Native { npm --prefix $ExtensionDir run build } +} + +Write-Step "Install complete" +Write-Host "" +Write-Host "Next:" +Write-Host " npm run dev" +Write-Host " npm run verify" +if ($dockerSkipped) { + Write-Host "" + Write-Host "Docker services were not started. Start Docker Desktop before running npm run dev." -ForegroundColor Yellow +} +if ($ollamaSkipped) { + Write-Host "" + Write-Host "Ollama models were not pulled. Start Ollama, then rerun npm run setup or add a cloud LLM key." -ForegroundColor Yellow +} diff --git a/scripts/patch-extension-local.ps1 b/scripts/patch-extension-local.ps1 new file mode 100644 index 0000000..ab46d0d --- /dev/null +++ b/scripts/patch-extension-local.ps1 @@ -0,0 +1,161 @@ +param( + [string]$ExtensionDir = "" +) + +$ErrorActionPreference = "Stop" + +if (-not $ExtensionDir) { + $Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path + $ExtensionDir = Join-Path $Root "repos\xmem-extension" +} + +$apiFile = Join-Path $ExtensionDir "src\api.ts" +if (-not (Test-Path $apiFile)) { + throw "Could not find extension API file at $apiFile" +} + +function New-HollowXIcon { + param( + [int]$Size, + [string]$Path + ) + + Add-Type -AssemblyName System.Drawing + $bitmap = New-Object System.Drawing.Bitmap $Size, $Size + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + $graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias + $graphics.Clear([System.Drawing.Color]::White) + + $margin = [Math]::Max(3, [int]($Size * 0.22)) + $outerWidth = [Math]::Max(4, [int]($Size * 0.22)) + $innerWidth = [Math]::Max(2, [int]($Size * 0.105)) + + $outerPen = New-Object System.Drawing.Pen ([System.Drawing.Color]::Black), $outerWidth + $outerPen.StartCap = [System.Drawing.Drawing2D.LineCap]::Round + $outerPen.EndCap = [System.Drawing.Drawing2D.LineCap]::Round + $outerPen.LineJoin = [System.Drawing.Drawing2D.LineJoin]::Round + + $innerPen = New-Object System.Drawing.Pen ([System.Drawing.Color]::White), $innerWidth + $innerPen.StartCap = [System.Drawing.Drawing2D.LineCap]::Round + $innerPen.EndCap = [System.Drawing.Drawing2D.LineCap]::Round + $innerPen.LineJoin = [System.Drawing.Drawing2D.LineJoin]::Round + + $graphics.DrawLine($outerPen, $margin, $margin, $Size - $margin, $Size - $margin) + $graphics.DrawLine($outerPen, $Size - $margin, $margin, $margin, $Size - $margin) + $graphics.DrawLine($innerPen, $margin, $margin, $Size - $margin, $Size - $margin) + $graphics.DrawLine($innerPen, $Size - $margin, $margin, $margin, $Size - $margin) + + $bitmap.Save($Path, [System.Drawing.Imaging.ImageFormat]::Png) + $outerPen.Dispose() + $innerPen.Dispose() + $graphics.Dispose() + $bitmap.Dispose() +} + +$iconDir = Join-Path $ExtensionDir "icons" +New-Item -ItemType Directory -Force -Path $iconDir | Out-Null +New-HollowXIcon -Size 16 -Path (Join-Path $iconDir "icon16.png") +New-HollowXIcon -Size 48 -Path (Join-Path $iconDir "icon48.png") +New-HollowXIcon -Size 128 -Path (Join-Path $iconDir "icon128.png") +New-HollowXIcon -Size 128 -Path (Join-Path $iconDir "logo.png") +$sourceFiles = @( + "src\api.ts", + "src\background.ts", + "src\content.ts" +) + +foreach ($relativePath in $sourceFiles) { + $sourceFile = Join-Path $ExtensionDir $relativePath + if (Test-Path $sourceFile) { + $source = Get-Content -Raw -Path $sourceFile + $source = $source.Replace("https://api.xmem.in", "http://localhost:8000") + $source = $source.Replace( + "new XMemClient(API_BASE_URL, config.apiKey, config.userId)", + "new XMemClient(API_BASE_URL, config.apiKey)" + ) + $source = $source.Replace( + ".replace(/[^\\w.\\-@]+/g, '_')", + ".replace(/[^A-Za-z0-9_.@-]+/g, '_')" + ) + Set-Content -Path $sourceFile -Value $source -NoNewline + } +} + +$content = Get-Content -Raw -Path $apiFile + +if ($content -notmatch "function normalizeUserId") { + $content = [regex]::Replace( + $content, + "(const API_BASE_URL = 'http://localhost:8000';\r?\n)", + "`$1`r`nfunction normalizeUserId(userId: string): string {`r`n const normalized = (userId || '')`r`n .trim()`r`n .replace(/[^A-Za-z0-9_.@-]+/g, '_')`r`n .replace(/^_+|_+$/g, '');`r`n return normalized || 'xmem-local-user';`r`n}`r`n", + 1 + ) +} + +$content = $content.Replace( + "userId: data.xmem_user_id || '',", + "userId: normalizeUserId(data.xmem_user_id || '')," +) + +$backgroundFile = Join-Path $ExtensionDir "src\background.ts" +if (Test-Path $backgroundFile) { + $background = Get-Content -Raw -Path $backgroundFile + if ($background -notmatch "function normalizeUserId") { + $background = [regex]::Replace( + $background, + "(interface XMemConfig \{\r?\n apiKey: string;\r?\n userId: string;\r?\n\}\r?\n)", + "`$1`r`nfunction normalizeUserId(userId: string): string {`r`n const normalized = (userId || '')`r`n .trim()`r`n .replace(/[^A-Za-z0-9_.@-]+/g, '_')`r`n .replace(/^_+|_+$/g, '');`r`n return normalized || 'xmem-local-user';`r`n}`r`n", + 1 + ) + } + + $background = $background.Replace( + "userId: data.xmem_user_id || '',", + "userId: normalizeUserId(data.xmem_user_id || '')," + ) + Set-Content -Path $backgroundFile -Value $background -NoNewline +} + +$replacement = @' +export async function validateCredentials(apiKey: string, username: string): Promise { + const url = `${API_BASE_URL}/auth/verify-key`; + try { + const response = await fetch(url, { + headers: { + 'Authorization': `Bearer ${apiKey}` + } + }); + + if (!response.ok) { + console.log('[XMem] Validation failed: HTTP', response.status); + return false; + } + + const data = await response.json(); + console.log('[XMem] Validated user data:', data); + + // Local dev static keys do not always map to a real username. If the local + // API accepted the key, allow any non-empty local user id from the popup. + if (API_BASE_URL.includes('localhost') || API_BASE_URL.includes('127.0.0.1')) { + return Boolean(username && username.trim()); + } + + return Boolean(data.username && data.username.toLowerCase() === username.toLowerCase()); + } catch (err) { + console.error('[XMem] Credential validation network error:', err); + return false; + } +} + +// +'@ + +$pattern = "export async function validateCredentials[\s\S]*?\r?\n}\r?\n\r?\n//" +$patched = [regex]::Replace($content, $pattern, $replacement, 1) + +if ($patched -eq $content -and $content -notmatch "http://localhost:8000") { + throw "Extension patch did not apply." +} + +Set-Content -Path $apiFile -Value $patched -NoNewline +Write-Host "[xmem] Patched extension API for http://localhost:8000" diff --git a/scripts/start.ps1 b/scripts/start.ps1 new file mode 100644 index 0000000..f7f4b58 --- /dev/null +++ b/scripts/start.ps1 @@ -0,0 +1,230 @@ +param( + [string]$ReposDir = "", + [switch]$SkipDocker +) + +$ErrorActionPreference = "Stop" + +function Invoke-Native { + param([scriptblock]$Command) + & $Command + if ($LASTEXITCODE -ne 0) { + throw "Command failed with exit code $LASTEXITCODE" + } +} + +function Test-Command { + param([string]$Name) + return [bool](Get-Command $Name -ErrorAction SilentlyContinue) +} + +function Assert-DockerRunning { + if (-not (Test-Command "docker")) { + Write-Host "[xmem] Docker was not found." -ForegroundColor Yellow + Write-Host "[xmem] Install Docker Desktop or rerun npm run start -- -SkipDocker if local databases are already running elsewhere." + exit 2 + } + + $oldErrorActionPreference = $ErrorActionPreference + $oldNativePreference = $null + $hasNativePreference = Get-Variable -Name PSNativeCommandUseErrorActionPreference -ErrorAction SilentlyContinue + + try { + $ErrorActionPreference = "Continue" + if ($hasNativePreference) { + $oldNativePreference = $PSNativeCommandUseErrorActionPreference + $PSNativeCommandUseErrorActionPreference = $false + } + + $null = & docker info 2>&1 + $dockerExitCode = $LASTEXITCODE + } finally { + $ErrorActionPreference = $oldErrorActionPreference + if ($hasNativePreference) { + $PSNativeCommandUseErrorActionPreference = $oldNativePreference + } + } + + if ($dockerExitCode -ne 0) { + Write-Host "[xmem] Docker Desktop is installed but not running." -ForegroundColor Yellow + Write-Host "[xmem] Start Docker Desktop, wait until it says Docker is running, then rerun npm run dev." + Write-Host "[xmem] Temporary escape hatch: rerun npm run start -- -SkipDocker if local databases are already running elsewhere." + exit 2 + } +} + +function Assert-OllamaRunning { + if (-not (Test-Command "ollama")) { + Write-Host "[xmem] Ollama was not found." -ForegroundColor Yellow + Write-Host "[xmem] Install Ollama, or add a cloud LLM key to .env and rerun." + exit 2 + } + + $oldErrorActionPreference = $ErrorActionPreference + $oldNativePreference = $null + $hasNativePreference = Get-Variable -Name PSNativeCommandUseErrorActionPreference -ErrorAction SilentlyContinue + + try { + $ErrorActionPreference = "Continue" + if ($hasNativePreference) { + $oldNativePreference = $PSNativeCommandUseErrorActionPreference + $PSNativeCommandUseErrorActionPreference = $false + } + + $null = & ollama list 2>&1 + $ollamaExitCode = $LASTEXITCODE + } finally { + $ErrorActionPreference = $oldErrorActionPreference + if ($hasNativePreference) { + $PSNativeCommandUseErrorActionPreference = $oldNativePreference + } + } + + if ($ollamaExitCode -ne 0) { + Write-Host "[xmem] XMem is configured to use local Ollama, but Ollama is not running." -ForegroundColor Yellow + Write-Host "[xmem] Start Ollama, or add a cloud LLM key to .env and rerun." + exit 2 + } +} + +function Test-XMemUsesOllama { + param([string]$EnvPath) + if (-not (Test-Path $EnvPath)) { + return $true + } + return [bool]((Get-Content -Raw -Path $EnvPath) -match "(?m)^\s*FALLBACK_ORDER\s*=.*ollama") +} + +function Get-DotEnvValue { + param( + [string]$Path, + [string]$Name, + [string]$Default = "" + ) + + if (-not (Test-Path $Path)) { + return $Default + } + + $pattern = "^\s*$([regex]::Escape($Name))\s*=\s*(.*)\s*$" + foreach ($line in Get-Content -Path $Path) { + if ($line -match $pattern) { + $value = $Matches[1].Trim() + if ( + ($value.StartsWith('"') -and $value.EndsWith('"')) -or + ($value.StartsWith("'") -and $value.EndsWith("'")) + ) { + $value = $value.Substring(1, $value.Length - 2) + } + if ($value) { + return $value + } + } + } + + return $Default +} + +function Assert-OllamaModels { + param([string]$EnvPath) + + $chatModel = Get-DotEnvValue -Path $EnvPath -Name "OLLAMA_MODEL" -Default "qwen2.5:1.5b" + $embeddingModel = Get-DotEnvValue -Path $EnvPath -Name "OLLAMA_EMBEDDING_MODEL" -Default "nomic-embed-text" + $installed = (& ollama list 2>$null | Select-Object -Skip 1) -join "`n" + $missing = @() + + foreach ($model in @($chatModel, $embeddingModel)) { + if (-not $model) { + continue + } + $escaped = [regex]::Escape($model) + $hasModel = ($installed -match "(?m)^$escaped(\s|:latest\s)") + if (-not $hasModel) { + $missing += $model + } + } + + if ($missing.Count -gt 0) { + Write-Host "[xmem] Ollama is running, but required local model(s) are missing." -ForegroundColor Yellow + foreach ($model in $missing) { + Write-Host "[xmem] Pull it with: ollama pull $model" + } + Write-Host "[xmem] Or add a cloud LLM key to .env so XMem does not use Ollama." + exit 2 + } +} + +function Wait-ContainerHealthy { + param( + [string[]]$ContainerNames, + [int]$TimeoutSeconds = 180 + ) + + $deadline = (Get-Date).AddSeconds($TimeoutSeconds) + $pending = @($ContainerNames) + + while ((Get-Date) -lt $deadline) { + $stillPending = @() + foreach ($name in $pending) { + $status = (& docker inspect --format "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}" $name 2>$null) + if ($LASTEXITCODE -ne 0) { + $stillPending += $name + continue + } + + $status = ($status | Select-Object -First 1).Trim() + if ($status -in @("healthy", "running")) { + continue + } + + if ($status -eq "unhealthy") { + throw "Container $name is unhealthy. Run npm run doctor or inspect it with: docker logs $name" + } + + $stillPending += $name + } + + if ($stillPending.Count -eq 0) { + return + } + + Write-Host "[xmem] Waiting for local database containers: $($stillPending -join ', ')" + $pending = $stillPending + Start-Sleep -Seconds 5 + } + + throw "Timed out waiting for local database containers: $($pending -join ', '). Run npm run doctor for details." +} + +$Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +if (-not $ReposDir) { + $ReposDir = Join-Path $Root "repos" +} + +$XmemDir = $Root + +$envTarget = Join-Path $XmemDir ".env" +if (-not (Test-Path $envTarget)) { + throw "XMem .env not found at $envTarget. Run npm run setup first." +} + +Invoke-Native { powershell -ExecutionPolicy Bypass -File (Join-Path $Root "scripts\configure-xmem-env.ps1") -EnvPath $envTarget } +if (Test-XMemUsesOllama -EnvPath $envTarget) { + Assert-OllamaRunning + Assert-OllamaModels -EnvPath $envTarget +} + +if (-not $SkipDocker) { + Assert-DockerRunning + Invoke-Native { docker compose -f (Join-Path $Root "docker-compose.local.yml") up -d --remove-orphans } + Wait-ContainerHealthy -ContainerNames @("xmem-postgres", "xmem-mongo", "xmem-neo4j") +} + +$pythonExe = Join-Path $XmemDir ".venv\Scripts\python.exe" +if (-not (Test-Path $pythonExe)) { + $pythonExe = "python" +} + +Set-Location $XmemDir +Write-Host "[xmem] Starting XMem API at http://localhost:8000" +Invoke-Native { & $pythonExe -m uvicorn src.api.app:create_app --factory --host 0.0.0.0 --port 8000 } diff --git a/scripts/verify.ps1 b/scripts/verify.ps1 new file mode 100644 index 0000000..a513f8a --- /dev/null +++ b/scripts/verify.ps1 @@ -0,0 +1,140 @@ +param( + [string]$BaseUrl = "http://localhost:8000", + [string]$ApiKey = "dev-xmem-key", + [string]$UserId = "xmem-local-user", + [int]$TimeoutSeconds = 180 +) + +$ErrorActionPreference = "Stop" + +function Write-Step { + param([string]$Message) + Write-Host "[xmem] $Message" +} + +function Get-ResponseErrorMessage { + param([object]$ErrorRecord) + + if ($ErrorRecord.ErrorDetails.Message) { + return $ErrorRecord.ErrorDetails.Message + } + + return $ErrorRecord.Exception.Message +} + +function Invoke-XMemJson { + param( + [string]$Uri, + [string]$Method, + [hashtable]$Headers = @{}, + [string]$Body = "", + [int]$TimeoutSec = 60 + ) + + try { + if ($Body) { + return Invoke-RestMethod -Uri $Uri -Method $Method -Headers $Headers -Body $Body -TimeoutSec $TimeoutSec + } + + return Invoke-RestMethod -Uri $Uri -Method $Method -Headers $Headers -TimeoutSec $TimeoutSec + } catch { + $message = Get-ResponseErrorMessage $_ + throw "Request failed: $Method $Uri`n$message" + } +} + +function Test-HealthReady { + param([object]$Health) + + if (-not $Health) { + return $false + } + + if ($Health.data) { + return [bool]$Health.data.pipelines_ready + } + + return [bool]$Health.pipelines_ready +} + +function Get-HealthSummary { + param([object]$Health) + + if ($Health.data) { + return "status=$($Health.data.status), pipelines_ready=$($Health.data.pipelines_ready), error=$($Health.data.error)" + } + + return "status=$($Health.status), pipelines_ready=$($Health.pipelines_ready), error=$($Health.error)" +} + +$deadline = (Get-Date).AddSeconds($TimeoutSeconds) +$health = $null + +Write-Step "Waiting for API health at $BaseUrl/health" +while ((Get-Date) -lt $deadline) { + try { + $health = Invoke-RestMethod -Uri "$BaseUrl/health" -Method GET -TimeoutSec 10 + if (Test-HealthReady $health) { + break + } + } catch { + Start-Sleep -Seconds 3 + } +} + +if (-not $health) { + throw "XMem API did not become reachable within $TimeoutSeconds seconds." +} + +Write-Step "Health: $(Get-HealthSummary $health)" +if (-not (Test-HealthReady $health)) { + throw "XMem API is reachable but pipelines are not ready." +} + +$headers = @{ + "Authorization" = "Bearer $ApiKey" + "Content-Type" = "application/json" +} + +$memoryText = "Remember that XMem local mode runs directly from the main XMem repository." + +$ingestBody = @{ + user_query = $memoryText + agent_response = "Got it. I will remember that XMem local mode runs from the main repository." + user_id = $UserId + effort_level = "low" +} | ConvertTo-Json + +Write-Step "Ingesting a smoke-test memory" +$ingest = Invoke-XMemJson -Uri "$BaseUrl/v1/memory/ingest" -Method POST -Headers $headers -Body $ingestBody -TimeoutSec 650 +Write-Step "Ingest status: $($ingest.status)" + +$searchBody = @{ + query = "What is XMem local mode?" + user_id = $UserId + domains = @("profile", "temporal", "summary") + top_k = 5 +} | ConvertTo-Json + +Write-Step "Searching memory" +$search = Invoke-XMemJson -Uri "$BaseUrl/v1/memory/search" -Method POST -Headers $headers -Body $searchBody -TimeoutSec 180 +$resultCount = 0 +if ($search.data -and $search.data.results) { + $resultCount = @($search.data.results).Count +} +Write-Step "Search result count: $resultCount" + +$retrieveBody = @{ + query = "Where does XMem local mode run from?" + user_id = $UserId + top_k = 5 +} | ConvertTo-Json + +Write-Step "Retrieving answer" +$retrieve = Invoke-XMemJson -Uri "$BaseUrl/v1/memory/retrieve" -Method POST -Headers $headers -Body $retrieveBody -TimeoutSec 240 + +Write-Host "" +Write-Host "Answer:" +Write-Host $retrieve.data.answer +Write-Host "" +Write-Step "Verification complete" diff --git a/scripts/xmem.js b/scripts/xmem.js new file mode 100644 index 0000000..834317c --- /dev/null +++ b/scripts/xmem.js @@ -0,0 +1,160 @@ +#!/usr/bin/env node + +const fs = require("node:fs"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); + +const root = path.resolve(__dirname, ".."); +const command = process.argv[2] || "help"; +const passthroughArgs = process.argv.slice(3); + +const commands = { + setup: "install.ps1", + start: "start.ps1", + verify: "verify.ps1", + doctor: "doctor.ps1", + "context:export": "context-export.ps1", + "context:import": "context-import.ps1", + "context:sync": "context-sync.ps1", +}; + +function log(message) { + console.log(`[xmem] ${message}`); +} + +function usage(exitCode = 0) { + console.log(`XMem local workspace + +Usage: + npm run dev + npm run setup + npm run start + npm run verify + npm run doctor + npm run context:export + npm run context:import -- --file .\\exports\\xmem-context.json + npm run context:sync -- --file .\\exports\\xmem-context.json --server https://api.xmem.in --api-key + +Power-user flags can be passed after --, for example: + npm run setup -- -IncludeMcp + npm run start -- -SkipDocker +`); + process.exit(exitCode); +} + +function powershellExecutable() { + if (process.env.XMEM_POWERSHELL) { + return process.env.XMEM_POWERSHELL; + } + return process.platform === "win32" ? "powershell.exe" : "pwsh"; +} + +function powershellArgs(scriptPath, extraArgs) { + const args = ["-NoProfile"]; + if (process.platform === "win32") { + args.push("-ExecutionPolicy", "Bypass"); + } + args.push("-File", scriptPath, ...extraArgs); + return args; +} + +function runPowerShellScript(scriptName, extraArgs = []) { + const scriptPath = path.join(root, "scripts", scriptName); + if (!fs.existsSync(scriptPath)) { + console.error(`[xmem] Missing script: ${scriptPath}`); + process.exit(1); + } + + const executable = powershellExecutable(); + const result = spawnSync(executable, powershellArgs(scriptPath, extraArgs), { + cwd: root, + stdio: "inherit", + shell: false, + }); + + if (result.error) { + const installHint = + process.platform === "win32" + ? "PowerShell should be available on Windows. Reopen the terminal and try again." + : "Install PowerShell 7+ (`pwsh`) and try again."; + console.error(`[xmem] Could not start ${executable}: ${result.error.message}`); + console.error(`[xmem] ${installHint}`); + process.exit(1); + } + + process.exitCode = result.status || 0; + return process.exitCode; +} + +function getOptionValue(args, optionName) { + const wanted = optionName.toLowerCase(); + for (let index = 0; index < args.length; index += 1) { + if (args[index].toLowerCase() === wanted) { + return args[index + 1] || ""; + } + } + return ""; +} + +function hasSwitch(args, switchName) { + const wanted = switchName.toLowerCase(); + return args.some((arg) => arg.toLowerCase() === wanted); +} + +function startCompatibleArgs(args) { + const next = []; + const reposDir = getOptionValue(args, "-ReposDir"); + if (reposDir) { + next.push("-ReposDir", reposDir); + } + if (hasSwitch(args, "-SkipDocker")) { + next.push("-SkipDocker"); + } + return next; +} + +function setupLooksComplete(reposDir) { + function existsInRoot(relativePath) { + return fs.existsSync(path.join(root, relativePath)); + } + + const pythonVenv = + process.platform === "win32" + ? ".venv/Scripts/python.exe" + : ".venv/bin/python"; + + return ( + existsInRoot("pyproject.toml") && + existsInRoot(".env") && + existsInRoot(pythonVenv) && + fs.existsSync(path.join(reposDir, "xmem-extension", ".git")) && + fs.existsSync(path.join(reposDir, "xmem-extension", "dist", "manifest.json")) + ); +} + +function runDev() { + const reposDir = path.resolve(root, getOptionValue(passthroughArgs, "-ReposDir") || "repos"); + + if (!setupLooksComplete(reposDir)) { + log("First run detected; running setup before starting XMem."); + const setupStatus = runPowerShellScript(commands.setup, passthroughArgs); + if (setupStatus !== 0) { + process.exit(setupStatus); + } + } + + return runPowerShellScript(commands.start, startCompatibleArgs(passthroughArgs)); +} + +if (command === "help" || command === "--help" || command === "-h") { + usage(0); +} + +if (command === "dev") { + runDev(); +} else if (commands[command]) { + runPowerShellScript(commands[command], passthroughArgs); +} else { + console.error(`[xmem] Unknown command: ${command}`); + usage(1); +} diff --git a/src/agents/base.py b/src/agents/base.py index f4e8c3c..d6f61f0 100644 --- a/src/agents/base.py +++ b/src/agents/base.py @@ -5,6 +5,8 @@ from langchain_core.language_models import BaseChatModel import time +from src.config import settings + @dataclass class BaseAgent(ABC): model: BaseChatModel @@ -34,7 +36,16 @@ def _build_messages(self, user_message: str) -> list: async def _call_model(self, messages: list) -> str: import asyncio start = time.perf_counter() - response = await asyncio.wait_for(self.model.ainvoke(messages), timeout=45.0) + timeout = float(getattr(settings, "llm_timeout_seconds", 45.0) or 45.0) + try: + response = await asyncio.wait_for(self.model.ainvoke(messages), timeout=timeout) + except asyncio.TimeoutError as exc: + model_name = getattr(self.model, "model", getattr(self.model, "model_name", type(self.model).__name__)) + raise TimeoutError( + f"LLM call timed out after {timeout:.0f}s in agent '{self.name}' " + f"using model '{model_name}'. For local Ollama, increase " + "LLM_TIMEOUT_SECONDS or configure a cloud LLM key." + ) from exc elapsed = time.perf_counter() - start content = response.content diff --git a/src/api/app.py b/src/api/app.py index 889151e..4cdaeb3 100644 --- a/src/api/app.py +++ b/src/api/app.py @@ -10,7 +10,8 @@ import traceback from contextlib import asynccontextmanager -from fastapi import FastAPI, Request +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, Response @@ -63,6 +64,57 @@ def _init_pipelines_sync() -> tuple: return ingest, retrieval +def _detail_to_text(detail) -> str: + if isinstance(detail, str): + return detail + if isinstance(detail, list): + return "; ".join(str(item) for item in detail) + if isinstance(detail, dict): + return detail.get("message") or detail.get("error") or str(detail) + return str(detail) + + +def _field_name(loc) -> str: + parts = [str(part) for part in loc if part not in {"body", "query", "path"}] + return ".".join(parts) or "request" + + +def _friendly_validation_error(error: dict) -> str: + field = _field_name(error.get("loc", [])) + kind = error.get("type", "") + message = error.get("msg", "Invalid value") + + if kind == "missing": + return f"{field} is required." + if kind == "string_too_short": + return f"{field} cannot be empty." + if kind == "string_too_long": + limit = (error.get("ctx") or {}).get("max_length") + return f"{field} is too long" + (f" (max {limit} characters)." if limit else ".") + if kind in {"int_parsing", "float_parsing"}: + return f"{field} must be a number." + if kind in {"greater_than_equal", "less_than_equal"}: + return f"{field}: {message}" + if kind == "value_error": + ctx = error.get("ctx") or {} + return f"{field}: {ctx.get('error') or message}" + + return f"{field}: {message}" + + +def _public_exception_message(exc: Exception) -> str: + message = str(exc).strip() + if isinstance(exc, TimeoutError): + return message or "The request timed out while waiting for an LLM response." + if isinstance(exc, (ValueError, RuntimeError, ConnectionError)): + return message or type(exc).__name__ + + if settings.environment.lower() in {"development", "dev", "local", "test"}: + return message or type(exc).__name__ + + return "Internal server error. Check the server logs with the request_id for details." + + async def _boot_pipelines() -> None: loop = asyncio.get_running_loop() try: @@ -204,6 +256,32 @@ async def sentry_debug(): # ── Global exception handler ────────────────────────────────────── + @app.exception_handler(RequestValidationError) + async def _validation_exception(request: Request, exc: RequestValidationError): + request_id = getattr(request.state, "request_id", None) + details = [_friendly_validation_error(error) for error in exc.errors()] + body = APIResponse( + status=StatusEnum.ERROR, + request_id=request_id, + error="Invalid request: " + " ".join(details), + data={"details": details}, + ) + return JSONResponse(content=body.model_dump(), status_code=422) + + @app.exception_handler(HTTPException) + async def _http_exception(request: Request, exc: HTTPException): + request_id = getattr(request.state, "request_id", None) + body = APIResponse( + status=StatusEnum.ERROR, + request_id=request_id, + error=_detail_to_text(exc.detail), + ) + return JSONResponse( + content=body.model_dump(), + status_code=exc.status_code, + headers=exc.headers, + ) + @app.exception_handler(Exception) async def _unhandled_exception(request: Request, exc: Exception): request_id = getattr(request.state, "request_id", None) @@ -214,7 +292,9 @@ async def _unhandled_exception(request: Request, exc: Exception): capture_exception(exc) body = APIResponse( - status=StatusEnum.ERROR, request_id=request_id, error="Internal server error.", + status=StatusEnum.ERROR, + request_id=request_id, + error=_public_exception_message(exc), ) return JSONResponse(content=body.model_dump(), status_code=500) diff --git a/src/api/routes/memory.py b/src/api/routes/memory.py index ab40c07..7268a6a 100644 --- a/src/api/routes/memory.py +++ b/src/api/routes/memory.py @@ -48,6 +48,7 @@ import re from playwright.sync_api import sync_playwright +from src.config import settings from src.jobs.durable import ( QUEUED, get_default_job_store, @@ -133,7 +134,11 @@ def _error( return JSONResponse(content=body.model_dump(), status_code=code) -def _current_user_id(user: dict) -> str: +def _current_user_id(user: dict, requested_user_id: str = "") -> str: + if requested_user_id and ( + user.get("email") == "static@xmem.ai" or user.get("name") == "Static Key User" + ): + return requested_user_id return user.get("username") or user.get("name") or user["id"] @@ -677,13 +682,13 @@ async def _scrape_chat_share(url: str) -> Dict[str, Any]: ) async def ingest_memory(req: IngestRequest, request: Request, user: dict = Depends(require_api_key)): start = time.perf_counter() - user_id = _current_user_id(user) + user_id = _current_user_id(user, req.user_id) payload = req.model_dump() try: data = await asyncio.wait_for( _run_ingest_payload(payload, user_id), - timeout=120.0, + timeout=float(settings.memory_ingest_timeout_seconds), ) elapsed = round((time.perf_counter() - start) * 1000, 2) return _wrap(request, data, elapsed) @@ -702,7 +707,7 @@ async def ingest_memory(req: IngestRequest, request: Request, user: dict = Depen ) async def ingest_memory_v2(req: IngestRequest, request: Request, user: dict = Depends(require_api_key)): start = time.perf_counter() - user_id = _current_user_id(user) + user_id = _current_user_id(user, req.user_id) payload = req.model_dump() payload["user_id"] = user_id @@ -721,7 +726,7 @@ async def ingest_memory_v2(req: IngestRequest, request: Request, user: dict = De "effort_level": req.effort_level, }, user_id=user_id, - timeout_seconds=120.0, + timeout_seconds=float(settings.memory_ingest_timeout_seconds), max_attempts=3, ) _schedule_job( @@ -797,14 +802,14 @@ async def memory_job_status(job_id: str, request: Request, user: dict = Depends( ) async def batch_ingest_memory(req: BatchIngestRequest, request: Request, user: dict = Depends(require_api_key)): start = time.perf_counter() - user_id = _current_user_id(user) + user_id = _current_user_id(user, req.items[0].user_id if req.items else "") try: results = [] for item in req.items: data = await asyncio.wait_for( _run_ingest_payload(item.model_dump(), user_id), - timeout=120.0, + timeout=float(settings.memory_ingest_timeout_seconds), ) results.append(IngestResponse(**data)) @@ -826,7 +831,7 @@ async def batch_ingest_memory(req: BatchIngestRequest, request: Request, user: d ) async def batch_ingest_memory_v2(req: BatchIngestRequest, request: Request, user: dict = Depends(require_api_key)): start = time.perf_counter() - user_id = _current_user_id(user) + user_id = _current_user_id(user, req.items[0].user_id if req.items else "") payload = req.model_dump() payload["user_id"] = user_id @@ -841,7 +846,10 @@ async def batch_ingest_memory_v2(req: BatchIngestRequest, request: Request, user "items": payload["items"], }, user_id=user_id, - timeout_seconds=max(120.0, min(len(req.items) * 120.0, 3600.0)), + timeout_seconds=max( + float(settings.memory_ingest_timeout_seconds), + min(len(req.items) * float(settings.memory_ingest_timeout_seconds), 3600.0), + ), max_attempts=3, ) _schedule_job( @@ -875,7 +883,7 @@ async def retrieve_memory(req: RetrieveRequest, request: Request, user: dict = D pipeline = get_retrieval_pipeline() # Get username from authenticated user - user_id = user.get("username") or user.get("name") or user["id"] + user_id = _current_user_id(user, req.user_id) try: result = await pipeline.run(query=req.query, user_id=user_id, top_k=req.top_k) @@ -911,7 +919,7 @@ async def search_memory(req: SearchRequest, request: Request, user: dict = Depen pipeline = get_retrieval_pipeline() # Get username from authenticated user - user_id = user.get("username") or user.get("name") or user["id"] + user_id = _current_user_id(user, req.user_id) try: all_results: List[SourceRecord] = [] diff --git a/src/api/schemas.py b/src/api/schemas.py index b7ee122..ff10e6d 100644 --- a/src/api/schemas.py +++ b/src/api/schemas.py @@ -9,11 +9,29 @@ from datetime import datetime from enum import Enum +import re from typing import Any, Dict, List, Optional from pydantic import BaseModel, Field, field_validator +def normalize_user_id(value: Any) -> str: + """Convert friendly user input into XMem's canonical storage id.""" + text = str(value or "").strip() + text = re.sub(r"[^A-Za-z0-9_.@-]+", "_", text) + text = re.sub(r"_+", "_", text).strip("_") + return text[:256] + + +class UserScopedModel(BaseModel): + """Base model for requests that scope data to a user.""" + + @field_validator("user_id", mode="before", check_fields=False) + @classmethod + def normalize_user_id_field(cls, v: Any) -> str: + return normalize_user_id(v) + + # ── Shared envelope ──────────────────────────────────────────────────────── class StatusEnum(str, Enum): @@ -42,7 +60,7 @@ class HealthResponse(BaseModel): # ── Ingest (save memory) ────────────────────────────────────────────────── -class IngestRequest(BaseModel): +class IngestRequest(UserScopedModel): """Store a new memory from a conversation turn.""" user_query: str = Field( ..., min_length=1, max_length=10_000, @@ -53,8 +71,8 @@ class IngestRequest(BaseModel): description="The assistant's reply (used for summary extraction)", ) user_id: str = Field( - ..., min_length=1, max_length=256, pattern=r"^[\w.\-@]+$", - description="Unique user identifier (alphanumeric, dots, hyphens, underscores, @)", + ..., min_length=1, max_length=256, + description="User identifier. Friendly names are normalized internally.", ) session_datetime: str = Field( default="", @@ -74,7 +92,6 @@ class IngestRequest(BaseModel): def strip_query(cls, v: str) -> str: return v.strip() - class OperationDetail(BaseModel): type: str content: str @@ -117,14 +134,14 @@ class BatchIngestResponse(BaseModel): # ── Retrieve (answer a question from memory) ────────────────────────────── -class RetrieveRequest(BaseModel): +class RetrieveRequest(UserScopedModel): """Ask a question answered from stored memories.""" query: str = Field( ..., min_length=1, max_length=5_000, description="The question to answer from memory", ) user_id: str = Field( - ..., min_length=1, max_length=256, pattern=r"^[\w.\-@]+$", + ..., min_length=1, max_length=256, ) top_k: int = Field(default=5, ge=1, le=50) @@ -133,7 +150,6 @@ class RetrieveRequest(BaseModel): def strip_query(cls, v: str) -> str: return v.strip() - class SourceRecord(BaseModel): domain: str content: str @@ -150,13 +166,13 @@ class RetrieveResponse(BaseModel): # ── Search (raw vector / graph search without LLM answer) ───────────────── -class SearchRequest(BaseModel): +class SearchRequest(UserScopedModel): """Raw semantic search across memory domains.""" query: str = Field( ..., min_length=1, max_length=5_000, ) user_id: str = Field( - ..., min_length=1, max_length=256, pattern=r"^[\w.\-@]+$", + ..., min_length=1, max_length=256, ) domains: List[str] = Field( default=["profile", "temporal", "summary"], @@ -199,7 +215,7 @@ class ScrapeResponse(BaseModel): # ── Code retrieval (IDE mode) ───────────────────────────────────────────── -class CodeQueryRequest(BaseModel): +class CodeQueryRequest(UserScopedModel): """Query a codebase via the code retrieval pipeline.""" org_id: str = Field(..., min_length=1, max_length=256) repo: str = Field(..., min_length=1, max_length=256) @@ -219,7 +235,7 @@ class CodeQueryResponse(BaseModel): confidence: float = 0.0 -class ExecuteToolRequest(BaseModel): +class ExecuteToolRequest(UserScopedModel): """Execute a specific raw code retrieval tool natively.""" org_id: str = Field(..., min_length=1, max_length=256) repo: str = Field(..., min_length=1, max_length=256) diff --git a/src/config/settings.py b/src/config/settings.py index d846b66..b910d43 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -104,6 +104,14 @@ class Settings(BaseSettings): default=0.4, description="LLM temperature for generation" ) + llm_timeout_seconds: float = Field( + default=45.0, + description="Per-agent LLM call timeout in seconds", + ) + memory_ingest_timeout_seconds: float = Field( + default=120.0, + description="Overall memory ingest timeout in seconds", + ) fallback_order: List[str] = Field( default=["openrouter", "gemini", "claude", "openai"], description="Order of LLM providers to try on failure" diff --git a/templates/xmem.env.local b/templates/xmem.env.local new file mode 100644 index 0000000..1588f34 --- /dev/null +++ b/templates/xmem.env.local @@ -0,0 +1,92 @@ +# XMem local environment. +# Copy to .env. + +ENVIRONMENT=development +API_HOST=0.0.0.0 +API_PORT=8000 +API_KEYS='["dev-xmem-key"]' +CORS_ORIGINS='["http://localhost:3000","http://localhost:5173","http://localhost:4173","http://localhost:8000"]' +RATE_LIMIT=120 +MAX_REQUEST_BODY_BYTES=10485760 +ENABLE_PROMETHEUS=true +ENABLE_ANALYTICS=false + +# LLM routing. +# scripts/configure-xmem-env.ps1 rewrites FALLBACK_ORDER before install/start: +# - any real cloud LLM key below means XMem will not call Ollama +# - no cloud LLM key means XMem will use local Ollama +FALLBACK_ORDER='["ollama"]' +OLLAMA_BASE_URL=http://localhost:11434 +OLLAMA_MODEL=qwen2.5:1.5b +OLLAMA_VISION_MODEL=llava:latest +TEMPERATURE=0.3 +LLM_TIMEOUT_SECONDS=180 +MEMORY_INGEST_TIMEOUT_SECONDS=600 + +# Optional cloud providers. Add one key here to avoid Ollama calls. +GEMINI_API_KEY= +GEMINI_MODEL=gemini-2.5-flash +GEMINI_VISION_MODEL=gemini-2.5-flash-lite +CLAUDE_API_KEY= +CLAUDE_MODEL=claude-3-5-sonnet +OPENAI_API_KEY= +OPENAI_MODEL=gpt-4.1-mini +OPENROUTER_API_KEY= +OPENROUTER_MODEL=google/gemini-2.5-flash +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +BEDROCK_REGION=us-east-1 +BEDROCK_MODEL=us.amazon.nova-lite-v1:0 + +# Local embeddings and vector storage. +# With a cloud LLM key, scripts switch embeddings to fastembed so Ollama is not used. +EMBEDDING_PROVIDER=ollama +OLLAMA_EMBEDDING_MODEL=nomic-embed-text +FASTEMBED_MODEL=BAAI/bge-small-en-v1.5 +EMBEDDING_MODEL=nomic-embed-text +VECTOR_STORE_PROVIDER=pgvector +PGVECTOR_URL=postgresql://xmem:xmem@localhost:15432/xmem +PGVECTOR_TABLE=xmem_vectors + +# Pinecone remains unset for local mode. +PINECONE_API_KEY= +PINECONE_INDEX_NAME=xmem-local +PINECONE_NAMESPACE=local +PINECONE_DIMENSION=768 +PINECONE_METRIC=cosine +PINECONE_CLOUD=aws +PINECONE_REGION=us-east-1 + +# App metadata and graph stores. +APP_STORE_PROVIDER=postgres +APP_POSTGRES_URL=postgresql://xmem:xmem@localhost:15432/xmem +MONGODB_URI=mongodb://localhost:27018 +MONGODB_DATABASE=xmem + +NEO4J_URI=bolt://localhost:17687 +NEO4J_USERNAME=neo4j +NEO4J_PASSWORD=local-password +NEO4J_DATABASE=neo4j +NEO4J_TRANSPORT=auto +NEO4J_CONNECTION_TIMEOUT=60 + +# Auth and app URLs. +GOOGLE_CLIENT_ID= +GOOGLE_CLIENT_SECRET= +GOOGLE_REDIRECT_URI=http://localhost:8000/auth/callback +JWT_SECRET_KEY=dev-only-change-me +JWT_ALGORITHM=HS256 +JWT_EXPIRATION_DAYS=7 +FRONTEND_URL=http://localhost:5173 +XMEM_SERVER_URL=http://localhost:8000 + +# Optional integrations. +SENTRY_DSN= +OPIK_API_KEY= +OPIK_WORKSPACE=xmem +OPIK_PROJECT=xmem-local +GITHUB_TOKEN= +GITHUB_REPO_OWNER=XortexAI +GITHUB_REPO_NAME=XMem +GMAIL_APP_PASSWORD= +GMAIL_SENDER_EMAIL=xmemlabs@gmail.com From 6429b2ff8c73f6e13e47c84f86f0ff952b8f50cd Mon Sep 17 00:00:00 2001 From: Ankit Kotnala Date: Sat, 23 May 2026 16:22:55 +0530 Subject: [PATCH 2/6] Add macOS local setup support --- README.md | 10 +- scripts/configure-xmem-env.sh | 150 ++++++++++++++++++ scripts/context-export.sh | 12 ++ scripts/context-import.sh | 12 ++ scripts/context-sync.sh | 12 ++ scripts/doctor.sh | 161 +++++++++++++++++++ scripts/install.ps1 | 1 + scripts/install.sh | 246 ++++++++++++++++++++++++++++++ scripts/patch-extension-local.js | 221 +++++++++++++++++++++++++++ scripts/patch-extension-local.ps1 | 153 +------------------ scripts/patch-extension-local.sh | 20 +++ scripts/start.sh | 133 ++++++++++++++++ scripts/verify.py | 137 +++++++++++++++++ scripts/verify.sh | 18 +++ scripts/xmem.js | 43 ++++-- 15 files changed, 1164 insertions(+), 165 deletions(-) create mode 100755 scripts/configure-xmem-env.sh create mode 100755 scripts/context-export.sh create mode 100755 scripts/context-import.sh create mode 100755 scripts/context-sync.sh create mode 100755 scripts/doctor.sh create mode 100755 scripts/install.sh create mode 100644 scripts/patch-extension-local.js create mode 100755 scripts/patch-extension-local.sh create mode 100755 scripts/start.sh create mode 100644 scripts/verify.py create mode 100755 scripts/verify.sh diff --git a/README.md b/README.md index efba9bf..2933fcf 100644 --- a/README.md +++ b/README.md @@ -317,7 +317,15 @@ cd xmem npm run dev ``` -This creates a local XMem workspace, installs the backend, starts local storage, builds the Chrome extension, and launches the API at `http://localhost:8000`. +This works on Windows, macOS, and Linux. It creates a local XMem workspace, installs the backend, starts local storage, builds the Chrome extension, and launches the API at `http://localhost:8000`. + +Local prerequisites: + +- Git +- Node.js 20+ +- Python 3.11+ +- Docker Desktop +- Ollama, unless you add a cloud LLM key to `.env` After setup, load the extension from: diff --git a/scripts/configure-xmem-env.sh b/scripts/configure-xmem-env.sh new file mode 100755 index 0000000..7d410f3 --- /dev/null +++ b/scripts/configure-xmem-env.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ENV_PATH="$ROOT/.env" +QUIET=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --env-path|-EnvPath) + ENV_PATH="$2" + shift 2 + ;; + --quiet|-Quiet) + QUIET=1 + shift + ;; + *) + echo "[xmem] Unknown configure option: $1" >&2 + exit 1 + ;; + esac +done + +if command -v python3 >/dev/null 2>&1; then + PYTHON_BIN=python3 +elif command -v python >/dev/null 2>&1; then + PYTHON_BIN=python +else + echo "[xmem] python3 is required. Install Python 3.11+ and rerun." >&2 + exit 1 +fi + +XMEM_ENV_PATH="$ENV_PATH" XMEM_QUIET="$QUIET" "$PYTHON_BIN" - <<'PY' +from __future__ import annotations + +import os +import re +from pathlib import Path + +env_path = Path(os.environ["XMEM_ENV_PATH"]) +quiet = os.environ.get("XMEM_QUIET") == "1" + +if not env_path.exists(): + raise SystemExit(f"XMem .env not found at {env_path}") + + +def log(message: str) -> None: + if not quiet: + print(f"[xmem] {message}") + + +def read_values(path: Path) -> tuple[list[str], dict[str, str]]: + lines = path.read_text(encoding="utf-8").splitlines() + values: dict[str, str] = {} + for line in lines: + if not line.strip() or line.lstrip().startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + value = value.strip().strip('"').strip("'") + values[key.strip()] = value.strip() + return lines, values + + +def is_secret(value: str | None) -> bool: + if not value: + return False + value = value.strip() + if not value: + return False + placeholders = [ + r"^your[_-]", + r"your_.*_key", + r"example", + r"sample", + r"placeholder", + r"change[-_]?me", + r"^dummy([-_].*)?$", + r"^fake([-_].*)?$", + r"^test([-_].*)?$", + ] + return not any(re.search(pattern, value, re.IGNORECASE) for pattern in placeholders) + + +def configured(values: dict[str, str], name: str) -> str: + env_value = os.environ.get(name) + if is_secret(env_value): + return env_value or "" + file_value = values.get(name) + if is_secret(file_value): + return file_value or "" + return "" + + +def set_value(lines: list[str], name: str, value: str) -> list[str]: + pattern = re.compile(rf"^\s*{re.escape(name)}\s*=") + replaced = False + next_lines: list[str] = [] + for line in lines: + if pattern.match(line): + next_lines.append(f"{name}={value}") + replaced = True + else: + next_lines.append(line) + if not replaced: + next_lines.append(f"{name}={value}") + return next_lines + + +lines, values = read_values(env_path) +providers: list[str] = [] +for key, provider in [ + ("OPENROUTER_API_KEY", "openrouter"), + ("GEMINI_API_KEY", "gemini"), + ("CLAUDE_API_KEY", "claude"), + ("OPENAI_API_KEY", "openai"), +]: + if configured(values, key): + providers.append(provider) + +if configured(values, "AWS_ACCESS_KEY_ID") and configured(values, "AWS_SECRET_ACCESS_KEY"): + providers.append("bedrock") + +if providers: + quoted = ",".join(f'"{provider}"' for provider in providers) + updates = { + "FALLBACK_ORDER": f"'[{quoted}]'", + "EMBEDDING_PROVIDER": "fastembed", + "FASTEMBED_MODEL": "BAAI/bge-small-en-v1.5", + "EMBEDDING_MODEL": "BAAI/bge-small-en-v1.5", + "PINECONE_DIMENSION": "384", + } + log(f"Detected cloud LLM provider(s): {', '.join(providers)}") + log("Configured XMem to avoid Ollama for LLM and embedding calls.") +else: + updates = { + "FALLBACK_ORDER": "'[\"ollama\"]'", + "EMBEDDING_PROVIDER": "ollama", + "OLLAMA_EMBEDDING_MODEL": "nomic-embed-text", + "EMBEDDING_MODEL": "nomic-embed-text", + "PINECONE_DIMENSION": "768", + } + log("No cloud LLM provider keys detected.") + log("Configured XMem to use local Ollama for LLM and embedding calls.") + +for key, value in updates.items(): + lines = set_value(lines, key, value) + +env_path.write_text("\n".join(lines) + "\n", encoding="utf-8") +PY diff --git a/scripts/context-export.sh b/scripts/context-export.sh new file mode 100755 index 0000000..4f2b3e6 --- /dev/null +++ b/scripts/context-export.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PYTHON_BIN="$ROOT/.venv/bin/python" + +if [[ ! -x "$PYTHON_BIN" ]]; then + echo "[xmem] XMem virtualenv not found. Run npm run setup first." >&2 + exit 1 +fi + +"$PYTHON_BIN" "$ROOT/scripts/context.py" export "$@" diff --git a/scripts/context-import.sh b/scripts/context-import.sh new file mode 100755 index 0000000..3c2dfab --- /dev/null +++ b/scripts/context-import.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PYTHON_BIN="$ROOT/.venv/bin/python" + +if [[ ! -x "$PYTHON_BIN" ]]; then + echo "[xmem] XMem virtualenv not found. Run npm run setup first." >&2 + exit 1 +fi + +"$PYTHON_BIN" "$ROOT/scripts/context.py" import "$@" diff --git a/scripts/context-sync.sh b/scripts/context-sync.sh new file mode 100755 index 0000000..5a19bef --- /dev/null +++ b/scripts/context-sync.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PYTHON_BIN="$ROOT/.venv/bin/python" + +if [[ ! -x "$PYTHON_BIN" ]]; then + echo "[xmem] XMem virtualenv not found. Run npm run setup first." >&2 + exit 1 +fi + +"$PYTHON_BIN" "$ROOT/scripts/context.py" sync "$@" diff --git a/scripts/doctor.sh b/scripts/doctor.sh new file mode 100755 index 0000000..669656d --- /dev/null +++ b/scripts/doctor.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REPOS_DIR="$ROOT/repos" +BASE_URL="http://localhost:8000" +FAILURES=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --base-url|-BaseUrl) + BASE_URL="$2" + shift 2 + ;; + --repos-dir|-ReposDir) + REPOS_DIR="$2" + shift 2 + ;; + *) + echo "[xmem] Unknown doctor option: $1" >&2 + exit 1 + ;; + esac +done + +check() { + local name="$1" + local ok="$2" + local message="$3" + local fix="${4:-}" + if [[ "$ok" == "1" ]]; then + echo "[OK] $name - $message" + else + echo "[FIX] $name - $message" + [[ -n "$fix" ]] && echo " $fix" + FAILURES=$((FAILURES + 1)) + fi +} + +has_command() { + command -v "$1" >/dev/null 2>&1 +} + +env_value() { + local key="$1" + local default="$2" + if [[ ! -f "$ROOT/.env" ]]; then + echo "$default" + return + fi + local value + value="$(grep -E "^[[:space:]]*$key[[:space:]]*=" "$ROOT/.env" | tail -n 1 | sed -E "s/^[^=]+=//; s/^['\"]//; s/['\"]$//")" + if [[ -n "$value" ]]; then + echo "$value" + else + echo "$default" + fi +} + +uses_ollama() { + [[ ! -f "$ROOT/.env" ]] && return 0 + grep -Eq '^[[:space:]]*FALLBACK_ORDER[[:space:]]*=.*ollama' "$ROOT/.env" +} + +echo "[xmem] Doctor report" +echo "" + +for cmd in git node npm; do + if has_command "$cmd"; then + check "$cmd" 1 "command lookup" + else + check "$cmd" 0 "command lookup" "Install $cmd and reopen this terminal." + fi +done + +if has_command python3 || has_command python; then + check "python" 1 "command lookup" +else + check "python" 0 "command lookup" "Install Python 3.11+ and reopen this terminal." +fi + +if has_command docker && docker info >/dev/null 2>&1; then + check "Docker" 1 "local database runtime" +else + check "Docker" 0 "local database runtime" "Start Docker Desktop, then rerun npm run dev." +fi + +if [[ -f "$ROOT/pyproject.toml" ]]; then + check "XMem repo" 1 "$ROOT" +else + check "XMem repo" 0 "$ROOT" "Run this from the XMem repository root." +fi + +if [[ -d "$REPOS_DIR/xmem-extension" ]]; then + check "Extension repo" 1 "$REPOS_DIR/xmem-extension" +else + check "Extension repo" 0 "$REPOS_DIR/xmem-extension" "Run npm run setup." +fi + +if [[ -f "$ROOT/.env" ]]; then + check "XMem .env" 1 "$ROOT/.env" +else + check "XMem .env" 0 "$ROOT/.env" "Run npm run setup to create it from templates/xmem.env.local." +fi + +if [[ -f "$ROOT/.env" ]]; then + if uses_ollama; then + if has_command ollama && ollama list >/dev/null 2>&1; then + check "Ollama" 1 "required because no cloud LLM key is configured" + installed="$(ollama list 2>/dev/null | tail -n +2 || true)" + for model in "$(env_value OLLAMA_MODEL qwen2.5:1.5b)" "$(env_value OLLAMA_EMBEDDING_MODEL nomic-embed-text)"; do + [[ -z "$model" ]] && continue + if printf "%s\n" "$installed" | grep -Eq "^$(printf '%s' "$model" | sed 's/[][\.^$*+?{}|()]/\\&/g')([[:space:]]|:latest[[:space:]])"; then + check "Ollama model $model" 1 "local model availability" + else + check "Ollama model $model" 0 "local model availability" "Run: ollama pull $model" + fi + done + else + check "Ollama" 0 "required because no cloud LLM key is configured" "Start Ollama, or add a cloud LLM key to .env." + fi + else + check "LLM routing" 1 "cloud key detected; Ollama is not required" + fi +fi + +if has_command python3; then + PYTHON_BIN=python3 +elif has_command python; then + PYTHON_BIN=python +else + PYTHON_BIN="" +fi + +if [[ -n "$PYTHON_BIN" ]]; then + if "$PYTHON_BIN" - "$BASE_URL" <<'PY' >/dev/null 2>&1 +import json +import sys +import urllib.request + +base = sys.argv[1].rstrip("/") +with urllib.request.urlopen(f"{base}/health", timeout=5) as response: + health = json.loads(response.read().decode("utf-8")) +data = health.get("data") or health +raise SystemExit(0 if data.get("pipelines_ready") else 1) +PY + then + check "XMem API" 1 "$BASE_URL/health" + else + check "XMem API" 0 "$BASE_URL is not ready" "Start it with npm run dev and wait for pipelines_ready=true." + fi +else + check "XMem API" 0 "$BASE_URL not checked" "Install Python 3.11+." +fi + +echo "" +if [[ "$FAILURES" -eq 0 ]]; then + echo "[xmem] Everything looks ready." +else + echo "[xmem] Found $FAILURES setup item(s) to fix." +fi diff --git a/scripts/install.ps1 b/scripts/install.ps1 index 2dca0e4..f729969 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -164,6 +164,7 @@ function Sync-Repo { throw "$target exists but is not a git checkout." } Write-Step "Updating $Name" + Invoke-Native { git -C $target reset --hard } Invoke-Native { git -C $target fetch origin } Invoke-Native { git -C $target checkout $Branch } Invoke-Native { git -C $target pull --ff-only origin $Branch } diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000..9f49baf --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,246 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REPOS_DIR="$ROOT/repos" +INCLUDE_MCP=0 +INCLUDE_SDK=0 +SKIP_MODEL_PULL=0 +SKIP_PYTHON_INSTALL=0 +SKIP_NODE_INSTALL=0 +SKIP_DOCKER=0 + +log() { + echo "[xmem] $*" +} + +has_command() { + command -v "$1" >/dev/null 2>&1 +} + +python_cmd() { + if has_command python3; then + echo "python3" + elif has_command python; then + echo "python" + else + echo "[xmem] python3 is required. Install Python 3.11+ and rerun." >&2 + exit 1 + fi +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --repos-dir|-ReposDir) + REPOS_DIR="$2" + shift 2 + ;; + --include-mcp|-IncludeMcp) + INCLUDE_MCP=1 + shift + ;; + --include-sdk|-IncludeSdk) + INCLUDE_SDK=1 + shift + ;; + --skip-model-pull|-SkipModelPull) + SKIP_MODEL_PULL=1 + shift + ;; + --skip-python-install|-SkipPythonInstall) + SKIP_PYTHON_INSTALL=1 + shift + ;; + --skip-node-install|-SkipNodeInstall) + SKIP_NODE_INSTALL=1 + shift + ;; + --skip-docker|-SkipDocker) + SKIP_DOCKER=1 + shift + ;; + *) + echo "[xmem] Unknown setup option: $1" >&2 + exit 1 + ;; + esac +done + +for cmd in git node npm; do + if ! has_command "$cmd"; then + echo "[xmem] $cmd is required. Install it, then run this script again." >&2 + exit 1 + fi +done + +PYTHON_BIN="$(python_cmd)" +mkdir -p "$REPOS_DIR" + +sync_repo() { + local name="$1" + local url="$2" + local branch="$3" + local target="$REPOS_DIR/$name" + + if [[ -d "$target" ]]; then + if [[ ! -d "$target/.git" ]]; then + echo "[xmem] $target exists but is not a git checkout." >&2 + exit 1 + fi + log "Updating $name" + git -C "$target" reset --hard + git -C "$target" fetch origin + git -C "$target" checkout "$branch" + git -C "$target" pull --ff-only origin "$branch" + else + log "Cloning $name" + git clone --branch "$branch" "$url" "$target" + fi +} + +docker_running() { + has_command docker && docker info >/dev/null 2>&1 +} + +ollama_running() { + has_command ollama && ollama list >/dev/null 2>&1 +} + +uses_ollama() { + [[ ! -f "$ROOT/.env" ]] && return 0 + grep -Eq '^[[:space:]]*FALLBACK_ORDER[[:space:]]*=.*ollama' "$ROOT/.env" +} + +env_value() { + local key="$1" + local default="$2" + if [[ ! -f "$ROOT/.env" ]]; then + echo "$default" + return + fi + local value + value="$(grep -E "^[[:space:]]*$key[[:space:]]*=" "$ROOT/.env" | tail -n 1 | sed -E "s/^[^=]+=//; s/^['\"]//; s/['\"]$//")" + if [[ -n "$value" ]]; then + echo "$value" + else + echo "$default" + fi +} + +wait_containers() { + local deadline=$((SECONDS + 180)) + local pending=("$@") + while (( SECONDS < deadline )); do + local next=() + for name in "${pending[@]}"; do + local status + status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$name" 2>/dev/null || true)" + if [[ "$status" == "healthy" || "$status" == "running" ]]; then + continue + fi + if [[ "$status" == "unhealthy" ]]; then + echo "[xmem] Container $name is unhealthy. Run npm run doctor or inspect it with: docker logs $name" >&2 + exit 1 + fi + next+=("$name") + done + + if [[ ${#next[@]} -eq 0 ]]; then + return + fi + + log "Waiting for local database containers: ${next[*]}" + pending=("${next[@]}") + sleep 5 + done + + echo "[xmem] Timed out waiting for local database containers: ${pending[*]}. Run npm run doctor for details." >&2 + exit 1 +} + +sync_repo "xmem-extension" "https://github.com/XortexAI/xmem-extension.git" "main" + +if [[ "$INCLUDE_MCP" == "1" ]]; then + sync_repo "xmem-mcp" "https://github.com/XortexAI/xmem-mcp.git" "main" +fi + +if [[ "$INCLUDE_SDK" == "1" ]]; then + sync_repo "xmem-sdk" "https://github.com/XortexAI/xmem-sdk.git" "master" +fi + +if [[ ! -f "$ROOT/.env" ]]; then + cp "$ROOT/templates/xmem.env.local" "$ROOT/.env" + log "Created .env from local template" +else + log ".env already exists; leaving it unchanged" +fi + +bash "$ROOT/scripts/configure-xmem-env.sh" --env-path "$ROOT/.env" + +docker_skipped=0 +ollama_skipped=0 + +if [[ "$SKIP_MODEL_PULL" != "1" ]]; then + if uses_ollama; then + if ollama_running; then + log "Pulling Ollama chat model" + ollama pull "$(env_value OLLAMA_MODEL qwen2.5:1.5b)" + log "Pulling Ollama embedding model" + ollama pull "$(env_value OLLAMA_EMBEDDING_MODEL nomic-embed-text)" + else + echo "[xmem] Ollama was not found or is not running." >&2 + echo "[xmem] Start Ollama, or add a cloud LLM key to .env and rerun." >&2 + ollama_skipped=1 + fi + else + log "Cloud LLM provider key detected; skipping Ollama model pulls" + fi +fi + +if [[ "$SKIP_DOCKER" != "1" ]]; then + if docker_running; then + log "Starting local Docker services" + docker compose -f "$ROOT/docker-compose.local.yml" up -d --remove-orphans + wait_containers xmem-postgres xmem-mongo xmem-neo4j + else + echo "[xmem] Docker Desktop is installed but not running, or Docker was not found." >&2 + echo "[xmem] Start Docker Desktop, then rerun this script." >&2 + docker_skipped=1 + fi +fi + +if [[ "$SKIP_PYTHON_INSTALL" != "1" ]]; then + VENV_PYTHON="$ROOT/.venv/bin/python" + if [[ ! -x "$VENV_PYTHON" ]]; then + log "Creating XMem virtualenv" + "$PYTHON_BIN" -m venv "$ROOT/.venv" + fi + log "Installing XMem local dependencies" + "$VENV_PYTHON" -m pip install --upgrade pip + "$VENV_PYTHON" -m pip install -e "$ROOT[local,dev]" +fi + +log "Patching extension for local API" +bash "$ROOT/scripts/patch-extension-local.sh" --extension-dir "$REPOS_DIR/xmem-extension" + +if [[ "$SKIP_NODE_INSTALL" != "1" ]]; then + log "Installing and building Chrome extension" + npm --prefix "$REPOS_DIR/xmem-extension" install + npm --prefix "$REPOS_DIR/xmem-extension" run build +fi + +log "Install complete" +echo "" +echo "Next:" +echo " npm run dev" +echo " npm run verify" + +if [[ "$docker_skipped" == "1" ]]; then + echo "" + echo "[xmem] Docker services were not started. Start Docker Desktop before running npm run dev." >&2 +fi + +if [[ "$ollama_skipped" == "1" ]]; then + echo "" + echo "[xmem] Ollama models were not pulled. Start Ollama, then rerun npm run setup or add a cloud LLM key." >&2 +fi diff --git a/scripts/patch-extension-local.js b/scripts/patch-extension-local.js new file mode 100644 index 0000000..ab36229 --- /dev/null +++ b/scripts/patch-extension-local.js @@ -0,0 +1,221 @@ +#!/usr/bin/env node + +const fs = require("node:fs"); +const path = require("node:path"); +const zlib = require("node:zlib"); + +function argValue(names, fallback = "") { + const args = process.argv.slice(2); + for (let index = 0; index < args.length; index += 1) { + if (names.includes(args[index])) { + return args[index + 1] || fallback; + } + } + return fallback; +} + +const root = path.resolve(__dirname, ".."); +const extensionDir = path.resolve( + argValue(["--extension-dir", "-ExtensionDir"], path.join(root, "repos", "xmem-extension")), +); + +function crc32(buffer) { + let crc = 0xffffffff; + for (const byte of buffer) { + crc ^= byte; + for (let index = 0; index < 8; index += 1) { + crc = (crc >>> 1) ^ (crc & 1 ? 0xedb88320 : 0); + } + } + return (crc ^ 0xffffffff) >>> 0; +} + +function pngChunk(type, data) { + const typeBuffer = Buffer.from(type); + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length, 0); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(crc32(Buffer.concat([typeBuffer, data])), 0); + return Buffer.concat([length, typeBuffer, data, crc]); +} + +function distanceToSegment(px, py, ax, ay, bx, by) { + const dx = bx - ax; + const dy = by - ay; + const lengthSquared = dx * dx + dy * dy; + if (lengthSquared === 0) { + return Math.hypot(px - ax, py - ay); + } + + const t = Math.max(0, Math.min(1, ((px - ax) * dx + (py - ay) * dy) / lengthSquared)); + const x = ax + t * dx; + const y = ay + t * dy; + return Math.hypot(px - x, py - y); +} + +function writeHollowXIcon(size, outPath) { + const margin = Math.max(3, Math.round(size * 0.22)); + const outerRadius = Math.max(2, Math.round(size * 0.11)); + const innerRadius = Math.max(1, Math.round(size * 0.052)); + const rows = []; + + for (let y = 0; y < size; y += 1) { + const row = Buffer.alloc(1 + size * 4); + row[0] = 0; + for (let x = 0; x < size; x += 1) { + const px = x + 0.5; + const py = y + 0.5; + const d1 = distanceToSegment(px, py, margin, margin, size - margin, size - margin); + const d2 = distanceToSegment(px, py, size - margin, margin, margin, size - margin); + const distance = Math.min(d1, d2); + const color = distance <= outerRadius && distance > innerRadius ? 0 : 255; + const offset = 1 + x * 4; + row[offset] = color; + row[offset + 1] = color; + row[offset + 2] = color; + row[offset + 3] = 255; + } + rows.push(row); + } + + const header = Buffer.alloc(13); + header.writeUInt32BE(size, 0); + header.writeUInt32BE(size, 4); + header[8] = 8; + header[9] = 6; + header[10] = 0; + header[11] = 0; + header[12] = 0; + + const png = Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + pngChunk("IHDR", header), + pngChunk("IDAT", zlib.deflateSync(Buffer.concat(rows))), + pngChunk("IEND", Buffer.alloc(0)), + ]); + + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + fs.writeFileSync(outPath, png); +} + +function patchFile(relativePath, patcher) { + const file = path.join(extensionDir, relativePath); + if (!fs.existsSync(file)) { + return; + } + const previous = fs.readFileSync(file, "utf8"); + const next = patcher(previous); + if (next !== previous) { + fs.writeFileSync(file, next); + } +} + +function normalizeSource(source) { + return source + .replaceAll("https://api.xmem.in", "http://localhost:8000") + .replaceAll( + "new XMemClient(API_BASE_URL, config.apiKey, config.userId)", + "new XMemClient(API_BASE_URL, config.apiKey)", + ) + .replaceAll(".replace(/[^\\\\w.\\\\-@]+/g, '_')", ".replace(/[^A-Za-z0-9_.@-]+/g, '_')"); +} + +function ensureApiNormalizeUserId(source) { + if (!source.includes("function normalizeUserId")) { + source = source.replace( + /(const API_BASE_URL = 'http:\/\/localhost:8000';\r?\n)/, + `$1 +function normalizeUserId(userId: string): string { + const normalized = (userId || '') + .trim() + .replace(/[^A-Za-z0-9_.@-]+/g, '_') + .replace(/^_+|_+$/g, ''); + return normalized || 'xmem-local-user'; +} +`, + ); + } + + return source.replaceAll( + "userId: data.xmem_user_id || '',", + "userId: normalizeUserId(data.xmem_user_id || ''),", + ); +} + +function ensureBackgroundNormalizeUserId(source) { + if (!source.includes("function normalizeUserId")) { + source = source.replace( + /(interface XMemConfig \{\r?\n apiKey: string;\r?\n userId: string;\r?\n\}\r?\n)/, + `$1 +function normalizeUserId(userId: string): string { + const normalized = (userId || '') + .trim() + .replace(/[^A-Za-z0-9_.@-]+/g, '_') + .replace(/^_+|_+$/g, ''); + return normalized || 'xmem-local-user'; +} +`, + ); + } + + return source.replaceAll( + "userId: data.xmem_user_id || '',", + "userId: normalizeUserId(data.xmem_user_id || ''),", + ); +} + +function patchValidateCredentials(source) { + const replacement = `export async function validateCredentials(apiKey: string, username: string): Promise { + const url = \`\${API_BASE_URL}/auth/verify-key\`; + try { + const response = await fetch(url, { + headers: { + 'Authorization': \`Bearer \${apiKey}\` + } + }); + + if (!response.ok) { + console.log('[XMem] Validation failed: HTTP', response.status); + return false; + } + + const data = await response.json(); + console.log('[XMem] Validated user data:', data); + + // Local dev static keys do not always map to a real username. If the local + // API accepted the key, allow any non-empty local user id from the popup. + if (API_BASE_URL.includes('localhost') || API_BASE_URL.includes('127.0.0.1')) { + return Boolean(username && username.trim()); + } + + return Boolean(data.username && data.username.toLowerCase() === username.toLowerCase()); + } catch (err) { + console.error('[XMem] Credential validation network error:', err); + return false; + } +} + +//`; + + return source.replace(/export async function validateCredentials[\s\S]*?\r?\n}\r?\n\r?\n\/\//, replacement); +} + +const apiFile = path.join(extensionDir, "src", "api.ts"); +if (!fs.existsSync(apiFile)) { + throw new Error(`Could not find extension API file at ${apiFile}`); +} + +const iconDir = path.join(extensionDir, "icons"); +writeHollowXIcon(16, path.join(iconDir, "icon16.png")); +writeHollowXIcon(48, path.join(iconDir, "icon48.png")); +writeHollowXIcon(128, path.join(iconDir, "icon128.png")); +writeHollowXIcon(128, path.join(iconDir, "logo.png")); + +for (const file of ["src/api.ts", "src/background.ts", "src/content.ts"]) { + patchFile(file, normalizeSource); +} + +patchFile("src/api.ts", (source) => patchValidateCredentials(ensureApiNormalizeUserId(source))); +patchFile("src/background.ts", ensureBackgroundNormalizeUserId); + +console.log("[xmem] Patched extension API for http://localhost:8000"); diff --git a/scripts/patch-extension-local.ps1 b/scripts/patch-extension-local.ps1 index ab46d0d..b577799 100644 --- a/scripts/patch-extension-local.ps1 +++ b/scripts/patch-extension-local.ps1 @@ -9,153 +9,8 @@ if (-not $ExtensionDir) { $ExtensionDir = Join-Path $Root "repos\xmem-extension" } -$apiFile = Join-Path $ExtensionDir "src\api.ts" -if (-not (Test-Path $apiFile)) { - throw "Could not find extension API file at $apiFile" +$scriptPath = Join-Path $PSScriptRoot "patch-extension-local.js" +& node $scriptPath --extension-dir $ExtensionDir +if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE } - -function New-HollowXIcon { - param( - [int]$Size, - [string]$Path - ) - - Add-Type -AssemblyName System.Drawing - $bitmap = New-Object System.Drawing.Bitmap $Size, $Size - $graphics = [System.Drawing.Graphics]::FromImage($bitmap) - $graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias - $graphics.Clear([System.Drawing.Color]::White) - - $margin = [Math]::Max(3, [int]($Size * 0.22)) - $outerWidth = [Math]::Max(4, [int]($Size * 0.22)) - $innerWidth = [Math]::Max(2, [int]($Size * 0.105)) - - $outerPen = New-Object System.Drawing.Pen ([System.Drawing.Color]::Black), $outerWidth - $outerPen.StartCap = [System.Drawing.Drawing2D.LineCap]::Round - $outerPen.EndCap = [System.Drawing.Drawing2D.LineCap]::Round - $outerPen.LineJoin = [System.Drawing.Drawing2D.LineJoin]::Round - - $innerPen = New-Object System.Drawing.Pen ([System.Drawing.Color]::White), $innerWidth - $innerPen.StartCap = [System.Drawing.Drawing2D.LineCap]::Round - $innerPen.EndCap = [System.Drawing.Drawing2D.LineCap]::Round - $innerPen.LineJoin = [System.Drawing.Drawing2D.LineJoin]::Round - - $graphics.DrawLine($outerPen, $margin, $margin, $Size - $margin, $Size - $margin) - $graphics.DrawLine($outerPen, $Size - $margin, $margin, $margin, $Size - $margin) - $graphics.DrawLine($innerPen, $margin, $margin, $Size - $margin, $Size - $margin) - $graphics.DrawLine($innerPen, $Size - $margin, $margin, $margin, $Size - $margin) - - $bitmap.Save($Path, [System.Drawing.Imaging.ImageFormat]::Png) - $outerPen.Dispose() - $innerPen.Dispose() - $graphics.Dispose() - $bitmap.Dispose() -} - -$iconDir = Join-Path $ExtensionDir "icons" -New-Item -ItemType Directory -Force -Path $iconDir | Out-Null -New-HollowXIcon -Size 16 -Path (Join-Path $iconDir "icon16.png") -New-HollowXIcon -Size 48 -Path (Join-Path $iconDir "icon48.png") -New-HollowXIcon -Size 128 -Path (Join-Path $iconDir "icon128.png") -New-HollowXIcon -Size 128 -Path (Join-Path $iconDir "logo.png") -$sourceFiles = @( - "src\api.ts", - "src\background.ts", - "src\content.ts" -) - -foreach ($relativePath in $sourceFiles) { - $sourceFile = Join-Path $ExtensionDir $relativePath - if (Test-Path $sourceFile) { - $source = Get-Content -Raw -Path $sourceFile - $source = $source.Replace("https://api.xmem.in", "http://localhost:8000") - $source = $source.Replace( - "new XMemClient(API_BASE_URL, config.apiKey, config.userId)", - "new XMemClient(API_BASE_URL, config.apiKey)" - ) - $source = $source.Replace( - ".replace(/[^\\w.\\-@]+/g, '_')", - ".replace(/[^A-Za-z0-9_.@-]+/g, '_')" - ) - Set-Content -Path $sourceFile -Value $source -NoNewline - } -} - -$content = Get-Content -Raw -Path $apiFile - -if ($content -notmatch "function normalizeUserId") { - $content = [regex]::Replace( - $content, - "(const API_BASE_URL = 'http://localhost:8000';\r?\n)", - "`$1`r`nfunction normalizeUserId(userId: string): string {`r`n const normalized = (userId || '')`r`n .trim()`r`n .replace(/[^A-Za-z0-9_.@-]+/g, '_')`r`n .replace(/^_+|_+$/g, '');`r`n return normalized || 'xmem-local-user';`r`n}`r`n", - 1 - ) -} - -$content = $content.Replace( - "userId: data.xmem_user_id || '',", - "userId: normalizeUserId(data.xmem_user_id || '')," -) - -$backgroundFile = Join-Path $ExtensionDir "src\background.ts" -if (Test-Path $backgroundFile) { - $background = Get-Content -Raw -Path $backgroundFile - if ($background -notmatch "function normalizeUserId") { - $background = [regex]::Replace( - $background, - "(interface XMemConfig \{\r?\n apiKey: string;\r?\n userId: string;\r?\n\}\r?\n)", - "`$1`r`nfunction normalizeUserId(userId: string): string {`r`n const normalized = (userId || '')`r`n .trim()`r`n .replace(/[^A-Za-z0-9_.@-]+/g, '_')`r`n .replace(/^_+|_+$/g, '');`r`n return normalized || 'xmem-local-user';`r`n}`r`n", - 1 - ) - } - - $background = $background.Replace( - "userId: data.xmem_user_id || '',", - "userId: normalizeUserId(data.xmem_user_id || '')," - ) - Set-Content -Path $backgroundFile -Value $background -NoNewline -} - -$replacement = @' -export async function validateCredentials(apiKey: string, username: string): Promise { - const url = `${API_BASE_URL}/auth/verify-key`; - try { - const response = await fetch(url, { - headers: { - 'Authorization': `Bearer ${apiKey}` - } - }); - - if (!response.ok) { - console.log('[XMem] Validation failed: HTTP', response.status); - return false; - } - - const data = await response.json(); - console.log('[XMem] Validated user data:', data); - - // Local dev static keys do not always map to a real username. If the local - // API accepted the key, allow any non-empty local user id from the popup. - if (API_BASE_URL.includes('localhost') || API_BASE_URL.includes('127.0.0.1')) { - return Boolean(username && username.trim()); - } - - return Boolean(data.username && data.username.toLowerCase() === username.toLowerCase()); - } catch (err) { - console.error('[XMem] Credential validation network error:', err); - return false; - } -} - -// -'@ - -$pattern = "export async function validateCredentials[\s\S]*?\r?\n}\r?\n\r?\n//" -$patched = [regex]::Replace($content, $pattern, $replacement, 1) - -if ($patched -eq $content -and $content -notmatch "http://localhost:8000") { - throw "Extension patch did not apply." -} - -Set-Content -Path $apiFile -Value $patched -NoNewline -Write-Host "[xmem] Patched extension API for http://localhost:8000" diff --git a/scripts/patch-extension-local.sh b/scripts/patch-extension-local.sh new file mode 100755 index 0000000..4eadaa1 --- /dev/null +++ b/scripts/patch-extension-local.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +EXTENSION_DIR="$ROOT/repos/xmem-extension" + +while [[ $# -gt 0 ]]; do + case "$1" in + --extension-dir|-ExtensionDir) + EXTENSION_DIR="$2" + shift 2 + ;; + *) + echo "[xmem] Unknown extension patch option: $1" >&2 + exit 1 + ;; + esac +done + +node "$ROOT/scripts/patch-extension-local.js" --extension-dir "$EXTENSION_DIR" diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100755 index 0000000..605eac4 --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REPOS_DIR="$ROOT/repos" +SKIP_DOCKER=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --repos-dir|-ReposDir) + REPOS_DIR="$2" + shift 2 + ;; + --skip-docker|-SkipDocker) + SKIP_DOCKER=1 + shift + ;; + *) + echo "[xmem] Unknown start option: $1" >&2 + exit 1 + ;; + esac +done + +has_command() { + command -v "$1" >/dev/null 2>&1 +} + +env_value() { + local key="$1" + local default="$2" + if [[ ! -f "$ROOT/.env" ]]; then + echo "$default" + return + fi + local value + value="$(grep -E "^[[:space:]]*$key[[:space:]]*=" "$ROOT/.env" | tail -n 1 | sed -E "s/^[^=]+=//; s/^['\"]//; s/['\"]$//")" + if [[ -n "$value" ]]; then + echo "$value" + else + echo "$default" + fi +} + +uses_ollama() { + [[ ! -f "$ROOT/.env" ]] && return 0 + grep -Eq '^[[:space:]]*FALLBACK_ORDER[[:space:]]*=.*ollama' "$ROOT/.env" +} + +docker_running() { + has_command docker && docker info >/dev/null 2>&1 +} + +wait_containers() { + local deadline=$((SECONDS + 180)) + local pending=("$@") + while (( SECONDS < deadline )); do + local next=() + for name in "${pending[@]}"; do + local status + status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$name" 2>/dev/null || true)" + if [[ "$status" == "healthy" || "$status" == "running" ]]; then + continue + fi + if [[ "$status" == "unhealthy" ]]; then + echo "[xmem] Container $name is unhealthy. Run npm run doctor or inspect it with: docker logs $name" >&2 + exit 1 + fi + next+=("$name") + done + + if [[ ${#next[@]} -eq 0 ]]; then + return + fi + + echo "[xmem] Waiting for local database containers: ${next[*]}" + pending=("${next[@]}") + sleep 5 + done + + echo "[xmem] Timed out waiting for local database containers: ${pending[*]}. Run npm run doctor for details." >&2 + exit 1 +} + +if [[ ! -f "$ROOT/.env" ]]; then + echo "[xmem] XMem .env not found at $ROOT/.env. Run npm run setup first." >&2 + exit 1 +fi + +bash "$ROOT/scripts/configure-xmem-env.sh" --env-path "$ROOT/.env" + +if uses_ollama; then + if ! has_command ollama || ! ollama list >/dev/null 2>&1; then + echo "[xmem] XMem is configured to use local Ollama, but Ollama is not running." >&2 + echo "[xmem] Start Ollama, or add a cloud LLM key to .env and rerun." >&2 + exit 2 + fi + + installed="$(ollama list 2>/dev/null | tail -n +2 || true)" + for model in "$(env_value OLLAMA_MODEL qwen2.5:1.5b)" "$(env_value OLLAMA_EMBEDDING_MODEL nomic-embed-text)"; do + [[ -z "$model" ]] && continue + if ! printf "%s\n" "$installed" | grep -Eq "^$(printf '%s' "$model" | sed 's/[][\.^$*+?{}|()]/\\&/g')([[:space:]]|:latest[[:space:]])"; then + echo "[xmem] Ollama model $model is missing. Run: ollama pull $model" >&2 + exit 2 + fi + done +fi + +if [[ "$SKIP_DOCKER" != "1" ]]; then + if ! docker_running; then + echo "[xmem] Docker Desktop is installed but not running, or Docker was not found." >&2 + echo "[xmem] Start Docker Desktop, wait until it says Docker is running, then rerun npm run dev." >&2 + exit 2 + fi + docker compose -f "$ROOT/docker-compose.local.yml" up -d --remove-orphans + wait_containers xmem-postgres xmem-mongo xmem-neo4j +fi + +PYTHON_BIN="$ROOT/.venv/bin/python" +if [[ ! -x "$PYTHON_BIN" ]]; then + if has_command python3; then + PYTHON_BIN=python3 + elif has_command python; then + PYTHON_BIN=python + else + echo "[xmem] python3 is required. Install Python 3.11+ and rerun." >&2 + exit 1 + fi +fi + +cd "$ROOT" +echo "[xmem] Starting XMem API at http://localhost:8000" +exec "$PYTHON_BIN" -m uvicorn src.api.app:create_app --factory --host 0.0.0.0 --port 8000 diff --git a/scripts/verify.py b/scripts/verify.py new file mode 100644 index 0000000..cf53825 --- /dev/null +++ b/scripts/verify.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import argparse +import json +import sys +import time +import urllib.error +import urllib.request +from typing import Any + + +def log(message: str) -> None: + print(f"[xmem] {message}") + + +def request_json( + url: str, + method: str = "GET", + headers: dict[str, str] | None = None, + body: dict[str, Any] | None = None, + timeout: int = 60, +) -> dict[str, Any]: + payload = json.dumps(body).encode("utf-8") if body is not None else None + req = urllib.request.Request(url, data=payload, headers=headers or {}, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + text = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"Request failed: {method} {url}\nHTTP {exc.code}\n{text}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"Request failed: {method} {url}\n{exc}") from exc + + +def health_ready(health: dict[str, Any]) -> bool: + data = health.get("data") or health + return bool(data.get("pipelines_ready")) + + +def health_summary(health: dict[str, Any]) -> str: + data = health.get("data") or health + return ( + f"status={data.get('status')}, " + f"pipelines_ready={data.get('pipelines_ready')}, " + f"error={data.get('error')}" + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Verify a local XMem API") + parser.add_argument("--base-url", "-BaseUrl", default="http://localhost:8000") + parser.add_argument("--api-key", "-ApiKey", default="dev-xmem-key") + parser.add_argument("--user-id", "-UserId", default="xmem-local-user") + parser.add_argument("--timeout-seconds", "-TimeoutSeconds", type=int, default=180) + args = parser.parse_args() + + deadline = time.time() + args.timeout_seconds + health: dict[str, Any] | None = None + log(f"Waiting for API health at {args.base_url}/health") + while time.time() < deadline: + try: + health = request_json(f"{args.base_url}/health", timeout=10) + if health_ready(health): + break + except Exception: + time.sleep(3) + + if not health: + raise RuntimeError(f"XMem API did not become reachable within {args.timeout_seconds} seconds.") + + log(f"Health: {health_summary(health)}") + if not health_ready(health): + raise RuntimeError("XMem API is reachable but pipelines are not ready.") + + headers = { + "Authorization": f"Bearer {args.api_key}", + "Content-Type": "application/json", + } + + log("Ingesting a smoke-test memory") + ingest = request_json( + f"{args.base_url}/v1/memory/ingest", + method="POST", + headers=headers, + body={ + "user_query": "Remember that XMem local mode runs directly from the main XMem repository.", + "agent_response": "Got it. I will remember that XMem local mode runs from the main repository.", + "user_id": args.user_id, + "effort_level": "low", + }, + timeout=650, + ) + log(f"Ingest status: {ingest.get('status')}") + + log("Searching memory") + search = request_json( + f"{args.base_url}/v1/memory/search", + method="POST", + headers=headers, + body={ + "query": "What is XMem local mode?", + "user_id": args.user_id, + "domains": ["profile", "temporal", "summary"], + "top_k": 5, + }, + timeout=180, + ) + result_count = len((search.get("data") or {}).get("results") or []) + log(f"Search result count: {result_count}") + + log("Retrieving answer") + retrieve = request_json( + f"{args.base_url}/v1/memory/retrieve", + method="POST", + headers=headers, + body={ + "query": "Where does XMem local mode run from?", + "user_id": args.user_id, + "top_k": 5, + }, + timeout=240, + ) + print("\nAnswer:") + print((retrieve.get("data") or {}).get("answer")) + print("") + log("Verification complete") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except KeyboardInterrupt: + raise SystemExit(130) + except Exception as exc: + print(str(exc), file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/verify.sh b/scripts/verify.sh new file mode 100755 index 0000000..bb9e886 --- /dev/null +++ b/scripts/verify.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PYTHON_BIN="$ROOT/.venv/bin/python" + +if [[ ! -x "$PYTHON_BIN" ]]; then + if command -v python3 >/dev/null 2>&1; then + PYTHON_BIN=python3 + elif command -v python >/dev/null 2>&1; then + PYTHON_BIN=python + else + echo "[xmem] python3 is required. Install Python 3.11+ and rerun." >&2 + exit 1 + fi +fi + +"$PYTHON_BIN" "$ROOT/scripts/verify.py" "$@" diff --git a/scripts/xmem.js b/scripts/xmem.js index 834317c..6278053 100644 --- a/scripts/xmem.js +++ b/scripts/xmem.js @@ -9,13 +9,13 @@ const command = process.argv[2] || "help"; const passthroughArgs = process.argv.slice(3); const commands = { - setup: "install.ps1", - start: "start.ps1", - verify: "verify.ps1", - doctor: "doctor.ps1", - "context:export": "context-export.ps1", - "context:import": "context-import.ps1", - "context:sync": "context-sync.ps1", + setup: "install", + start: "start", + verify: "verify", + doctor: "doctor", + "context:export": "context-export", + "context:import": "context-import", + "context:sync": "context-sync", }; function log(message) { @@ -58,15 +58,28 @@ function powershellArgs(scriptPath, extraArgs) { return args; } -function runPowerShellScript(scriptName, extraArgs = []) { - const scriptPath = path.join(root, "scripts", scriptName); +function shellArgs(scriptPath, extraArgs) { + return [scriptPath, ...extraArgs]; +} + +function runScript(scriptName, extraArgs = []) { + const extension = process.platform === "win32" ? ".ps1" : ".sh"; + const executable = + process.platform === "win32" + ? powershellExecutable() + : process.env.XMEM_SHELL || "bash"; + const scriptPath = path.join(root, "scripts", `${scriptName}${extension}`); if (!fs.existsSync(scriptPath)) { console.error(`[xmem] Missing script: ${scriptPath}`); process.exit(1); } - const executable = powershellExecutable(); - const result = spawnSync(executable, powershellArgs(scriptPath, extraArgs), { + const args = + process.platform === "win32" + ? powershellArgs(scriptPath, extraArgs) + : shellArgs(scriptPath, extraArgs); + + const result = spawnSync(executable, args, { cwd: root, stdio: "inherit", shell: false, @@ -76,7 +89,7 @@ function runPowerShellScript(scriptName, extraArgs = []) { const installHint = process.platform === "win32" ? "PowerShell should be available on Windows. Reopen the terminal and try again." - : "Install PowerShell 7+ (`pwsh`) and try again."; + : "Install bash, or set XMEM_SHELL to a compatible shell."; console.error(`[xmem] Could not start ${executable}: ${result.error.message}`); console.error(`[xmem] ${installHint}`); process.exit(1); @@ -137,13 +150,13 @@ function runDev() { if (!setupLooksComplete(reposDir)) { log("First run detected; running setup before starting XMem."); - const setupStatus = runPowerShellScript(commands.setup, passthroughArgs); + const setupStatus = runScript(commands.setup, passthroughArgs); if (setupStatus !== 0) { process.exit(setupStatus); } } - return runPowerShellScript(commands.start, startCompatibleArgs(passthroughArgs)); + return runScript(commands.start, startCompatibleArgs(passthroughArgs)); } if (command === "help" || command === "--help" || command === "-h") { @@ -153,7 +166,7 @@ if (command === "help" || command === "--help" || command === "-h") { if (command === "dev") { runDev(); } else if (commands[command]) { - runPowerShellScript(commands[command], passthroughArgs); + runScript(commands[command], passthroughArgs); } else { console.error(`[xmem] Unknown command: ${command}`); usage(1); From 60c2f3784f0b63fa4209e9f19940b8563b856a37 Mon Sep 17 00:00:00 2001 From: Ankit Kotnala Date: Sat, 23 May 2026 16:47:34 +0530 Subject: [PATCH 3/6] Polish cross-platform local setup --- .gitattributes | 17 + scripts/configure-xmem-env.ps1 | 174 ------- scripts/configure-xmem-env.sh | 150 ------ scripts/context-export.ps1 | 20 - scripts/context-export.sh | 12 - scripts/context-import.ps1 | 20 - scripts/context-import.sh | 12 - scripts/context-sync.ps1 | 20 - scripts/context-sync.sh | 12 - scripts/doctor.ps1 | 155 ------ scripts/doctor.sh | 161 ------ scripts/install.ps1 | 274 ---------- scripts/install.sh | 246 --------- scripts/patch-extension-local.ps1 | 16 - scripts/patch-extension-local.sh | 20 - scripts/start.ps1 | 230 --------- scripts/start.sh | 133 ----- scripts/verify.ps1 | 140 ------ scripts/verify.sh | 18 - scripts/xmem.js | 799 ++++++++++++++++++++++++++---- templates/xmem.env.local | 2 +- 21 files changed, 726 insertions(+), 1905 deletions(-) create mode 100644 .gitattributes delete mode 100644 scripts/configure-xmem-env.ps1 delete mode 100755 scripts/configure-xmem-env.sh delete mode 100644 scripts/context-export.ps1 delete mode 100755 scripts/context-export.sh delete mode 100644 scripts/context-import.ps1 delete mode 100755 scripts/context-import.sh delete mode 100644 scripts/context-sync.ps1 delete mode 100755 scripts/context-sync.sh delete mode 100644 scripts/doctor.ps1 delete mode 100755 scripts/doctor.sh delete mode 100644 scripts/install.ps1 delete mode 100755 scripts/install.sh delete mode 100644 scripts/patch-extension-local.ps1 delete mode 100755 scripts/patch-extension-local.sh delete mode 100644 scripts/start.ps1 delete mode 100755 scripts/start.sh delete mode 100644 scripts/verify.ps1 delete mode 100755 scripts/verify.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..0a6b31d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,17 @@ +* text=auto + +*.js text eol=lf +*.json text eol=lf +*.py text eol=lf +*.md text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.toml text eol=lf +*.env text eol=lf +*.env.* text eol=lf + +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.ico binary diff --git a/scripts/configure-xmem-env.ps1 b/scripts/configure-xmem-env.ps1 deleted file mode 100644 index 236d635..0000000 --- a/scripts/configure-xmem-env.ps1 +++ /dev/null @@ -1,174 +0,0 @@ -param( - [string]$EnvPath = "", - [switch]$Quiet -) - -$ErrorActionPreference = "Stop" - -function Write-Step { - param([string]$Message) - if (-not $Quiet) { - Write-Host "[xmem] $Message" - } -} - -function Get-DotEnvValue { - param( - [string]$Path, - [string]$Name - ) - - if (-not (Test-Path $Path)) { - return "" - } - - $pattern = "^\s*$([regex]::Escape($Name))\s*=\s*(.*)\s*$" - foreach ($line in Get-Content -Path $Path) { - if ($line -match $pattern) { - $value = $Matches[1].Trim() - if ( - ($value.StartsWith('"') -and $value.EndsWith('"')) -or - ($value.StartsWith("'") -and $value.EndsWith("'")) - ) { - $value = $value.Substring(1, $value.Length - 2) - } - return $value.Trim() - } - } - - return "" -} - -function Set-DotEnvValue { - param( - [string]$Path, - [string]$Name, - [string]$Value - ) - - $lines = @() - if (Test-Path $Path) { - $lines = @(Get-Content -Path $Path) - } - - $pattern = "^\s*$([regex]::Escape($Name))\s*=" - $updated = $false - $next = foreach ($line in $lines) { - if ($line -match $pattern) { - $updated = $true - "$Name=$Value" - } else { - $line - } - } - - if (-not $updated) { - $next += "$Name=$Value" - } - - Set-Content -Path $Path -Value $next -} - -function Test-SecretValue { - param([string]$Value) - - if (-not $Value) { - return $false - } - - $trimmed = $Value.Trim() - if (-not $trimmed) { - return $false - } - - $placeholderPatterns = @( - "^your[_-]", - "your_.*_key", - "example", - "sample", - "placeholder", - "change[-_]?me", - "^dummy([-_].*)?$", - "^fake([-_].*)?$", - "^test([-_].*)?$" - ) - - foreach ($pattern in $placeholderPatterns) { - if ($trimmed -match $pattern) { - return $false - } - } - - return $true -} - -function Get-ConfiguredValue { - param( - [string]$Path, - [string]$Name - ) - - $envValue = [Environment]::GetEnvironmentVariable($Name) - if (Test-SecretValue $envValue) { - return $envValue - } - - $fileValue = Get-DotEnvValue -Path $Path -Name $Name - if (Test-SecretValue $fileValue) { - return $fileValue - } - - return "" -} - -if (-not $EnvPath) { - $Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path - $EnvPath = Join-Path $Root ".env" -} - -if (-not (Test-Path $EnvPath)) { - throw "XMem .env not found at $EnvPath" -} - -$providers = @() -if (Get-ConfiguredValue -Path $EnvPath -Name "OPENROUTER_API_KEY") { - $providers += "openrouter" -} -if (Get-ConfiguredValue -Path $EnvPath -Name "GEMINI_API_KEY") { - $providers += "gemini" -} -if (Get-ConfiguredValue -Path $EnvPath -Name "CLAUDE_API_KEY") { - $providers += "claude" -} -if (Get-ConfiguredValue -Path $EnvPath -Name "OPENAI_API_KEY") { - $providers += "openai" -} - -$awsAccessKey = Get-ConfiguredValue -Path $EnvPath -Name "AWS_ACCESS_KEY_ID" -$awsSecretKey = Get-ConfiguredValue -Path $EnvPath -Name "AWS_SECRET_ACCESS_KEY" -if ($awsAccessKey -and $awsSecretKey) { - $providers += "bedrock" -} - -if ($providers.Count -gt 0) { - $providerJson = "[" + (($providers | ForEach-Object { '"' + $_ + '"' }) -join ",") + "]" - Set-DotEnvValue -Path $EnvPath -Name "FALLBACK_ORDER" -Value "'$providerJson'" - - # Keep embeddings local and non-Ollama when a cloud LLM key is available. - Set-DotEnvValue -Path $EnvPath -Name "EMBEDDING_PROVIDER" -Value "fastembed" - Set-DotEnvValue -Path $EnvPath -Name "FASTEMBED_MODEL" -Value "BAAI/bge-small-en-v1.5" - Set-DotEnvValue -Path $EnvPath -Name "EMBEDDING_MODEL" -Value "BAAI/bge-small-en-v1.5" - Set-DotEnvValue -Path $EnvPath -Name "PINECONE_DIMENSION" -Value "384" - - Write-Step "Detected cloud LLM provider(s): $($providers -join ', ')" - Write-Step "Configured XMem to avoid Ollama for LLM and embedding calls." -} else { - Set-DotEnvValue -Path $EnvPath -Name "FALLBACK_ORDER" -Value "'[`"ollama`"]'" - Set-DotEnvValue -Path $EnvPath -Name "EMBEDDING_PROVIDER" -Value "ollama" - Set-DotEnvValue -Path $EnvPath -Name "OLLAMA_EMBEDDING_MODEL" -Value "nomic-embed-text" - Set-DotEnvValue -Path $EnvPath -Name "EMBEDDING_MODEL" -Value "nomic-embed-text" - Set-DotEnvValue -Path $EnvPath -Name "PINECONE_DIMENSION" -Value "768" - - Write-Step "No cloud LLM provider keys detected." - Write-Step "Configured XMem to use local Ollama for LLM and embedding calls." -} diff --git a/scripts/configure-xmem-env.sh b/scripts/configure-xmem-env.sh deleted file mode 100755 index 7d410f3..0000000 --- a/scripts/configure-xmem-env.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -ENV_PATH="$ROOT/.env" -QUIET=0 - -while [[ $# -gt 0 ]]; do - case "$1" in - --env-path|-EnvPath) - ENV_PATH="$2" - shift 2 - ;; - --quiet|-Quiet) - QUIET=1 - shift - ;; - *) - echo "[xmem] Unknown configure option: $1" >&2 - exit 1 - ;; - esac -done - -if command -v python3 >/dev/null 2>&1; then - PYTHON_BIN=python3 -elif command -v python >/dev/null 2>&1; then - PYTHON_BIN=python -else - echo "[xmem] python3 is required. Install Python 3.11+ and rerun." >&2 - exit 1 -fi - -XMEM_ENV_PATH="$ENV_PATH" XMEM_QUIET="$QUIET" "$PYTHON_BIN" - <<'PY' -from __future__ import annotations - -import os -import re -from pathlib import Path - -env_path = Path(os.environ["XMEM_ENV_PATH"]) -quiet = os.environ.get("XMEM_QUIET") == "1" - -if not env_path.exists(): - raise SystemExit(f"XMem .env not found at {env_path}") - - -def log(message: str) -> None: - if not quiet: - print(f"[xmem] {message}") - - -def read_values(path: Path) -> tuple[list[str], dict[str, str]]: - lines = path.read_text(encoding="utf-8").splitlines() - values: dict[str, str] = {} - for line in lines: - if not line.strip() or line.lstrip().startswith("#") or "=" not in line: - continue - key, value = line.split("=", 1) - value = value.strip().strip('"').strip("'") - values[key.strip()] = value.strip() - return lines, values - - -def is_secret(value: str | None) -> bool: - if not value: - return False - value = value.strip() - if not value: - return False - placeholders = [ - r"^your[_-]", - r"your_.*_key", - r"example", - r"sample", - r"placeholder", - r"change[-_]?me", - r"^dummy([-_].*)?$", - r"^fake([-_].*)?$", - r"^test([-_].*)?$", - ] - return not any(re.search(pattern, value, re.IGNORECASE) for pattern in placeholders) - - -def configured(values: dict[str, str], name: str) -> str: - env_value = os.environ.get(name) - if is_secret(env_value): - return env_value or "" - file_value = values.get(name) - if is_secret(file_value): - return file_value or "" - return "" - - -def set_value(lines: list[str], name: str, value: str) -> list[str]: - pattern = re.compile(rf"^\s*{re.escape(name)}\s*=") - replaced = False - next_lines: list[str] = [] - for line in lines: - if pattern.match(line): - next_lines.append(f"{name}={value}") - replaced = True - else: - next_lines.append(line) - if not replaced: - next_lines.append(f"{name}={value}") - return next_lines - - -lines, values = read_values(env_path) -providers: list[str] = [] -for key, provider in [ - ("OPENROUTER_API_KEY", "openrouter"), - ("GEMINI_API_KEY", "gemini"), - ("CLAUDE_API_KEY", "claude"), - ("OPENAI_API_KEY", "openai"), -]: - if configured(values, key): - providers.append(provider) - -if configured(values, "AWS_ACCESS_KEY_ID") and configured(values, "AWS_SECRET_ACCESS_KEY"): - providers.append("bedrock") - -if providers: - quoted = ",".join(f'"{provider}"' for provider in providers) - updates = { - "FALLBACK_ORDER": f"'[{quoted}]'", - "EMBEDDING_PROVIDER": "fastembed", - "FASTEMBED_MODEL": "BAAI/bge-small-en-v1.5", - "EMBEDDING_MODEL": "BAAI/bge-small-en-v1.5", - "PINECONE_DIMENSION": "384", - } - log(f"Detected cloud LLM provider(s): {', '.join(providers)}") - log("Configured XMem to avoid Ollama for LLM and embedding calls.") -else: - updates = { - "FALLBACK_ORDER": "'[\"ollama\"]'", - "EMBEDDING_PROVIDER": "ollama", - "OLLAMA_EMBEDDING_MODEL": "nomic-embed-text", - "EMBEDDING_MODEL": "nomic-embed-text", - "PINECONE_DIMENSION": "768", - } - log("No cloud LLM provider keys detected.") - log("Configured XMem to use local Ollama for LLM and embedding calls.") - -for key, value in updates.items(): - lines = set_value(lines, key, value) - -env_path.write_text("\n".join(lines) + "\n", encoding="utf-8") -PY diff --git a/scripts/context-export.ps1 b/scripts/context-export.ps1 deleted file mode 100644 index 46037c2..0000000 --- a/scripts/context-export.ps1 +++ /dev/null @@ -1,20 +0,0 @@ -param( - [string]$ReposDir = "" -) - -$ErrorActionPreference = "Stop" - -$Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path -if (-not $ReposDir) { - $ReposDir = Join-Path $Root "repos" -} - -$pythonExe = Join-Path $Root ".venv\Scripts\python.exe" -if (-not (Test-Path $pythonExe)) { - throw "XMem virtualenv not found. Run npm run setup first." -} - -& $pythonExe (Join-Path $Root "scripts\context.py") export @args -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} diff --git a/scripts/context-export.sh b/scripts/context-export.sh deleted file mode 100755 index 4f2b3e6..0000000 --- a/scripts/context-export.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -PYTHON_BIN="$ROOT/.venv/bin/python" - -if [[ ! -x "$PYTHON_BIN" ]]; then - echo "[xmem] XMem virtualenv not found. Run npm run setup first." >&2 - exit 1 -fi - -"$PYTHON_BIN" "$ROOT/scripts/context.py" export "$@" diff --git a/scripts/context-import.ps1 b/scripts/context-import.ps1 deleted file mode 100644 index 6879f88..0000000 --- a/scripts/context-import.ps1 +++ /dev/null @@ -1,20 +0,0 @@ -param( - [string]$ReposDir = "" -) - -$ErrorActionPreference = "Stop" - -$Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path -if (-not $ReposDir) { - $ReposDir = Join-Path $Root "repos" -} - -$pythonExe = Join-Path $Root ".venv\Scripts\python.exe" -if (-not (Test-Path $pythonExe)) { - throw "XMem virtualenv not found. Run npm run setup first." -} - -& $pythonExe (Join-Path $Root "scripts\context.py") import @args -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} diff --git a/scripts/context-import.sh b/scripts/context-import.sh deleted file mode 100755 index 3c2dfab..0000000 --- a/scripts/context-import.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -PYTHON_BIN="$ROOT/.venv/bin/python" - -if [[ ! -x "$PYTHON_BIN" ]]; then - echo "[xmem] XMem virtualenv not found. Run npm run setup first." >&2 - exit 1 -fi - -"$PYTHON_BIN" "$ROOT/scripts/context.py" import "$@" diff --git a/scripts/context-sync.ps1 b/scripts/context-sync.ps1 deleted file mode 100644 index 0a7d741..0000000 --- a/scripts/context-sync.ps1 +++ /dev/null @@ -1,20 +0,0 @@ -param( - [string]$ReposDir = "" -) - -$ErrorActionPreference = "Stop" - -$Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path -if (-not $ReposDir) { - $ReposDir = Join-Path $Root "repos" -} - -$pythonExe = Join-Path $Root ".venv\Scripts\python.exe" -if (-not (Test-Path $pythonExe)) { - throw "XMem virtualenv not found. Run npm run setup first." -} - -& $pythonExe (Join-Path $Root "scripts\context.py") sync @args -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} diff --git a/scripts/context-sync.sh b/scripts/context-sync.sh deleted file mode 100755 index 5a19bef..0000000 --- a/scripts/context-sync.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -PYTHON_BIN="$ROOT/.venv/bin/python" - -if [[ ! -x "$PYTHON_BIN" ]]; then - echo "[xmem] XMem virtualenv not found. Run npm run setup first." >&2 - exit 1 -fi - -"$PYTHON_BIN" "$ROOT/scripts/context.py" sync "$@" diff --git a/scripts/doctor.ps1 b/scripts/doctor.ps1 deleted file mode 100644 index c8805f9..0000000 --- a/scripts/doctor.ps1 +++ /dev/null @@ -1,155 +0,0 @@ -param( - [string]$BaseUrl = "http://localhost:8000", - [string]$ReposDir = "" -) - -$ErrorActionPreference = "Stop" - -function Write-Check { - param( - [string]$Name, - [bool]$Ok, - [string]$Message, - [string]$Fix = "" - ) - - $label = if ($Ok) { "OK" } else { "FIX" } - $color = if ($Ok) { "Green" } else { "Yellow" } - Write-Host "[$label] $Name - $Message" -ForegroundColor $color - if (-not $Ok -and $Fix) { - Write-Host " $Fix" - } -} - -function Test-CommandExists { - param([string]$Name) - return [bool](Get-Command $Name -ErrorAction SilentlyContinue) -} - -function Get-DotEnvValue { - param( - [string]$Path, - [string]$Name, - [string]$Default = "" - ) - - if (-not (Test-Path $Path)) { - return $Default - } - - $pattern = "^\s*$([regex]::Escape($Name))\s*=\s*(.*)\s*$" - foreach ($line in Get-Content -Path $Path) { - if ($line -match $pattern) { - $value = $Matches[1].Trim() - if ( - ($value.StartsWith('"') -and $value.EndsWith('"')) -or - ($value.StartsWith("'") -and $value.EndsWith("'")) - ) { - $value = $value.Substring(1, $value.Length - 2) - } - return $value - } - } - - return $Default -} - -function Test-NativeOk { - param([scriptblock]$Command) - - $oldErrorActionPreference = $ErrorActionPreference - $oldNativePreference = $null - $hasNativePreference = Get-Variable -Name PSNativeCommandUseErrorActionPreference -ErrorAction SilentlyContinue - - try { - $ErrorActionPreference = "Continue" - if ($hasNativePreference) { - $oldNativePreference = $PSNativeCommandUseErrorActionPreference - $PSNativeCommandUseErrorActionPreference = $false - } - $null = & $Command 2>&1 - return ($LASTEXITCODE -eq 0) - } finally { - $ErrorActionPreference = $oldErrorActionPreference - if ($hasNativePreference) { - $PSNativeCommandUseErrorActionPreference = $oldNativePreference - } - } -} - -$Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path -if (-not $ReposDir) { - $ReposDir = Join-Path $Root "repos" -} - -$xmemDir = $Root -$extensionDir = Join-Path $ReposDir "xmem-extension" -$envPath = Join-Path $xmemDir ".env" -$failures = 0 - -Write-Host "[xmem] Doctor report" -Write-Host "" - -foreach ($cmd in @("git", "python", "node", "npm")) { - $ok = Test-CommandExists $cmd - if (-not $ok) { $failures++ } - Write-Check $cmd $ok "command lookup" "Install $cmd and reopen this terminal." -} - -$dockerCommand = Test-CommandExists "docker" -$dockerRunning = $dockerCommand -and (Test-NativeOk { docker info }) -if (-not $dockerCommand -or -not $dockerRunning) { $failures++ } -Write-Check "Docker" $dockerRunning "local database runtime" "Start Docker Desktop, then rerun npm run dev." - -$xmemExists = Test-Path (Join-Path $xmemDir "pyproject.toml") -if (-not $xmemExists) { $failures++ } -Write-Check "XMem repo" $xmemExists $xmemDir "Run this from the XMem repository root." - -$extensionExists = Test-Path $extensionDir -if (-not $extensionExists) { $failures++ } -Write-Check "Extension repo" $extensionExists $extensionDir "Run npm run setup." - -$envExists = Test-Path $envPath -if (-not $envExists) { $failures++ } -Write-Check "XMem .env" $envExists $envPath "Run npm run setup to create it from templates/xmem.env.local." - -if ($envExists) { - $usesOllama = [bool]((Get-Content -Raw -Path $envPath) -match "(?m)^\s*FALLBACK_ORDER\s*=.*ollama") - if ($usesOllama) { - $ollamaCommand = Test-CommandExists "ollama" - $ollamaRunning = $ollamaCommand -and (Test-NativeOk { ollama list }) - if (-not $ollamaCommand -or -not $ollamaRunning) { $failures++ } - Write-Check "Ollama" $ollamaRunning "required because no cloud LLM key is configured" "Start Ollama, or add a cloud LLM key to .env." - - if ($ollamaRunning) { - $chatModel = Get-DotEnvValue -Path $envPath -Name "OLLAMA_MODEL" -Default "qwen2.5:1.5b" - $embeddingModel = Get-DotEnvValue -Path $envPath -Name "OLLAMA_EMBEDDING_MODEL" -Default "nomic-embed-text" - $installed = (& ollama list 2>$null | Select-Object -Skip 1) -join "`n" - foreach ($model in @($chatModel, $embeddingModel)) { - $escaped = [regex]::Escape($model) - $ok = ($installed -match "(?m)^$escaped(\s|:latest\s)") - if (-not $ok) { $failures++ } - Write-Check "Ollama model $model" $ok "local model availability" "Run: ollama pull $model" - } - } - } else { - Write-Check "LLM routing" $true "cloud key detected; Ollama is not required" - } -} - -try { - $health = Invoke-RestMethod -Uri "$BaseUrl/health" -Method GET -TimeoutSec 5 - $ready = if ($health.data) { [bool]$health.data.pipelines_ready } else { [bool]$health.pipelines_ready } - if (-not $ready) { $failures++ } - Write-Check "XMem API" $ready "$BaseUrl/health" "Start it with npm run dev and wait for pipelines_ready=true." -} catch { - $failures++ - Write-Check "XMem API" $false "$BaseUrl is not reachable" "Start it with npm run dev." -} - -Write-Host "" -if ($failures -eq 0) { - Write-Host "[xmem] Everything looks ready." -ForegroundColor Green -} else { - Write-Host "[xmem] Found $failures setup item(s) to fix." -ForegroundColor Yellow -} diff --git a/scripts/doctor.sh b/scripts/doctor.sh deleted file mode 100755 index 669656d..0000000 --- a/scripts/doctor.sh +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -REPOS_DIR="$ROOT/repos" -BASE_URL="http://localhost:8000" -FAILURES=0 - -while [[ $# -gt 0 ]]; do - case "$1" in - --base-url|-BaseUrl) - BASE_URL="$2" - shift 2 - ;; - --repos-dir|-ReposDir) - REPOS_DIR="$2" - shift 2 - ;; - *) - echo "[xmem] Unknown doctor option: $1" >&2 - exit 1 - ;; - esac -done - -check() { - local name="$1" - local ok="$2" - local message="$3" - local fix="${4:-}" - if [[ "$ok" == "1" ]]; then - echo "[OK] $name - $message" - else - echo "[FIX] $name - $message" - [[ -n "$fix" ]] && echo " $fix" - FAILURES=$((FAILURES + 1)) - fi -} - -has_command() { - command -v "$1" >/dev/null 2>&1 -} - -env_value() { - local key="$1" - local default="$2" - if [[ ! -f "$ROOT/.env" ]]; then - echo "$default" - return - fi - local value - value="$(grep -E "^[[:space:]]*$key[[:space:]]*=" "$ROOT/.env" | tail -n 1 | sed -E "s/^[^=]+=//; s/^['\"]//; s/['\"]$//")" - if [[ -n "$value" ]]; then - echo "$value" - else - echo "$default" - fi -} - -uses_ollama() { - [[ ! -f "$ROOT/.env" ]] && return 0 - grep -Eq '^[[:space:]]*FALLBACK_ORDER[[:space:]]*=.*ollama' "$ROOT/.env" -} - -echo "[xmem] Doctor report" -echo "" - -for cmd in git node npm; do - if has_command "$cmd"; then - check "$cmd" 1 "command lookup" - else - check "$cmd" 0 "command lookup" "Install $cmd and reopen this terminal." - fi -done - -if has_command python3 || has_command python; then - check "python" 1 "command lookup" -else - check "python" 0 "command lookup" "Install Python 3.11+ and reopen this terminal." -fi - -if has_command docker && docker info >/dev/null 2>&1; then - check "Docker" 1 "local database runtime" -else - check "Docker" 0 "local database runtime" "Start Docker Desktop, then rerun npm run dev." -fi - -if [[ -f "$ROOT/pyproject.toml" ]]; then - check "XMem repo" 1 "$ROOT" -else - check "XMem repo" 0 "$ROOT" "Run this from the XMem repository root." -fi - -if [[ -d "$REPOS_DIR/xmem-extension" ]]; then - check "Extension repo" 1 "$REPOS_DIR/xmem-extension" -else - check "Extension repo" 0 "$REPOS_DIR/xmem-extension" "Run npm run setup." -fi - -if [[ -f "$ROOT/.env" ]]; then - check "XMem .env" 1 "$ROOT/.env" -else - check "XMem .env" 0 "$ROOT/.env" "Run npm run setup to create it from templates/xmem.env.local." -fi - -if [[ -f "$ROOT/.env" ]]; then - if uses_ollama; then - if has_command ollama && ollama list >/dev/null 2>&1; then - check "Ollama" 1 "required because no cloud LLM key is configured" - installed="$(ollama list 2>/dev/null | tail -n +2 || true)" - for model in "$(env_value OLLAMA_MODEL qwen2.5:1.5b)" "$(env_value OLLAMA_EMBEDDING_MODEL nomic-embed-text)"; do - [[ -z "$model" ]] && continue - if printf "%s\n" "$installed" | grep -Eq "^$(printf '%s' "$model" | sed 's/[][\.^$*+?{}|()]/\\&/g')([[:space:]]|:latest[[:space:]])"; then - check "Ollama model $model" 1 "local model availability" - else - check "Ollama model $model" 0 "local model availability" "Run: ollama pull $model" - fi - done - else - check "Ollama" 0 "required because no cloud LLM key is configured" "Start Ollama, or add a cloud LLM key to .env." - fi - else - check "LLM routing" 1 "cloud key detected; Ollama is not required" - fi -fi - -if has_command python3; then - PYTHON_BIN=python3 -elif has_command python; then - PYTHON_BIN=python -else - PYTHON_BIN="" -fi - -if [[ -n "$PYTHON_BIN" ]]; then - if "$PYTHON_BIN" - "$BASE_URL" <<'PY' >/dev/null 2>&1 -import json -import sys -import urllib.request - -base = sys.argv[1].rstrip("/") -with urllib.request.urlopen(f"{base}/health", timeout=5) as response: - health = json.loads(response.read().decode("utf-8")) -data = health.get("data") or health -raise SystemExit(0 if data.get("pipelines_ready") else 1) -PY - then - check "XMem API" 1 "$BASE_URL/health" - else - check "XMem API" 0 "$BASE_URL is not ready" "Start it with npm run dev and wait for pipelines_ready=true." - fi -else - check "XMem API" 0 "$BASE_URL not checked" "Install Python 3.11+." -fi - -echo "" -if [[ "$FAILURES" -eq 0 ]]; then - echo "[xmem] Everything looks ready." -else - echo "[xmem] Found $FAILURES setup item(s) to fix." -fi diff --git a/scripts/install.ps1 b/scripts/install.ps1 deleted file mode 100644 index f729969..0000000 --- a/scripts/install.ps1 +++ /dev/null @@ -1,274 +0,0 @@ -param( - [string]$ReposDir = "", - [switch]$IncludeMcp, - [switch]$IncludeSdk, - [switch]$SkipModelPull, - [switch]$SkipPythonInstall, - [switch]$SkipNodeInstall, - [switch]$SkipDocker -) - -$ErrorActionPreference = "Stop" - -function Write-Step { - param([string]$Message) - Write-Host "[xmem] $Message" -} - -function Test-Command { - param([string]$Name) - return [bool](Get-Command $Name -ErrorAction SilentlyContinue) -} - -function Invoke-Native { - param([scriptblock]$Command) - & $Command - if ($LASTEXITCODE -ne 0) { - throw "Command failed with exit code $LASTEXITCODE" - } -} - -function Test-DockerRunning { - if (-not (Test-Command "docker")) { - Write-Host "[xmem] Docker was not found." -ForegroundColor Yellow - Write-Host "[xmem] Install Docker Desktop or rerun npm run setup -- -SkipDocker to skip local database startup." - return $false - } - - $oldErrorActionPreference = $ErrorActionPreference - $oldNativePreference = $null - $hasNativePreference = Get-Variable -Name PSNativeCommandUseErrorActionPreference -ErrorAction SilentlyContinue - - try { - $ErrorActionPreference = "Continue" - if ($hasNativePreference) { - $oldNativePreference = $PSNativeCommandUseErrorActionPreference - $PSNativeCommandUseErrorActionPreference = $false - } - - $null = & docker info 2>&1 - $dockerExitCode = $LASTEXITCODE - } finally { - $ErrorActionPreference = $oldErrorActionPreference - if ($hasNativePreference) { - $PSNativeCommandUseErrorActionPreference = $oldNativePreference - } - } - - if ($dockerExitCode -ne 0) { - Write-Host "[xmem] Docker Desktop is installed but not running." -ForegroundColor Yellow - Write-Host "[xmem] Start Docker Desktop, wait until it says Docker is running, then rerun this script." - Write-Host "[xmem] Temporary escape hatch: rerun npm run setup -- -SkipDocker to continue cloning/building without local databases." - return $false - } - - return $true -} - -function Test-OllamaRunning { - if (-not (Test-Command "ollama")) { - Write-Host "[xmem] Ollama was not found." -ForegroundColor Yellow - Write-Host "[xmem] Install Ollama, or add a cloud LLM key to .env and rerun." - return $false - } - - $oldErrorActionPreference = $ErrorActionPreference - $oldNativePreference = $null - $hasNativePreference = Get-Variable -Name PSNativeCommandUseErrorActionPreference -ErrorAction SilentlyContinue - - try { - $ErrorActionPreference = "Continue" - if ($hasNativePreference) { - $oldNativePreference = $PSNativeCommandUseErrorActionPreference - $PSNativeCommandUseErrorActionPreference = $false - } - - $null = & ollama list 2>&1 - $ollamaExitCode = $LASTEXITCODE - } finally { - $ErrorActionPreference = $oldErrorActionPreference - if ($hasNativePreference) { - $PSNativeCommandUseErrorActionPreference = $oldNativePreference - } - } - - if ($ollamaExitCode -ne 0) { - Write-Host "[xmem] Ollama is installed but not running." -ForegroundColor Yellow - Write-Host "[xmem] Start Ollama, or add a cloud LLM key to .env and rerun." - return $false - } - - return $true -} - -function Test-XMemUsesOllama { - param([string]$EnvPath) - if (-not (Test-Path $EnvPath)) { - return $true - } - return [bool]((Get-Content -Raw -Path $EnvPath) -match "(?m)^\s*FALLBACK_ORDER\s*=.*ollama") -} - -function Wait-ContainerHealthy { - param( - [string[]]$ContainerNames, - [int]$TimeoutSeconds = 180 - ) - - $deadline = (Get-Date).AddSeconds($TimeoutSeconds) - $pending = @($ContainerNames) - - while ((Get-Date) -lt $deadline) { - $stillPending = @() - foreach ($name in $pending) { - $status = (& docker inspect --format "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}" $name 2>$null) - if ($LASTEXITCODE -ne 0) { - $stillPending += $name - continue - } - - $status = ($status | Select-Object -First 1).Trim() - if ($status -in @("healthy", "running")) { - continue - } - - if ($status -eq "unhealthy") { - throw "Container $name is unhealthy. Run npm run doctor or inspect it with: docker logs $name" - } - - $stillPending += $name - } - - if ($stillPending.Count -eq 0) { - return - } - - Write-Host "[xmem] Waiting for local database containers: $($stillPending -join ', ')" - $pending = $stillPending - Start-Sleep -Seconds 5 - } - - throw "Timed out waiting for local database containers: $($pending -join ', '). Run npm run doctor for details." -} - -function Sync-Repo { - param( - [string]$Name, - [string]$Url, - [string]$Branch - ) - - $target = Join-Path $ReposDir $Name - if (Test-Path $target) { - if (-not (Test-Path (Join-Path $target ".git"))) { - throw "$target exists but is not a git checkout." - } - Write-Step "Updating $Name" - Invoke-Native { git -C $target reset --hard } - Invoke-Native { git -C $target fetch origin } - Invoke-Native { git -C $target checkout $Branch } - Invoke-Native { git -C $target pull --ff-only origin $Branch } - } else { - Write-Step "Cloning $Name" - Invoke-Native { git clone --branch $Branch $Url $target } - } -} - -$Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path -if (-not $ReposDir) { - $ReposDir = Join-Path $Root "repos" -} - -New-Item -ItemType Directory -Force -Path $ReposDir | Out-Null - -foreach ($cmd in @("git", "python", "node", "npm")) { - if (-not (Test-Command $cmd)) { - throw "$cmd is required. Install it, then run this script again." - } -} - -Sync-Repo "xmem-extension" "https://github.com/XortexAI/xmem-extension.git" "main" - -if ($IncludeMcp) { - Sync-Repo "xmem-mcp" "https://github.com/XortexAI/xmem-mcp.git" "main" -} - -if ($IncludeSdk) { - Sync-Repo "xmem-sdk" "https://github.com/XortexAI/xmem-sdk.git" "master" -} - -$XmemDir = $Root -$ExtensionDir = Join-Path $ReposDir "xmem-extension" - -$envTemplate = Join-Path $Root "templates\xmem.env.local" -$envTarget = Join-Path $XmemDir ".env" -if (-not (Test-Path $envTarget)) { - Copy-Item $envTemplate $envTarget - Write-Step "Created .env from local template" -} else { - Write-Step ".env already exists; leaving it unchanged" -} - -Invoke-Native { powershell -ExecutionPolicy Bypass -File (Join-Path $Root "scripts\configure-xmem-env.ps1") -EnvPath $envTarget } -$usesOllama = Test-XMemUsesOllama -EnvPath $envTarget -$dockerSkipped = $false -$ollamaSkipped = $false - -if (-not $SkipModelPull) { - if ($usesOllama) { - if (Test-OllamaRunning) { - Write-Step "Pulling Ollama chat model" - Invoke-Native { ollama pull qwen2.5:1.5b } - Write-Step "Pulling Ollama embedding model" - Invoke-Native { ollama pull nomic-embed-text } - } else { - $ollamaSkipped = $true - } - } else { - Write-Step "Cloud LLM provider key detected; skipping Ollama model pulls" - } -} - -if (-not $SkipDocker) { - if (Test-DockerRunning) { - Write-Step "Starting local Docker services" - Invoke-Native { docker compose -f (Join-Path $Root "docker-compose.local.yml") up -d --remove-orphans } - Wait-ContainerHealthy -ContainerNames @("xmem-postgres", "xmem-mongo", "xmem-neo4j") - } else { - $dockerSkipped = $true - } -} - -if (-not $SkipPythonInstall) { - $venvPython = Join-Path $XmemDir ".venv\Scripts\python.exe" - if (-not (Test-Path $venvPython)) { - Write-Step "Creating XMem virtualenv" - Invoke-Native { python -m venv (Join-Path $XmemDir ".venv") } - } - Write-Step "Installing XMem local dependencies" - Invoke-Native { & $venvPython -m pip install --upgrade pip } - Invoke-Native { & $venvPython -m pip install -e "$XmemDir[local,dev]" } -} - -Write-Step "Patching extension for local API" -Invoke-Native { powershell -ExecutionPolicy Bypass -File (Join-Path $Root "scripts\patch-extension-local.ps1") -ExtensionDir $ExtensionDir } - -if (-not $SkipNodeInstall) { - Write-Step "Installing and building Chrome extension" - Invoke-Native { npm --prefix $ExtensionDir install } - Invoke-Native { npm --prefix $ExtensionDir run build } -} - -Write-Step "Install complete" -Write-Host "" -Write-Host "Next:" -Write-Host " npm run dev" -Write-Host " npm run verify" -if ($dockerSkipped) { - Write-Host "" - Write-Host "Docker services were not started. Start Docker Desktop before running npm run dev." -ForegroundColor Yellow -} -if ($ollamaSkipped) { - Write-Host "" - Write-Host "Ollama models were not pulled. Start Ollama, then rerun npm run setup or add a cloud LLM key." -ForegroundColor Yellow -} diff --git a/scripts/install.sh b/scripts/install.sh deleted file mode 100755 index 9f49baf..0000000 --- a/scripts/install.sh +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -REPOS_DIR="$ROOT/repos" -INCLUDE_MCP=0 -INCLUDE_SDK=0 -SKIP_MODEL_PULL=0 -SKIP_PYTHON_INSTALL=0 -SKIP_NODE_INSTALL=0 -SKIP_DOCKER=0 - -log() { - echo "[xmem] $*" -} - -has_command() { - command -v "$1" >/dev/null 2>&1 -} - -python_cmd() { - if has_command python3; then - echo "python3" - elif has_command python; then - echo "python" - else - echo "[xmem] python3 is required. Install Python 3.11+ and rerun." >&2 - exit 1 - fi -} - -while [[ $# -gt 0 ]]; do - case "$1" in - --repos-dir|-ReposDir) - REPOS_DIR="$2" - shift 2 - ;; - --include-mcp|-IncludeMcp) - INCLUDE_MCP=1 - shift - ;; - --include-sdk|-IncludeSdk) - INCLUDE_SDK=1 - shift - ;; - --skip-model-pull|-SkipModelPull) - SKIP_MODEL_PULL=1 - shift - ;; - --skip-python-install|-SkipPythonInstall) - SKIP_PYTHON_INSTALL=1 - shift - ;; - --skip-node-install|-SkipNodeInstall) - SKIP_NODE_INSTALL=1 - shift - ;; - --skip-docker|-SkipDocker) - SKIP_DOCKER=1 - shift - ;; - *) - echo "[xmem] Unknown setup option: $1" >&2 - exit 1 - ;; - esac -done - -for cmd in git node npm; do - if ! has_command "$cmd"; then - echo "[xmem] $cmd is required. Install it, then run this script again." >&2 - exit 1 - fi -done - -PYTHON_BIN="$(python_cmd)" -mkdir -p "$REPOS_DIR" - -sync_repo() { - local name="$1" - local url="$2" - local branch="$3" - local target="$REPOS_DIR/$name" - - if [[ -d "$target" ]]; then - if [[ ! -d "$target/.git" ]]; then - echo "[xmem] $target exists but is not a git checkout." >&2 - exit 1 - fi - log "Updating $name" - git -C "$target" reset --hard - git -C "$target" fetch origin - git -C "$target" checkout "$branch" - git -C "$target" pull --ff-only origin "$branch" - else - log "Cloning $name" - git clone --branch "$branch" "$url" "$target" - fi -} - -docker_running() { - has_command docker && docker info >/dev/null 2>&1 -} - -ollama_running() { - has_command ollama && ollama list >/dev/null 2>&1 -} - -uses_ollama() { - [[ ! -f "$ROOT/.env" ]] && return 0 - grep -Eq '^[[:space:]]*FALLBACK_ORDER[[:space:]]*=.*ollama' "$ROOT/.env" -} - -env_value() { - local key="$1" - local default="$2" - if [[ ! -f "$ROOT/.env" ]]; then - echo "$default" - return - fi - local value - value="$(grep -E "^[[:space:]]*$key[[:space:]]*=" "$ROOT/.env" | tail -n 1 | sed -E "s/^[^=]+=//; s/^['\"]//; s/['\"]$//")" - if [[ -n "$value" ]]; then - echo "$value" - else - echo "$default" - fi -} - -wait_containers() { - local deadline=$((SECONDS + 180)) - local pending=("$@") - while (( SECONDS < deadline )); do - local next=() - for name in "${pending[@]}"; do - local status - status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$name" 2>/dev/null || true)" - if [[ "$status" == "healthy" || "$status" == "running" ]]; then - continue - fi - if [[ "$status" == "unhealthy" ]]; then - echo "[xmem] Container $name is unhealthy. Run npm run doctor or inspect it with: docker logs $name" >&2 - exit 1 - fi - next+=("$name") - done - - if [[ ${#next[@]} -eq 0 ]]; then - return - fi - - log "Waiting for local database containers: ${next[*]}" - pending=("${next[@]}") - sleep 5 - done - - echo "[xmem] Timed out waiting for local database containers: ${pending[*]}. Run npm run doctor for details." >&2 - exit 1 -} - -sync_repo "xmem-extension" "https://github.com/XortexAI/xmem-extension.git" "main" - -if [[ "$INCLUDE_MCP" == "1" ]]; then - sync_repo "xmem-mcp" "https://github.com/XortexAI/xmem-mcp.git" "main" -fi - -if [[ "$INCLUDE_SDK" == "1" ]]; then - sync_repo "xmem-sdk" "https://github.com/XortexAI/xmem-sdk.git" "master" -fi - -if [[ ! -f "$ROOT/.env" ]]; then - cp "$ROOT/templates/xmem.env.local" "$ROOT/.env" - log "Created .env from local template" -else - log ".env already exists; leaving it unchanged" -fi - -bash "$ROOT/scripts/configure-xmem-env.sh" --env-path "$ROOT/.env" - -docker_skipped=0 -ollama_skipped=0 - -if [[ "$SKIP_MODEL_PULL" != "1" ]]; then - if uses_ollama; then - if ollama_running; then - log "Pulling Ollama chat model" - ollama pull "$(env_value OLLAMA_MODEL qwen2.5:1.5b)" - log "Pulling Ollama embedding model" - ollama pull "$(env_value OLLAMA_EMBEDDING_MODEL nomic-embed-text)" - else - echo "[xmem] Ollama was not found or is not running." >&2 - echo "[xmem] Start Ollama, or add a cloud LLM key to .env and rerun." >&2 - ollama_skipped=1 - fi - else - log "Cloud LLM provider key detected; skipping Ollama model pulls" - fi -fi - -if [[ "$SKIP_DOCKER" != "1" ]]; then - if docker_running; then - log "Starting local Docker services" - docker compose -f "$ROOT/docker-compose.local.yml" up -d --remove-orphans - wait_containers xmem-postgres xmem-mongo xmem-neo4j - else - echo "[xmem] Docker Desktop is installed but not running, or Docker was not found." >&2 - echo "[xmem] Start Docker Desktop, then rerun this script." >&2 - docker_skipped=1 - fi -fi - -if [[ "$SKIP_PYTHON_INSTALL" != "1" ]]; then - VENV_PYTHON="$ROOT/.venv/bin/python" - if [[ ! -x "$VENV_PYTHON" ]]; then - log "Creating XMem virtualenv" - "$PYTHON_BIN" -m venv "$ROOT/.venv" - fi - log "Installing XMem local dependencies" - "$VENV_PYTHON" -m pip install --upgrade pip - "$VENV_PYTHON" -m pip install -e "$ROOT[local,dev]" -fi - -log "Patching extension for local API" -bash "$ROOT/scripts/patch-extension-local.sh" --extension-dir "$REPOS_DIR/xmem-extension" - -if [[ "$SKIP_NODE_INSTALL" != "1" ]]; then - log "Installing and building Chrome extension" - npm --prefix "$REPOS_DIR/xmem-extension" install - npm --prefix "$REPOS_DIR/xmem-extension" run build -fi - -log "Install complete" -echo "" -echo "Next:" -echo " npm run dev" -echo " npm run verify" - -if [[ "$docker_skipped" == "1" ]]; then - echo "" - echo "[xmem] Docker services were not started. Start Docker Desktop before running npm run dev." >&2 -fi - -if [[ "$ollama_skipped" == "1" ]]; then - echo "" - echo "[xmem] Ollama models were not pulled. Start Ollama, then rerun npm run setup or add a cloud LLM key." >&2 -fi diff --git a/scripts/patch-extension-local.ps1 b/scripts/patch-extension-local.ps1 deleted file mode 100644 index b577799..0000000 --- a/scripts/patch-extension-local.ps1 +++ /dev/null @@ -1,16 +0,0 @@ -param( - [string]$ExtensionDir = "" -) - -$ErrorActionPreference = "Stop" - -if (-not $ExtensionDir) { - $Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path - $ExtensionDir = Join-Path $Root "repos\xmem-extension" -} - -$scriptPath = Join-Path $PSScriptRoot "patch-extension-local.js" -& node $scriptPath --extension-dir $ExtensionDir -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} diff --git a/scripts/patch-extension-local.sh b/scripts/patch-extension-local.sh deleted file mode 100755 index 4eadaa1..0000000 --- a/scripts/patch-extension-local.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -EXTENSION_DIR="$ROOT/repos/xmem-extension" - -while [[ $# -gt 0 ]]; do - case "$1" in - --extension-dir|-ExtensionDir) - EXTENSION_DIR="$2" - shift 2 - ;; - *) - echo "[xmem] Unknown extension patch option: $1" >&2 - exit 1 - ;; - esac -done - -node "$ROOT/scripts/patch-extension-local.js" --extension-dir "$EXTENSION_DIR" diff --git a/scripts/start.ps1 b/scripts/start.ps1 deleted file mode 100644 index f7f4b58..0000000 --- a/scripts/start.ps1 +++ /dev/null @@ -1,230 +0,0 @@ -param( - [string]$ReposDir = "", - [switch]$SkipDocker -) - -$ErrorActionPreference = "Stop" - -function Invoke-Native { - param([scriptblock]$Command) - & $Command - if ($LASTEXITCODE -ne 0) { - throw "Command failed with exit code $LASTEXITCODE" - } -} - -function Test-Command { - param([string]$Name) - return [bool](Get-Command $Name -ErrorAction SilentlyContinue) -} - -function Assert-DockerRunning { - if (-not (Test-Command "docker")) { - Write-Host "[xmem] Docker was not found." -ForegroundColor Yellow - Write-Host "[xmem] Install Docker Desktop or rerun npm run start -- -SkipDocker if local databases are already running elsewhere." - exit 2 - } - - $oldErrorActionPreference = $ErrorActionPreference - $oldNativePreference = $null - $hasNativePreference = Get-Variable -Name PSNativeCommandUseErrorActionPreference -ErrorAction SilentlyContinue - - try { - $ErrorActionPreference = "Continue" - if ($hasNativePreference) { - $oldNativePreference = $PSNativeCommandUseErrorActionPreference - $PSNativeCommandUseErrorActionPreference = $false - } - - $null = & docker info 2>&1 - $dockerExitCode = $LASTEXITCODE - } finally { - $ErrorActionPreference = $oldErrorActionPreference - if ($hasNativePreference) { - $PSNativeCommandUseErrorActionPreference = $oldNativePreference - } - } - - if ($dockerExitCode -ne 0) { - Write-Host "[xmem] Docker Desktop is installed but not running." -ForegroundColor Yellow - Write-Host "[xmem] Start Docker Desktop, wait until it says Docker is running, then rerun npm run dev." - Write-Host "[xmem] Temporary escape hatch: rerun npm run start -- -SkipDocker if local databases are already running elsewhere." - exit 2 - } -} - -function Assert-OllamaRunning { - if (-not (Test-Command "ollama")) { - Write-Host "[xmem] Ollama was not found." -ForegroundColor Yellow - Write-Host "[xmem] Install Ollama, or add a cloud LLM key to .env and rerun." - exit 2 - } - - $oldErrorActionPreference = $ErrorActionPreference - $oldNativePreference = $null - $hasNativePreference = Get-Variable -Name PSNativeCommandUseErrorActionPreference -ErrorAction SilentlyContinue - - try { - $ErrorActionPreference = "Continue" - if ($hasNativePreference) { - $oldNativePreference = $PSNativeCommandUseErrorActionPreference - $PSNativeCommandUseErrorActionPreference = $false - } - - $null = & ollama list 2>&1 - $ollamaExitCode = $LASTEXITCODE - } finally { - $ErrorActionPreference = $oldErrorActionPreference - if ($hasNativePreference) { - $PSNativeCommandUseErrorActionPreference = $oldNativePreference - } - } - - if ($ollamaExitCode -ne 0) { - Write-Host "[xmem] XMem is configured to use local Ollama, but Ollama is not running." -ForegroundColor Yellow - Write-Host "[xmem] Start Ollama, or add a cloud LLM key to .env and rerun." - exit 2 - } -} - -function Test-XMemUsesOllama { - param([string]$EnvPath) - if (-not (Test-Path $EnvPath)) { - return $true - } - return [bool]((Get-Content -Raw -Path $EnvPath) -match "(?m)^\s*FALLBACK_ORDER\s*=.*ollama") -} - -function Get-DotEnvValue { - param( - [string]$Path, - [string]$Name, - [string]$Default = "" - ) - - if (-not (Test-Path $Path)) { - return $Default - } - - $pattern = "^\s*$([regex]::Escape($Name))\s*=\s*(.*)\s*$" - foreach ($line in Get-Content -Path $Path) { - if ($line -match $pattern) { - $value = $Matches[1].Trim() - if ( - ($value.StartsWith('"') -and $value.EndsWith('"')) -or - ($value.StartsWith("'") -and $value.EndsWith("'")) - ) { - $value = $value.Substring(1, $value.Length - 2) - } - if ($value) { - return $value - } - } - } - - return $Default -} - -function Assert-OllamaModels { - param([string]$EnvPath) - - $chatModel = Get-DotEnvValue -Path $EnvPath -Name "OLLAMA_MODEL" -Default "qwen2.5:1.5b" - $embeddingModel = Get-DotEnvValue -Path $EnvPath -Name "OLLAMA_EMBEDDING_MODEL" -Default "nomic-embed-text" - $installed = (& ollama list 2>$null | Select-Object -Skip 1) -join "`n" - $missing = @() - - foreach ($model in @($chatModel, $embeddingModel)) { - if (-not $model) { - continue - } - $escaped = [regex]::Escape($model) - $hasModel = ($installed -match "(?m)^$escaped(\s|:latest\s)") - if (-not $hasModel) { - $missing += $model - } - } - - if ($missing.Count -gt 0) { - Write-Host "[xmem] Ollama is running, but required local model(s) are missing." -ForegroundColor Yellow - foreach ($model in $missing) { - Write-Host "[xmem] Pull it with: ollama pull $model" - } - Write-Host "[xmem] Or add a cloud LLM key to .env so XMem does not use Ollama." - exit 2 - } -} - -function Wait-ContainerHealthy { - param( - [string[]]$ContainerNames, - [int]$TimeoutSeconds = 180 - ) - - $deadline = (Get-Date).AddSeconds($TimeoutSeconds) - $pending = @($ContainerNames) - - while ((Get-Date) -lt $deadline) { - $stillPending = @() - foreach ($name in $pending) { - $status = (& docker inspect --format "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}" $name 2>$null) - if ($LASTEXITCODE -ne 0) { - $stillPending += $name - continue - } - - $status = ($status | Select-Object -First 1).Trim() - if ($status -in @("healthy", "running")) { - continue - } - - if ($status -eq "unhealthy") { - throw "Container $name is unhealthy. Run npm run doctor or inspect it with: docker logs $name" - } - - $stillPending += $name - } - - if ($stillPending.Count -eq 0) { - return - } - - Write-Host "[xmem] Waiting for local database containers: $($stillPending -join ', ')" - $pending = $stillPending - Start-Sleep -Seconds 5 - } - - throw "Timed out waiting for local database containers: $($pending -join ', '). Run npm run doctor for details." -} - -$Root = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path -if (-not $ReposDir) { - $ReposDir = Join-Path $Root "repos" -} - -$XmemDir = $Root - -$envTarget = Join-Path $XmemDir ".env" -if (-not (Test-Path $envTarget)) { - throw "XMem .env not found at $envTarget. Run npm run setup first." -} - -Invoke-Native { powershell -ExecutionPolicy Bypass -File (Join-Path $Root "scripts\configure-xmem-env.ps1") -EnvPath $envTarget } -if (Test-XMemUsesOllama -EnvPath $envTarget) { - Assert-OllamaRunning - Assert-OllamaModels -EnvPath $envTarget -} - -if (-not $SkipDocker) { - Assert-DockerRunning - Invoke-Native { docker compose -f (Join-Path $Root "docker-compose.local.yml") up -d --remove-orphans } - Wait-ContainerHealthy -ContainerNames @("xmem-postgres", "xmem-mongo", "xmem-neo4j") -} - -$pythonExe = Join-Path $XmemDir ".venv\Scripts\python.exe" -if (-not (Test-Path $pythonExe)) { - $pythonExe = "python" -} - -Set-Location $XmemDir -Write-Host "[xmem] Starting XMem API at http://localhost:8000" -Invoke-Native { & $pythonExe -m uvicorn src.api.app:create_app --factory --host 0.0.0.0 --port 8000 } diff --git a/scripts/start.sh b/scripts/start.sh deleted file mode 100755 index 605eac4..0000000 --- a/scripts/start.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -REPOS_DIR="$ROOT/repos" -SKIP_DOCKER=0 - -while [[ $# -gt 0 ]]; do - case "$1" in - --repos-dir|-ReposDir) - REPOS_DIR="$2" - shift 2 - ;; - --skip-docker|-SkipDocker) - SKIP_DOCKER=1 - shift - ;; - *) - echo "[xmem] Unknown start option: $1" >&2 - exit 1 - ;; - esac -done - -has_command() { - command -v "$1" >/dev/null 2>&1 -} - -env_value() { - local key="$1" - local default="$2" - if [[ ! -f "$ROOT/.env" ]]; then - echo "$default" - return - fi - local value - value="$(grep -E "^[[:space:]]*$key[[:space:]]*=" "$ROOT/.env" | tail -n 1 | sed -E "s/^[^=]+=//; s/^['\"]//; s/['\"]$//")" - if [[ -n "$value" ]]; then - echo "$value" - else - echo "$default" - fi -} - -uses_ollama() { - [[ ! -f "$ROOT/.env" ]] && return 0 - grep -Eq '^[[:space:]]*FALLBACK_ORDER[[:space:]]*=.*ollama' "$ROOT/.env" -} - -docker_running() { - has_command docker && docker info >/dev/null 2>&1 -} - -wait_containers() { - local deadline=$((SECONDS + 180)) - local pending=("$@") - while (( SECONDS < deadline )); do - local next=() - for name in "${pending[@]}"; do - local status - status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$name" 2>/dev/null || true)" - if [[ "$status" == "healthy" || "$status" == "running" ]]; then - continue - fi - if [[ "$status" == "unhealthy" ]]; then - echo "[xmem] Container $name is unhealthy. Run npm run doctor or inspect it with: docker logs $name" >&2 - exit 1 - fi - next+=("$name") - done - - if [[ ${#next[@]} -eq 0 ]]; then - return - fi - - echo "[xmem] Waiting for local database containers: ${next[*]}" - pending=("${next[@]}") - sleep 5 - done - - echo "[xmem] Timed out waiting for local database containers: ${pending[*]}. Run npm run doctor for details." >&2 - exit 1 -} - -if [[ ! -f "$ROOT/.env" ]]; then - echo "[xmem] XMem .env not found at $ROOT/.env. Run npm run setup first." >&2 - exit 1 -fi - -bash "$ROOT/scripts/configure-xmem-env.sh" --env-path "$ROOT/.env" - -if uses_ollama; then - if ! has_command ollama || ! ollama list >/dev/null 2>&1; then - echo "[xmem] XMem is configured to use local Ollama, but Ollama is not running." >&2 - echo "[xmem] Start Ollama, or add a cloud LLM key to .env and rerun." >&2 - exit 2 - fi - - installed="$(ollama list 2>/dev/null | tail -n +2 || true)" - for model in "$(env_value OLLAMA_MODEL qwen2.5:1.5b)" "$(env_value OLLAMA_EMBEDDING_MODEL nomic-embed-text)"; do - [[ -z "$model" ]] && continue - if ! printf "%s\n" "$installed" | grep -Eq "^$(printf '%s' "$model" | sed 's/[][\.^$*+?{}|()]/\\&/g')([[:space:]]|:latest[[:space:]])"; then - echo "[xmem] Ollama model $model is missing. Run: ollama pull $model" >&2 - exit 2 - fi - done -fi - -if [[ "$SKIP_DOCKER" != "1" ]]; then - if ! docker_running; then - echo "[xmem] Docker Desktop is installed but not running, or Docker was not found." >&2 - echo "[xmem] Start Docker Desktop, wait until it says Docker is running, then rerun npm run dev." >&2 - exit 2 - fi - docker compose -f "$ROOT/docker-compose.local.yml" up -d --remove-orphans - wait_containers xmem-postgres xmem-mongo xmem-neo4j -fi - -PYTHON_BIN="$ROOT/.venv/bin/python" -if [[ ! -x "$PYTHON_BIN" ]]; then - if has_command python3; then - PYTHON_BIN=python3 - elif has_command python; then - PYTHON_BIN=python - else - echo "[xmem] python3 is required. Install Python 3.11+ and rerun." >&2 - exit 1 - fi -fi - -cd "$ROOT" -echo "[xmem] Starting XMem API at http://localhost:8000" -exec "$PYTHON_BIN" -m uvicorn src.api.app:create_app --factory --host 0.0.0.0 --port 8000 diff --git a/scripts/verify.ps1 b/scripts/verify.ps1 deleted file mode 100644 index a513f8a..0000000 --- a/scripts/verify.ps1 +++ /dev/null @@ -1,140 +0,0 @@ -param( - [string]$BaseUrl = "http://localhost:8000", - [string]$ApiKey = "dev-xmem-key", - [string]$UserId = "xmem-local-user", - [int]$TimeoutSeconds = 180 -) - -$ErrorActionPreference = "Stop" - -function Write-Step { - param([string]$Message) - Write-Host "[xmem] $Message" -} - -function Get-ResponseErrorMessage { - param([object]$ErrorRecord) - - if ($ErrorRecord.ErrorDetails.Message) { - return $ErrorRecord.ErrorDetails.Message - } - - return $ErrorRecord.Exception.Message -} - -function Invoke-XMemJson { - param( - [string]$Uri, - [string]$Method, - [hashtable]$Headers = @{}, - [string]$Body = "", - [int]$TimeoutSec = 60 - ) - - try { - if ($Body) { - return Invoke-RestMethod -Uri $Uri -Method $Method -Headers $Headers -Body $Body -TimeoutSec $TimeoutSec - } - - return Invoke-RestMethod -Uri $Uri -Method $Method -Headers $Headers -TimeoutSec $TimeoutSec - } catch { - $message = Get-ResponseErrorMessage $_ - throw "Request failed: $Method $Uri`n$message" - } -} - -function Test-HealthReady { - param([object]$Health) - - if (-not $Health) { - return $false - } - - if ($Health.data) { - return [bool]$Health.data.pipelines_ready - } - - return [bool]$Health.pipelines_ready -} - -function Get-HealthSummary { - param([object]$Health) - - if ($Health.data) { - return "status=$($Health.data.status), pipelines_ready=$($Health.data.pipelines_ready), error=$($Health.data.error)" - } - - return "status=$($Health.status), pipelines_ready=$($Health.pipelines_ready), error=$($Health.error)" -} - -$deadline = (Get-Date).AddSeconds($TimeoutSeconds) -$health = $null - -Write-Step "Waiting for API health at $BaseUrl/health" -while ((Get-Date) -lt $deadline) { - try { - $health = Invoke-RestMethod -Uri "$BaseUrl/health" -Method GET -TimeoutSec 10 - if (Test-HealthReady $health) { - break - } - } catch { - Start-Sleep -Seconds 3 - } -} - -if (-not $health) { - throw "XMem API did not become reachable within $TimeoutSeconds seconds." -} - -Write-Step "Health: $(Get-HealthSummary $health)" -if (-not (Test-HealthReady $health)) { - throw "XMem API is reachable but pipelines are not ready." -} - -$headers = @{ - "Authorization" = "Bearer $ApiKey" - "Content-Type" = "application/json" -} - -$memoryText = "Remember that XMem local mode runs directly from the main XMem repository." - -$ingestBody = @{ - user_query = $memoryText - agent_response = "Got it. I will remember that XMem local mode runs from the main repository." - user_id = $UserId - effort_level = "low" -} | ConvertTo-Json - -Write-Step "Ingesting a smoke-test memory" -$ingest = Invoke-XMemJson -Uri "$BaseUrl/v1/memory/ingest" -Method POST -Headers $headers -Body $ingestBody -TimeoutSec 650 -Write-Step "Ingest status: $($ingest.status)" - -$searchBody = @{ - query = "What is XMem local mode?" - user_id = $UserId - domains = @("profile", "temporal", "summary") - top_k = 5 -} | ConvertTo-Json - -Write-Step "Searching memory" -$search = Invoke-XMemJson -Uri "$BaseUrl/v1/memory/search" -Method POST -Headers $headers -Body $searchBody -TimeoutSec 180 -$resultCount = 0 -if ($search.data -and $search.data.results) { - $resultCount = @($search.data.results).Count -} -Write-Step "Search result count: $resultCount" - -$retrieveBody = @{ - query = "Where does XMem local mode run from?" - user_id = $UserId - top_k = 5 -} | ConvertTo-Json - -Write-Step "Retrieving answer" -$retrieve = Invoke-XMemJson -Uri "$BaseUrl/v1/memory/retrieve" -Method POST -Headers $headers -Body $retrieveBody -TimeoutSec 240 - -Write-Host "" -Write-Host "Answer:" -Write-Host $retrieve.data.answer -Write-Host "" -Write-Step "Verification complete" diff --git a/scripts/verify.sh b/scripts/verify.sh deleted file mode 100755 index bb9e886..0000000 --- a/scripts/verify.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -PYTHON_BIN="$ROOT/.venv/bin/python" - -if [[ ! -x "$PYTHON_BIN" ]]; then - if command -v python3 >/dev/null 2>&1; then - PYTHON_BIN=python3 - elif command -v python >/dev/null 2>&1; then - PYTHON_BIN=python - else - echo "[xmem] python3 is required. Install Python 3.11+ and rerun." >&2 - exit 1 - fi -fi - -"$PYTHON_BIN" "$ROOT/scripts/verify.py" "$@" diff --git a/scripts/xmem.js b/scripts/xmem.js index 6278053..de8f1ba 100644 --- a/scripts/xmem.js +++ b/scripts/xmem.js @@ -5,23 +5,39 @@ const path = require("node:path"); const { spawnSync } = require("node:child_process"); const root = path.resolve(__dirname, ".."); +const scriptsDir = path.join(root, "scripts"); const command = process.argv[2] || "help"; const passthroughArgs = process.argv.slice(3); +const isWindows = process.platform === "win32"; -const commands = { - setup: "install", - start: "start", - verify: "verify", - doctor: "doctor", - "context:export": "context-export", - "context:import": "context-import", - "context:sync": "context-sync", -}; +const managedRepos = [ + { + flag: "includeMcp", + name: "xmem-mcp", + url: "https://github.com/XortexAI/xmem-mcp.git", + branch: "main", + }, + { + flag: "includeSdk", + name: "xmem-sdk", + url: "https://github.com/XortexAI/xmem-sdk.git", + branch: "master", + }, +]; function log(message) { console.log(`[xmem] ${message}`); } +function warn(message) { + console.warn(`[xmem] ${message}`); +} + +function fail(message, exitCode = 1) { + console.error(`[xmem] ${message}`); + process.exit(exitCode); +} + function usage(exitCode = 0) { console.log(`XMem local workspace @@ -32,142 +48,743 @@ Usage: npm run verify npm run doctor npm run context:export - npm run context:import -- --file .\\exports\\xmem-context.json - npm run context:sync -- --file .\\exports\\xmem-context.json --server https://api.xmem.in --api-key + npm run context:import -- --file ./exports/xmem-context.json + npm run context:sync -- --file ./exports/xmem-context.json --server https://api.xmem.in --api-key Power-user flags can be passed after --, for example: + npm run setup -- --include-mcp + npm run setup -- --skip-model-pull + npm run start -- --skip-docker + +Windows-style flags are also accepted: npm run setup -- -IncludeMcp npm run start -- -SkipDocker `); process.exit(exitCode); } -function powershellExecutable() { - if (process.env.XMEM_POWERSHELL) { - return process.env.XMEM_POWERSHELL; +function executable(commandName) { + if (isWindows && ["npm", "npx"].includes(commandName)) { + return `${commandName}.cmd`; } - return process.platform === "win32" ? "powershell.exe" : "pwsh"; + return commandName; } -function powershellArgs(scriptPath, extraArgs) { - const args = ["-NoProfile"]; - if (process.platform === "win32") { - args.push("-ExecutionPolicy", "Bypass"); +function run(commandName, args = [], options = {}) { + const result = spawnSync(executable(commandName), args, { + cwd: options.cwd || root, + env: options.env || process.env, + encoding: options.capture ? "utf8" : undefined, + stdio: options.capture ? "pipe" : "inherit", + shell: false, + }); + + if (result.error) { + if (options.allowFailure) { + return result; + } + throw new Error(`Could not start ${commandName}: ${result.error.message}`); + } + + if (result.status !== 0 && !options.allowFailure) { + throw new Error(`${commandName} ${args.join(" ")} failed with exit code ${result.status}`); } - args.push("-File", scriptPath, ...extraArgs); - return args; + + return result; +} + +function commandExists(commandName) { + const checker = isWindows + ? ["where.exe", [commandName]] + : ["which", [commandName]]; + const result = run(checker[0], checker[1], { + capture: true, + allowFailure: true, + }); + return result.status === 0; } -function shellArgs(scriptPath, extraArgs) { - return [scriptPath, ...extraArgs]; +function sleep(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); } -function runScript(scriptName, extraArgs = []) { - const extension = process.platform === "win32" ? ".ps1" : ".sh"; - const executable = - process.platform === "win32" - ? powershellExecutable() - : process.env.XMEM_SHELL || "bash"; - const scriptPath = path.join(root, "scripts", `${scriptName}${extension}`); - if (!fs.existsSync(scriptPath)) { - console.error(`[xmem] Missing script: ${scriptPath}`); - process.exit(1); +function optionParser(spec) { + const aliases = new Map(); + for (const [key, config] of Object.entries(spec)) { + for (const alias of config.aliases || []) { + aliases.set(alias.toLowerCase(), { key, ...config }); + } } - const args = - process.platform === "win32" - ? powershellArgs(scriptPath, extraArgs) - : shellArgs(scriptPath, extraArgs); + return function parse(args) { + const values = {}; + for (const [key, config] of Object.entries(spec)) { + values[key] = config.type === "flag" ? false : config.default || ""; + } - const result = spawnSync(executable, args, { - cwd: root, - stdio: "inherit", - shell: false, - }); + for (let index = 0; index < args.length; index += 1) { + let arg = args[index]; + let inlineValue = ""; + const equalsIndex = arg.indexOf("="); + if (equalsIndex > -1) { + inlineValue = arg.slice(equalsIndex + 1); + arg = arg.slice(0, equalsIndex); + } - if (result.error) { - const installHint = - process.platform === "win32" - ? "PowerShell should be available on Windows. Reopen the terminal and try again." - : "Install bash, or set XMEM_SHELL to a compatible shell."; - console.error(`[xmem] Could not start ${executable}: ${result.error.message}`); - console.error(`[xmem] ${installHint}`); - process.exit(1); - } + const match = aliases.get(arg.toLowerCase()); + if (!match) { + throw new Error(`Unknown option: ${args[index]}`); + } + + if (match.type === "flag") { + values[match.key] = true; + continue; + } - process.exitCode = result.status || 0; - return process.exitCode; + const value = inlineValue || args[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error(`${arg} requires a value.`); + } + values[match.key] = value; + if (!inlineValue) { + index += 1; + } + } + + return values; + }; } -function getOptionValue(args, optionName) { - const wanted = optionName.toLowerCase(); +const parseSetupOptions = optionParser({ + reposDir: { type: "value", default: "repos", aliases: ["--repos-dir", "-ReposDir"] }, + includeMcp: { type: "flag", aliases: ["--include-mcp", "-IncludeMcp"] }, + includeSdk: { type: "flag", aliases: ["--include-sdk", "-IncludeSdk"] }, + skipModelPull: { type: "flag", aliases: ["--skip-model-pull", "-SkipModelPull"] }, + skipPythonInstall: { type: "flag", aliases: ["--skip-python-install", "-SkipPythonInstall"] }, + skipNodeInstall: { type: "flag", aliases: ["--skip-node-install", "-SkipNodeInstall"] }, + skipDocker: { type: "flag", aliases: ["--skip-docker", "-SkipDocker"] }, +}); + +const parseStartOptions = optionParser({ + reposDir: { type: "value", default: "repos", aliases: ["--repos-dir", "-ReposDir"] }, + skipDocker: { type: "flag", aliases: ["--skip-docker", "-SkipDocker"] }, +}); + +const parseDoctorOptions = optionParser({ + baseUrl: { type: "value", default: "http://localhost:8000", aliases: ["--base-url", "-BaseUrl"] }, + reposDir: { type: "value", default: "repos", aliases: ["--repos-dir", "-ReposDir"] }, +}); + +function readOption(args, names, fallback = "") { for (let index = 0; index < args.length; index += 1) { - if (args[index].toLowerCase() === wanted) { - return args[index + 1] || ""; + const arg = args[index]; + const equalsIndex = arg.indexOf("="); + const name = equalsIndex > -1 ? arg.slice(0, equalsIndex) : arg; + if (!names.map((item) => item.toLowerCase()).includes(name.toLowerCase())) { + continue; + } + if (equalsIndex > -1) { + return arg.slice(equalsIndex + 1); } + return args[index + 1] || fallback; } - return ""; + return fallback; } -function hasSwitch(args, switchName) { - const wanted = switchName.toLowerCase(); - return args.some((arg) => arg.toLowerCase() === wanted); +function hasSwitch(args, names) { + const wanted = names.map((name) => name.toLowerCase()); + return args.some((arg) => wanted.includes(arg.split("=")[0].toLowerCase())); } function startCompatibleArgs(args) { const next = []; - const reposDir = getOptionValue(args, "-ReposDir"); + const reposDir = readOption(args, ["--repos-dir", "-ReposDir"]); if (reposDir) { - next.push("-ReposDir", reposDir); + next.push("--repos-dir", reposDir); } - if (hasSwitch(args, "-SkipDocker")) { - next.push("-SkipDocker"); + if (hasSwitch(args, ["--skip-docker", "-SkipDocker"])) { + next.push("--skip-docker"); } return next; } -function setupLooksComplete(reposDir) { - function existsInRoot(relativePath) { - return fs.existsSync(path.join(root, relativePath)); +function systemPythonCommand() { + if (!isWindows && commandExists("python3")) { + return "python3"; + } + if (commandExists("python")) { + return "python"; + } + fail("Python 3.11+ is required. Install Python, reopen your terminal, and rerun this command."); +} + +function venvPythonPath() { + return path.join(root, ".venv", isWindows ? "Scripts/python.exe" : "bin/python"); +} + +function pythonForRuntime() { + const venvPython = venvPythonPath(); + if (fs.existsSync(venvPython)) { + return venvPython; + } + warn("XMem virtualenv was not found; using system Python. Run npm run setup if startup fails."); + return systemPythonCommand(); +} + +function stripQuotes(value) { + const trimmed = String(value || "").trim(); + if ( + (trimmed.startsWith("'") && trimmed.endsWith("'")) || + (trimmed.startsWith('"') && trimmed.endsWith('"')) + ) { + return trimmed.slice(1, -1); + } + return trimmed; +} + +function readDotEnv(envPath) { + const values = {}; + if (!fs.existsSync(envPath)) { + return values; + } + + for (const rawLine of fs.readFileSync(envPath, "utf8").split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith("#") || !line.includes("=")) { + continue; + } + const [key, ...rest] = line.split("="); + values[key.trim()] = stripQuotes(rest.join("=")); + } + return values; +} + +function setDotEnvValues(envPath, updates) { + const original = fs.existsSync(envPath) ? fs.readFileSync(envPath, "utf8") : ""; + const lines = original ? original.split(/\r?\n/) : []; + const updatedKeys = new Set(); + const next = lines.map((line) => { + for (const [key, value] of Object.entries(updates)) { + const pattern = new RegExp(`^\\s*${key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\s*=`); + if (pattern.test(line)) { + updatedKeys.add(key); + return `${key}=${value}`; + } + } + return line; + }); + + for (const [key, value] of Object.entries(updates)) { + if (!updatedKeys.has(key)) { + next.push(`${key}=${value}`); + } + } + + fs.writeFileSync(envPath, `${next.join("\n").replace(/\n+$/g, "")}\n`); +} + +function isRealSecret(value) { + const text = stripQuotes(value).trim(); + if (!text) { + return false; + } + + return ![ + /^your[_-]/i, + /your_.*_key/i, + /example/i, + /sample/i, + /placeholder/i, + /change[-_]?me/i, + /^dummy([-_].*)?$/i, + /^fake([-_].*)?$/i, + /^test([-_].*)?$/i, + ].some((pattern) => pattern.test(text)); +} + +function configuredValue(envPath, name) { + const envValue = process.env[name]; + if (isRealSecret(envValue)) { + return envValue; + } + const fileValue = readDotEnv(envPath)[name]; + return isRealSecret(fileValue) ? fileValue : ""; +} + +function configuredProviders(envPath) { + const providers = []; + if (configuredValue(envPath, "OPENROUTER_API_KEY")) providers.push("openrouter"); + if (configuredValue(envPath, "GEMINI_API_KEY")) providers.push("gemini"); + if (configuredValue(envPath, "CLAUDE_API_KEY")) providers.push("claude"); + if (configuredValue(envPath, "OPENAI_API_KEY")) providers.push("openai"); + if ( + configuredValue(envPath, "AWS_ACCESS_KEY_ID") && + configuredValue(envPath, "AWS_SECRET_ACCESS_KEY") + ) { + providers.push("bedrock"); + } + return providers; +} + +function configureEnv(envPath, quiet = false) { + if (!fs.existsSync(envPath)) { + throw new Error(`XMem .env not found at ${envPath}. Run npm run setup first.`); } - const pythonVenv = - process.platform === "win32" - ? ".venv/Scripts/python.exe" - : ".venv/bin/python"; + const providers = configuredProviders(envPath); + + if (providers.length > 0) { + setDotEnvValues(envPath, { + FALLBACK_ORDER: `'${JSON.stringify(providers)}'`, + EMBEDDING_PROVIDER: "fastembed", + FASTEMBED_MODEL: "BAAI/bge-small-en-v1.5", + EMBEDDING_MODEL: "BAAI/bge-small-en-v1.5", + PINECONE_DIMENSION: "384", + }); + if (!quiet) { + log(`Detected cloud LLM provider(s): ${providers.join(", ")}`); + log("Configured XMem to avoid Ollama for LLM and embedding calls."); + } + return providers; + } + setDotEnvValues(envPath, { + FALLBACK_ORDER: `'["ollama"]'`, + EMBEDDING_PROVIDER: "ollama", + OLLAMA_EMBEDDING_MODEL: "nomic-embed-text", + EMBEDDING_MODEL: "nomic-embed-text", + PINECONE_DIMENSION: "768", + }); + if (!quiet) { + log("No cloud LLM provider keys detected."); + log("Configured XMem to use local Ollama for LLM and embedding calls."); + } + return []; +} + +function dotEnvValue(envPath, name, fallback = "") { + return readDotEnv(envPath)[name] || fallback; +} + +function usesOllama(envPath) { + if (!fs.existsSync(envPath)) { + return true; + } + return /ollama/i.test(readDotEnv(envPath).FALLBACK_ORDER || ""); +} + +function syncRepo(reposDir, name, url, branch) { + const target = path.join(reposDir, name); + if (fs.existsSync(target)) { + if (!fs.existsSync(path.join(target, ".git"))) { + throw new Error(`${target} exists but is not a git checkout.`); + } + log(`Updating ${name}`); + run("git", ["-C", target, "reset", "--hard"]); + run("git", ["-C", target, "fetch", "origin"]); + run("git", ["-C", target, "checkout", branch]); + run("git", ["-C", target, "pull", "--ff-only", "origin", branch]); + return; + } + + log(`Cloning ${name}`); + run("git", ["clone", "--branch", branch, url, target]); +} + +function dockerRunning() { + return commandExists("docker") && run("docker", ["info"], { capture: true, allowFailure: true }).status === 0; +} + +function ollamaRunning() { + return commandExists("ollama") && run("ollama", ["list"], { capture: true, allowFailure: true }).status === 0; +} + +function waitForContainers(names, timeoutSeconds = 180) { + const pending = new Set(names); + const deadline = Date.now() + timeoutSeconds * 1000; + + while (Date.now() < deadline) { + for (const name of [...pending]) { + const result = run( + "docker", + [ + "inspect", + "--format", + "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}", + name, + ], + { capture: true, allowFailure: true }, + ); + + if (result.status !== 0) { + continue; + } + + const status = String(result.stdout || "").trim(); + if (status === "healthy" || status === "running") { + pending.delete(name); + } else if (status === "unhealthy") { + throw new Error(`Container ${name} is unhealthy. Run npm run doctor or inspect it with: docker logs ${name}`); + } + } + + if (pending.size === 0) { + return; + } + + log(`Waiting for local database containers: ${[...pending].join(", ")}`); + sleep(5000); + } + + throw new Error(`Timed out waiting for local database containers: ${[...pending].join(", ")}. Run npm run doctor for details.`); +} + +function startDockerServices() { + if (!dockerRunning()) { + return false; + } + log("Starting local Docker services"); + run("docker", ["compose", "-f", path.join(root, "docker-compose.local.yml"), "up", "-d", "--remove-orphans"]); + waitForContainers(["xmem-postgres", "xmem-mongo", "xmem-neo4j"]); + return true; +} + +function installedOllamaModels() { + const result = run("ollama", ["list"], { capture: true, allowFailure: true }); + if (result.status !== 0) { + return new Set(); + } + return new Set( + String(result.stdout || "") + .split(/\r?\n/) + .slice(1) + .map((line) => line.trim().split(/\s+/)[0]) + .filter(Boolean), + ); +} + +function hasOllamaModel(model, installed) { + if (!model) { + return true; + } + return installed.has(model) || (!model.includes(":") && installed.has(`${model}:latest`)); +} + +function assertOllamaReady(envPath) { + if (!commandExists("ollama")) { + fail("Ollama was not found. Install Ollama, or add a cloud LLM key to .env and rerun.", 2); + } + if (!ollamaRunning()) { + fail("XMem is configured to use local Ollama, but Ollama is not running. Start Ollama, or add a cloud LLM key to .env and rerun.", 2); + } + + const chatModel = dotEnvValue(envPath, "OLLAMA_MODEL", "qwen2.5:1.5b"); + const embeddingModel = dotEnvValue(envPath, "OLLAMA_EMBEDDING_MODEL", "nomic-embed-text"); + const installed = installedOllamaModels(); + const missing = [chatModel, embeddingModel].filter((model) => !hasOllamaModel(model, installed)); + if (missing.length > 0) { + for (const model of missing) { + warn(`Ollama model ${model} is missing. Run: ollama pull ${model}`); + } + fail("Required Ollama model(s) are missing, or add a cloud LLM key to .env so XMem does not use Ollama.", 2); + } +} + +function ensurePrerequisites(skipPython = false) { + for (const required of ["git", "node", "npm"]) { + if (!commandExists(required)) { + fail(`${required} is required. Install it, reopen your terminal, and rerun this command.`); + } + } + if (!skipPython) { + systemPythonCommand(); + } +} + +function setupLooksComplete(reposDir) { return ( - existsInRoot("pyproject.toml") && - existsInRoot(".env") && - existsInRoot(pythonVenv) && + fs.existsSync(path.join(root, "pyproject.toml")) && + fs.existsSync(path.join(root, ".env")) && + fs.existsSync(venvPythonPath()) && fs.existsSync(path.join(reposDir, "xmem-extension", ".git")) && fs.existsSync(path.join(reposDir, "xmem-extension", "dist", "manifest.json")) ); } -function runDev() { - const reposDir = path.resolve(root, getOptionValue(passthroughArgs, "-ReposDir") || "repos"); +function runSetup(args) { + const options = parseSetupOptions(args); + const reposDir = path.resolve(root, options.reposDir); + const extensionDir = path.join(reposDir, "xmem-extension"); + let dockerSkipped = false; + let ollamaSkipped = false; + + ensurePrerequisites(options.skipPythonInstall); + fs.mkdirSync(reposDir, { recursive: true }); + + syncRepo(reposDir, "xmem-extension", "https://github.com/XortexAI/xmem-extension.git", "main"); + for (const repo of managedRepos) { + if (options[repo.flag]) { + syncRepo(reposDir, repo.name, repo.url, repo.branch); + } + } + + const envTemplate = path.join(root, "templates", "xmem.env.local"); + const envTarget = path.join(root, ".env"); + if (!fs.existsSync(envTarget)) { + fs.copyFileSync(envTemplate, envTarget); + log("Created .env from local template"); + } else { + log(".env already exists; leaving it unchanged"); + } + + configureEnv(envTarget); + + if (!options.skipModelPull) { + if (usesOllama(envTarget)) { + if (ollamaRunning()) { + const chatModel = dotEnvValue(envTarget, "OLLAMA_MODEL", "qwen2.5:1.5b"); + const embeddingModel = dotEnvValue(envTarget, "OLLAMA_EMBEDDING_MODEL", "nomic-embed-text"); + log("Pulling Ollama chat model"); + run("ollama", ["pull", chatModel]); + log("Pulling Ollama embedding model"); + run("ollama", ["pull", embeddingModel]); + } else { + warn("Ollama was not found or is not running."); + warn("Start Ollama, or add a cloud LLM key to .env and rerun."); + ollamaSkipped = true; + } + } else { + log("Cloud LLM provider key detected; skipping Ollama model pulls"); + } + } + + if (!options.skipDocker) { + if (!startDockerServices()) { + warn("Docker Desktop is installed but not running, or Docker was not found."); + warn("Start Docker Desktop, wait until it says Docker is running, then rerun this command."); + warn("Temporary escape hatch: rerun npm run setup -- --skip-docker to continue without local databases."); + dockerSkipped = true; + } + } + + if (!options.skipPythonInstall) { + const venvPython = venvPythonPath(); + if (!fs.existsSync(venvPython)) { + log("Creating XMem virtualenv"); + run(systemPythonCommand(), ["-m", "venv", path.join(root, ".venv")]); + } + log("Installing XMem local dependencies"); + run(venvPython, ["-m", "pip", "install", "--upgrade", "pip"]); + run(venvPython, ["-m", "pip", "install", "-e", `${root}[local,dev]`]); + } + + log("Patching extension for local API"); + run(process.execPath, [path.join(scriptsDir, "patch-extension-local.js"), "--extension-dir", extensionDir]); + + if (!options.skipNodeInstall) { + log("Installing and building Chrome extension"); + run("npm", ["--prefix", extensionDir, "install"]); + run("npm", ["--prefix", extensionDir, "run", "build"]); + } + log("Install complete"); + console.log(""); + console.log("Next:"); + console.log(" npm run dev"); + console.log(" npm run verify"); + if (dockerSkipped) { + console.log(""); + warn("Docker services were not started. Start Docker Desktop before running npm run dev."); + } + if (ollamaSkipped) { + console.log(""); + warn("Ollama models were not pulled. Start Ollama, then rerun npm run setup or add a cloud LLM key."); + } +} + +function runStart(args) { + const options = parseStartOptions(args); + const envTarget = path.join(root, ".env"); + + if (!fs.existsSync(envTarget)) { + fail(`XMem .env not found at ${envTarget}. Run npm run setup first.`); + } + + configureEnv(envTarget); + + if (usesOllama(envTarget)) { + assertOllamaReady(envTarget); + } + + if (!options.skipDocker) { + if (!startDockerServices()) { + fail("Docker Desktop is installed but not running, or Docker was not found. Start Docker Desktop, then rerun npm run dev.", 2); + } + } + + const python = pythonForRuntime(); + log("Starting XMem API at http://localhost:8000"); + run(python, ["-m", "uvicorn", "src.api.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8000"]); +} + +function runDev(args) { + const reposDir = path.resolve(root, readOption(args, ["--repos-dir", "-ReposDir"], "repos")); if (!setupLooksComplete(reposDir)) { log("First run detected; running setup before starting XMem."); - const setupStatus = runScript(commands.setup, passthroughArgs); - if (setupStatus !== 0) { - process.exit(setupStatus); - } + runSetup(args); + } + runStart(startCompatibleArgs(args)); +} + +function runVerify(args) { + const python = pythonForRuntime(); + run(python, [path.join(scriptsDir, "verify.py"), ...args]); +} + +function runContext(subcommand, args) { + const venvPython = venvPythonPath(); + if (!fs.existsSync(venvPython)) { + fail("XMem virtualenv not found. Run npm run setup first."); + } + run(venvPython, [path.join(scriptsDir, "context.py"), subcommand, ...args]); +} + +function writeCheck(name, ok, message, fix = "") { + const label = ok ? "OK" : "FIX"; + console.log(`[${label}] ${name} - ${message}`); + if (!ok && fix) { + console.log(` ${fix}`); } +} - return runScript(commands.start, startCompatibleArgs(passthroughArgs)); +async function fetchHealth(baseUrl) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5000); + try { + const response = await fetch(`${baseUrl.replace(/\/+$/, "")}/health`, { + signal: controller.signal, + }); + if (!response.ok) { + return { ok: false, message: `HTTP ${response.status}` }; + } + const body = await response.json(); + const data = body.data || body; + return { + ok: Boolean(data.pipelines_ready), + message: `${baseUrl}/health`, + }; + } catch { + return { ok: false, message: `${baseUrl} is not reachable` }; + } finally { + clearTimeout(timeout); + } } -if (command === "help" || command === "--help" || command === "-h") { - usage(0); +async function runDoctor(args) { + const options = parseDoctorOptions(args); + const reposDir = path.resolve(root, options.reposDir); + const envPath = path.join(root, ".env"); + const extensionDir = path.join(reposDir, "xmem-extension"); + let failures = 0; + + console.log("[xmem] Doctor report"); + console.log(""); + + for (const cmd of ["git", "node", "npm"]) { + const ok = commandExists(cmd); + if (!ok) failures += 1; + writeCheck(cmd, ok, "command lookup", `Install ${cmd} and reopen this terminal.`); + } + + const pythonOk = commandExists("python") || (!isWindows && commandExists("python3")); + if (!pythonOk) failures += 1; + writeCheck("Python", pythonOk, "Python 3.11+ lookup", "Install Python 3.11+ and reopen this terminal."); + + const dockerOk = dockerRunning(); + if (!dockerOk) failures += 1; + writeCheck("Docker", dockerOk, "local database runtime", "Start Docker Desktop, then rerun npm run dev."); + + const xmemExists = fs.existsSync(path.join(root, "pyproject.toml")); + if (!xmemExists) failures += 1; + writeCheck("XMem repo", xmemExists, root, "Run this from the XMem repository root."); + + const extensionExists = fs.existsSync(extensionDir); + if (!extensionExists) failures += 1; + writeCheck("Extension repo", extensionExists, extensionDir, "Run npm run setup."); + + const extensionBuildExists = fs.existsSync(path.join(extensionDir, "dist", "manifest.json")); + if (!extensionBuildExists) failures += 1; + writeCheck("Extension build", extensionBuildExists, "repos/xmem-extension/dist", "Run npm run setup."); + + const envExists = fs.existsSync(envPath); + if (!envExists) failures += 1; + writeCheck("XMem .env", envExists, envPath, "Run npm run setup to create it from templates/xmem.env.local."); + + if (envExists) { + const providers = configuredProviders(envPath); + if (providers.length > 0) { + writeCheck("LLM routing", true, `cloud key detected: ${providers.join(", ")}; Ollama is not required`); + } else if (usesOllama(envPath)) { + const ollamaOk = ollamaRunning(); + if (!ollamaOk) failures += 1; + writeCheck("Ollama", ollamaOk, "required because no cloud LLM key is configured", "Start Ollama, or add a cloud LLM key to .env."); + + if (ollamaOk) { + const installed = installedOllamaModels(); + for (const model of [ + dotEnvValue(envPath, "OLLAMA_MODEL", "qwen2.5:1.5b"), + dotEnvValue(envPath, "OLLAMA_EMBEDDING_MODEL", "nomic-embed-text"), + ]) { + const ok = hasOllamaModel(model, installed); + if (!ok) failures += 1; + writeCheck(`Ollama model ${model}`, ok, "local model availability", `Run: ollama pull ${model}`); + } + } + } else { + failures += 1; + writeCheck("LLM routing", false, "no cloud key detected and FALLBACK_ORDER does not include Ollama", "Run npm run setup to repair .env routing."); + } + } + + const health = await fetchHealth(options.baseUrl); + if (!health.ok) failures += 1; + writeCheck("XMem API", health.ok, health.message, "Start it with npm run dev and wait for pipelines_ready=true."); + + console.log(""); + if (failures === 0) { + log("Everything looks ready."); + } else { + warn(`Found ${failures} setup item(s) to fix.`); + } } -if (command === "dev") { - runDev(); -} else if (commands[command]) { - runScript(commands[command], passthroughArgs); -} else { - console.error(`[xmem] Unknown command: ${command}`); - usage(1); +async function main() { + try { + if (command === "help" || command === "--help" || command === "-h") { + usage(0); + } else if (command === "setup") { + runSetup(passthroughArgs); + } else if (command === "dev") { + runDev(passthroughArgs); + } else if (command === "start") { + runStart(passthroughArgs); + } else if (command === "verify") { + runVerify(passthroughArgs); + } else if (command === "doctor") { + await runDoctor(passthroughArgs); + } else if (command === "context:export") { + runContext("export", passthroughArgs); + } else if (command === "context:import") { + runContext("import", passthroughArgs); + } else if (command === "context:sync") { + runContext("sync", passthroughArgs); + } else { + console.error(`[xmem] Unknown command: ${command}`); + usage(1); + } + } catch (error) { + fail(error.message || String(error)); + } } + +main(); diff --git a/templates/xmem.env.local b/templates/xmem.env.local index 1588f34..45f2088 100644 --- a/templates/xmem.env.local +++ b/templates/xmem.env.local @@ -12,7 +12,7 @@ ENABLE_PROMETHEUS=true ENABLE_ANALYTICS=false # LLM routing. -# scripts/configure-xmem-env.ps1 rewrites FALLBACK_ORDER before install/start: +# npm run setup/start rewrites FALLBACK_ORDER before local startup: # - any real cloud LLM key below means XMem will not call Ollama # - no cloud LLM key means XMem will use local Ollama FALLBACK_ORDER='["ollama"]' From 1b471f1f76ac2c679bc0a8761f8eabd37f8b5d0b Mon Sep 17 00:00:00 2001 From: Ankit Kotnala Date: Sat, 23 May 2026 17:25:11 +0530 Subject: [PATCH 4/6] Fix npm invocation in local setup --- scripts/xmem.js | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/scripts/xmem.js b/scripts/xmem.js index de8f1ba..d9158de 100644 --- a/scripts/xmem.js +++ b/scripts/xmem.js @@ -63,20 +63,38 @@ Windows-style flags are also accepted: process.exit(exitCode); } -function executable(commandName) { +function commandInvocation(commandName, args) { + if (commandName === "npm" && process.env.npm_execpath) { + return { + command: process.execPath, + args: [process.env.npm_execpath, ...args], + shell: false, + }; + } + if (isWindows && ["npm", "npx"].includes(commandName)) { - return `${commandName}.cmd`; + return { + command: commandName, + args, + shell: true, + }; } - return commandName; + + return { + command: commandName, + args, + shell: false, + }; } function run(commandName, args = [], options = {}) { - const result = spawnSync(executable(commandName), args, { + const invocation = commandInvocation(commandName, args); + const result = spawnSync(invocation.command, invocation.args, { cwd: options.cwd || root, env: options.env || process.env, encoding: options.capture ? "utf8" : undefined, stdio: options.capture ? "pipe" : "inherit", - shell: false, + shell: invocation.shell, }); if (result.error) { From 955216a6d478c813f7c087e2a13a766b5695a1b3 Mon Sep 17 00:00:00 2001 From: Ankit Kotnala Date: Sat, 23 May 2026 18:02:09 +0530 Subject: [PATCH 5/6] Made changes as per bot reviews --- .github/workflows/security-scan.yml | 3 +- .gitignore | 1 - CHANGELOG.md | 6 +++ packages/create-xmem/bin/create-xmem.js | 23 ++++++++-- packages/create-xmem/package.json | 2 +- scripts/context.py | 32 +++++++++---- src/api/app.py | 16 +++++-- src/api/routes/memory.py | 36 +++++++++++---- tests/api/test_app_errors.py | 23 ++++++++++ tests/api/test_memory_versioning.py | 61 ++++++++++++++++++++++++- tests/unit/test_schemas.py | 15 ++++++ 11 files changed, 189 insertions(+), 29 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 tests/api/test_app_errors.py diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index ccfd1f5..db83312 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -29,7 +29,7 @@ jobs: cache: pip - name: Install bandit - run: pip install bandit[toml] + run: pip install "bandit[toml,sarif]" - name: Run Bandit run: | @@ -41,6 +41,7 @@ jobs: continue-on-error: true - name: Upload Bandit SARIF + if: always() && hashFiles('bandit-results.sarif') != '' uses: github/codeql-action/upload-sarif@v3 with: sarif_file: bandit-results.sarif diff --git a/.gitignore b/.gitignore index 02e560a..37c88b7 100644 --- a/.gitignore +++ b/.gitignore @@ -62,7 +62,6 @@ rust/ # Empty root files Makefile -CHANGELOG.md ruff.toml # GitHub templates (empty) diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1251ec3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,6 @@ +# Changelog + +## Unreleased + +- Add local XMem setup through `npx create-xmem@latest` and `npm run dev`. +- Add local Docker storage, Chrome extension build patching, diagnostics, verification, and context export/import/sync commands. diff --git a/packages/create-xmem/bin/create-xmem.js b/packages/create-xmem/bin/create-xmem.js index fc98de8..94ebd5c 100644 --- a/packages/create-xmem/bin/create-xmem.js +++ b/packages/create-xmem/bin/create-xmem.js @@ -32,6 +32,16 @@ function parseArgs(argv) { repo: process.env.XMEM_REPO || DEFAULT_REPO, branch: process.env.XMEM_BRANCH || DEFAULT_BRANCH, }; + let targetSet = false; + + function readOptionValue(index, name) { + const value = argv[index + 1]; + if (!value || value.startsWith("-")) { + console.error(`[create-xmem] ${name} requires a value.`); + usage(1); + } + return value; + } for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; @@ -41,13 +51,13 @@ function parseArgs(argv) { } if (arg === "--repo") { - options.repo = argv[index + 1]; + options.repo = readOptionValue(index, arg); index += 1; continue; } if (arg === "--branch") { - options.branch = argv[index + 1]; + options.branch = readOptionValue(index, arg); index += 1; continue; } @@ -57,7 +67,14 @@ function parseArgs(argv) { usage(1); } - options.target = arg; + if (!targetSet) { + options.target = arg; + targetSet = true; + continue; + } + + console.error(`[create-xmem] Unexpected extra argument: ${arg}`); + usage(1); } if (!options.repo || !options.branch) { diff --git a/packages/create-xmem/package.json b/packages/create-xmem/package.json index 7b1e189..3d0ca6b 100644 --- a/packages/create-xmem/package.json +++ b/packages/create-xmem/package.json @@ -1,6 +1,6 @@ { "name": "create-xmem", - "version": "0.1.1", + "version": "0.1.2", "description": "Create a local XMem workspace.", "bin": { "create-xmem": "bin/create-xmem.js" diff --git a/scripts/context.py b/scripts/context.py index d95e276..1379685 100644 --- a/scripts/context.py +++ b/scripts/context.py @@ -60,6 +60,17 @@ def connect_postgres(env: dict[str, str]): return psycopg.connect(env.get("PGVECTOR_URL") or "postgresql://xmem:xmem@localhost:15432/xmem") +def pgvector_table_identifier(env: dict[str, str]): + from psycopg import sql + + table = env.get("PGVECTOR_TABLE") or "xmem_vectors" + parts = table.split(".") + for part in parts: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", part): + raise SystemExit(f"Invalid PGVECTOR_TABLE name: {table}") + return sql.Identifier(*parts) + + def user_filter_values(user_id: str | None) -> list[str]: if not user_id: return [] @@ -68,24 +79,27 @@ def user_filter_values(user_id: str | None) -> list[str]: def export_pgvector(env: dict[str, str], user_id: str | None) -> list[dict[str, Any]]: - table = env.get("PGVECTOR_TABLE") or "xmem_vectors" + from psycopg import sql + filters = user_filter_values(user_id) params: list[Any] = [] - where = "" + where_sql = sql.SQL("") if filters: - where = "WHERE metadata->>'user_id' = ANY(%s)" + where_sql = sql.SQL("WHERE metadata->>'user_id' = ANY(%s)") params.append(filters) with connect_postgres(env) as conn: with conn.cursor() as cur: cur.execute( - f""" + sql.SQL( + """ SELECT namespace, id, content, embedding::text AS embedding, metadata, created_at, updated_at FROM {table} {where} ORDER BY created_at, namespace, id - """, + """ + ).format(table=pgvector_table_identifier(env), where=where_sql), params, ) rows = cur.fetchall() @@ -185,9 +199,9 @@ def export_context(args: argparse.Namespace) -> None: def import_pgvector(env: dict[str, str], rows: list[dict[str, Any]], user_id: str | None) -> int: if not rows: return 0 + from psycopg import sql from psycopg.types.json import Jsonb - table = env.get("PGVECTOR_TABLE") or "xmem_vectors" with connect_postgres(env) as conn: conn.autocommit = True with conn.cursor() as cur: @@ -196,7 +210,8 @@ def import_pgvector(env: dict[str, str], rows: list[dict[str, Any]], user_id: st if user_id: metadata["user_id"] = normalize_user_id(user_id) cur.execute( - f""" + sql.SQL( + """ INSERT INTO {table}(namespace, id, content, embedding, metadata, created_at, updated_at) VALUES (%s, %s, %s, %s::vector, %s, COALESCE(%s::timestamptz, now()), COALESCE(%s::timestamptz, now())) ON CONFLICT(namespace, id) DO UPDATE SET @@ -204,7 +219,8 @@ def import_pgvector(env: dict[str, str], rows: list[dict[str, Any]], user_id: st embedding = excluded.embedding, metadata = excluded.metadata, updated_at = now() - """, + """ + ).format(table=pgvector_table_identifier(env)), ( row["namespace"], row["id"], diff --git a/src/api/app.py b/src/api/app.py index 4cdaeb3..585c848 100644 --- a/src/api/app.py +++ b/src/api/app.py @@ -104,13 +104,21 @@ def _friendly_validation_error(error: dict) -> str: def _public_exception_message(exc: Exception) -> str: message = str(exc).strip() - if isinstance(exc, TimeoutError): + is_local = settings.environment.lower() in {"development", "dev", "local", "test"} + + if is_local and isinstance(exc, TimeoutError): return message or "The request timed out while waiting for an LLM response." - if isinstance(exc, (ValueError, RuntimeError, ConnectionError)): + if is_local: return message or type(exc).__name__ - if settings.environment.lower() in {"development", "dev", "local", "test"}: - return message or type(exc).__name__ + if isinstance(exc, TimeoutError): + return "The request timed out while waiting for an LLM response." + if isinstance(exc, ValueError): + return message or "Invalid request." + if isinstance(exc, ConnectionError): + return "A backend service is unavailable. Check the server logs with the request_id for details." + if isinstance(exc, RuntimeError): + return "The request could not be completed. Check the server logs with the request_id." return "Internal server error. Check the server logs with the request_id for details." diff --git a/src/api/routes/memory.py b/src/api/routes/memory.py index 7268a6a..cab1d9a 100644 --- a/src/api/routes/memory.py +++ b/src/api/routes/memory.py @@ -59,6 +59,7 @@ logger = logging.getLogger("xmem.api.routes.memory") _ingest_semaphore = asyncio.Semaphore(5) +_LOCAL_ENVIRONMENTS = {"development", "dev", "local", "test"} router = APIRouter( prefix="/v1/memory", @@ -125,6 +126,8 @@ def _error( code: int, elapsed_ms: float = 0, ) -> JSONResponse: + if code >= 500 and settings.environment.lower() not in _LOCAL_ENVIRONMENTS: + detail = "The request could not be completed. Check the server logs with the request_id." body = APIResponse( status=StatusEnum.ERROR, request_id=getattr(request.state, "request_id", None), @@ -134,14 +137,26 @@ def _error( return JSONResponse(content=body.model_dump(), status_code=code) +def _is_static_key_user(user: dict) -> bool: + return user.get("email") == "static@xmem.ai" or user.get("name") == "Static Key User" + + def _current_user_id(user: dict, requested_user_id: str = "") -> str: - if requested_user_id and ( - user.get("email") == "static@xmem.ai" or user.get("name") == "Static Key User" + if ( + requested_user_id + and settings.environment.lower() in _LOCAL_ENVIRONMENTS + and _is_static_key_user(user) ): return requested_user_id return user.get("username") or user.get("name") or user["id"] +def _scoped_ingest_payload(user: dict, item: IngestRequest) -> Dict[str, Any]: + payload = item.model_dump() + payload["user_id"] = _current_user_id(user, payload.get("user_id", "")) + return payload + + def _job_status_data(job: Dict[str, Any]) -> Dict[str, Any]: public = serialize_job(job) or {} return { @@ -217,11 +232,11 @@ async def _run_ingest_payload( async def _run_batch_ingest_payload( payload: Dict[str, Any], - user_id: str, ) -> Dict[str, Any]: results = [] for item in payload["items"]: - results.append(await _run_ingest_payload(item, user_id)) + item_user_id = item.get("user_id") or payload["user_id"] + results.append(await _run_ingest_payload(item, item_user_id)) return {"results": results} @@ -708,6 +723,7 @@ async def ingest_memory(req: IngestRequest, request: Request, user: dict = Depen async def ingest_memory_v2(req: IngestRequest, request: Request, user: dict = Depends(require_api_key)): start = time.perf_counter() user_id = _current_user_id(user, req.user_id) + job_user_id = _current_user_id(user) payload = req.model_dump() payload["user_id"] = user_id @@ -725,7 +741,7 @@ async def ingest_memory_v2(req: IngestRequest, request: Request, user: dict = De "image_url": req.image_url, "effort_level": req.effort_level, }, - user_id=user_id, + user_id=job_user_id, timeout_seconds=float(settings.memory_ingest_timeout_seconds), max_attempts=3, ) @@ -802,13 +818,14 @@ async def memory_job_status(job_id: str, request: Request, user: dict = Depends( ) async def batch_ingest_memory(req: BatchIngestRequest, request: Request, user: dict = Depends(require_api_key)): start = time.perf_counter() - user_id = _current_user_id(user, req.items[0].user_id if req.items else "") + user_id = _current_user_id(user) try: results = [] for item in req.items: + payload = _scoped_ingest_payload(user, item) data = await asyncio.wait_for( - _run_ingest_payload(item.model_dump(), user_id), + _run_ingest_payload(payload, payload["user_id"]), timeout=float(settings.memory_ingest_timeout_seconds), ) results.append(IngestResponse(**data)) @@ -831,9 +848,10 @@ async def batch_ingest_memory(req: BatchIngestRequest, request: Request, user: d ) async def batch_ingest_memory_v2(req: BatchIngestRequest, request: Request, user: dict = Depends(require_api_key)): start = time.perf_counter() - user_id = _current_user_id(user, req.items[0].user_id if req.items else "") + user_id = _current_user_id(user) payload = req.model_dump() payload["user_id"] = user_id + payload["items"] = [_scoped_ingest_payload(user, item) for item in req.items] try: store = get_default_job_store() @@ -854,7 +872,7 @@ async def batch_ingest_memory_v2(req: BatchIngestRequest, request: Request, user ) _schedule_job( job, - lambda: _run_batch_ingest_payload(payload, user_id), + lambda: _run_batch_ingest_payload(payload), ) elapsed = round((time.perf_counter() - start) * 1000, 2) return _job_accepted( diff --git a/tests/api/test_app_errors.py b/tests/api/test_app_errors.py new file mode 100644 index 0000000..2815e3c --- /dev/null +++ b/tests/api/test_app_errors.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from src.api import app as api_app + + +def test_public_exception_message_redacts_connection_details_in_production(monkeypatch): + monkeypatch.setattr(api_app.settings, "environment", "production", raising=False) + + message = api_app._public_exception_message( + ConnectionError("postgresql://user:password@internal-db:5432/xmem") + ) + + assert "password" not in message + assert "internal-db" not in message + assert "backend service is unavailable" in message + + +def test_public_exception_message_keeps_timeout_detail_in_local(monkeypatch): + monkeypatch.setattr(api_app.settings, "environment", "development", raising=False) + + message = api_app._public_exception_message(TimeoutError("LLM timed out after 180 seconds")) + + assert message == "LLM timed out after 180 seconds" diff --git a/tests/api/test_memory_versioning.py b/tests/api/test_memory_versioning.py index a7091cb..11ebbcc 100644 --- a/tests/api/test_memory_versioning.py +++ b/tests/api/test_memory_versioning.py @@ -46,14 +46,15 @@ def get(self, job_id): return self.jobs.get(job_id) -def _build_app(monkeypatch): +def _build_app(monkeypatch, user=None): ingest = FakeIngestPipeline() deps._init_error = None deps._pipelines_ready.set() deps.set_pipelines(ingest, FakeRetrievalPipeline()) + auth_user = user or {"id": "user-1", "username": "hunter"} async def fake_user(): - return {"id": "user-1", "username": "hunter"} + return auth_user async def fake_ready(): return None @@ -72,6 +73,16 @@ async def fake_rate_limit(): return app, ingest +def test_static_key_user_id_override_is_local_only(monkeypatch): + static_user = {"id": "static-key", "name": "Static Key User", "email": "static@xmem.ai"} + + monkeypatch.setattr(memory.settings, "environment", "development", raising=False) + assert memory._current_user_id(static_user, "friendly-user") == "friendly-user" + + monkeypatch.setattr(memory.settings, "environment", "production", raising=False) + assert memory._current_user_id(static_user, "friendly-user") == "Static Key User" + + def test_v1_ingest_keeps_synchronous_response_contract(monkeypatch): app, ingest = _build_app(monkeypatch) payload = { @@ -119,3 +130,49 @@ def test_v2_ingest_returns_durable_job_envelope(monkeypatch): } assert scheduled == ["memory_ingest:fake"] assert ingest.calls == [] + + +def test_v1_batch_ingest_scopes_each_item_for_local_static_key(monkeypatch): + monkeypatch.setattr(memory.settings, "environment", "development", raising=False) + static_user = {"id": "static-key", "name": "Static Key User", "email": "static@xmem.ai"} + app, ingest = _build_app(monkeypatch, user=static_user) + payload = { + "items": [ + {"user_query": "remember alpha", "agent_response": "done", "user_id": "alice"}, + {"user_query": "remember beta", "agent_response": "done", "user_id": "bob"}, + ], + } + + response = TestClient(app).post("/v1/memory/batch-ingest", json=payload) + + assert response.status_code == 200 + assert [call["user_id"] for call in ingest.calls] == ["alice", "bob"] + + +def test_v2_batch_ingest_queues_scoped_items_for_local_static_key(monkeypatch): + monkeypatch.setattr(memory.settings, "environment", "development", raising=False) + static_user = {"id": "static-key", "name": "Static Key User", "email": "static@xmem.ai"} + app, ingest = _build_app(monkeypatch, user=static_user) + store = FakeJobStore() + scheduled = [] + monkeypatch.setattr(memory, "get_default_job_store", lambda: store) + monkeypatch.setattr( + memory, + "_schedule_job", + lambda job, handler: scheduled.append(job["job_id"]), + ) + payload = { + "items": [ + {"user_query": "remember alpha", "agent_response": "done", "user_id": "alice"}, + {"user_query": "remember beta", "agent_response": "done", "user_id": "bob"}, + ], + } + + response = TestClient(app).post("/v2/memory/batch-ingest", json=payload) + + assert response.status_code == 200 + job = store.jobs["memory_batch_ingest:fake"] + assert job["user_id"] == "Static Key User" + assert [item["user_id"] for item in job["payload"]["items"]] == ["alice", "bob"] + assert scheduled == ["memory_batch_ingest:fake"] + assert ingest.calls == [] diff --git a/tests/unit/test_schemas.py b/tests/unit/test_schemas.py index 3905c53..8499977 100644 --- a/tests/unit/test_schemas.py +++ b/tests/unit/test_schemas.py @@ -3,6 +3,7 @@ import pytest from pydantic import ValidationError +from src.api.schemas import IngestRequest, normalize_user_id from src.schemas.code import ( AnnotationSeverity, AnnotationType, @@ -84,3 +85,17 @@ def test_code_schema_enums_and_namespace_helpers(): assert symbols_namespace("acme", "payments") == "acme:payments:symbols" assert annotations_namespace("acme") == "acme:annotations" assert snippets_namespace("user-1") == "user-1:snippets" + + +def test_api_user_ids_are_normalized_before_validation(): + assert normalize_user_id(" Ankit Kotnala! ") == "Ankit_Kotnala" + + request = IngestRequest( + user_query="remember this", + agent_response="done", + user_id="Ankit Kotnala!", + ) + assert request.user_id == "Ankit_Kotnala" + + with pytest.raises(ValidationError): + IngestRequest(user_query="remember this", agent_response="done", user_id=" !!! ") From f7ee0af3aca89e8598da20b2455aca1adf649845 Mon Sep 17 00:00:00 2001 From: Ankit Kotnala Date: Sat, 23 May 2026 19:43:23 +0530 Subject: [PATCH 6/6] Redact production ValueError responses --- src/api/app.py | 2 +- tests/api/test_app_errors.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/api/app.py b/src/api/app.py index 585c848..79bce88 100644 --- a/src/api/app.py +++ b/src/api/app.py @@ -114,7 +114,7 @@ def _public_exception_message(exc: Exception) -> str: if isinstance(exc, TimeoutError): return "The request timed out while waiting for an LLM response." if isinstance(exc, ValueError): - return message or "Invalid request." + return "Invalid request." if isinstance(exc, ConnectionError): return "A backend service is unavailable. Check the server logs with the request_id for details." if isinstance(exc, RuntimeError): diff --git a/tests/api/test_app_errors.py b/tests/api/test_app_errors.py index 2815e3c..0dea010 100644 --- a/tests/api/test_app_errors.py +++ b/tests/api/test_app_errors.py @@ -15,6 +15,18 @@ def test_public_exception_message_redacts_connection_details_in_production(monke assert "backend service is unavailable" in message +def test_public_exception_message_redacts_value_error_in_production(monkeypatch): + monkeypatch.setattr(api_app.settings, "environment", "production", raising=False) + + message = api_app._public_exception_message( + ValueError("vector mismatch for postgresql://user:password@internal-db:5432/xmem") + ) + + assert message == "Invalid request." + assert "password" not in message + assert "internal-db" not in message + + def test_public_exception_message_keeps_timeout_detail_in_local(monkeypatch): monkeypatch.setattr(api_app.settings, "environment", "development", raising=False)