From 4d65cdc4e6242bbb3596a4f7c6b43b1418bfcbb7 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Thu, 14 May 2026 13:14:15 -0500 Subject: [PATCH 01/17] chore: prototype security without RLS penalty --- proto/create.sh | 13 + proto/create.sql | 609 +++++++++++++++++++++++++++++++++++++++++++++++ proto/create.ts | 26 ++ 3 files changed, 648 insertions(+) create mode 100755 proto/create.sh create mode 100644 proto/create.sql create mode 100644 proto/create.ts diff --git a/proto/create.sh b/proto/create.sh new file mode 100755 index 0000000..57fc072 --- /dev/null +++ b/proto/create.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_dir="$(dirname -- "$script_dir")" + +: "${DATABASE_URL:?DATABASE_URL is required}" + +"$repo_dir/bun" "$script_dir/create.ts" | psql -d "$DATABASE_URL" \ + --single-transaction \ + -v ON_ERROR_STOP=1 \ + -f - diff --git a/proto/create.sql b/proto/create.sql new file mode 100644 index 0000000..a57e804 --- /dev/null +++ b/proto/create.sql @@ -0,0 +1,609 @@ + +------------------------------------------------------------------------------- +-- check database version +------------------------------------------------------------------------------- +select current_setting('server_version_num')::int < 180000 as bad_pg_version +\gset +\if :bad_pg_version +\warn postgres 18 or greater required +\q +\endif + +------------------------------------------------------------------------------- +-- check database version +------------------------------------------------------------------------------- +select current_setting('server_version_num')::int < 180000 as bad_pg_version +\gset +\if :bad_pg_version +\warn postgres 18 or greater required +\q +\endif + +------------------------------------------------------------------------------- +-- ensure extensions installed +------------------------------------------------------------------------------- +create extension if not exists citext; +create extension if not exists ltree; +create extension if not exists vector; +create extension if not exists pg_textsearch; + +------------------------------------------------------------------------------- +-- database roles +------------------------------------------------------------------------------- +do $block$ +declare + _roles text[] = array['me_ro', 'me_rw', 'me_embed']; + _role text; + _sql text; +begin + for _role in select * from unnest(_roles) loop + perform + from pg_roles r + where r.rolname = _role; + if found then + continue; + end if; + _sql = format($sql$create role %I nologin$sql$, _role); + execute _sql; + _sql = format($sql$grant %I to %I$sql$, _role, current_user); + execute _sql; + end loop; +end; +$block$; + +------------------------------------------------------------------------------- +-- engine schema +------------------------------------------------------------------------------- +drop schema if exists {{schema}} cascade; +create schema {{schema}}; + +------------------------------------------------------------------------------- +-- grant usage on engine schema to roles +------------------------------------------------------------------------------- +do $block$ +declare + _roles text[] = array['me_ro', 'me_rw', 'me_embed']; + _role text; + _sql text; +begin + for _role in select * from unnest(_roles) + loop + _sql = format($sql$grant usage on schema %I to %I$sql$, '{{schema}}', _role); + execute _sql; + end loop; +end; +$block$; + +------------------------------------------------------------------------------- +-- generic updated_at trigger +------------------------------------------------------------------------------- +create or replace function {{schema}}.update_updated_at() +returns trigger +as $func$ +begin + new.updated_at = pg_catalog.now(); + return new; +end; +$func$ language plpgsql volatile security definer +set search_path to {{schema}}, pg_temp; + +------------------------------------------------------------------------------- +-- users +------------------------------------------------------------------------------- +-- User: thing that accesses memories, or a role (can_login = false) +-- identity_id is a soft FK to accounts.identity (nullable for service users) +-- Note: "user" is a reserved word, must be quoted +create table {{schema}}."user" +( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) +, name citext not null unique +, identity_id uuid check (identity_id is null or uuid_extract_version(identity_id) = 7) -- soft FK to accounts.identity +, can_login boolean not null default true -- false = role (grant container) +, superuser boolean not null default false +, createrole boolean not null default false -- can create other users/roles +, created_at timestamptz not null default now() +, updated_at timestamptz +); + +create index idx_user_identity_id on {{schema}}."user" (identity_id) where identity_id is not null; + +create trigger user_updated_at +before update on {{schema}}."user" +for each row +execute function {{schema}}.update_updated_at() +; + +------------------------------------------------------------------------------- +-- memory +------------------------------------------------------------------------------- +create table {{schema}}.memory +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, meta jsonb not null default '{}' +, tree ltree not null default ''::ltree +, temporal tstzrange +, content text not null +, embedding halfvec({{embedding_dimensions}}) +, embedding_version int4 not null default 1 +, created_at timestamptz not null default now() +, created_by uuid references {{schema}}."user" (id) on delete set null +, updated_at timestamptz +); + +grant select on {{schema}}.memory to me_ro; +grant select, insert, update, delete on {{schema}}.memory to me_rw; +grant select, update on {{schema}}.memory to me_embed; + +-- index for faceted search +create index memory_meta_gin_idx on {{schema}}.memory using gin (meta); + +-- index for temporal search +create index memory_temporal_gist_idx on {{schema}}.memory using gist (temporal) where (temporal is not null); + +-- index for BM25 text search +create index memory_content_bm25_idx on {{schema}}.memory using bm25 (content) +with (text_config = {{bm25_text_config}}, k1 = {{bm25_k1}}, b = {{bm25_b}}); + +-- index for vector similarity search +create index memory_embedding_hnsw_idx on {{schema}}.memory using hnsw (embedding halfvec_cosine_ops) +with (m = {{hnsw_m}}, ef_construction = {{hnsw_ef_construction}}); + +-- index for hierarchical organization +create index memory_tree_gist_idx on {{schema}}.memory using gist (tree); + +-- make sure the metadata is an object +alter table {{schema}}.memory add check (jsonb_typeof(meta) = 'object'); + +/* +enforce consistent temporal range conventions: +- point-in-time events: lower = upper with inclusive bounds '[same,same]' +- time periods: lower < upper with inclusive-exclusive bounds '[start,end)' +*/ +alter table {{schema}}.memory add constraint temporal_bounds_convention check +( + temporal is null + or ( + -- point-in-time: both bounds equal and inclusive + (lower(temporal) = upper(temporal) and lower_inc(temporal) and upper_inc(temporal)) + or + -- time range: start before end, inclusive-exclusive + (lower(temporal) < upper(temporal) and lower_inc(temporal) and not upper_inc(temporal)) + ) +); + +------------------------------------------------------------------------------- +-- memory triggers +------------------------------------------------------------------------------- +create or replace function {{schema}}.memory_before_update() +returns trigger +as $func$ +begin + -- always update the timestamp + new.updated_at = pg_catalog.now(); + + -- content changed -> new embedding needs to be generated + if old.content is distinct from new.content + and old.embedding is not distinct from new.embedding + then + new.embedding = null; + new.embedding_version = old.embedding_version operator(pg_catalog.+) 1; + end if; + + return new; +end; +$func$ language plpgsql volatile security definer +set search_path to {{schema}}, public, pg_temp; -- public required for pgvector's `is not distinct from` + +create trigger memory_before_update_trg +before update on {{schema}}.memory +for each row +execute function {{schema}}.memory_before_update(); + + +------------------------------------------------------------------------------- +-- embedding queue +------------------------------------------------------------------------------- +-- per-engine embedding queue table +create table {{schema}}.embedding_queue +( id bigint generated always as identity primary key +, memory_id uuid not null references {{schema}}.memory(id) on delete cascade +, embedding_version int not null +, vt timestamptz not null default now() +, outcome text check (outcome is null or outcome in ('completed', 'failed', 'cancelled')) +, attempts int not null default 0 +, last_error text +, created_at timestamptz not null default now() +); + +-- index to find items to claim +create index embedding_queue_claim_idx on {{schema}}.embedding_queue (vt) where outcome is null; +-- index also used in finding items to claim. used to ensure there aren't any items for the same memory with a newer version +create index embedding_queue_memory_idx on {{schema}}.embedding_queue (memory_id, embedding_version desc) where outcome is null; +-- index to find items that have resolved to an outcome. these can be pruned +create index embedding_queue_archive_idx on {{schema}}.embedding_queue (created_at) where outcome is not null; + +grant select, update, delete on {{schema}}.embedding_queue to me_embed; + +------------------------------------------------------------------------------- +-- enqueue_embedding +------------------------------------------------------------------------------- +-- this must be security definer because we won't allow me_rw to access queue directly +create or replace function {{schema}}.enqueue_embedding() +returns trigger +as $func$ +begin + insert into {{schema}}.embedding_queue (memory_id, embedding_version) + values (new.id, new.embedding_version); + return new; +end; +$func$ +language plpgsql volatile security definer +set search_path to pg_catalog, {{schema}}, pg_temp +; + +------------------------------------------------------------------------------- +-- enqueuing triggers +------------------------------------------------------------------------------- +create trigger memory_enqueue_embedding_insert +after insert on {{schema}}.memory +for each row +when (new.embedding is null) -- it's possible to insert WITH an embedding +execute function {{schema}}.enqueue_embedding() +; + +create trigger memory_enqueue_embedding_update +after update on {{schema}}.memory +for each row +when +( old.content is distinct from new.content + and new.embedding is null +) +execute function {{schema}}.enqueue_embedding() +; + +------------------------------------------------------------------------------- +-- claim_embedding_batch +------------------------------------------------------------------------------- +create or replace function {{schema}}.claim_embedding_batch +( batch_size int default 10 +, lock_duration interval default '5 minutes' +, max_attempts int default 3 +) +returns table +( queue_id bigint +, memory_id uuid +, embedding_version int +, content text +) +as $func$ +declare + rec record; + mem record; + claimed_count int = 0; +begin + -- bulk-cancel visible queue rows superseded by a newer row for the same memory + update {{schema}}.embedding_queue eq + set outcome = 'cancelled' + where eq.outcome is null + and eq.vt <= now() + and exists + ( + select 1 + from {{schema}}.embedding_queue newer + where newer.memory_id = eq.memory_id + and newer.embedding_version > eq.embedding_version + and newer.outcome is null + ); + + -- sweep: finalize exhausted rows orphaned by worker crash + -- (attempts reached max but outcome was never written back) + update {{schema}}.embedding_queue + set + outcome = 'failed' + , last_error = coalesce(last_error, 'exceeded max attempts (worker crash)') + where outcome is null + and vt <= now() + and attempts >= max_attempts + ; + + for rec in + ( + select + eq.id + , eq.memory_id + , eq.embedding_version + from {{schema}}.embedding_queue eq + where eq.outcome is null + and eq.vt <= now() + and eq.attempts < max_attempts + order by eq.vt + for update skip locked + ) + loop + -- check memory still exists + current version + select m.content, m.embedding_version + into mem + from {{schema}}.memory m + where m.id = rec.memory_id + ; + + if not found or mem.content is null then + -- memory deleted or empty → cancel queue row + update {{schema}}.embedding_queue + set outcome = 'cancelled' + where id = rec.id; + continue; + end if; + + if rec.embedding_version != mem.embedding_version then + -- stale version → cancel + update {{schema}}.embedding_queue + set outcome = 'cancelled' + where id = rec.id; + continue; + end if; + + -- claim this row + update {{schema}}.embedding_queue q set + vt = now() + lock_duration + , attempts = q.attempts + 1 + where id = rec.id; + + queue_id = rec.id; + memory_id = rec.memory_id; + embedding_version = rec.embedding_version; + content = mem.content; + return next; + + claimed_count = claimed_count + 1; + exit when claimed_count >= batch_size; + end loop; +end; +$func$ +language plpgsql volatile +set search_path to pg_catalog, {{schema}}, pg_temp +; + +grant execute on function {{schema}}.claim_embedding_batch(int, interval, int) to me_embed; + +------------------------------------------------------------------------------- +-- prune embedding queue +------------------------------------------------------------------------------- +-- prune terminal queue rows older than the retention window. +-- runs opportunistically from the worker on engines that returned no +-- claimable work, so the queue table doesn't grow unbounded. +-- +-- relies on embedding_queue_archive_idx (created_at) where outcome is not null +-- from migration 005, so the no-op case is cheap. +create or replace function {{schema}}.prune_embedding_queue +( retention interval default '7 days' +) +returns bigint +as $func$ +declare + pruned bigint; +begin + delete from {{schema}}.embedding_queue + where outcome is not null + and created_at < now() - retention + ; + get diagnostics pruned = row_count; + return pruned; +end; +$func$ +language plpgsql volatile +set search_path to pg_catalog, {{schema}}, pg_temp +; + +-- me_embed already has DELETE on embedding_queue (granted in 005); +-- this just exposes the function entrypoint. +grant execute on function {{schema}}.prune_embedding_queue(interval) to me_embed; + + +------------------------------------------------------------------------------- +-- api keys +------------------------------------------------------------------------------- +-- Engine-scoped, user-scoped authentication +create table {{schema}}.api_key +( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) +, user_id uuid not null references {{schema}}."user" on delete cascade +, lookup_id text unique not null check (lookup_id ~ '^[A-Za-z0-9_-]{16}$') +, key_hash text not null +, name text not null +, expires_at timestamptz +, created_at timestamptz not null default now() +, revoked_at timestamptz +); + +create index idx_api_key_user on {{schema}}.api_key (user_id); +create index idx_api_key_lookup on {{schema}}.api_key (lookup_id) where revoked_at is null; + +------------------------------------------------------------------------------- +-- tree grants +------------------------------------------------------------------------------- +create table {{schema}}.tree_grant +( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) +, user_id uuid not null references {{schema}}."user"(id) on delete cascade +, tree_path ltree not null +, actions text[] not null +, granted_by uuid references {{schema}}."user"(id) +, created_at timestamptz not null default now() +, with_grant_option boolean not null default false +, constraint valid_actions check ( + actions <@ '{read,create,update,delete}'::text[] + ) +); + +create unique index idx_tree_grant_unique on {{schema}}.tree_grant (user_id, tree_path); +create index idx_tree_grant_user on {{schema}}.tree_grant using btree (user_id); +create index idx_tree_grant_path on {{schema}}.tree_grant using gist (tree_path); + +------------------------------------------------------------------------------- +-- role membership +------------------------------------------------------------------------------- +create table {{schema}}.role_membership +( role_id uuid not null references {{schema}}."user"(id) on delete cascade +, member_id uuid not null references {{schema}}."user"(id) on delete cascade +, with_admin_option boolean not null default false +, created_at timestamptz not null default now() +, primary key (role_id, member_id) +, constraint no_self_membership check (role_id <> member_id) +); + +create index idx_role_membership_member on {{schema}}.role_membership(member_id); + +-- ===== Cycle Detection ===== +create function {{schema}}.would_create_cycle +( _role_id uuid +, _member_id uuid +) +returns boolean +as $func$ + with recursive ancestors(id) as ( + select rm.role_id + from {{schema}}.role_membership rm + where rm.member_id = _role_id + union + select rm.role_id + from {{schema}}.role_membership rm + inner join ancestors a on a.id = rm.member_id + ) + select _member_id = _role_id + or exists + ( + select 1 + from ancestors + where id = _member_id + ) +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; + +------------------------------------------------------------------------------- +-- tree ownership +------------------------------------------------------------------------------- +create table {{schema}}.tree_owner +( tree_path ltree primary key +, user_id uuid not null references {{schema}}."user"(id) on delete cascade +, created_by uuid references {{schema}}."user"(id) +, created_at timestamptz not null default now() +); + +create index idx_tree_owner_user on {{schema}}.tree_owner (user_id); +create index idx_tree_owner_gist on {{schema}}.tree_owner using gist (tree_path); + +------------------------------------------------------------------------------- +-- tree access +------------------------------------------------------------------------------- +-- Returns set of tree paths the user can access for the given action. +-- Superusers get ''::ltree (empty root) which matches all paths via <@. +create function {{schema}}.tree_access +( _user_id uuid +, _action text +) +returns setof ltree +as $func$ + with recursive effective_roles(user_id) as + ( + select _user_id + union + select rm.role_id + from {{schema}}.role_membership rm + inner join effective_roles er on (er.user_id = rm.member_id) + ) + select distinct tree_path + from + ( + -- superuser: empty ltree matches everything via <@ + select ''::ltree as tree_path + from {{schema}}."user" u + inner join effective_roles er on (u.id = er.user_id) + where u.superuser + union + -- ownership grants full access + select o.tree_path + from {{schema}}.tree_owner o + inner join effective_roles er on (er.user_id = o.user_id) + union + -- explicit grants for the requested action + select g.tree_path + from {{schema}}.tree_grant g + inner join effective_roles er on (er.user_id = g.user_id) + where _action = any(g.actions) + ) +$func$ +language sql stable security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +revoke all on function {{schema}}.tree_access(uuid, text) from public; +grant execute on function {{schema}}.tree_access(uuid, text) to me_ro, me_rw; + +-- defense in depth: revoke PUBLIC access on auth tables +revoke all on {{schema}}."user" from public; +revoke all on {{schema}}.api_key from public; +revoke all on {{schema}}.tree_grant from public; +revoke all on {{schema}}.role_membership from public; +revoke all on {{schema}}.tree_owner from public; + +-- ===== RLS on memory ===== +alter table {{schema}}.memory enable row level security; + +create policy memory_select on {{schema}}.memory + for select to me_ro, me_rw + using + ( + exists + ( + select true + from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'read') ta(tree_path) + where tree <@ ta.tree_path + ) + ); + +create policy memory_insert on {{schema}}.memory + for insert to me_rw + with check + ( + exists + ( + select true + from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'create') ta(tree_path) + where tree <@ ta.tree_path + ) + ); + +create policy memory_update on {{schema}}.memory + for update to me_rw + using + ( + exists + ( + select true + from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'update') ta(tree_path) + where tree <@ ta.tree_path + ) + ) + with check + ( + exists + ( + select true + from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'update') ta(tree_path) + where tree <@ ta.tree_path + ) + ); + +create policy memory_delete on {{schema}}.memory + for delete to me_rw + using + ( + exists + ( + select true + from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'delete') ta(tree_path) + where tree <@ ta.tree_path + ) + ); + +-- ===== Memory FK ===== +alter table {{schema}}.memory add constraint memory_created_by_fk + foreign key (created_by) references {{schema}}."user"(id) on delete set null; diff --git a/proto/create.ts b/proto/create.ts new file mode 100644 index 0000000..fca9dfa --- /dev/null +++ b/proto/create.ts @@ -0,0 +1,26 @@ +const variables: Record = { + schema: "me_01", + embedding_dimensions: "1536", + bm25_text_config: "english", + bm25_k1: "1.2", + bm25_b: "0.75", + hnsw_m: "16", + hnsw_ef_construction: "64", +}; + +const input = await Bun.file(new URL("./create.sql", import.meta.url)).text(); + +const output = input.replace(/\{\{([a-zA-Z0-9_]+)\}\}/g, (match, name) => { + const value = variables[name]; + if (value === undefined) { + throw new Error(`No value configured for placeholder ${match}`); + } + return value; +}); + +const unresolved = output.match(/\{\{[a-zA-Z0-9_]+\}\}/g); +if (unresolved) { + throw new Error(`Unresolved placeholders: ${unresolved.join(", ")}`); +} + +process.stdout.write(output); From ca4d7d5bd2396bbf9e6c881966aeb8b96a2af90c Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Thu, 14 May 2026 14:09:26 -0500 Subject: [PATCH 02/17] progress --- proto/.gitignore | 1 + proto/copy-memory.sh | 8 ++ proto/copy-memory.ts | 119 ++++++++++++++++++++++++ proto/create.sql | 209 ++++++++++++++++++++++++++----------------- 4 files changed, 254 insertions(+), 83 deletions(-) create mode 100644 proto/.gitignore create mode 100755 proto/copy-memory.sh create mode 100644 proto/copy-memory.ts diff --git a/proto/.gitignore b/proto/.gitignore new file mode 100644 index 0000000..f5f242b --- /dev/null +++ b/proto/.gitignore @@ -0,0 +1 @@ +copy.sh diff --git a/proto/copy-memory.sh b/proto/copy-memory.sh new file mode 100755 index 0000000..cb635bf --- /dev/null +++ b/proto/copy-memory.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_dir="$(dirname -- "$script_dir")" + +exec "$repo_dir/bun" "$script_dir/copy-memory.ts" diff --git a/proto/copy-memory.ts b/proto/copy-memory.ts new file mode 100644 index 0000000..79b8931 --- /dev/null +++ b/proto/copy-memory.ts @@ -0,0 +1,119 @@ +import { $ } from "bun"; + +const columns = [ + "id", + "meta", + "tree", + "temporal", + "content", + "embedding", + "embedding_version", + "created_at", + "created_by", + "updated_at", +].join(", "); + +function requireEnv(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +function parseBatchSize(): number { + const value = process.env.BATCH_SIZE ?? "1000"; + const parsed = Number.parseInt(value, 10); + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`BATCH_SIZE must be a positive integer, got: ${value}`); + } + return parsed; +} + +function quoteIdent(name: string, value: string): string { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(value)) { + throw new Error(`${name} must be a simple SQL identifier, got: ${value}`); + } + return `"${value}"`; +} + +function quoteUuid(value: string): string { + if ( + !/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/.test( + value, + ) + ) { + throw new Error(`Expected UUID boundary, got: ${value}`); + } + return `'${value}'::uuid`; +} + +async function psqlJson(databaseUrl: string, sql: string): Promise { + const output = + await $`psql ${databaseUrl} -X -q -t -A -v ON_ERROR_STOP=1 -c ${sql}`.text(); + return JSON.parse(output.trim()) as T; +} + +function progress(copied: number, total: number, startedAt: number): string { + const pct = total === 0 ? 100 : (copied / total) * 100; + const elapsedSeconds = (performance.now() - startedAt) / 1000; + const rowsPerSecond = elapsedSeconds > 0 ? copied / elapsedSeconds : 0; + return `${copied}/${total} (${pct.toFixed(1)}%, ${rowsPerSecond.toFixed(0)} rows/s)`; +} + +const databaseUrlFrom = requireEnv("DATABASE_URL_FROM"); +const databaseUrlTo = requireEnv("DATABASE_URL_TO"); +const schemaFrom = quoteIdent("SCHEMA_FROM", requireEnv("SCHEMA_FROM")); +const schemaTo = quoteIdent("SCHEMA_TO", requireEnv("SCHEMA_TO")); +const batchSize = parseBatchSize(); + +type Stats = { min_id: string | null; count: string }; +type Boundary = { last_id: string | null; count: string }; + +const stats = await psqlJson( + databaseUrlFrom, + `select json_build_object('min_id', (select id::text from ${schemaFrom}.memory order by id limit 1), 'count', count(*)::text)::text from ${schemaFrom}.memory`, +); +const total = Number.parseInt(stats.count, 10); +if (!Number.isSafeInteger(total)) { + throw new Error(`Source memory count is not a safe integer: ${stats.count}`); +} + +console.error( + `Copying ${total} memories from ${schemaFrom}.memory to ${schemaTo}.memory in batches of ${batchSize}`, +); +if (stats.min_id) console.error(`First source id: ${stats.min_id}`); + +let copied = 0; +let batch = 0; +let lastId: string | null = null; +const startedAt = performance.now(); + +while (copied < total) { + const whereAfterLast: string = lastId + ? `where id > ${quoteUuid(lastId)}` + : ""; + const boundary: Boundary = await psqlJson( + databaseUrlFrom, + `with batch as (select id from ${schemaFrom}.memory ${whereAfterLast} order by id limit ${batchSize}) select json_build_object('last_id', (select id::text from batch order by id desc limit 1), 'count', (select count(*)::text from batch))::text`, + ); + const batchRows = Number.parseInt(boundary.count, 10); + if (!Number.isSafeInteger(batchRows)) { + throw new Error(`Batch count is not a safe integer: ${boundary.count}`); + } + if (batchRows === 0 || !boundary.last_id) break; + + const lowerBound = lastId ? `id > ${quoteUuid(lastId)} and ` : ""; + const upperBound = `id <= ${quoteUuid(boundary.last_id)}`; + const sourceCopy = `\\copy (select ${columns} from ${schemaFrom}.memory where ${lowerBound}${upperBound} order by id) to stdout with (format binary)`; + const targetCopy = `\\copy ${schemaTo}.memory (${columns}) from stdin with (format binary)`; + + batch++; + await $`psql ${databaseUrlFrom} -X -q -v ON_ERROR_STOP=1 -c ${sourceCopy} | psql ${databaseUrlTo} -X -q -v ON_ERROR_STOP=1 -c ${targetCopy}`; + + copied += batchRows; + lastId = boundary.last_id; + console.error( + `Batch ${batch}: copied ${batchRows} rows, ${progress(copied, total, startedAt)}`, + ); +} + +console.error(`Finished copying ${copied} memories in ${batch} batches.`); diff --git a/proto/create.sql b/proto/create.sql index a57e804..bcf9812 100644 --- a/proto/create.sql +++ b/proto/create.sql @@ -112,6 +112,125 @@ for each row execute function {{schema}}.update_updated_at() ; +revoke all on {{schema}}."user" from public; +grant select on {{schema}}."user" to me_ro; +grant select, insert, update, delete on {{schema}}."user" to me_rw; + +------------------------------------------------------------------------------- +-- role membership +------------------------------------------------------------------------------- +create table {{schema}}.role_membership +( role_id uuid not null references {{schema}}."user"(id) on delete cascade +, member_id uuid not null references {{schema}}."user"(id) on delete cascade +, with_admin_option boolean not null default false +, created_at timestamptz not null default now() +, primary key (role_id, member_id) +, constraint no_self_membership check (role_id != member_id) +); + +create index idx_role_membership_member on {{schema}}.role_membership(member_id); + +revoke all on {{schema}}.role_membership from public; +grant select on {{schema}}.role_membership to me_ro; +grant select, insert, update, delete on {{schema}}.role_membership to me_rw; + +------------------------------------------------------------------------------- +-- would_create_cycle +------------------------------------------------------------------------------- +create function {{schema}}.would_create_cycle +( _role_id uuid +, _member_id uuid +) +returns boolean +as $func$ + with recursive ancestors(id) as + ( + select rm.role_id + from {{schema}}.role_membership rm + where rm.member_id = _role_id + union all + select rm.role_id + from {{schema}}.role_membership rm + inner join ancestors a on a.id = rm.member_id + ) + select _member_id = _role_id + or exists + ( + select 1 + from ancestors + where id = _member_id + ) +$func$ language sql stable security invoker +parallel safe +set search_path to pg_catalog, {{schema}}, pg_temp +; + +revoke all on {{schema}}.would_create_cycle(uuid, uuid) from public; +grant execute on {{schema}}.would_create_cycle(uuid, uuid) to me_rw; + +-- Prevent role membership cycles for ordinary writes. +-- Note: this check observes the current transaction snapshot. Concurrent +-- transactions that insert/update related role edges can still race unless the +-- caller uses stronger locking or serializable transactions around +-- role_membership writes. +create function {{schema}}.role_membership_before_write() +returns trigger +as $func$ +begin + if {{schema}}.would_create_cycle(new.role_id, new.member_id) then + raise exception 'role membership would create a cycle: role_id %, member_id %', new.role_id, new.member_id + using errcode = 'integrity_constraint_violation'; + end if; + + return new; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; + +create trigger role_membership_before_write_trg +before insert or update of role_id, member_id on {{schema}}.role_membership +for each row +execute function {{schema}}.role_membership_before_write() +; + +------------------------------------------------------------------------------- +-- tree ownership +------------------------------------------------------------------------------- +create table {{schema}}.tree_owner +( tree_path ltree not null primary key +, user_id uuid not null references {{schema}}."user" (id) on delete cascade +, created_by uuid references {{schema}}."user" (id) +, created_at timestamptz not null default now() +); + +create index idx_tree_owner_user on {{schema}}.tree_owner (user_id); +create index idx_tree_owner_gist on {{schema}}.tree_owner using gist (tree_path); + +revoke all on {{schema}}.tree_owner from public; +grant select on {{schema}}.tree_owner to me_ro; +grant select, insert, update, delete on {{schema}}.tree_owner to me_rw; + +------------------------------------------------------------------------------- +-- tree grants +------------------------------------------------------------------------------- +create table {{schema}}.tree_grant +( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) +, user_id uuid not null references {{schema}}."user"(id) on delete cascade +, tree_path ltree not null +, actions text[] not null check (actions <@ '{read,create,update,delete}'::text[]) +, granted_by uuid references {{schema}}."user"(id) +, created_at timestamptz not null default now() +, with_grant_option boolean not null default false +); + +create unique index idx_tree_grant_unique on {{schema}}.tree_grant (user_id, tree_path); +create index idx_tree_grant_path on {{schema}}.tree_grant using gist (tree_path); + +revoke all on {{schema}}.tree_grant from public; +grant select on {{schema}}.tree_grant to me_ro; +grant select, insert, update, delete on {{schema}}.tree_grant to me_rw; + ------------------------------------------------------------------------------- -- memory ------------------------------------------------------------------------------- @@ -124,10 +243,11 @@ create table {{schema}}.memory , embedding halfvec({{embedding_dimensions}}) , embedding_version int4 not null default 1 , created_at timestamptz not null default now() -, created_by uuid references {{schema}}."user" (id) on delete set null +, created_by uuid , updated_at timestamptz ); +revoke all on {{schema}}.memory from public; grant select on {{schema}}.memory to me_ro; grant select, insert, update, delete on {{schema}}.memory to me_rw; grant select, update on {{schema}}.memory to me_embed; @@ -398,6 +518,9 @@ set search_path to pg_catalog, {{schema}}, pg_temp grant execute on function {{schema}}.prune_embedding_queue(interval) to me_embed; + + + ------------------------------------------------------------------------------- -- api keys ------------------------------------------------------------------------------- @@ -416,79 +539,6 @@ create table {{schema}}.api_key create index idx_api_key_user on {{schema}}.api_key (user_id); create index idx_api_key_lookup on {{schema}}.api_key (lookup_id) where revoked_at is null; -------------------------------------------------------------------------------- --- tree grants -------------------------------------------------------------------------------- -create table {{schema}}.tree_grant -( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) -, user_id uuid not null references {{schema}}."user"(id) on delete cascade -, tree_path ltree not null -, actions text[] not null -, granted_by uuid references {{schema}}."user"(id) -, created_at timestamptz not null default now() -, with_grant_option boolean not null default false -, constraint valid_actions check ( - actions <@ '{read,create,update,delete}'::text[] - ) -); - -create unique index idx_tree_grant_unique on {{schema}}.tree_grant (user_id, tree_path); -create index idx_tree_grant_user on {{schema}}.tree_grant using btree (user_id); -create index idx_tree_grant_path on {{schema}}.tree_grant using gist (tree_path); - -------------------------------------------------------------------------------- --- role membership -------------------------------------------------------------------------------- -create table {{schema}}.role_membership -( role_id uuid not null references {{schema}}."user"(id) on delete cascade -, member_id uuid not null references {{schema}}."user"(id) on delete cascade -, with_admin_option boolean not null default false -, created_at timestamptz not null default now() -, primary key (role_id, member_id) -, constraint no_self_membership check (role_id <> member_id) -); - -create index idx_role_membership_member on {{schema}}.role_membership(member_id); - --- ===== Cycle Detection ===== -create function {{schema}}.would_create_cycle -( _role_id uuid -, _member_id uuid -) -returns boolean -as $func$ - with recursive ancestors(id) as ( - select rm.role_id - from {{schema}}.role_membership rm - where rm.member_id = _role_id - union - select rm.role_id - from {{schema}}.role_membership rm - inner join ancestors a on a.id = rm.member_id - ) - select _member_id = _role_id - or exists - ( - select 1 - from ancestors - where id = _member_id - ) -$func$ language sql stable security invoker -set search_path to pg_catalog, {{schema}}, pg_temp -; - -------------------------------------------------------------------------------- --- tree ownership -------------------------------------------------------------------------------- -create table {{schema}}.tree_owner -( tree_path ltree primary key -, user_id uuid not null references {{schema}}."user"(id) on delete cascade -, created_by uuid references {{schema}}."user"(id) -, created_at timestamptz not null default now() -); - -create index idx_tree_owner_user on {{schema}}.tree_owner (user_id); -create index idx_tree_owner_gist on {{schema}}.tree_owner using gist (tree_path); ------------------------------------------------------------------------------- -- tree access @@ -537,13 +587,8 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp revoke all on function {{schema}}.tree_access(uuid, text) from public; grant execute on function {{schema}}.tree_access(uuid, text) to me_ro, me_rw; --- defense in depth: revoke PUBLIC access on auth tables -revoke all on {{schema}}."user" from public; -revoke all on {{schema}}.api_key from public; -revoke all on {{schema}}.tree_grant from public; -revoke all on {{schema}}.role_membership from public; -revoke all on {{schema}}.tree_owner from public; +/* -- ===== RLS on memory ===== alter table {{schema}}.memory enable row level security; @@ -604,6 +649,4 @@ create policy memory_delete on {{schema}}.memory ) ); --- ===== Memory FK ===== -alter table {{schema}}.memory add constraint memory_created_by_fk - foreign key (created_by) references {{schema}}."user"(id) on delete set null; +*/ From 4c24387c4fa3ead5346ed7f13105a2b1841466f3 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Thu, 14 May 2026 15:57:27 -0500 Subject: [PATCH 03/17] chore: more progress --- proto/create.sql | 156 +++++++++++++++++++++++++++++++++++------------ proto/create.ts | 2 +- proto/data.sql | 47 ++++++++++++++ 3 files changed, 165 insertions(+), 40 deletions(-) create mode 100644 proto/data.sql diff --git a/proto/create.sql b/proto/create.sql index bcf9812..1ff3536 100644 --- a/proto/create.sql +++ b/proto/create.sql @@ -22,10 +22,10 @@ select current_setting('server_version_num')::int < 180000 as bad_pg_version ------------------------------------------------------------------------------- -- ensure extensions installed ------------------------------------------------------------------------------- -create extension if not exists citext; -create extension if not exists ltree; -create extension if not exists vector; -create extension if not exists pg_textsearch; +create extension if not exists citext with schema public version '1.6'; +create extension if not exists ltree with schema public version '1.3'; +create extension if not exists vector with schema public version '0.8.2'; +create extension if not exists pg_textsearch with schema public version '1.1.0'; ------------------------------------------------------------------------------- -- database roles @@ -97,9 +97,7 @@ create table {{schema}}."user" ( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) , name citext not null unique , identity_id uuid check (identity_id is null or uuid_extract_version(identity_id) = 7) -- soft FK to accounts.identity -, can_login boolean not null default true -- false = role (grant container) , superuser boolean not null default false -, createrole boolean not null default false -- can create other users/roles , created_at timestamptz not null default now() , updated_at timestamptz ); @@ -124,11 +122,11 @@ create table {{schema}}.role_membership , member_id uuid not null references {{schema}}."user"(id) on delete cascade , with_admin_option boolean not null default false , created_at timestamptz not null default now() -, primary key (role_id, member_id) +, primary key (member_id, role_id) , constraint no_self_membership check (role_id != member_id) ); -create index idx_role_membership_member on {{schema}}.role_membership(member_id); +create index idx_role_membership_role on {{schema}}.role_membership(role_id) include (member_id); revoke all on {{schema}}.role_membership from public; grant select on {{schema}}.role_membership to me_ro; @@ -148,7 +146,7 @@ as $func$ select rm.role_id from {{schema}}.role_membership rm where rm.member_id = _role_id - union all + union select rm.role_id from {{schema}}.role_membership rm inner join ancestors a on a.id = rm.member_id @@ -165,9 +163,12 @@ parallel safe set search_path to pg_catalog, {{schema}}, pg_temp ; -revoke all on {{schema}}.would_create_cycle(uuid, uuid) from public; -grant execute on {{schema}}.would_create_cycle(uuid, uuid) to me_rw; +revoke all on function {{schema}}.would_create_cycle(uuid, uuid) from public; +grant execute on function {{schema}}.would_create_cycle(uuid, uuid) to me_rw; +------------------------------------------------------------------------------- +-- role_membership_before_write trigger +------------------------------------------------------------------------------- -- Prevent role membership cycles for ordinary writes. -- Note: this check observes the current transaction snapshot. Concurrent -- transactions that insert/update related role edges can still race unless the @@ -181,7 +182,6 @@ begin raise exception 'role membership would create a cycle: role_id %, member_id %', new.role_id, new.member_id using errcode = 'integrity_constraint_violation'; end if; - return new; end; $func$ language plpgsql volatile security invoker @@ -194,6 +194,86 @@ for each row execute function {{schema}}.role_membership_before_write() ; +------------------------------------------------------------------------------- +-- explode_role_membership +------------------------------------------------------------------------------- +create or replace function {{schema}}.explode_role_membership(_user_id uuid) +returns table +( role_id uuid +, superuser bool +, dist int4 +) +as $func$ + with recursive ancestors(id, dist) as + ( + select rm.role_id, 1::int4 + from {{schema}}.role_membership rm + where rm.member_id = _user_id + union + select rm.role_id, a.dist + 1 + from {{schema}}.role_membership rm + inner join ancestors a on a.id = rm.member_id + ) + select + u.id + , u.superuser + , 0::int4 + from {{schema}}."user" u + where u.id = _user_id + union + select + u.id + , u.superuser + , a.dist + from {{schema}}."user" u + inner join ancestors a on (u.id = a.id) +$func$ language sql stable security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; + +------------------------------------------------------------------------------- +-- grant_role_membership +------------------------------------------------------------------------------- +create or replace function {{schema}}.grant_role_membership(_role_id uuid, _member_id uuid, _with_admin_option bool default false) +returns void +as $func$ +begin + -- exclusive write access required to fully ensure against cycle creation by concurrent writes + lock table {{schema}}.role_membership in share row exclusive mode; + -- role_membership_before_write_trg protects against cycles in the graph + insert into {{schema}}.role_membership + ( role_id + , member_id + , with_admin_option + ) + values + ( _role_id + , _member_id + , _with_admin_option + ); +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; + +------------------------------------------------------------------------------- +-- revoke_role_membership +------------------------------------------------------------------------------- +create or replace function {{schema}}.revoke_role_membership(_role uuid, _member_id uuid) +returns void +as $func$ +begin + lock table {{schema}}.role_membership in share row exclusive mode; + + delete from {{schema}}.role_membership d + where d.role_id = _role_id + and d.member_id = _member_id + ; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; + ------------------------------------------------------------------------------- -- tree ownership ------------------------------------------------------------------------------- @@ -383,9 +463,9 @@ execute function {{schema}}.enqueue_embedding() -- claim_embedding_batch ------------------------------------------------------------------------------- create or replace function {{schema}}.claim_embedding_batch -( batch_size int default 10 -, lock_duration interval default '5 minutes' -, max_attempts int default 3 +( _batch_size int default 10 +, _lock_duration interval default '5 minutes' +, _max_attempts int default 3 ) returns table ( queue_id bigint @@ -395,9 +475,9 @@ returns table ) as $func$ declare - rec record; - mem record; - claimed_count int = 0; + _rec record; + _mem record; + _claimed_count int = 0; begin -- bulk-cancel visible queue rows superseded by a newer row for the same memory update {{schema}}.embedding_queue eq @@ -421,10 +501,10 @@ begin , last_error = coalesce(last_error, 'exceeded max attempts (worker crash)') where outcome is null and vt <= now() - and attempts >= max_attempts + and attempts >= _max_attempts ; - for rec in + for _rec in ( select eq.id @@ -433,48 +513,48 @@ begin from {{schema}}.embedding_queue eq where eq.outcome is null and eq.vt <= now() - and eq.attempts < max_attempts + and eq.attempts < _max_attempts order by eq.vt for update skip locked ) loop -- check memory still exists + current version select m.content, m.embedding_version - into mem + into _mem from {{schema}}.memory m - where m.id = rec.memory_id + where m.id = _rec.memory_id ; - if not found or mem.content is null then + if not found or _mem.content is null then -- memory deleted or empty → cancel queue row update {{schema}}.embedding_queue set outcome = 'cancelled' - where id = rec.id; + where id = _rec.id; continue; end if; - if rec.embedding_version != mem.embedding_version then + if _rec.embedding_version != _mem.embedding_version then -- stale version → cancel update {{schema}}.embedding_queue set outcome = 'cancelled' - where id = rec.id; + where id = _rec.id; continue; end if; -- claim this row update {{schema}}.embedding_queue q set - vt = now() + lock_duration + vt = now() + _lock_duration , attempts = q.attempts + 1 - where id = rec.id; + where id = _rec.id; - queue_id = rec.id; - memory_id = rec.memory_id; - embedding_version = rec.embedding_version; - content = mem.content; + queue_id = _rec.id; + memory_id = _rec.memory_id; + embedding_version = _rec.embedding_version; + content = _mem.content; return next; - claimed_count = claimed_count + 1; - exit when claimed_count >= batch_size; + _claimed_count = _claimed_count + 1; + exit when _claimed_count >= _batch_size; end loop; end; $func$ @@ -493,9 +573,7 @@ grant execute on function {{schema}}.claim_embedding_batch(int, interval, int) t -- -- relies on embedding_queue_archive_idx (created_at) where outcome is not null -- from migration 005, so the no-op case is cheap. -create or replace function {{schema}}.prune_embedding_queue -( retention interval default '7 days' -) +create or replace function {{schema}}.prune_embedding_queue(_retention interval default '7 days') returns bigint as $func$ declare @@ -503,7 +581,7 @@ declare begin delete from {{schema}}.embedding_queue where outcome is not null - and created_at < now() - retention + and created_at < now() - _retention ; get diagnostics pruned = row_count; return pruned; diff --git a/proto/create.ts b/proto/create.ts index fca9dfa..bdd2884 100644 --- a/proto/create.ts +++ b/proto/create.ts @@ -1,5 +1,5 @@ const variables: Record = { - schema: "me_01", + schema: "me", embedding_dimensions: "1536", bm25_text_config: "english", bm25_k1: "1.2", diff --git a/proto/data.sql b/proto/data.sql new file mode 100644 index 0000000..dc18633 --- /dev/null +++ b/proto/data.sql @@ -0,0 +1,47 @@ + + +------------------------------------------------------------------------------- +-- testing data +------------------------------------------------------------------------------- +begin; + +insert into me."user" (id, name, superuser) +values + ('019e2833-f217-7457-ba8b-f110393b6d1c', 'user_0', true) +, ('019e2833-f217-74a0-84ca-49e9655ed2e2', 'user_1', true) +, ('019e2833-f217-74a9-9860-fde559ebc44f', 'user_2', false) +, ('019e2833-f217-74af-b5f5-c0cd0beb78ab', 'user_3', false) +, ('019e2833-f217-74b6-97f5-bad28152696d', 'user_4', false) +, ('019e2833-f217-74bc-9ae6-54f009c08d3e', 'user_5', false) +, ('019e2833-f217-74c2-a612-9ccc17e11380', 'user_6', false) +, ('019e2833-f217-74c8-8eab-bcfcedc95d29', 'user_7', false) +, ('019e2833-f217-74ce-8820-6ea10aebd123', 'user_8', false) +, ('019e2833-f217-74d5-b22f-0091833bf484', 'user_9', false) +; + +insert into me."user" (id, name, superuser) +values + ('019e2835-3ece-7cbc-a450-4abb1d3437c2','role_0', true ) +, ('019e2835-3ece-7d03-91bb-61c94fa959a5','role_0.1', false) +, ('019e2835-3ece-7d0b-bf85-7b8707750774','role_1', false) +, ('019e2835-3ece-7d11-aaa5-24414460784f','role_1.1', false) +, ('019e2835-3ece-7d17-8d9d-7291258c8d0b','role_1.2', false) +, ('019e2835-3ece-7d1e-9acd-b4597611d70c','role_1.2.1', false) +; + +-- roles to roles +select me.grant_role_membership('019e2835-3ece-7cbc-a450-4abb1d3437c2', '019e2835-3ece-7d03-91bb-61c94fa959a5'); +select me.grant_role_membership('019e2835-3ece-7d0b-bf85-7b8707750774', '019e2835-3ece-7d11-aaa5-24414460784f'); +select me.grant_role_membership('019e2835-3ece-7d0b-bf85-7b8707750774', '019e2835-3ece-7d17-8d9d-7291258c8d0b'); +select me.grant_role_membership('019e2835-3ece-7d17-8d9d-7291258c8d0b', '019e2835-3ece-7d1e-9acd-b4597611d70c'); + +-- add users to roles +select me.grant_role_membership('019e2835-3ece-7cbc-a450-4abb1d3437c2', '019e2833-f217-74a9-9860-fde559ebc44f'); +select me.grant_role_membership('019e2835-3ece-7d03-91bb-61c94fa959a5', '019e2833-f217-74af-b5f5-c0cd0beb78ab'); +select me.grant_role_membership('019e2835-3ece-7d0b-bf85-7b8707750774', '019e2833-f217-74b6-97f5-bad28152696d'); +select me.grant_role_membership('019e2835-3ece-7d11-aaa5-24414460784f', '019e2833-f217-74bc-9ae6-54f009c08d3e'); +select me.grant_role_membership('019e2835-3ece-7d17-8d9d-7291258c8d0b', '019e2833-f217-74c2-a612-9ccc17e11380'); +select me.grant_role_membership('019e2835-3ece-7d1e-9acd-b4597611d70c', '019e2833-f217-74c8-8eab-bcfcedc95d29'); +select me.grant_role_membership('019e2835-3ece-7d1e-9acd-b4597611d70c', '019e2833-f217-74ce-8820-6ea10aebd123'); +select me.grant_role_membership('019e2835-3ece-7d1e-9acd-b4597611d70c', '019e2833-f217-74d5-b22f-0091833bf484'); +commit; From 55423c2a429d53db86060deaafdd26ef8a4b7fab Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Fri, 15 May 2026 14:20:27 -0500 Subject: [PATCH 04/17] chore: more progress --- proto/create.sql | 338 ++++++++++++++++++++++++++++------------------- proto/data.sql | 35 +++-- 2 files changed, 228 insertions(+), 145 deletions(-) diff --git a/proto/create.sql b/proto/create.sql index 1ff3536..65d6f95 100644 --- a/proto/create.sql +++ b/proto/create.sql @@ -96,23 +96,20 @@ set search_path to {{schema}}, pg_temp; create table {{schema}}."user" ( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) , name citext not null unique -, identity_id uuid check (identity_id is null or uuid_extract_version(identity_id) = 7) -- soft FK to accounts.identity , superuser boolean not null default false , created_at timestamptz not null default now() , updated_at timestamptz ); -create index idx_user_identity_id on {{schema}}."user" (identity_id) where identity_id is not null; - -create trigger user_updated_at +create or replace trigger user_updated_at before update on {{schema}}."user" for each row execute function {{schema}}.update_updated_at() ; revoke all on {{schema}}."user" from public; -grant select on {{schema}}."user" to me_ro; -grant select, insert, update, delete on {{schema}}."user" to me_rw; +grant select on {{schema}}."user" to me_ro, me_rw; +-- NO direct writes on the table. must go through functions ------------------------------------------------------------------------------- -- role membership @@ -129,13 +126,13 @@ create table {{schema}}.role_membership create index idx_role_membership_role on {{schema}}.role_membership(role_id) include (member_id); revoke all on {{schema}}.role_membership from public; -grant select on {{schema}}.role_membership to me_ro; -grant select, insert, update, delete on {{schema}}.role_membership to me_rw; +grant select on {{schema}}.role_membership to me_ro, me_rw; +-- NO direct writes on the table. must go through functions ------------------------------------------------------------------------------- -- would_create_cycle ------------------------------------------------------------------------------- -create function {{schema}}.would_create_cycle +create or replace function {{schema}}.would_create_cycle ( _role_id uuid , _member_id uuid ) @@ -158,7 +155,7 @@ as $func$ from ancestors where id = _member_id ) -$func$ language sql stable security invoker +$func$ language sql stable security definer parallel safe set search_path to pg_catalog, {{schema}}, pg_temp ; @@ -174,7 +171,7 @@ grant execute on function {{schema}}.would_create_cycle(uuid, uuid) to me_rw; -- transactions that insert/update related role edges can still race unless the -- caller uses stronger locking or serializable transactions around -- role_membership writes. -create function {{schema}}.role_membership_before_write() +create or replace function {{schema}}.role_membership_before_write() returns trigger as $func$ begin @@ -184,11 +181,11 @@ begin end if; return new; end; -$func$ language plpgsql volatile security invoker +$func$ language plpgsql volatile security definer set search_path to pg_catalog, {{schema}}, pg_temp ; -create trigger role_membership_before_write_trg +create or replace trigger role_membership_before_write_trg before insert or update of role_id, member_id on {{schema}}.role_membership for each row execute function {{schema}}.role_membership_before_write() @@ -228,18 +225,67 @@ as $func$ from {{schema}}."user" u inner join ancestors a on (u.id = a.id) $func$ language sql stable security invoker +; + +revoke all on function {{schema}}.explode_role_membership(uuid) from public; +grant execute on function {{schema}}.explode_role_membership(uuid) to me_ro, me_rw; + +------------------------------------------------------------------------------- +-- is_superuser +------------------------------------------------------------------------------- +create or replace function {{schema}}.is_superuser(_user_id uuid) +returns boolean +as $func$ + select exists + ( + select 1 + from {{schema}}.explode_role_membership(_user_id) x + where x.superuser + ) +$func$ language sql stable security definer set search_path to pg_catalog, {{schema}}, pg_temp ; +revoke all on function {{schema}}.is_superuser(uuid) from public; +grant execute on function {{schema}}.is_superuser(uuid) to me_ro, me_rw; + ------------------------------------------------------------------------------- -- grant_role_membership ------------------------------------------------------------------------------- -create or replace function {{schema}}.grant_role_membership(_role_id uuid, _member_id uuid, _with_admin_option bool default false) +create or replace function {{schema}}.grant_role_membership +( _grantor_id uuid +, _role_id uuid +, _member_id uuid +, _with_admin_option bool default false +) returns void as $func$ +declare + _allowed bool; begin -- exclusive write access required to fully ensure against cycle creation by concurrent writes lock table {{schema}}.role_membership in share row exclusive mode; + + -- is grantor allowed to do this? + select + exists + ( + -- does the grantor have with admin privilege directly on this role? + select 1 + from {{schema}}.role_membership rm + where rm.role_id = _role_id + and rm.member_id = _grantor_id + and rm.with_admin_option + ) + or {{schema}}.is_superuser(_grantor_id) -- or are they a superuser (even indirectly)? + into strict _allowed + ; + + if not _allowed then + raise exception 'grantor must be a superuser or have with admin option on role: grantor_id % role_id %', _grantor_id, _role_id + using errcode = 'insufficient_privilege'; + end if; + -- role_membership_before_write_trg protects against cycles in the graph insert into {{schema}}.role_membership ( role_id @@ -250,21 +296,53 @@ begin ( _role_id , _member_id , _with_admin_option - ); + ) + on conflict (member_id, role_id) + do update set with_admin_option = _with_admin_option + ; end; -$func$ language plpgsql volatile security invoker +$func$ language plpgsql volatile security definer set search_path to pg_catalog, {{schema}}, pg_temp ; +revoke all on function {{schema}}.grant_role_membership(uuid, uuid, uuid, bool) from public; +grant execute on function {{schema}}.grant_role_membership(uuid, uuid, uuid, bool) to me_rw; + ------------------------------------------------------------------------------- -- revoke_role_membership ------------------------------------------------------------------------------- -create or replace function {{schema}}.revoke_role_membership(_role uuid, _member_id uuid) +create or replace function {{schema}}.revoke_role_membership +( _revoker_id uuid +, _role_id uuid +, _member_id uuid +) returns void as $func$ +declare + _allowed bool; begin lock table {{schema}}.role_membership in share row exclusive mode; + -- is revoker allowed to do this? + select + exists + ( + -- does the revoker have with admin privilege directly on this role? + select 1 + from {{schema}}.role_membership rm + where rm.role_id = _role_id + and rm.member_id = _revoker_id + and rm.with_admin_option + ) + or {{schema}}.is_superuser(_revoker_id) -- or are they a superuser (even indirectly)? + into strict _allowed + ; + + if not _allowed then + raise exception 'revoker must be a superuser or have with admin option on role: revoker_id % role_id %', _revoker_id, _role_id + using errcode = 'insufficient_privilege'; + end if; + delete from {{schema}}.role_membership d where d.role_id = _role_id and d.member_id = _member_id @@ -274,6 +352,9 @@ $func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, pg_temp ; +revoke all on function {{schema}}.grant_role_membership(uuid, uuid, uuid, bool) from public; +grant execute on function {{schema}}.grant_role_membership(uuid, uuid, uuid, bool) to me_rw; + ------------------------------------------------------------------------------- -- tree ownership ------------------------------------------------------------------------------- @@ -311,6 +392,110 @@ revoke all on {{schema}}.tree_grant from public; grant select on {{schema}}.tree_grant to me_ro; grant select, insert, update, delete on {{schema}}.tree_grant to me_rw; +------------------------------------------------------------------------------- +-- calc_tree_privileges +------------------------------------------------------------------------------- +create or replace function {{schema}}.calc_tree_privileges(_user_id uuid) +returns table +( role_id uuid +, tree_path ltree +, actions text[] +, reason text +) +as $func$ + with r as + ( + -- the user + select + u.id as role_id + , u.superuser + from me."user" u + where u.id = _user_id + union + -- the roles they belong to + select + x.role_id + , x.superuser + from {{schema}}.explode_role_membership(_user_id) x + ) + -- superuser + select + r.role_id + , ''::ltree as tree_path + , array['read', 'create', 'update', 'delete'] as actions + , 'superuser' as reason + from r + where r.superuser + union all + -- ownership + select + r.role_id + , o.tree_path + , array['read', 'create', 'update', 'delete'] as actions + , 'owner' as reason + from r + inner join {{schema}}.tree_owner o on (r.role_id = o.user_id) + union all + -- grants + select + r.role_id + , g.tree_path + , g.actions + , 'grant' as reason + from r + inner join {{schema}}.tree_grant g on (r.role_id = g.user_id) +$func$ language sql stable security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +revoke all on function {{schema}}.calc_tree_privileges(uuid) from public; +grant execute on function {{schema}}.calc_tree_privileges(uuid) to me_ro, me_rw; + +------------------------------------------------------------------------------- +-- has_tree_privilege +------------------------------------------------------------------------------- +create or replace function {{schema}}.has_tree_privilege +( _user_id uuid +, _tree_path ltree +, _actions text[] +) +returns bool +as $func$ + select + -- is the user a superuser? + exists + ( + select 1 + from {{schema}}.explode_role_membership(_user_id) x + where x.superuser + ) + -- or do they own the branch of the tree? + or exists + ( + select 1 + from {{schema}}.explode_role_membership(_user_id) x + inner join {{schema}}.tree_owner o + on (x.role_id = o.user_id and o.tree_path @> _tree_path) + ) + -- or do they have an explicit grant to the branch? + or exists + ( + select 1 + from {{schema}}.explode_role_membership(_user_id) x + inner join {{schema}}.tree_grant g + on + ( x.role_id = g.user_id + and g.tree_path @> _tree_path + and g.actions @> _actions + ) + ) +$func$ language sql stable security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +revoke all on function {{schema}}.has_tree_privilege(uuid, ltree, text[]) from public; +grant execute on function {{schema}}.has_tree_privilege(uuid, ltree, text[]) to me_ro, me_rw; + ------------------------------------------------------------------------------- -- memory ------------------------------------------------------------------------------- @@ -392,12 +577,11 @@ end; $func$ language plpgsql volatile security definer set search_path to {{schema}}, public, pg_temp; -- public required for pgvector's `is not distinct from` -create trigger memory_before_update_trg +create or replace trigger memory_before_update_trg before update on {{schema}}.memory for each row execute function {{schema}}.memory_before_update(); - ------------------------------------------------------------------------------- -- embedding queue ------------------------------------------------------------------------------- @@ -442,14 +626,14 @@ set search_path to pg_catalog, {{schema}}, pg_temp ------------------------------------------------------------------------------- -- enqueuing triggers ------------------------------------------------------------------------------- -create trigger memory_enqueue_embedding_insert +create or replace trigger memory_enqueue_embedding_insert after insert on {{schema}}.memory for each row when (new.embedding is null) -- it's possible to insert WITH an embedding execute function {{schema}}.enqueue_embedding() ; -create trigger memory_enqueue_embedding_update +create or replace trigger memory_enqueue_embedding_update after update on {{schema}}.memory for each row when @@ -616,115 +800,3 @@ create table {{schema}}.api_key create index idx_api_key_user on {{schema}}.api_key (user_id); create index idx_api_key_lookup on {{schema}}.api_key (lookup_id) where revoked_at is null; - - -------------------------------------------------------------------------------- --- tree access -------------------------------------------------------------------------------- --- Returns set of tree paths the user can access for the given action. --- Superusers get ''::ltree (empty root) which matches all paths via <@. -create function {{schema}}.tree_access -( _user_id uuid -, _action text -) -returns setof ltree -as $func$ - with recursive effective_roles(user_id) as - ( - select _user_id - union - select rm.role_id - from {{schema}}.role_membership rm - inner join effective_roles er on (er.user_id = rm.member_id) - ) - select distinct tree_path - from - ( - -- superuser: empty ltree matches everything via <@ - select ''::ltree as tree_path - from {{schema}}."user" u - inner join effective_roles er on (u.id = er.user_id) - where u.superuser - union - -- ownership grants full access - select o.tree_path - from {{schema}}.tree_owner o - inner join effective_roles er on (er.user_id = o.user_id) - union - -- explicit grants for the requested action - select g.tree_path - from {{schema}}.tree_grant g - inner join effective_roles er on (er.user_id = g.user_id) - where _action = any(g.actions) - ) -$func$ -language sql stable security definer -set search_path to pg_catalog, {{schema}}, public, pg_temp -; - -revoke all on function {{schema}}.tree_access(uuid, text) from public; -grant execute on function {{schema}}.tree_access(uuid, text) to me_ro, me_rw; - - -/* --- ===== RLS on memory ===== -alter table {{schema}}.memory enable row level security; - -create policy memory_select on {{schema}}.memory - for select to me_ro, me_rw - using - ( - exists - ( - select true - from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'read') ta(tree_path) - where tree <@ ta.tree_path - ) - ); - -create policy memory_insert on {{schema}}.memory - for insert to me_rw - with check - ( - exists - ( - select true - from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'create') ta(tree_path) - where tree <@ ta.tree_path - ) - ); - -create policy memory_update on {{schema}}.memory - for update to me_rw - using - ( - exists - ( - select true - from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'update') ta(tree_path) - where tree <@ ta.tree_path - ) - ) - with check - ( - exists - ( - select true - from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'update') ta(tree_path) - where tree <@ ta.tree_path - ) - ); - -create policy memory_delete on {{schema}}.memory - for delete to me_rw - using - ( - exists - ( - select true - from {{schema}}.tree_access(current_setting('me.user_id', true)::uuid, 'delete') ta(tree_path) - where tree <@ ta.tree_path - ) - ); - -*/ diff --git a/proto/data.sql b/proto/data.sql index dc18633..8ab14c8 100644 --- a/proto/data.sql +++ b/proto/data.sql @@ -30,18 +30,29 @@ values ; -- roles to roles -select me.grant_role_membership('019e2835-3ece-7cbc-a450-4abb1d3437c2', '019e2835-3ece-7d03-91bb-61c94fa959a5'); -select me.grant_role_membership('019e2835-3ece-7d0b-bf85-7b8707750774', '019e2835-3ece-7d11-aaa5-24414460784f'); -select me.grant_role_membership('019e2835-3ece-7d0b-bf85-7b8707750774', '019e2835-3ece-7d17-8d9d-7291258c8d0b'); -select me.grant_role_membership('019e2835-3ece-7d17-8d9d-7291258c8d0b', '019e2835-3ece-7d1e-9acd-b4597611d70c'); +select me.grant_role_membership('019e2833-f217-7457-ba8b-f110393b6d1c', '019e2835-3ece-7cbc-a450-4abb1d3437c2', '019e2835-3ece-7d03-91bb-61c94fa959a5'); +select me.grant_role_membership('019e2833-f217-7457-ba8b-f110393b6d1c', '019e2835-3ece-7d0b-bf85-7b8707750774', '019e2835-3ece-7d11-aaa5-24414460784f'); +select me.grant_role_membership('019e2833-f217-7457-ba8b-f110393b6d1c', '019e2835-3ece-7d0b-bf85-7b8707750774', '019e2835-3ece-7d17-8d9d-7291258c8d0b'); +select me.grant_role_membership('019e2833-f217-7457-ba8b-f110393b6d1c', '019e2835-3ece-7d17-8d9d-7291258c8d0b', '019e2835-3ece-7d1e-9acd-b4597611d70c'); -- add users to roles -select me.grant_role_membership('019e2835-3ece-7cbc-a450-4abb1d3437c2', '019e2833-f217-74a9-9860-fde559ebc44f'); -select me.grant_role_membership('019e2835-3ece-7d03-91bb-61c94fa959a5', '019e2833-f217-74af-b5f5-c0cd0beb78ab'); -select me.grant_role_membership('019e2835-3ece-7d0b-bf85-7b8707750774', '019e2833-f217-74b6-97f5-bad28152696d'); -select me.grant_role_membership('019e2835-3ece-7d11-aaa5-24414460784f', '019e2833-f217-74bc-9ae6-54f009c08d3e'); -select me.grant_role_membership('019e2835-3ece-7d17-8d9d-7291258c8d0b', '019e2833-f217-74c2-a612-9ccc17e11380'); -select me.grant_role_membership('019e2835-3ece-7d1e-9acd-b4597611d70c', '019e2833-f217-74c8-8eab-bcfcedc95d29'); -select me.grant_role_membership('019e2835-3ece-7d1e-9acd-b4597611d70c', '019e2833-f217-74ce-8820-6ea10aebd123'); -select me.grant_role_membership('019e2835-3ece-7d1e-9acd-b4597611d70c', '019e2833-f217-74d5-b22f-0091833bf484'); +select me.grant_role_membership('019e2833-f217-7457-ba8b-f110393b6d1c', '019e2835-3ece-7cbc-a450-4abb1d3437c2', '019e2833-f217-74a9-9860-fde559ebc44f'); +select me.grant_role_membership('019e2833-f217-7457-ba8b-f110393b6d1c', '019e2835-3ece-7d03-91bb-61c94fa959a5', '019e2833-f217-74af-b5f5-c0cd0beb78ab'); +select me.grant_role_membership('019e2833-f217-7457-ba8b-f110393b6d1c', '019e2835-3ece-7d0b-bf85-7b8707750774', '019e2833-f217-74b6-97f5-bad28152696d'); +select me.grant_role_membership('019e2833-f217-7457-ba8b-f110393b6d1c', '019e2835-3ece-7d11-aaa5-24414460784f', '019e2833-f217-74bc-9ae6-54f009c08d3e'); +select me.grant_role_membership('019e2833-f217-7457-ba8b-f110393b6d1c', '019e2835-3ece-7d17-8d9d-7291258c8d0b', '019e2833-f217-74c2-a612-9ccc17e11380'); +select me.grant_role_membership('019e2833-f217-7457-ba8b-f110393b6d1c', '019e2835-3ece-7d1e-9acd-b4597611d70c', '019e2833-f217-74c8-8eab-bcfcedc95d29'); +select me.grant_role_membership('019e2833-f217-7457-ba8b-f110393b6d1c', '019e2835-3ece-7d1e-9acd-b4597611d70c', '019e2833-f217-74ce-8820-6ea10aebd123'); +select me.grant_role_membership('019e2833-f217-7457-ba8b-f110393b6d1c', '019e2835-3ece-7d1e-9acd-b4597611d70c', '019e2833-f217-74d5-b22f-0091833bf484'); commit; + + +insert into me.tree_grant +( user_id +, tree_path +, actions +, granted_by +) +values + ('019e2833-f217-74ce-8820-6ea10aebd123', 'wikipedia.air_force', array['read'], '019e2833-f217-7457-ba8b-f110393b6d1c') +; From b292bd9a99826692bfe53307d9f97cd1c9605ad2 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Mon, 18 May 2026 13:20:50 -0500 Subject: [PATCH 05/17] feat: working on alternate engine backend --- packages/engine-core/index.ts | 0 packages/engine-core/migrate/bootstrap.ts | 145 ++++++++++ packages/engine-core/migrate/migrate.ts | 265 ++++++++++++++++++ .../migrate/migrations/001_updated_at.sql | 0 .../engine-core/migrate/migrations/sql.d.ts | 4 + packages/engine-core/package.json | 6 + packages/engine-core/slug.ts | 18 ++ packages/engine-core/tsconfig.json | 4 + 8 files changed, 442 insertions(+) create mode 100644 packages/engine-core/index.ts create mode 100644 packages/engine-core/migrate/bootstrap.ts create mode 100644 packages/engine-core/migrate/migrate.ts create mode 100644 packages/engine-core/migrate/migrations/001_updated_at.sql create mode 100644 packages/engine-core/migrate/migrations/sql.d.ts create mode 100644 packages/engine-core/package.json create mode 100644 packages/engine-core/slug.ts create mode 100644 packages/engine-core/tsconfig.json diff --git a/packages/engine-core/index.ts b/packages/engine-core/index.ts new file mode 100644 index 0000000..e69de29 diff --git a/packages/engine-core/migrate/bootstrap.ts b/packages/engine-core/migrate/bootstrap.ts new file mode 100644 index 0000000..d548bfe --- /dev/null +++ b/packages/engine-core/migrate/bootstrap.ts @@ -0,0 +1,145 @@ +import { SQL, semver } from "bun"; + +export async function bootstrapEngineDatabase( + sql: SQL, + statementTimeout: string = "20s", + lockTimeout: string = "5s", + transactionTimeout: string = "30s", + idleInTransactionSessionTimeout: string = "30s", + shardId?: number, +): Promise { + await sql.begin(async (tx) => { + if (shardId !== undefined) { + await tx.unsafe(`set local pgdog.shard to ${String(shardId)}`); + } + await ensurePostgresVersion(tx); + await acquireAdvisoryLock(tx); + await tx`select set_config('statement_timeout', ${statementTimeout}, true)`; + await tx`select set_config('lock_timeout', ${lockTimeout}, true)`; + await tx`select set_config('transaction_timeout', ${transactionTimeout}, true)`; + await tx`select set_config('idle_in_transaction_session_timeout', ${idleInTransactionSessionTimeout}, true)`; + await ensureExtension(tx, "citext", "1.6"); + await ensureExtension(tx, "ltree", "1.3"); + await ensureExtension(tx, "vector", "0.8.2"); + await ensureExtension(tx, "pg_textsearch", "1.1.0"); + await ensureRoles(tx); + }); +} + +const MAX_LOCK_RETRIES = 5; +const BASE_DELAY_MS = 100; +const BOOTSTRAP_LOCK_ID = 1982010637711; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function acquireAdvisoryLock(tx: SQL): Promise { + let acquired = false; + for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { + const [result] = await tx` + select pg_try_advisory_xact_lock(${BOOTSTRAP_LOCK_ID}) as acquired + `; + if (result.acquired) { + acquired = true; + break; + } + if (attempt < MAX_LOCK_RETRIES - 1) { + await sleep(BASE_DELAY_MS * 2 ** attempt); + } + } + + if (!acquired) { + throw new Error(`Failed to acquire advisory lock`); + } +} + +async function ensurePostgresVersion(tx: SQL): Promise { + const [{ server_version_num }] = await tx` + select current_setting('server_version_num')::int as server_version_num + `; + if (server_version_num < 180000) { + throw new Error( + `PostgreSQL version 18 or higher is required (found ${server_version_num})`, + ); + } +} + +async function ensureExtension( + tx: SQL, + name: string, + minVersion: string, +): Promise { + const [installed] = await tx` + select x.extversion, n.nspname + from pg_extension x + inner join pg_namespace n on (x.extnamespace = n.oid) + where x.extname = ${name} + `; + + if (installed) { + if ( + installed.nspname === "public" && + semver.order(installed.extversion, minVersion) >= 0 + ) { + return; + } + throw new Error( + `Extension "${name}" version ${minVersion} or higher is required in the "public" schema (found ${installed.extversion} installed in "${installed.nspname}")`, + ); + } + + const [available] = await tx` + select default_version + from pg_available_extensions + where name = ${name} + `; + + if (!available || semver.order(available.default_version, minVersion) < 0) { + const found = available + ? `found ${available.default_version} available` + : "not available"; + throw new Error( + `Extension "${name}" version ${minVersion} or higher is required (${found})`, + ); + } + + try { + await tx`create extension if not exists ${tx(name)} with schema public`; + } catch (error: unknown) { + // Ignore duplicate extension errors (race condition in concurrent calls) + if ( + error instanceof SQL.PostgresError && + error.errno === "23505" && + error.constraint === "pg_extension_name_index" + ) { + return; + } + throw error; + } +} + +async function ensureRoles(tx: SQL): Promise { + await tx.unsafe(` + do $block$ + declare + _roles text[] = array['me_ro', 'me_rw', 'me_embed']; + _role text; + _sql text; + begin + for _role in select * from unnest(_roles) loop + perform + from pg_roles r + where r.rolname = _role; + if found then + continue; + end if; + _sql = format($sql$create role %I nologin$sql$, _role); + execute _sql; + _sql = format($sql$grant %I to %I$sql$, _role, current_user); + execute _sql; + end loop; + end; + $block$; + `); +} diff --git a/packages/engine-core/migrate/migrate.ts b/packages/engine-core/migrate/migrate.ts new file mode 100644 index 0000000..7ba9ff0 --- /dev/null +++ b/packages/engine-core/migrate/migrate.ts @@ -0,0 +1,265 @@ +import { createHash } from "node:crypto"; +import type { SQL } from "bun"; +import { semver } from "bun"; +import { isValidSlug, slugToSchema } from "../slug"; + +import migration001 from "./migrations/001_updated_at.sql" with { + type: "text", +}; + +interface Migration { + name: string; + sql: string; +} + +const migrations: Migration[] = [{ name: "001_updated_at", sql: migration001 }]; + +export interface MigrateEngineOptions { + slug: string; + targetVersion: string; + shardId?: number; + embeddingDimensions?: number; + bm25TextConfig?: string; + bm25K1?: number; + bm25B?: number; + hnswM?: number; + hnswEfConstruction?: number; + statementTimeout?: string; + lockTimeout?: string; + transactionTimeout?: string; + idleInTransactionSessionTimeout?: string; +} + +interface NormalizedMigrateEngineOptions { + slug: string; + targetVersion: string; + shardId?: number; + embeddingDimensions: number; + bm25TextConfig: string; + bm25K1: number; + bm25B: number; + hnswM: number; + hnswEfConstruction: number; + statementTimeout: string; + lockTimeout: string; + transactionTimeout: string; + idleInTransactionSessionTimeout: string; +} + +export async function migrateEngine( + sql: SQL, + options: MigrateEngineOptions, +): Promise { + const opts = normalizeMigrateEngineOptions(options); + + if (!isValidSlug(opts.slug)) { + throw new Error( + `Invalid engine slug: "${opts.slug}" — must be 12 lowercase alphanumeric characters`, + ); + } + if (!semver.satisfies(opts.targetVersion, "*")) { + throw new Error(`Invalid target version: "${opts.targetVersion}"`); + } + const schema = slugToSchema(opts.slug); + const [key1, key2] = advisoryLockKey(`memory-engine:schema:${schema}`); + + await sql.begin(async (tx) => { + if (opts.shardId !== undefined) { + if (!Number.isSafeInteger(opts.shardId)) { + throw new Error(`shardId must be a safe integer, got: ${opts.shardId}`); + } + await tx.unsafe(`set local pgdog.shard to ${String(opts.shardId)}`); + } + await tx`select set_config('statement_timeout', ${opts.statementTimeout}, true)`; + await tx`select set_config('lock_timeout', ${opts.lockTimeout}, true)`; + await tx`select set_config('transaction_timeout', ${opts.transactionTimeout}, true)`; + await tx`select set_config('idle_in_transaction_session_timeout', ${opts.idleInTransactionSessionTimeout}, true)`; + if (!(await acquireAdvisoryLock(tx, key1, key2))) { + throw new Error( + `Unable to acquire lock for engine slug ${opts.slug} migrations.`, + ); + } + + if (!(await doesEngineExist(tx, schema))) { + await provisionEngine(tx, schema); + } + await runMigrations(tx, schema, opts); + }); +} + +function normalizeMigrateEngineOptions( + options: MigrateEngineOptions, +): NormalizedMigrateEngineOptions { + return { + slug: options.slug, + targetVersion: options.targetVersion, + shardId: options.shardId, + embeddingDimensions: options.embeddingDimensions ?? 1536, + bm25TextConfig: options.bm25TextConfig ?? "english", + bm25K1: options.bm25K1 ?? 1.2, + bm25B: options.bm25B ?? 0.75, + hnswM: options.hnswM ?? 16, + hnswEfConstruction: options.hnswEfConstruction ?? 64, + statementTimeout: options.statementTimeout ?? "20s", + lockTimeout: options.lockTimeout ?? "5s", + transactionTimeout: options.transactionTimeout ?? "1m", + idleInTransactionSessionTimeout: + options.idleInTransactionSessionTimeout ?? "5s", + }; +} + +function advisoryLockKey(schema: string): [number, number] { + const digest = createHash("sha256").update(schema).digest(); + return [digest.readInt32BE(0), digest.readInt32BE(4)]; +} + +const MAX_LOCK_RETRIES = 5; +const BASE_DELAY_MS = 100; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function acquireAdvisoryLock( + tx: SQL, + key1: number, + key2: number, +): Promise { + let acquired = false; + for (let attempt = 0; attempt < MAX_LOCK_RETRIES; attempt++) { + const [result] = await tx` + select pg_try_advisory_xact_lock(${key1}, ${key2}) as acquired + `; + if (result.acquired) { + acquired = true; + break; + } + if (attempt < MAX_LOCK_RETRIES - 1) { + await sleep(BASE_DELAY_MS * 2 ** attempt); + } + } + return acquired; +} + +async function doesEngineExist(tx: SQL, schema: string): Promise { + const [{ engineExists }] = await tx` + select exists + ( + select 1 + from pg_namespace n + where n.nspname = ${schema} + ) as "engineExists" + `; + return engineExists; +} + +async function provisionEngine(tx: SQL, schema: string): Promise { + await tx`create schema ${tx(schema)}`; + + // grant usage to all roles + await tx`grant usage on schema ${tx(schema)} to me_ro, me_rw, me_embed`; + + // version tracking table (single row) + await tx` + create table ${tx(schema)}.version + ( version text not null + , at timestamptz not null default now() + ) + `; + await tx`create unique index version_singleton_idx on ${tx(schema)}.version ((true))`; // only allow one row + await tx`insert into ${tx(schema)}.version (version) values ('0.0.0')`; + + // migration tracking table + await tx` + create table ${tx(schema)}.migration + ( name text not null constraint migration_pkey primary key + , applied_at_version text not null + , applied_at timestamptz not null default pg_catalog.clock_timestamp() + ) + `; +} + +async function runMigrations( + tx: SQL, + schema: string, + options: NormalizedMigrateEngineOptions, +): Promise { + // check ownership + await assertSchemaOwnership(tx, schema); + + // check version (reject downgrades) + await assertVersion(tx, schema, options.targetVersion); + + // run migrations + const sorted = [...migrations].sort((a, b) => a.name.localeCompare(b.name)); + + for (const migration of sorted) { + const [{ existing }] = await tx` + select exists + ( + select 1 + from ${tx(schema)}.migration + where name = ${migration.name} + ) as existing + `; + + if (existing) { + continue; + } + + const templateVars: Record = { + ...options, + schema, + }; + const renderedSql = template(migration.sql, templateVars); + await tx.unsafe(renderedSql); + await tx` + insert into ${tx(schema)}.migration (name, applied_at_version) + values (${migration.name}, ${options.targetVersion})`; + } + + // update version + await tx`update ${tx(schema)}.version set version = ${options.targetVersion}, at = now()`; +} + +async function assertSchemaOwnership(tx: SQL, schema: string): Promise { + const [result] = await tx` + select + n.nspowner = (select pg_catalog.to_regrole(current_user)::oid) as is_owner + from pg_catalog.pg_namespace n + where n.nspname = ${schema} + `; + + if (!result?.is_owner) { + throw new Error( + `Only the owner of the ${schema} schema can run database migrations`, + ); + } +} + +async function assertVersion( + tx: SQL, + schema: string, + targetVersion: string, +): Promise { + const [{ version: dbVersion }] = await tx` + select version from ${tx(schema)}.version + `; + + const cmp = semver.order(targetVersion, dbVersion); + if (cmp < 0) { + throw new Error( + `Target version (${targetVersion}) is older than database version (${dbVersion}). ` + + "Please upgrade the server.", + ); + } +} + +function template(sql: string, vars: Record): string { + return sql.replace(/\{\{(\w+)\}\}/g, (_, key) => { + if (!(key in vars)) { + throw new Error(`Missing template variable: ${key}`); + } + return String(vars[key]); + }); +} diff --git a/packages/engine-core/migrate/migrations/001_updated_at.sql b/packages/engine-core/migrate/migrations/001_updated_at.sql new file mode 100644 index 0000000..e69de29 diff --git a/packages/engine-core/migrate/migrations/sql.d.ts b/packages/engine-core/migrate/migrations/sql.d.ts new file mode 100644 index 0000000..89b092e --- /dev/null +++ b/packages/engine-core/migrate/migrations/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const content: string; + export default content; +} diff --git a/packages/engine-core/package.json b/packages/engine-core/package.json new file mode 100644 index 0000000..13708aa --- /dev/null +++ b/packages/engine-core/package.json @@ -0,0 +1,6 @@ +{ + "name": "@memory.build/engine-core", + "version": "0.0.0", + "private": true, + "type": "module" +} diff --git a/packages/engine-core/slug.ts b/packages/engine-core/slug.ts new file mode 100644 index 0000000..233910c --- /dev/null +++ b/packages/engine-core/slug.ts @@ -0,0 +1,18 @@ +const ENGINE_SCHEMA_RE = /^me_[a-z0-9]{12}$/; +const SLUG_RE = /^[a-z0-9]{12}$/; + +export function isValidEngineSchema(name: string): boolean { + return ENGINE_SCHEMA_RE.test(name); +} + +export function isValidSlug(slug: string): boolean { + return SLUG_RE.test(slug); +} + +export function slugToSchema(slug: string): string { + return `me_${slug}`; +} + +export function schemaToSlug(schema: string): string { + return schema.slice(3); +} diff --git a/packages/engine-core/tsconfig.json b/packages/engine-core/tsconfig.json new file mode 100644 index 0000000..23b1d27 --- /dev/null +++ b/packages/engine-core/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["**/*.ts", "**/*.d.ts"] +} From 9ca411103339f67316dc7ff92f69c51bd5ee8ea3 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Mon, 18 May 2026 13:35:00 -0500 Subject: [PATCH 06/17] test: add engine core bootstrap integration tests --- .../migrate/bootstrap.integration.test.ts | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 packages/engine-core/migrate/bootstrap.integration.test.ts diff --git a/packages/engine-core/migrate/bootstrap.integration.test.ts b/packages/engine-core/migrate/bootstrap.integration.test.ts new file mode 100644 index 0000000..dce1f54 --- /dev/null +++ b/packages/engine-core/migrate/bootstrap.integration.test.ts @@ -0,0 +1,148 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { SQL, semver } from "bun"; +import { bootstrapEngineDatabase } from "./bootstrap"; + +const adminUrl = + process.env.ENGINE_CORE_TEST_DATABASE_URL ?? + "postgresql://postgres@localhost:5432/postgres"; + +// These tests expect the local Postgres image from docker/Dockerfile.postgres, +// usually started with `./bun run pg`, unless ENGINE_CORE_TEST_DATABASE_URL is set. + +let dbName: string | undefined; +let sql: SQL | undefined; + +function assertSafeIdentifier(name: string): void { + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + throw new Error(`Unsafe database identifier: ${name}`); + } +} + +async function createTestDatabase(): Promise { + dbName = `test_engine_core_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + assertSafeIdentifier(dbName); + + const admin = new SQL(adminUrl); + try { + await admin.unsafe(`create database ${dbName}`); + } finally { + await admin.close(); + } + + const url = new URL(adminUrl); + url.pathname = `/${dbName}`; + return url.toString(); +} + +async function dropTestDatabase(): Promise { + if (!dbName) return; + assertSafeIdentifier(dbName); + + const admin = new SQL(adminUrl); + try { + await admin` + select pg_terminate_backend(pid) + from pg_stat_activity + where datname = ${dbName} + and pid <> pg_backend_pid() + `; + await admin.unsafe(`drop database if exists ${dbName}`); + } finally { + await admin.close(); + dbName = undefined; + } +} + +function getSql(): SQL { + if (!sql) throw new Error("test database is not initialized"); + return sql; +} + +beforeAll(async () => { + const connectionString = await createTestDatabase(); + sql = new SQL(connectionString); +}); + +afterAll(async () => { + await sql?.close(); + sql = undefined; + await dropTestDatabase(); +}); + +describe("bootstrapEngineDatabase", () => { + test("creates required extensions in public", async () => { + await bootstrapEngineDatabase(getSql()); + + const rows = await getSql()` + select e.extname, e.extversion, n.nspname + from pg_extension e + inner join pg_namespace n on n.oid = e.extnamespace + where e.extname in ('citext', 'ltree', 'vector', 'pg_textsearch') + order by e.extname + `; + + expect(rows.map((row: { extname: string }) => row.extname)).toEqual([ + "citext", + "ltree", + "pg_textsearch", + "vector", + ]); + + const minimumVersions = new Map([ + ["citext", "1.6"], + ["ltree", "1.3"], + ["pg_textsearch", "1.1.0"], + ["vector", "0.8.2"], + ]); + for (const row of rows as Array<{ + extname: string; + extversion: string; + nspname: string; + }>) { + expect(row.nspname).toBe("public"); + expect( + semver.order(row.extversion, minimumVersions.get(row.extname)!), + ).toBeGreaterThanOrEqual(0); + } + }); + + test("creates required nologin roles", async () => { + await bootstrapEngineDatabase(getSql()); + + const rows = await getSql()` + select rolname, rolcanlogin + from pg_roles + where rolname in ('me_ro', 'me_rw', 'me_embed') + order by rolname + `; + + expect(rows).toHaveLength(3); + expect(rows.map((row: { rolname: string }) => row.rolname)).toEqual([ + "me_embed", + "me_ro", + "me_rw", + ]); + for (const row of rows as Array<{ rolcanlogin: boolean }>) { + expect(row.rolcanlogin).toBe(false); + } + }); + + test("is idempotent", async () => { + await bootstrapEngineDatabase(getSql()); + await bootstrapEngineDatabase(getSql()); + + const [{ extensionCount }] = await getSql()` + select count(*)::int as "extensionCount" + from pg_extension + where extname in ('citext', 'ltree', 'vector', 'pg_textsearch') + `; + const [{ roleCount }] = await getSql()` + select count(*)::int as "roleCount" + from pg_roles + where rolname in ('me_ro', 'me_rw', 'me_embed') + `; + + expect(extensionCount).toBe(4); + expect(roleCount).toBe(3); + }); +}); From 915be714ed4ed6a4d01b2d1936f09d1746e6b9be Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Mon, 18 May 2026 13:41:08 -0500 Subject: [PATCH 07/17] test: add engine core migration integration tests --- .../migrate/migrate.integration.test.ts | 207 ++++++++++++++++++ packages/engine-core/migrate/migrate.ts | 2 +- .../migrate/migrations/001_updated_at.sql | 12 + 3 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 packages/engine-core/migrate/migrate.integration.test.ts diff --git a/packages/engine-core/migrate/migrate.integration.test.ts b/packages/engine-core/migrate/migrate.integration.test.ts new file mode 100644 index 0000000..73bc144 --- /dev/null +++ b/packages/engine-core/migrate/migrate.integration.test.ts @@ -0,0 +1,207 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { SQL } from "bun"; +import { bootstrapEngineDatabase } from "./bootstrap"; +import { migrateEngine } from "./migrate"; + +const adminUrl = + process.env.ENGINE_CORE_TEST_DATABASE_URL ?? + "postgresql://postgres@localhost:5432/postgres"; + +// These tests expect the local Postgres image from docker/Dockerfile.postgres, +// usually started with `./bun run pg`, unless ENGINE_CORE_TEST_DATABASE_URL is set. + +let dbName: string | undefined; +let sql: SQL | undefined; + +function assertSafeIdentifier(name: string): void { + if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) { + throw new Error(`Unsafe database identifier: ${name}`); + } +} + +function getSql(): SQL { + if (!sql) throw new Error("test database is not initialized"); + return sql; +} + +function randomSlug(): string { + return `t${Math.random().toString(36).slice(2, 13).padEnd(11, "0")}`; +} + +function schemaFor(slug: string): string { + return `me_${slug}`; +} + +async function createTestDatabase(): Promise { + dbName = `test_engine_core_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + assertSafeIdentifier(dbName); + + const admin = new SQL(adminUrl); + try { + await admin.unsafe(`create database ${dbName}`); + } finally { + await admin.close(); + } + + const url = new URL(adminUrl); + url.pathname = `/${dbName}`; + return url.toString(); +} + +async function dropTestDatabase(): Promise { + if (!dbName) return; + assertSafeIdentifier(dbName); + + const admin = new SQL(adminUrl); + try { + await admin` + select pg_terminate_backend(pid) + from pg_stat_activity + where datname = ${dbName} + and pid <> pg_backend_pid() + `; + await admin.unsafe(`drop database if exists ${dbName}`); + } finally { + await admin.close(); + dbName = undefined; + } +} + +async function schemaExists(schema: string): Promise { + const [{ exists }] = await getSql()` + select exists ( + select 1 + from information_schema.schemata + where schema_name = ${schema} + ) as exists + `; + return exists; +} + +async function tableExists(schema: string, table: string): Promise { + const [{ exists }] = await getSql()` + select exists ( + select 1 + from information_schema.tables + where table_schema = ${schema} + and table_name = ${table} + ) as exists + `; + return exists; +} + +async function migrationCount(schema: string): Promise { + const [{ count }] = await getSql()` + select count(*)::int as count + from ${getSql()(schema)}.migration + `; + return count; +} + +async function engineVersion(schema: string): Promise { + const [{ version }] = await getSql()` + select version + from ${getSql()(schema)}.version + `; + return version; +} + +beforeAll(async () => { + const connectionString = await createTestDatabase(); + sql = new SQL(connectionString); + await bootstrapEngineDatabase(sql); +}); + +afterAll(async () => { + await sql?.close(); + sql = undefined; + await dropTestDatabase(); +}); + +describe("migrateEngine", () => { + test("provisions a new engine schema", async () => { + const slug = randomSlug(); + const schema = schemaFor(slug); + + await migrateEngine(getSql(), { slug, targetVersion: "0.1.0" }); + + expect(await schemaExists(schema)).toBe(true); + expect(await tableExists(schema, "version")).toBe(true); + expect(await tableExists(schema, "migration")).toBe(true); + expect(await engineVersion(schema)).toBe("0.1.0"); + + const rows = await getSql()` + select name, applied_at_version, applied_at + from ${getSql()(schema)}.migration + `; + expect(rows).toHaveLength(1); + expect(rows[0].name).toBe("001_updated_at"); + expect(rows[0].applied_at_version).toBe("0.1.0"); + expect(rows[0].applied_at).toBeTruthy(); + }); + + test("is idempotent", async () => { + const slug = randomSlug(); + const schema = schemaFor(slug); + + await migrateEngine(getSql(), { slug, targetVersion: "0.1.0" }); + await migrateEngine(getSql(), { slug, targetVersion: "0.1.0" }); + + expect(await migrationCount(schema)).toBe(1); + expect(await engineVersion(schema)).toBe("0.1.0"); + }); + + test("rejects invalid slug", async () => { + await expect( + migrateEngine(getSql(), { slug: "bad-slug", targetVersion: "0.1.0" }), + ).rejects.toThrow("Invalid engine slug"); + }); + + test("rejects invalid targetVersion", async () => { + await expect( + migrateEngine(getSql(), { slug: randomSlug(), targetVersion: "nope" }), + ).rejects.toThrow("Invalid target version"); + }); + + test("rejects downgrade", async () => { + const slug = randomSlug(); + + await migrateEngine(getSql(), { slug, targetVersion: "0.2.0" }); + + await expect( + migrateEngine(getSql(), { slug, targetVersion: "0.1.0" }), + ).rejects.toThrow("older than database version"); + }); + + test("allows equal current version rerun", async () => { + const slug = randomSlug(); + const schema = schemaFor(slug); + + await migrateEngine(getSql(), { slug, targetVersion: "0.2.0" }); + await migrateEngine(getSql(), { slug, targetVersion: "0.2.0" }); + + expect(await migrationCount(schema)).toBe(1); + expect(await engineVersion(schema)).toBe("0.2.0"); + }); + + test("allows upgrade without pending migrations", async () => { + const slug = randomSlug(); + const schema = schemaFor(slug); + + await migrateEngine(getSql(), { slug, targetVersion: "0.1.0" }); + await migrateEngine(getSql(), { slug, targetVersion: "0.2.0" }); + + expect(await migrationCount(schema)).toBe(1); + expect(await engineVersion(schema)).toBe("0.2.0"); + }); + + test("rejects unsafe shardId", async () => { + await expect( + migrateEngine(getSql(), { + slug: randomSlug(), + targetVersion: "0.1.0", + shardId: Number.MAX_SAFE_INTEGER + 1, + }), + ).rejects.toThrow("shardId must be a safe integer"); + }); +}); diff --git a/packages/engine-core/migrate/migrate.ts b/packages/engine-core/migrate/migrate.ts index 7ba9ff0..fc46ecd 100644 --- a/packages/engine-core/migrate/migrate.ts +++ b/packages/engine-core/migrate/migrate.ts @@ -102,7 +102,7 @@ function normalizeMigrateEngineOptions( hnswEfConstruction: options.hnswEfConstruction ?? 64, statementTimeout: options.statementTimeout ?? "20s", lockTimeout: options.lockTimeout ?? "5s", - transactionTimeout: options.transactionTimeout ?? "1m", + transactionTimeout: options.transactionTimeout ?? "1min", idleInTransactionSessionTimeout: options.idleInTransactionSessionTimeout ?? "5s", }; diff --git a/packages/engine-core/migrate/migrations/001_updated_at.sql b/packages/engine-core/migrate/migrations/001_updated_at.sql index e69de29..8e32112 100644 --- a/packages/engine-core/migrate/migrations/001_updated_at.sql +++ b/packages/engine-core/migrate/migrations/001_updated_at.sql @@ -0,0 +1,12 @@ +------------------------------------------------------------------------------- +-- generic updated_at trigger +------------------------------------------------------------------------------- +create or replace function {{schema}}.update_updated_at() +returns trigger +as $func$ +begin + new.updated_at = pg_catalog.now(); + return new; +end; +$func$ language plpgsql volatile security definer +set search_path to {{schema}}, pg_temp; From 24b363a2c9e33155d572cfe419ce4fd442a321f2 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Mon, 18 May 2026 13:43:26 -0500 Subject: [PATCH 08/17] test: add engine core slug unit tests --- packages/engine-core/slug.test.ts | 40 +++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 packages/engine-core/slug.test.ts diff --git a/packages/engine-core/slug.test.ts b/packages/engine-core/slug.test.ts new file mode 100644 index 0000000..a398df2 --- /dev/null +++ b/packages/engine-core/slug.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from "bun:test"; +import { + isValidEngineSchema, + isValidSlug, + schemaToSlug, + slugToSchema, +} from "./slug"; + +describe("engine slugs", () => { + test("validates slugs", () => { + expect(isValidSlug("abc123def456")).toBe(true); + expect(isValidSlug("000000000000")).toBe(true); + + expect(isValidSlug("abc123def45")).toBe(false); + expect(isValidSlug("abc123def4567")).toBe(false); + expect(isValidSlug("ABC123def456")).toBe(false); + expect(isValidSlug("abc123_def45")).toBe(false); + expect(isValidSlug("abc123-def45")).toBe(false); + }); + + test("validates engine schemas", () => { + expect(isValidEngineSchema("me_abc123def456")).toBe(true); + expect(isValidEngineSchema("me_000000000000")).toBe(true); + + expect(isValidEngineSchema("abc123def456")).toBe(false); + expect(isValidEngineSchema("me_abc123def45")).toBe(false); + expect(isValidEngineSchema("me_abc123def4567")).toBe(false); + expect(isValidEngineSchema("me_ABC123def456")).toBe(false); + expect(isValidEngineSchema("me_abc123_def45")).toBe(false); + }); + + test("converts between slugs and schemas", () => { + const slug = "abc123def456"; + const schema = "me_abc123def456"; + + expect(slugToSchema(slug)).toBe(schema); + expect(schemaToSlug(schema)).toBe(slug); + expect(schemaToSlug(slugToSchema(slug))).toBe(slug); + }); +}); From fc4aa568d0d0fb80ab1e467274c84a1fbbb1e2fc Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Mon, 18 May 2026 14:34:30 -0500 Subject: [PATCH 09/17] feat: split engine core migrations by execution mode --- .../idempotent/001_role_membership.sql | 208 ++++++++++++++++++ .../idempotent/002_tree_privileges.sql | 74 +++++++ .../migrate/idempotent/003_memory.sql | 28 +++ .../idempotent/004_embedding_queue.sql | 163 ++++++++++++++ .../{migrations => idempotent}/sql.d.ts | 0 .../migrate/incremental/001_user.sql | 13 ++ .../incremental/002_role_membership.sql | 13 ++ .../incremental/003_tree_ownership.sql | 11 + .../migrate/incremental/004_tree_grant.sql | 13 ++ .../migrate/incremental/005_memory.sql | 48 ++++ .../incremental/006_embedding_queue.sql | 21 ++ .../engine-core/migrate/incremental/sql.d.ts | 4 + .../migrate/migrate.integration.test.ts | 32 ++- packages/engine-core/migrate/migrate.ts | 121 +++++++--- .../migrate/migrations/001_updated_at.sql | 12 - 15 files changed, 711 insertions(+), 50 deletions(-) create mode 100644 packages/engine-core/migrate/idempotent/001_role_membership.sql create mode 100644 packages/engine-core/migrate/idempotent/002_tree_privileges.sql create mode 100644 packages/engine-core/migrate/idempotent/003_memory.sql create mode 100644 packages/engine-core/migrate/idempotent/004_embedding_queue.sql rename packages/engine-core/migrate/{migrations => idempotent}/sql.d.ts (100%) create mode 100644 packages/engine-core/migrate/incremental/001_user.sql create mode 100644 packages/engine-core/migrate/incremental/002_role_membership.sql create mode 100644 packages/engine-core/migrate/incremental/003_tree_ownership.sql create mode 100644 packages/engine-core/migrate/incremental/004_tree_grant.sql create mode 100644 packages/engine-core/migrate/incremental/005_memory.sql create mode 100644 packages/engine-core/migrate/incremental/006_embedding_queue.sql create mode 100644 packages/engine-core/migrate/incremental/sql.d.ts delete mode 100644 packages/engine-core/migrate/migrations/001_updated_at.sql diff --git a/packages/engine-core/migrate/idempotent/001_role_membership.sql b/packages/engine-core/migrate/idempotent/001_role_membership.sql new file mode 100644 index 0000000..9e451ec --- /dev/null +++ b/packages/engine-core/migrate/idempotent/001_role_membership.sql @@ -0,0 +1,208 @@ +------------------------------------------------------------------------------- +-- would_create_cycle +------------------------------------------------------------------------------- +create or replace function {{schema}}.would_create_cycle +( _role_id uuid +, _member_id uuid +) +returns boolean +as $func$ + with recursive ancestors(id) as + ( + select rm.role_id + from {{schema}}.role_membership rm + where rm.member_id = _role_id + union + select rm.role_id + from {{schema}}.role_membership rm + inner join ancestors a on a.id = rm.member_id + ) + select _member_id = _role_id + or exists + ( + select 1 + from ancestors + where id = _member_id + ) +$func$ language sql stable security invoker +parallel safe +; + +------------------------------------------------------------------------------- +-- role_membership_before_write trigger +------------------------------------------------------------------------------- +-- Prevent role membership cycles for ordinary writes. +-- Note: this check observes the current transaction snapshot. Concurrent +-- transactions that insert/update related role edges can still race unless the +-- caller uses stronger locking or serializable transactions around +-- role_membership writes. +create or replace function {{schema}}.role_membership_before_write() +returns trigger +as $func$ +begin + if {{schema}}.would_create_cycle(new.role_id, new.member_id) then + raise exception 'role membership would create a cycle: role_id %, member_id %', new.role_id, new.member_id + using errcode = 'integrity_constraint_violation'; + end if; + return new; +end; +$func$ language plpgsql volatile security definer +; + +create or replace trigger role_membership_before_write_trg +before insert or update of role_id, member_id on {{schema}}.role_membership +for each row +execute function {{schema}}.role_membership_before_write() +; + +------------------------------------------------------------------------------- +-- explode_role_membership +------------------------------------------------------------------------------- +create or replace function {{schema}}.explode_role_membership(_user_id uuid) +returns table +( role_id uuid +, superuser bool +, dist int4 +) +as $func$ + with recursive ancestors(id, dist) as + ( + select rm.role_id, 1::int4 + from {{schema}}.role_membership rm + where rm.member_id = _user_id + union + select rm.role_id, a.dist + 1 + from {{schema}}.role_membership rm + inner join ancestors a on a.id = rm.member_id + ) + select + u.id + , u.superuser + , 0::int4 + from {{schema}}."user" u + where u.id = _user_id + union + select + u.id + , u.superuser + , a.dist + from {{schema}}."user" u + inner join ancestors a on (u.id = a.id) +$func$ language sql stable security invoker +; + +------------------------------------------------------------------------------- +-- is_superuser +------------------------------------------------------------------------------- +create or replace function {{schema}}.is_superuser(_user_id uuid) +returns boolean +as $func$ + select exists + ( + select 1 + from {{schema}}.explode_role_membership(_user_id) x + where x.superuser + ) +$func$ language sql stable security definer +set search_path to pg_catalog, {{schema}}, pg_temp +; + +------------------------------------------------------------------------------- +-- grant_role_membership +------------------------------------------------------------------------------- +create or replace function {{schema}}.grant_role_membership +( _grantor_id uuid +, _role_id uuid +, _member_id uuid +, _with_admin_option bool default false +) +returns void +as $func$ +declare + _allowed bool; +begin + -- exclusive write access required to fully ensure against cycle creation by concurrent writes + lock table {{schema}}.role_membership in share row exclusive mode; + + -- is grantor allowed to do this? + select + exists + ( + -- does the grantor have with admin privilege directly on this role? + select 1 + from {{schema}}.role_membership rm + where rm.role_id = _role_id + and rm.member_id = _grantor_id + and rm.with_admin_option + ) + or {{schema}}.is_superuser(_grantor_id) -- or are they a superuser (even indirectly)? + into strict _allowed + ; + + if not _allowed then + raise exception 'grantor must be a superuser or have with admin option on role: grantor_id % role_id %', _grantor_id, _role_id + using errcode = 'insufficient_privilege'; + end if; + + -- role_membership_before_write_trg protects against cycles in the graph + insert into {{schema}}.role_membership + ( role_id + , member_id + , with_admin_option + ) + values + ( _role_id + , _member_id + , _with_admin_option + ) + on conflict (member_id, role_id) + do update set with_admin_option = _with_admin_option + ; +end; +$func$ language plpgsql volatile security definer +set search_path to pg_catalog, {{schema}}, pg_temp +; + +------------------------------------------------------------------------------- +-- revoke_role_membership +------------------------------------------------------------------------------- +create or replace function {{schema}}.revoke_role_membership +( _revoker_id uuid +, _role_id uuid +, _member_id uuid +) +returns void +as $func$ +declare + _allowed bool; +begin + lock table {{schema}}.role_membership in share row exclusive mode; + + -- is revoker allowed to do this? + select + exists + ( + -- does the revoker have with admin privilege directly on this role? + select 1 + from {{schema}}.role_membership rm + where rm.role_id = _role_id + and rm.member_id = _revoker_id + and rm.with_admin_option + ) + or {{schema}}.is_superuser(_revoker_id) -- or are they a superuser (even indirectly)? + into strict _allowed + ; + + if not _allowed then + raise exception 'revoker must be a superuser or have with admin option on role: revoker_id % role_id %', _revoker_id, _role_id + using errcode = 'insufficient_privilege'; + end if; + + delete from {{schema}}.role_membership d + where d.role_id = _role_id + and d.member_id = _member_id + ; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, pg_temp +; diff --git a/packages/engine-core/migrate/idempotent/002_tree_privileges.sql b/packages/engine-core/migrate/idempotent/002_tree_privileges.sql new file mode 100644 index 0000000..5b94da0 --- /dev/null +++ b/packages/engine-core/migrate/idempotent/002_tree_privileges.sql @@ -0,0 +1,74 @@ +------------------------------------------------------------------------------- +-- calc_tree_privileges +------------------------------------------------------------------------------- +create or replace function {{schema}}.calc_tree_privileges(_user_id uuid) +returns table +( role_id uuid +, tree_path ltree +, actions text[] +, reason text +) +as $func$ + with r as + ( + -- the user + select + u.id as role_id + , u.superuser + from {{schema}}."user" u + where u.id = _user_id + union + -- the roles they belong to + select + x.role_id + , x.superuser + from {{schema}}.explode_role_membership(_user_id) x + ) + -- superuser + select + r.role_id + , ''::ltree as tree_path + , array['read', 'create', 'update', 'delete'] as actions + , 'superuser' as reason + from r + where r.superuser + union all + -- ownership + select + r.role_id + , o.tree_path + , array['read', 'create', 'update', 'delete'] as actions + , 'owner' as reason + from r + inner join {{schema}}.tree_owner o on (r.role_id = o.user_id) + union all + -- grants + select + r.role_id + , g.tree_path + , g.actions + , 'grant' as reason + from r + inner join {{schema}}.tree_grant g on (r.role_id = g.user_id) +$func$ language sql stable security invoker +; + +------------------------------------------------------------------------------- +-- has_tree_privilege +------------------------------------------------------------------------------- +create or replace function {{schema}}.has_tree_privilege +( _user_id uuid +, _tree_path ltree +, _actions text[] +) +returns bool +as $func$ + select exists + ( + select 1 + from {{schema}}.calc_tree_privileges(_user_id) x + where x.tree_path @> _tree_path + and x.actions @> _actions + ) +$func$ language sql stable security definer +; diff --git a/packages/engine-core/migrate/idempotent/003_memory.sql b/packages/engine-core/migrate/idempotent/003_memory.sql new file mode 100644 index 0000000..3784018 --- /dev/null +++ b/packages/engine-core/migrate/idempotent/003_memory.sql @@ -0,0 +1,28 @@ +------------------------------------------------------------------------------- +-- memory triggers +------------------------------------------------------------------------------- +create or replace function {{schema}}.memory_before_update() +returns trigger +as $func$ +begin + -- always update the timestamp + new.updated_at = pg_catalog.now(); + + -- content changed -> new embedding needs to be generated + if old.content is distinct from new.content + and old.embedding is not distinct from new.embedding + then + new.embedding = null; + new.embedding_version = old.embedding_version operator(pg_catalog.+) 1; + end if; + + return new; +end; +$func$ language plpgsql volatile security definer +set search_path to {{schema}}, public, pg_temp -- public required for pgvector's `is not distinct from` +; + +create or replace trigger memory_before_update_trg +before update on {{schema}}.memory +for each row +execute function {{schema}}.memory_before_update(); diff --git a/packages/engine-core/migrate/idempotent/004_embedding_queue.sql b/packages/engine-core/migrate/idempotent/004_embedding_queue.sql new file mode 100644 index 0000000..1a9ab6d --- /dev/null +++ b/packages/engine-core/migrate/idempotent/004_embedding_queue.sql @@ -0,0 +1,163 @@ + +------------------------------------------------------------------------------- +-- enqueue_embedding +------------------------------------------------------------------------------- +create or replace function {{schema}}.enqueue_embedding() +returns trigger +as $func$ +begin + insert into {{schema}}.embedding_queue (memory_id, embedding_version) + values (new.id, new.embedding_version); + return new; +end; +$func$ +language plpgsql volatile security definer +set search_path to pg_catalog, {{schema}}, pg_temp +; + +create or replace trigger memory_enqueue_embedding_insert +after insert on {{schema}}.memory +for each row +when (new.embedding is null) -- it's possible to insert with an embedding +execute function {{schema}}.enqueue_embedding() +; + +create or replace trigger memory_enqueue_embedding_update +after update on {{schema}}.memory +for each row +when +( old.content is distinct from new.content + and new.embedding is null +) +execute function {{schema}}.enqueue_embedding() +; + +------------------------------------------------------------------------------- +-- claim_embedding_batch +------------------------------------------------------------------------------- +create or replace function {{schema}}.claim_embedding_batch +( _batch_size int default 10 +, _lock_duration interval default '5 minutes' +, _max_attempts int default 3 +) +returns table +( queue_id bigint +, memory_id uuid +, embedding_version int +, content text +) +as $func$ +declare + _rec record; + _mem record; + _claimed_count int = 0; +begin + -- bulk-cancel visible queue rows superseded by a newer row for the same memory + update {{schema}}.embedding_queue eq + set outcome = 'cancelled' + where eq.outcome is null + and eq.vt <= now() + and exists + ( + select 1 + from {{schema}}.embedding_queue newer + where newer.memory_id = eq.memory_id + and newer.embedding_version > eq.embedding_version + and newer.outcome is null + ); + + -- sweep: finalize exhausted rows orphaned by worker crash + -- (attempts reached max but outcome was never written back) + update {{schema}}.embedding_queue + set + outcome = 'failed' + , last_error = coalesce(last_error, 'exceeded max attempts') + where outcome is null + and vt <= now() + and attempts >= _max_attempts + ; + + for _rec in + ( + select + eq.id + , eq.memory_id + , eq.embedding_version + from {{schema}}.embedding_queue eq + where eq.outcome is null + and eq.vt <= now() + and eq.attempts < _max_attempts + order by eq.vt + for update skip locked + ) + loop + -- check memory still exists + current version + select m.content, m.embedding_version + into _mem + from {{schema}}.memory m + where m.id = _rec.memory_id + ; + + if not found or _mem.content is null then + -- memory deleted or empty → cancel queue row + update {{schema}}.embedding_queue + set outcome = 'cancelled' + where id = _rec.id; + continue; + end if; + + if _rec.embedding_version != _mem.embedding_version then + -- stale version → cancel + update {{schema}}.embedding_queue + set outcome = 'cancelled' + where id = _rec.id; + continue; + end if; + + -- claim this row + update {{schema}}.embedding_queue q set + vt = now() + _lock_duration + , attempts = q.attempts + 1 + where id = _rec.id; + + queue_id = _rec.id; + memory_id = _rec.memory_id; + embedding_version = _rec.embedding_version; + content = _mem.content; + return next; + + _claimed_count = _claimed_count + 1; + exit when _claimed_count >= _batch_size; + end loop; +end; +$func$ +language plpgsql volatile +set search_path to pg_catalog, {{schema}}, pg_temp +; + +------------------------------------------------------------------------------- +-- prune embedding queue +------------------------------------------------------------------------------- +-- prune terminal queue rows older than the retention window. +-- runs opportunistically from the worker on engines that returned no +-- claimable work, so the queue table doesn't grow unbounded. +-- +-- relies on embedding_queue_archive_idx (created_at) where outcome is not null +-- from migration 005, so the no-op case is cheap. +create or replace function {{schema}}.prune_embedding_queue(_retention interval default '7 days') +returns bigint +as $func$ +declare + pruned bigint; +begin + delete from {{schema}}.embedding_queue + where outcome is not null + and created_at < now() - _retention + ; + get diagnostics pruned = row_count; + return pruned; +end; +$func$ +language plpgsql volatile +set search_path to pg_catalog, {{schema}}, pg_temp +; diff --git a/packages/engine-core/migrate/migrations/sql.d.ts b/packages/engine-core/migrate/idempotent/sql.d.ts similarity index 100% rename from packages/engine-core/migrate/migrations/sql.d.ts rename to packages/engine-core/migrate/idempotent/sql.d.ts diff --git a/packages/engine-core/migrate/incremental/001_user.sql b/packages/engine-core/migrate/incremental/001_user.sql new file mode 100644 index 0000000..ecdba44 --- /dev/null +++ b/packages/engine-core/migrate/incremental/001_user.sql @@ -0,0 +1,13 @@ + +------------------------------------------------------------------------------- +-- users +------------------------------------------------------------------------------- +-- User: thing that accesses memories, or a role (can_login = false) +-- identity_id is a soft FK to accounts.identity (nullable for service users) +-- Note: "user" is a reserved word, must be quoted +create table {{schema}}."user" +( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) +, name citext not null unique +, superuser boolean not null default false +, created_at timestamptz not null default now() +); diff --git a/packages/engine-core/migrate/incremental/002_role_membership.sql b/packages/engine-core/migrate/incremental/002_role_membership.sql new file mode 100644 index 0000000..2257d6a --- /dev/null +++ b/packages/engine-core/migrate/incremental/002_role_membership.sql @@ -0,0 +1,13 @@ +------------------------------------------------------------------------------- +-- role membership +------------------------------------------------------------------------------- +create table {{schema}}.role_membership +( role_id uuid not null references {{schema}}."user"(id) on delete cascade +, member_id uuid not null references {{schema}}."user"(id) on delete cascade +, with_admin_option boolean not null default false +, created_at timestamptz not null default now() +, constraint pkey_role_membership primary key (member_id, role_id) +, constraint no_self_membership check (role_id != member_id) +); + +create index idx_role_membership_role on {{schema}}.role_membership(role_id) include (member_id); diff --git a/packages/engine-core/migrate/incremental/003_tree_ownership.sql b/packages/engine-core/migrate/incremental/003_tree_ownership.sql new file mode 100644 index 0000000..2eb4c17 --- /dev/null +++ b/packages/engine-core/migrate/incremental/003_tree_ownership.sql @@ -0,0 +1,11 @@ +------------------------------------------------------------------------------- +-- tree ownership +------------------------------------------------------------------------------- +create table {{schema}}.tree_owner +( tree_path ltree not null primary key +, user_id uuid not null references {{schema}}."user" (id) on delete cascade +, created_at timestamptz not null default now() +); + +create index idx_tree_owner_user on {{schema}}.tree_owner (user_id); +create index idx_tree_owner_gist on {{schema}}.tree_owner using gist (tree_path); diff --git a/packages/engine-core/migrate/incremental/004_tree_grant.sql b/packages/engine-core/migrate/incremental/004_tree_grant.sql new file mode 100644 index 0000000..a74cdd4 --- /dev/null +++ b/packages/engine-core/migrate/incremental/004_tree_grant.sql @@ -0,0 +1,13 @@ +------------------------------------------------------------------------------- +-- tree grant +------------------------------------------------------------------------------- +create table {{schema}}.tree_grant +( user_id uuid not null references {{schema}}."user"(id) on delete cascade +, tree_path ltree not null +, actions text[] not null check (actions <@ '{read,create,update,delete}'::text[]) +, with_grant_option boolean not null default false +, created_at timestamptz not null default now() +, constraint pkey_tree_grant primary key (user_id, tree_path) +); + +create index idx_tree_grant_path on {{schema}}.tree_grant using gist (tree_path); diff --git a/packages/engine-core/migrate/incremental/005_memory.sql b/packages/engine-core/migrate/incremental/005_memory.sql new file mode 100644 index 0000000..c66eea7 --- /dev/null +++ b/packages/engine-core/migrate/incremental/005_memory.sql @@ -0,0 +1,48 @@ +------------------------------------------------------------------------------- +-- memory +------------------------------------------------------------------------------- +create table {{schema}}.memory +( id uuid not null primary key default uuidv7() check (uuid_extract_version(id) = 7) +, meta jsonb not null default '{}' check (jsonb_typeof(meta) = 'object') +, tree ltree not null default ''::ltree +, temporal tstzrange +, content text not null +, embedding halfvec({{embedding_dimensions}}) +, embedding_version int4 not null default 1 +, created_at timestamptz not null default now() +, updated_at timestamptz +); + +-- index for faceted search +create index memory_meta_gin_idx on {{schema}}.memory using gin (meta); + +-- index for temporal search +create index memory_temporal_gist_idx on {{schema}}.memory using gist (temporal) where (temporal is not null); + +-- index for BM25 text search +create index memory_content_bm25_idx on {{schema}}.memory using bm25 (content) +with (text_config = {{bm25_text_config}}, k1 = {{bm25_k1}}, b = {{bm25_b}}); + +-- index for vector similarity search +create index memory_embedding_hnsw_idx on {{schema}}.memory using hnsw (embedding halfvec_cosine_ops) +with (m = {{hnsw_m}}, ef_construction = {{hnsw_ef_construction}}); + +-- index for hierarchical organization +create index memory_tree_gist_idx on {{schema}}.memory using gist (tree); + +/* +enforce consistent temporal range conventions: +- point-in-time events: lower = upper with inclusive bounds '[same,same]' +- time periods: lower < upper with inclusive-exclusive bounds '[start,end)' +*/ +alter table {{schema}}.memory add constraint temporal_bounds_convention check +( + temporal is null + or ( + -- point-in-time: both bounds equal and inclusive + (lower(temporal) = upper(temporal) and lower_inc(temporal) and upper_inc(temporal)) + or + -- time range: start before end, inclusive-exclusive + (lower(temporal) < upper(temporal) and lower_inc(temporal) and not upper_inc(temporal)) + ) +); diff --git a/packages/engine-core/migrate/incremental/006_embedding_queue.sql b/packages/engine-core/migrate/incremental/006_embedding_queue.sql new file mode 100644 index 0000000..dce748a --- /dev/null +++ b/packages/engine-core/migrate/incremental/006_embedding_queue.sql @@ -0,0 +1,21 @@ +------------------------------------------------------------------------------- +-- embedding queue +------------------------------------------------------------------------------- +-- per-engine embedding queue table +create table {{schema}}.embedding_queue +( id bigint generated always as identity primary key +, memory_id uuid not null references {{schema}}.memory(id) on delete cascade +, embedding_version int not null +, vt timestamptz not null default now() +, outcome text check (outcome is null or outcome in ('completed', 'failed', 'cancelled')) +, attempts int not null default 0 +, last_error text +, created_at timestamptz not null default now() +); + +-- index to find items to claim +create index embedding_queue_claim_idx on {{schema}}.embedding_queue (vt) where outcome is null; +-- index also used in finding items to claim. used to ensure there aren't any items for the same memory with a newer version +create index embedding_queue_memory_idx on {{schema}}.embedding_queue (memory_id, embedding_version desc) where outcome is null; +-- index to find items that have resolved to an outcome. these can be pruned +create index embedding_queue_archive_idx on {{schema}}.embedding_queue (created_at) where outcome is not null; diff --git a/packages/engine-core/migrate/incremental/sql.d.ts b/packages/engine-core/migrate/incremental/sql.d.ts new file mode 100644 index 0000000..89b092e --- /dev/null +++ b/packages/engine-core/migrate/incremental/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const content: string; + export default content; +} diff --git a/packages/engine-core/migrate/migrate.integration.test.ts b/packages/engine-core/migrate/migrate.integration.test.ts index 73bc144..748fe75 100644 --- a/packages/engine-core/migrate/migrate.integration.test.ts +++ b/packages/engine-core/migrate/migrate.integration.test.ts @@ -128,16 +128,34 @@ describe("migrateEngine", () => { expect(await schemaExists(schema)).toBe(true); expect(await tableExists(schema, "version")).toBe(true); expect(await tableExists(schema, "migration")).toBe(true); + expect(await tableExists(schema, "user")).toBe(true); + expect(await tableExists(schema, "role_membership")).toBe(true); + expect(await tableExists(schema, "tree_owner")).toBe(true); + expect(await tableExists(schema, "tree_grant")).toBe(true); + expect(await tableExists(schema, "memory")).toBe(true); + expect(await tableExists(schema, "embedding_queue")).toBe(true); expect(await engineVersion(schema)).toBe("0.1.0"); const rows = await getSql()` select name, applied_at_version, applied_at from ${getSql()(schema)}.migration + order by name `; - expect(rows).toHaveLength(1); - expect(rows[0].name).toBe("001_updated_at"); - expect(rows[0].applied_at_version).toBe("0.1.0"); - expect(rows[0].applied_at).toBeTruthy(); + expect(rows.map((row: { name: string }) => row.name)).toEqual([ + "001_user", + "002_role_membership", + "003_tree_ownership", + "004_tree_grant", + "005_memory", + "006_embedding_queue", + ]); + for (const row of rows as Array<{ + applied_at_version: string; + applied_at: Date; + }>) { + expect(row.applied_at_version).toBe("0.1.0"); + expect(row.applied_at).toBeTruthy(); + } }); test("is idempotent", async () => { @@ -147,7 +165,7 @@ describe("migrateEngine", () => { await migrateEngine(getSql(), { slug, targetVersion: "0.1.0" }); await migrateEngine(getSql(), { slug, targetVersion: "0.1.0" }); - expect(await migrationCount(schema)).toBe(1); + expect(await migrationCount(schema)).toBe(6); expect(await engineVersion(schema)).toBe("0.1.0"); }); @@ -180,7 +198,7 @@ describe("migrateEngine", () => { await migrateEngine(getSql(), { slug, targetVersion: "0.2.0" }); await migrateEngine(getSql(), { slug, targetVersion: "0.2.0" }); - expect(await migrationCount(schema)).toBe(1); + expect(await migrationCount(schema)).toBe(6); expect(await engineVersion(schema)).toBe("0.2.0"); }); @@ -191,7 +209,7 @@ describe("migrateEngine", () => { await migrateEngine(getSql(), { slug, targetVersion: "0.1.0" }); await migrateEngine(getSql(), { slug, targetVersion: "0.2.0" }); - expect(await migrationCount(schema)).toBe(1); + expect(await migrationCount(schema)).toBe(6); expect(await engineVersion(schema)).toBe("0.2.0"); }); diff --git a/packages/engine-core/migrate/migrate.ts b/packages/engine-core/migrate/migrate.ts index fc46ecd..318649c 100644 --- a/packages/engine-core/migrate/migrate.ts +++ b/packages/engine-core/migrate/migrate.ts @@ -3,16 +3,57 @@ import type { SQL } from "bun"; import { semver } from "bun"; import { isValidSlug, slugToSchema } from "../slug"; -import migration001 from "./migrations/001_updated_at.sql" with { +import incremental001 from "./incremental/001_user.sql" with { type: "text" }; +import incremental002 from "./incremental/002_role_membership.sql" with { + type: "text", +}; +import incremental003 from "./incremental/003_tree_ownership.sql" with { + type: "text", +}; +import incremental004 from "./incremental/004_tree_grant.sql" with { + type: "text", +}; +import incremental005 from "./incremental/005_memory.sql" with { type: "text" }; +import incremental006 from "./incremental/006_embedding_queue.sql" with { + type: "text", +}; + +interface Incremental { + name: string; + sql: string; +} + +const incrementals: Incremental[] = [ + { name: "001_user", sql: incremental001 }, + { name: "002_role_membership", sql: incremental002 }, + { name: "003_tree_ownership", sql: incremental003 }, + { name: "004_tree_grant", sql: incremental004 }, + { name: "005_memory", sql: incremental005 }, + { name: "006_embedding_queue", sql: incremental006 }, +]; + +import idempotent001 from "./idempotent/001_role_membership.sql" with { + type: "text", +}; +import idempotent002 from "./idempotent/002_tree_privileges.sql" with { + type: "text", +}; +import idempotent003 from "./idempotent/003_memory.sql" with { type: "text" }; +import idempotent004 from "./idempotent/004_embedding_queue.sql" with { type: "text", }; -interface Migration { +interface Idempotent { name: string; sql: string; } -const migrations: Migration[] = [{ name: "001_updated_at", sql: migration001 }]; +const idempotents: Idempotent[] = [ + { name: "001_role_membership", sql: idempotent001 }, + { name: "002_tree_privileges", sql: idempotent002 }, + { name: "003_memory", sql: idempotent003 }, + { name: "004_embedding_queue", sql: idempotent004 }, +]; export interface MigrateEngineOptions { slug: string; @@ -108,6 +149,22 @@ function normalizeMigrateEngineOptions( }; } +function templateVars( + schema: string, + options: NormalizedMigrateEngineOptions, +): Record { + return { + ...options, + schema, + embedding_dimensions: options.embeddingDimensions, + bm25_text_config: options.bm25TextConfig, + bm25_k1: options.bm25K1, + bm25_b: options.bm25B, + hnsw_m: options.hnswM, + hnsw_ef_construction: options.hnswEfConstruction, + }; +} + function advisoryLockKey(schema: string): [number, number] { const digest = createHash("sha256").update(schema).digest(); return [digest.readInt32BE(0), digest.readInt32BE(4)]; @@ -187,13 +244,29 @@ async function runMigrations( // check ownership await assertSchemaOwnership(tx, schema); - // check version (reject downgrades) - await assertVersion(tx, schema, options.targetVersion); + // check version + const [{ version: dbVersion }] = await tx` + select version from ${tx(schema)}.version + `; + const cmp = semver.order(options.targetVersion, dbVersion); + // abort if target is older than the database + if (cmp < 0) { + throw new Error( + `Target version (${options.targetVersion}) is older than database version (${dbVersion}). ` + + "Please upgrade the server.", + ); + } + if (cmp === 0) { + // version matches. no need to run migrations + return; + } - // run migrations - const sorted = [...migrations].sort((a, b) => a.name.localeCompare(b.name)); + // run incremental migrations + const sorted1 = [...incrementals].sort((a, b) => + a.name.localeCompare(b.name), + ); - for (const migration of sorted) { + for (const migration of sorted1) { const [{ existing }] = await tx` select exists ( @@ -207,17 +280,21 @@ async function runMigrations( continue; } - const templateVars: Record = { - ...options, - schema, - }; - const renderedSql = template(migration.sql, templateVars); + const renderedSql = template(migration.sql, templateVars(schema, options)); await tx.unsafe(renderedSql); await tx` insert into ${tx(schema)}.migration (name, applied_at_version) values (${migration.name}, ${options.targetVersion})`; } + // run idempotent migrations + const sorted2 = [...idempotents].sort((a, b) => a.name.localeCompare(b.name)); + + for (const migration of sorted2) { + const renderedSql = template(migration.sql, templateVars(schema, options)); + await tx.unsafe(renderedSql); + } + // update version await tx`update ${tx(schema)}.version set version = ${options.targetVersion}, at = now()`; } @@ -237,24 +314,6 @@ async function assertSchemaOwnership(tx: SQL, schema: string): Promise { } } -async function assertVersion( - tx: SQL, - schema: string, - targetVersion: string, -): Promise { - const [{ version: dbVersion }] = await tx` - select version from ${tx(schema)}.version - `; - - const cmp = semver.order(targetVersion, dbVersion); - if (cmp < 0) { - throw new Error( - `Target version (${targetVersion}) is older than database version (${dbVersion}). ` + - "Please upgrade the server.", - ); - } -} - function template(sql: string, vars: Record): string { return sql.replace(/\{\{(\w+)\}\}/g, (_, key) => { if (!(key in vars)) { diff --git a/packages/engine-core/migrate/migrations/001_updated_at.sql b/packages/engine-core/migrate/migrations/001_updated_at.sql deleted file mode 100644 index 8e32112..0000000 --- a/packages/engine-core/migrate/migrations/001_updated_at.sql +++ /dev/null @@ -1,12 +0,0 @@ -------------------------------------------------------------------------------- --- generic updated_at trigger -------------------------------------------------------------------------------- -create or replace function {{schema}}.update_updated_at() -returns trigger -as $func$ -begin - new.updated_at = pg_catalog.now(); - return new; -end; -$func$ language plpgsql volatile security definer -set search_path to {{schema}}, pg_temp; From 69b8d953c341678e52a1be019afe30ce02adc28d Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Tue, 19 May 2026 09:12:19 -0500 Subject: [PATCH 10/17] more db functions --- bun.lock | 22 +- .../idempotent/001_role_membership.sql | 11 +- .../idempotent/002_tree_privileges.sql | 13 +- .../migrate/idempotent/003_memory.sql | 384 +++++++++++++++++- .../idempotent/004_embedding_queue.sql | 4 +- .../migrate/idempotent/005_tree_ownership.sql | 97 +++++ .../migrate/idempotent/006_tree_grant.sql | 107 +++++ .../migrate/incremental/004_tree_grant.sql | 1 - .../migrate/migrate.integration.test.ts | 105 +++++ packages/engine-core/migrate/migrate.ts | 8 + 10 files changed, 724 insertions(+), 28 deletions(-) create mode 100644 packages/engine-core/migrate/idempotent/005_tree_ownership.sql create mode 100644 packages/engine-core/migrate/idempotent/006_tree_grant.sql diff --git a/bun.lock b/bun.lock index 10ba56f..f49e3f0 100644 --- a/bun.lock +++ b/bun.lock @@ -14,14 +14,14 @@ }, "packages/accounts": { "name": "@memory.build/accounts", - "version": "0.2.0", + "version": "0.2.5", "dependencies": { "@pydantic/logfire-node": "^0.13.1", }, }, "packages/cli": { "name": "@memory.build/cli", - "version": "0.2.5", + "version": "0.2.6", "bin": { "me": "./index.ts", }, @@ -38,7 +38,7 @@ }, "packages/client": { "name": "@memory.build/client", - "version": "0.2.5", + "version": "0.2.6", "dependencies": { "@memory.build/protocol": "workspace:*", }, @@ -83,7 +83,7 @@ }, "packages/embedding": { "name": "@memory.build/embedding", - "version": "0.2.0", + "version": "0.2.5", "dependencies": { "@ai-sdk/openai": "^3.0.0", "@pydantic/logfire-node": "^0.13.1", @@ -92,14 +92,18 @@ }, "packages/engine": { "name": "@memory.build/engine", - "version": "0.2.0", + "version": "0.2.5", "dependencies": { "@pydantic/logfire-node": "^0.13.1", }, }, + "packages/engine-core": { + "name": "@memory.build/engine-core", + "version": "0.0.0", + }, "packages/protocol": { "name": "@memory.build/protocol", - "version": "0.2.5", + "version": "0.2.6", "dependencies": { "zod": "^4.0.0", }, @@ -112,7 +116,7 @@ }, "packages/server": { "name": "memory-engine-server", - "version": "0.2.0", + "version": "0.2.5", "dependencies": { "@memory.build/accounts": "workspace:*", "@memory.build/embedding": "workspace:*", @@ -153,7 +157,7 @@ }, "packages/worker": { "name": "@memory.build/worker", - "version": "0.2.0", + "version": "0.2.5", "dependencies": { "@memory.build/embedding": "workspace:*", "@memory.build/engine": "workspace:*", @@ -375,6 +379,8 @@ "@memory.build/engine": ["@memory.build/engine@workspace:packages/engine"], + "@memory.build/engine-core": ["@memory.build/engine-core@workspace:packages/engine-core"], + "@memory.build/protocol": ["@memory.build/protocol@workspace:packages/protocol"], "@memory.build/web": ["@memory.build/web@workspace:packages/web"], diff --git a/packages/engine-core/migrate/idempotent/001_role_membership.sql b/packages/engine-core/migrate/idempotent/001_role_membership.sql index 9e451ec..a4fbc9d 100644 --- a/packages/engine-core/migrate/idempotent/001_role_membership.sql +++ b/packages/engine-core/migrate/idempotent/001_role_membership.sql @@ -24,8 +24,7 @@ as $func$ from ancestors where id = _member_id ) -$func$ language sql stable security invoker -parallel safe +$func$ language sql stable security invoker parallel safe ; ------------------------------------------------------------------------------- @@ -56,9 +55,9 @@ execute function {{schema}}.role_membership_before_write() ; ------------------------------------------------------------------------------- --- explode_role_membership +-- calc_role_membership ------------------------------------------------------------------------------- -create or replace function {{schema}}.explode_role_membership(_user_id uuid) +create or replace function {{schema}}.calc_role_membership(_user_id uuid) returns table ( role_id uuid , superuser bool @@ -100,7 +99,7 @@ as $func$ select exists ( select 1 - from {{schema}}.explode_role_membership(_user_id) x + from {{schema}}.calc_role_membership(_user_id) x where x.superuser ) $func$ language sql stable security definer @@ -203,6 +202,6 @@ begin and d.member_id = _member_id ; end; -$func$ language plpgsql volatile security invoker +$func$ language plpgsql volatile security definer set search_path to pg_catalog, {{schema}}, pg_temp ; diff --git a/packages/engine-core/migrate/idempotent/002_tree_privileges.sql b/packages/engine-core/migrate/idempotent/002_tree_privileges.sql index 5b94da0..94049a4 100644 --- a/packages/engine-core/migrate/idempotent/002_tree_privileges.sql +++ b/packages/engine-core/migrate/idempotent/002_tree_privileges.sql @@ -11,18 +11,11 @@ returns table as $func$ with r as ( - -- the user - select - u.id as role_id - , u.superuser - from {{schema}}."user" u - where u.id = _user_id - union - -- the roles they belong to + -- the user and the roles they belong to select x.role_id , x.superuser - from {{schema}}.explode_role_membership(_user_id) x + from {{schema}}.calc_role_membership(_user_id) x ) -- superuser select @@ -70,5 +63,5 @@ as $func$ where x.tree_path @> _tree_path and x.actions @> _actions ) -$func$ language sql stable security definer +$func$ language sql stable security invoker ; diff --git a/packages/engine-core/migrate/idempotent/003_memory.sql b/packages/engine-core/migrate/idempotent/003_memory.sql index 3784018..bdf99ed 100644 --- a/packages/engine-core/migrate/idempotent/003_memory.sql +++ b/packages/engine-core/migrate/idempotent/003_memory.sql @@ -19,10 +19,392 @@ begin return new; end; $func$ language plpgsql volatile security definer -set search_path to {{schema}}, public, pg_temp -- public required for pgvector's `is not distinct from` +set search_path to pg_catalog, {{schema}}, public, pg_temp -- public required for pgvector's `is not distinct from` ; create or replace trigger memory_before_update_trg before update on {{schema}}.memory for each row execute function {{schema}}.memory_before_update(); + +------------------------------------------------------------------------------- +-- create memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.create_memory +( _user_id uuid +, _tree ltree +, _content text +, _id uuid default null +, _meta jsonb default '{}' +, _temporal tstzrange default null +) +returns uuid +as $func$ +begin + if not {{schema}}.has_tree_privilege(_user_id, _tree, array['create']) then + raise exception 'user (%) must be a superuser or own or have create on the tree path %', _user_id, _tree + using errcode = 'insufficient_privilege'; + end if; + + insert into {{schema}}.memory + ( id + , tree + , meta + , temporal + , content + ) + values + ( coalesce(_id, uuidv7()) + , _tree + , coalesce(_meta, '{}'::jsonb) + , _temporal + , _content + ) + returning id into strict _id + ; + return _id; +end; +$func$ language plpgsql volatile security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- update memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.update_memory +( _user_id uuid +, _id uuid +, _tree ltree +, _content text +, _meta jsonb default '{}' +, _temporal tstzrange default null +) +returns bool +as $func$ +begin + if not {{schema}}.has_tree_privilege(_user_id, _tree, array['update']) then + raise exception 'user (%) must be a superuser or own or have update on the tree path %', _user_id, _tree + using errcode = 'insufficient_privilege'; + end if; + + update {{schema}}.memory set + tree = _tree + , meta = meta || _meta + , temporal = _temporal + , content = _content + where id = _id + ; + return found; +end; +$func$ language plpgsql volatile security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- move memories +------------------------------------------------------------------------------- +create or replace function {{schema}}.move_memories +( _user_id uuid +, _query lquery +, _to ltree +) +returns uuid[] +as $func$ +declare + _moved uuid[]; +begin + -- must have create on target tree path + if not {{schema}}.has_tree_privilege(_user_id, _to, array['create']) then + raise exception 'user (%) must be a superuser or own or have create on the tree path %', _user_id, _to + using errcode = 'insufficient_privilege'; + end if; + + with x as + ( + -- must have update on source tree paths + select + p.role_id + , p.tree_path + from {{schema}}.calc_tree_privileges(_user_id) p + where p.actions @> array['update'] + ) + , u as + ( + update {{schema}}.memory m set tree = _to + from x + where m.tree ~ _query + and x.tree_path @> m.tree + returning id + ) + select array_agg(u.id) into strict _moved + from u + ; + + return _moved; +end; +$func$ language plpgsql volatile security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- move memories +------------------------------------------------------------------------------- +create or replace function {{schema}}.move_memories +( _user_id uuid +, _query ltxtquery +, _to ltree +) +returns uuid[] +as $func$ +declare + _moved uuid[]; +begin + -- must have create on target tree path + if not {{schema}}.has_tree_privilege(_user_id, _to, array['create']) then + raise exception 'user (%) must be a superuser or own or have create on the tree path %', _user_id, _to + using errcode = 'insufficient_privilege'; + end if; + + with x as + ( + -- must have update on source tree paths + select + p.role_id + , p.tree_path + from {{schema}}.calc_tree_privileges(_user_id) p + where p.actions @> array['update'] + ) + , u as + ( + update {{schema}}.memory m set tree = _to + from x + where m.tree @ _query + and x.tree_path @> m.tree + returning id + ) + select array_agg(u.id) into strict _moved + from u + ; + + return _moved; +end; +$func$ language plpgsql volatile security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- copy memories +------------------------------------------------------------------------------- +create or replace function {{schema}}.copy_memories +( _user_id uuid +, _query lquery +, _to ltree +) +returns uuid[] +as $func$ +declare + _copied uuid[]; +begin + -- must have create on target tree path + if not {{schema}}.has_tree_privilege(_user_id, _to, array['create']) then + raise exception 'user (%) must be a superuser or own or have create on the tree path %', _user_id, _to + using errcode = 'insufficient_privilege'; + end if; + + with x as + ( + -- must have read on source tree paths + select + p.role_id + , p.tree_path + from {{schema}}.calc_tree_privileges(_user_id) p + where p.actions @> array['read'] + ) + , i as + ( + insert into {{schema}}.memory + ( meta + , tree + , temporal + , content + , embedding + , embedding_version + ) + select + m.meta + , _to + , m.temporal + , m.content + , m.embedding + , m.embedding_version + from {{schema}}.memory m + inner join x on (x.tree_path @> m.tree) + where m.tree ~ _query + returning id + ) + select array_agg(i.id) into strict _copied + from i + ; + + return _copied; +end; +$func$ language plpgsql volatile security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- copy memories +------------------------------------------------------------------------------- +create or replace function {{schema}}.copy_memories +( _user_id uuid +, _query ltxtquery +, _to ltree +) +returns uuid[] +as $func$ +declare + _copied uuid[]; +begin + -- must have create on target tree path + if not {{schema}}.has_tree_privilege(_user_id, _to, array['create']) then + raise exception 'user (%) must be a superuser or own or have create on the tree path %', _user_id, _to + using errcode = 'insufficient_privilege'; + end if; + + with x as + ( + -- must have read on source tree paths + select + p.role_id + , p.tree_path + from {{schema}}.calc_tree_privileges(_user_id) p + where p.actions @> array['read'] + ) + , i as + ( + insert into {{schema}}.memory + ( meta + , tree + , temporal + , content + , embedding + , embedding_version + ) + select + m.meta + , _to + , m.temporal + , m.content + , m.embedding + , m.embedding_version + from {{schema}}.memory m + inner join x on (x.tree_path @> m.tree) + where m.tree @ _query + returning id + ) + select array_agg(i.id) into strict _copied + from i + ; + + return _copied; +end; +$func$ language plpgsql volatile security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_memory +( _user_id uuid +, _id uuid +) +returns bool +as $func$ +declare + _tree ltree; +begin + select m.tree into _tree + from {{schema}}.memory m + where m.id = _id + ; + + if not found then + return false; + end if; + + if not {{schema}}.has_tree_privilege(_user_id, _tree, array['delete']) then + raise exception 'user (%) must be a superuser or own or have delete on the tree path %', _user_id, _tree + using errcode = 'insufficient_privilege'; + end if; + + delete from {{schema}}.memory + where id = _id + ; + return found; +end; +$func$ language plpgsql volatile security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete memories +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_memories +( _user_id uuid +, _query lquery +) +returns uuid[] +as $func$ + with x as + ( + select + p.role_id + , p.tree_path + from {{schema}}.calc_tree_privileges(_user_id) p + where p.actions @> array['delete'] + ) + , d as + ( + delete from {{schema}}.memory m + using x + where m.tree ~ _query + and x.tree_path @> m.tree + returning id + ) + select array_agg(d.id) + from d +$func$ language sql volatile security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- delete memories +------------------------------------------------------------------------------- +create or replace function {{schema}}.delete_memories +( _user_id uuid +, _query ltxtquery +) +returns uuid[] +as $func$ + with x as + ( + select + p.role_id + , p.tree_path + from {{schema}}.calc_tree_privileges(_user_id) p + where p.actions @> array['delete'] + ) + , d as + ( + delete from {{schema}}.memory m + using x + where m.tree @ _query + and x.tree_path @> m.tree + returning id + ) + select array_agg(d.id) + from d +$func$ language sql volatile security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/engine-core/migrate/idempotent/004_embedding_queue.sql b/packages/engine-core/migrate/idempotent/004_embedding_queue.sql index 1a9ab6d..fa19348 100644 --- a/packages/engine-core/migrate/idempotent/004_embedding_queue.sql +++ b/packages/engine-core/migrate/idempotent/004_embedding_queue.sql @@ -131,7 +131,7 @@ begin end loop; end; $func$ -language plpgsql volatile +language plpgsql volatile security definer set search_path to pg_catalog, {{schema}}, pg_temp ; @@ -158,6 +158,6 @@ begin return pruned; end; $func$ -language plpgsql volatile +language plpgsql volatile security definer set search_path to pg_catalog, {{schema}}, pg_temp ; diff --git a/packages/engine-core/migrate/idempotent/005_tree_ownership.sql b/packages/engine-core/migrate/idempotent/005_tree_ownership.sql new file mode 100644 index 0000000..6a8d8b8 --- /dev/null +++ b/packages/engine-core/migrate/idempotent/005_tree_ownership.sql @@ -0,0 +1,97 @@ + +------------------------------------------------------------------------------- +-- is_tree_owner +------------------------------------------------------------------------------- +create or replace function {{schema}}.is_tree_owner +( _user_id uuid +, _tree_path ltree +) +returns bool +as $func$ + with r as + ( + select * + from {{schema}}.calc_role_membership(_user_id) + ) + select + exists (select 1 from r where r.superuser) -- is user a superuser? + or exists + ( + -- does user own the path? + select 1 + from r + inner join {{schema}}.tree_owner o on (r.role_id = o.user_id) + where o.tree_path @> _tree_path + ) +$func$ language sql volatile security invoker parallel safe +; + +------------------------------------------------------------------------------- +-- grant_tree_ownership +------------------------------------------------------------------------------- +create or replace function {{schema}}.grant_tree_ownership +( _grantor_id uuid +, _tree_path ltree +, _owner_id uuid +) +returns void +as $func$ +begin + -- is grantor allowed to do this? + if not {{schema}}.is_tree_owner(_grantor_id, _tree_path) then + raise exception 'grantor (%) must be a superuser or own the tree path %', _grantor_id, _tree_path + using errcode = 'insufficient_privilege'; + end if; + + insert into {{schema}}.tree_owner + ( tree_path + , user_id + ) + values + ( _tree_path + , _owner_id + ) + on conflict (tree_path) do update + set user_id = _owner_id + ; +end; +$func$ language plpgsql volatile security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- revoke_tree_ownership +------------------------------------------------------------------------------- +create or replace function {{schema}}.revoke_tree_ownership +( _revoker_id uuid +, _tree_path ltree +, _owner_id uuid +) +returns void +as $func$ +begin + -- checking permissions is expensive (relatively) + -- ensure this operation even makes sense first + perform 1 + from {{schema}}.tree_owner o + where o.tree_path = _tree_path + and o.user_id = _owner_id + ; + if not found then + return; + end if; + + -- is revoker allowed to do this? + if not {{schema}}.is_tree_owner(_revoker_id, _tree_path) then + raise exception 'revoker (%) must be a superuser or own the tree path %', _revoker_id, _tree_path + using errcode = 'insufficient_privilege'; + end if; + + delete from {{schema}}.tree_owner + where tree_path = _tree_path + and user_id = _owner_id + ; +end; +$func$ language plpgsql volatile security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/engine-core/migrate/idempotent/006_tree_grant.sql b/packages/engine-core/migrate/idempotent/006_tree_grant.sql new file mode 100644 index 0000000..fc4b502 --- /dev/null +++ b/packages/engine-core/migrate/idempotent/006_tree_grant.sql @@ -0,0 +1,107 @@ + +------------------------------------------------------------------------------- +-- grant_tree_actions +------------------------------------------------------------------------------- +create or replace function {{schema}}.grant_tree_actions +( _grantor_id uuid +, _actions text[] +, _tree_path ltree +, _user_id uuid +) +returns void +as $func$ +begin + -- is grantor allowed to do this? + if not {{schema}}.is_tree_owner(_grantor_id, _tree_path) then + raise exception 'grantor (%) must be a superuser or own the tree path %', _grantor_id, _tree_path + using errcode = 'insufficient_privilege'; + end if; + + insert into {{schema}}.tree_grant as g + ( user_id + , tree_path + , actions + ) + values + ( _user_id + , _tree_path + , coalesce(array(select distinct a.action from unnest(_actions) a(action) order by a.action), '{}') + ) + on conflict (user_id, tree_path) do update + set actions = coalesce + ( + array + ( + select distinct a.action + from unnest(g.actions || excluded.actions) a(action) + order by a.action + ) + , '{}' + ) + ; +end; +$func$ language plpgsql volatile security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- revoke_tree_actions +------------------------------------------------------------------------------- +create or replace function {{schema}}.revoke_tree_actions +( _revoker_id uuid +, _actions text[] +, _tree_path ltree +, _user_id uuid +) +returns void +as $func$ +declare + _existing_actions text[]; + _remaining_actions text[]; + _remaining_action_count int8; +begin + -- checking permissions is expensive (relatively) + -- ensure this operation even makes sense first + select g.actions into _existing_actions + from {{schema}}.tree_grant g + where g.tree_path = _tree_path + and g.user_id = _user_id + and g.actions && _actions + ; + if not found then + return; + end if; + + -- is revoker allowed to do this? + if not {{schema}}.is_tree_owner(_revoker_id, _tree_path) then + raise exception 'revoker (%) must be a superuser or own the tree path %', _revoker_id, _tree_path + using errcode = 'insufficient_privilege'; + end if; + + -- calc remaining actions + select coalesce(array_agg(x.action order by x.action), '{}'), count(*) + into strict _remaining_actions, _remaining_action_count + from + ( + select unnest(_existing_actions) as action + except + select unnest(_actions) as action + ) x + ; + + if _remaining_action_count = 0 then + delete from {{schema}}.tree_grant g + where g.user_id = _user_id + and g.tree_path = _tree_path + ; + else + update {{schema}}.tree_grant g + set actions = _remaining_actions + where g.user_id = _user_id + and g.tree_path = _tree_path + ; + end if; +end; +$func$ language plpgsql volatile security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/engine-core/migrate/incremental/004_tree_grant.sql b/packages/engine-core/migrate/incremental/004_tree_grant.sql index a74cdd4..cb6164b 100644 --- a/packages/engine-core/migrate/incremental/004_tree_grant.sql +++ b/packages/engine-core/migrate/incremental/004_tree_grant.sql @@ -5,7 +5,6 @@ create table {{schema}}.tree_grant ( user_id uuid not null references {{schema}}."user"(id) on delete cascade , tree_path ltree not null , actions text[] not null check (actions <@ '{read,create,update,delete}'::text[]) -, with_grant_option boolean not null default false , created_at timestamptz not null default now() , constraint pkey_tree_grant primary key (user_id, tree_path) ); diff --git a/packages/engine-core/migrate/migrate.integration.test.ts b/packages/engine-core/migrate/migrate.integration.test.ts index 748fe75..fe4b60c 100644 --- a/packages/engine-core/migrate/migrate.integration.test.ts +++ b/packages/engine-core/migrate/migrate.integration.test.ts @@ -222,4 +222,109 @@ describe("migrateEngine", () => { }), ).rejects.toThrow("shardId must be a safe integer"); }); + + test("grants and revokes tree actions", async () => { + const slug = randomSlug(); + const schema = schemaFor(slug); + const db = getSql(); + + await migrateEngine(db, { slug, targetVersion: "0.1.0" }); + + const [{ id: ownerId }] = await db` + insert into ${db(schema)}."user" (name) + values (${`owner_${slug}`}) + returning id + `; + const [{ id: granteeId }] = await db` + insert into ${db(schema)}."user" (name) + values (${`grantee_${slug}`}) + returning id + `; + const [{ id: outsiderId }] = await db` + insert into ${db(schema)}."user" (name) + values (${`outsider_${slug}`}) + returning id + `; + + await db` + insert into ${db(schema)}.tree_owner (tree_path, user_id) + values ('project'::ltree, ${ownerId}::uuid) + `; + + try { + await db` + select ${db(schema)}.grant_tree_actions( + ${outsiderId}::uuid, + array['read']::text[], + 'project.alpha'::ltree, + ${granteeId}::uuid + ) + `; + throw new Error("expected grant_tree_actions to reject"); + } catch (error) { + expect(String(error)).toContain("must be a superuser or own the tree path"); + } + + await db` + select ${db(schema)}.grant_tree_actions( + ${ownerId}::uuid, + array['read']::text[], + 'project.alpha'::ltree, + ${granteeId}::uuid + ) + `; + await db` + select ${db(schema)}.grant_tree_actions( + ${ownerId}::uuid, + array['update']::text[], + 'project.alpha'::ltree, + ${granteeId}::uuid + ) + `; + + const [{ actions: grantedActions }] = await db` + select actions + from ${db(schema)}.tree_grant + where user_id = ${granteeId}::uuid + and tree_path = 'project.alpha'::ltree + `; + expect(grantedActions).toEqual(["read", "update"]); + + await db` + select ${db(schema)}.revoke_tree_actions( + ${ownerId}::uuid, + array['read']::text[], + 'project.alpha'::ltree, + ${granteeId}::uuid + ) + `; + + const [{ actions: remainingActions }] = await db` + select actions + from ${db(schema)}.tree_grant + where user_id = ${granteeId}::uuid + and tree_path = 'project.alpha'::ltree + `; + expect(remainingActions).toEqual(["update"]); + + await db` + select ${db(schema)}.revoke_tree_actions( + ${ownerId}::uuid, + array['update']::text[], + 'project.alpha'::ltree, + ${granteeId}::uuid + ) + `; + + const [{ exists }] = await db` + select exists + ( + select 1 + from ${db(schema)}.tree_grant + where user_id = ${granteeId}::uuid + and tree_path = 'project.alpha'::ltree + ) as exists + `; + expect(exists).toBe(false); + }, 30_000); }); diff --git a/packages/engine-core/migrate/migrate.ts b/packages/engine-core/migrate/migrate.ts index 318649c..2468c9c 100644 --- a/packages/engine-core/migrate/migrate.ts +++ b/packages/engine-core/migrate/migrate.ts @@ -42,6 +42,12 @@ import idempotent003 from "./idempotent/003_memory.sql" with { type: "text" }; import idempotent004 from "./idempotent/004_embedding_queue.sql" with { type: "text", }; +import idempotent005 from "./idempotent/005_tree_ownership.sql" with { + type: "text", +}; +import idempotent006 from "./idempotent/006_tree_grant.sql" with { + type: "text", +}; interface Idempotent { name: string; @@ -53,6 +59,8 @@ const idempotents: Idempotent[] = [ { name: "002_tree_privileges", sql: idempotent002 }, { name: "003_memory", sql: idempotent003 }, { name: "004_embedding_queue", sql: idempotent004 }, + { name: "005_tree_ownership", sql: idempotent005 }, + { name: "006_tree_grant", sql: idempotent006 }, ]; export interface MigrateEngineOptions { From 783df8f11fb4eb8eed5be5982576537477360ae4 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Tue, 19 May 2026 09:26:56 -0500 Subject: [PATCH 11/17] add engine core migration tracing --- bun.lock | 3 + packages/engine-core/migrate/bootstrap.ts | 70 ++++++-- .../migrate/migrate.integration.test.ts | 4 +- packages/engine-core/migrate/migrate.ts | 157 +++++++++++++----- packages/engine-core/package.json | 5 +- 5 files changed, 183 insertions(+), 56 deletions(-) diff --git a/bun.lock b/bun.lock index f49e3f0..b14c2ed 100644 --- a/bun.lock +++ b/bun.lock @@ -100,6 +100,9 @@ "packages/engine-core": { "name": "@memory.build/engine-core", "version": "0.0.0", + "dependencies": { + "@pydantic/logfire-node": "^0.13.1", + }, }, "packages/protocol": { "name": "@memory.build/protocol", diff --git a/packages/engine-core/migrate/bootstrap.ts b/packages/engine-core/migrate/bootstrap.ts index d548bfe..6fd0c4e 100644 --- a/packages/engine-core/migrate/bootstrap.ts +++ b/packages/engine-core/migrate/bootstrap.ts @@ -1,5 +1,13 @@ +import { info, reportError, span } from "@pydantic/logfire-node"; import { SQL, semver } from "bun"; +const REQUIRED_EXTENSIONS = [ + { name: "citext", minVersion: "1.6" }, + { name: "ltree", minVersion: "1.3" }, + { name: "vector", minVersion: "0.8.2" }, + { name: "pg_textsearch", minVersion: "1.1.0" }, +] as const; + export async function bootstrapEngineDatabase( sql: SQL, statementTimeout: string = "20s", @@ -8,21 +16,53 @@ export async function bootstrapEngineDatabase( idleInTransactionSessionTimeout: string = "30s", shardId?: number, ): Promise { - await sql.begin(async (tx) => { - if (shardId !== undefined) { - await tx.unsafe(`set local pgdog.shard to ${String(shardId)}`); - } - await ensurePostgresVersion(tx); - await acquireAdvisoryLock(tx); - await tx`select set_config('statement_timeout', ${statementTimeout}, true)`; - await tx`select set_config('lock_timeout', ${lockTimeout}, true)`; - await tx`select set_config('transaction_timeout', ${transactionTimeout}, true)`; - await tx`select set_config('idle_in_transaction_session_timeout', ${idleInTransactionSessionTimeout}, true)`; - await ensureExtension(tx, "citext", "1.6"); - await ensureExtension(tx, "ltree", "1.3"); - await ensureExtension(tx, "vector", "0.8.2"); - await ensureExtension(tx, "pg_textsearch", "1.1.0"); - await ensureRoles(tx); + const attributes = { + "db.shard": shardId, + "db.statement_timeout": statementTimeout, + "db.lock_timeout": lockTimeout, + "db.transaction_timeout": transactionTimeout, + "db.idle_in_transaction_session_timeout": idleInTransactionSessionTimeout, + "engine_core.required_extensions": REQUIRED_EXTENSIONS.map( + (extension) => `${extension.name}@>=${extension.minVersion}`, + ), + }; + + await span("engine_core.bootstrap", { + attributes, + callback: async () => { + try { + await sql.begin(async (tx) => { + if (shardId !== undefined) { + await tx.unsafe(`set local pgdog.shard to ${String(shardId)}`); + } + await ensurePostgresVersion(tx); + await span("engine_core.bootstrap.acquire_lock", { + callback: () => acquireAdvisoryLock(tx), + }); + await tx`select set_config('statement_timeout', ${statementTimeout}, true)`; + await tx`select set_config('lock_timeout', ${lockTimeout}, true)`; + await tx`select set_config('transaction_timeout', ${transactionTimeout}, true)`; + await tx`select set_config('idle_in_transaction_session_timeout', ${idleInTransactionSessionTimeout}, true)`; + for (const extension of REQUIRED_EXTENSIONS) { + await span("engine_core.bootstrap.ensure_extension", { + attributes: { + "db.extension": extension.name, + "db.extension_min_version": extension.minVersion, + }, + callback: () => + ensureExtension(tx, extension.name, extension.minVersion), + }); + } + await span("engine_core.bootstrap.ensure_roles", { + callback: () => ensureRoles(tx), + }); + }); + info("Engine core bootstrap completed", attributes); + } catch (error) { + reportError("Engine core bootstrap failed", error as Error, attributes); + throw error; + } + }, }); } diff --git a/packages/engine-core/migrate/migrate.integration.test.ts b/packages/engine-core/migrate/migrate.integration.test.ts index fe4b60c..764a076 100644 --- a/packages/engine-core/migrate/migrate.integration.test.ts +++ b/packages/engine-core/migrate/migrate.integration.test.ts @@ -262,7 +262,9 @@ describe("migrateEngine", () => { `; throw new Error("expected grant_tree_actions to reject"); } catch (error) { - expect(String(error)).toContain("must be a superuser or own the tree path"); + expect(String(error)).toContain( + "must be a superuser or own the tree path", + ); } await db` diff --git a/packages/engine-core/migrate/migrate.ts b/packages/engine-core/migrate/migrate.ts index 2468c9c..e548dbc 100644 --- a/packages/engine-core/migrate/migrate.ts +++ b/packages/engine-core/migrate/migrate.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { info, reportError, span } from "@pydantic/logfire-node"; import type { SQL } from "bun"; import { semver } from "bun"; import { isValidSlug, slugToSchema } from "../slug"; @@ -100,42 +101,83 @@ export async function migrateEngine( options: MigrateEngineOptions, ): Promise { const opts = normalizeMigrateEngineOptions(options); - - if (!isValidSlug(opts.slug)) { - throw new Error( - `Invalid engine slug: "${opts.slug}" — must be 12 lowercase alphanumeric characters`, - ); - } - if (!semver.satisfies(opts.targetVersion, "*")) { - throw new Error(`Invalid target version: "${opts.targetVersion}"`); - } - const schema = slugToSchema(opts.slug); - const [key1, key2] = advisoryLockKey(`memory-engine:schema:${schema}`); - - await sql.begin(async (tx) => { - if (opts.shardId !== undefined) { - if (!Number.isSafeInteger(opts.shardId)) { - throw new Error(`shardId must be a safe integer, got: ${opts.shardId}`); + const attributes = migrateAttributes(opts); + + await span("engine_core.migrate", { + attributes, + callback: async () => { + try { + if (!isValidSlug(opts.slug)) { + throw new Error( + `Invalid engine slug: "${opts.slug}" — must be 12 lowercase alphanumeric characters`, + ); + } + if (!semver.satisfies(opts.targetVersion, "*")) { + throw new Error(`Invalid target version: "${opts.targetVersion}"`); + } + const schema = slugToSchema(opts.slug); + const schemaAttributes = { ...attributes, "db.schema": schema }; + const [key1, key2] = advisoryLockKey(`memory-engine:schema:${schema}`); + + await sql.begin(async (tx) => { + if (opts.shardId !== undefined) { + if (!Number.isSafeInteger(opts.shardId)) { + throw new Error( + `shardId must be a safe integer, got: ${opts.shardId}`, + ); + } + await tx.unsafe(`set local pgdog.shard to ${String(opts.shardId)}`); + } + await tx`select set_config('statement_timeout', ${opts.statementTimeout}, true)`; + await tx`select set_config('lock_timeout', ${opts.lockTimeout}, true)`; + await tx`select set_config('transaction_timeout', ${opts.transactionTimeout}, true)`; + await tx`select set_config('idle_in_transaction_session_timeout', ${opts.idleInTransactionSessionTimeout}, true)`; + const acquired = await span("engine_core.migrate.acquire_lock", { + attributes: schemaAttributes, + callback: () => acquireAdvisoryLock(tx, key1, key2), + }); + if (!acquired) { + throw new Error( + `Unable to acquire lock for engine slug ${opts.slug} migrations.`, + ); + } + + if (!(await doesEngineExist(tx, schema))) { + await span("engine_core.migrate.provision", { + attributes: schemaAttributes, + callback: () => provisionEngine(tx, schema), + }); + info("Engine core schema provisioned", schemaAttributes); + } + await span("engine_core.migrate.run", { + attributes: schemaAttributes, + callback: () => runMigrations(tx, schema, opts), + }); + }); + info("Engine core migrations completed", schemaAttributes); + } catch (error) { + reportError("Engine core migration failed", error as Error, attributes); + throw error; } - await tx.unsafe(`set local pgdog.shard to ${String(opts.shardId)}`); - } - await tx`select set_config('statement_timeout', ${opts.statementTimeout}, true)`; - await tx`select set_config('lock_timeout', ${opts.lockTimeout}, true)`; - await tx`select set_config('transaction_timeout', ${opts.transactionTimeout}, true)`; - await tx`select set_config('idle_in_transaction_session_timeout', ${opts.idleInTransactionSessionTimeout}, true)`; - if (!(await acquireAdvisoryLock(tx, key1, key2))) { - throw new Error( - `Unable to acquire lock for engine slug ${opts.slug} migrations.`, - ); - } - - if (!(await doesEngineExist(tx, schema))) { - await provisionEngine(tx, schema); - } - await runMigrations(tx, schema, opts); + }, }); } +function migrateAttributes( + options: NormalizedMigrateEngineOptions, +): Record { + return { + "engine.slug": options.slug, + "engine.target_version": options.targetVersion, + "db.shard": options.shardId, + "db.statement_timeout": options.statementTimeout, + "db.lock_timeout": options.lockTimeout, + "db.transaction_timeout": options.transactionTimeout, + "db.idle_in_transaction_session_timeout": + options.idleInTransactionSessionTimeout, + }; +} + function normalizeMigrateEngineOptions( options: MigrateEngineOptions, ): NormalizedMigrateEngineOptions { @@ -266,6 +308,11 @@ async function runMigrations( } if (cmp === 0) { // version matches. no need to run migrations + info("Engine core migration skipped, version current", { + "db.schema": schema, + "engine.version": dbVersion, + "engine.target_version": options.targetVersion, + }); return; } @@ -288,19 +335,51 @@ async function runMigrations( continue; } - const renderedSql = template(migration.sql, templateVars(schema, options)); - await tx.unsafe(renderedSql); - await tx` - insert into ${tx(schema)}.migration (name, applied_at_version) - values (${migration.name}, ${options.targetVersion})`; + await span("engine_core.migrate.incremental", { + attributes: { + "db.schema": schema, + "engine.migration": migration.name, + "engine.migration_type": "incremental", + "engine.target_version": options.targetVersion, + }, + callback: async () => { + const renderedSql = template( + migration.sql, + templateVars(schema, options), + ); + await tx.unsafe(renderedSql); + await tx` + insert into ${tx(schema)}.migration (name, applied_at_version) + values (${migration.name}, ${options.targetVersion})`; + }, + }); + info("Engine core migration applied", { + "db.schema": schema, + "engine.migration": migration.name, + "engine.migration_type": "incremental", + "engine.target_version": options.targetVersion, + }); } // run idempotent migrations const sorted2 = [...idempotents].sort((a, b) => a.name.localeCompare(b.name)); for (const migration of sorted2) { - const renderedSql = template(migration.sql, templateVars(schema, options)); - await tx.unsafe(renderedSql); + await span("engine_core.migrate.idempotent", { + attributes: { + "db.schema": schema, + "engine.migration": migration.name, + "engine.migration_type": "idempotent", + "engine.target_version": options.targetVersion, + }, + callback: async () => { + const renderedSql = template( + migration.sql, + templateVars(schema, options), + ); + await tx.unsafe(renderedSql); + }, + }); } // update version diff --git a/packages/engine-core/package.json b/packages/engine-core/package.json index 13708aa..d8fd729 100644 --- a/packages/engine-core/package.json +++ b/packages/engine-core/package.json @@ -2,5 +2,8 @@ "name": "@memory.build/engine-core", "version": "0.0.0", "private": true, - "type": "module" + "type": "module", + "dependencies": { + "@pydantic/logfire-node": "^0.13.1" + } } From c69e1756437405f00abf80519bb64c1d2f9914bf Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Tue, 19 May 2026 15:48:49 -0500 Subject: [PATCH 12/17] foo --- package.json | 1 + .../migrate/idempotent/003_memory.sql | 415 ++++++++++-------- .../migrate/migrate.integration.test.ts | 192 ++++++++ scripts/migrate-engine-core.ts | 29 ++ 4 files changed, 457 insertions(+), 180 deletions(-) create mode 100644 scripts/migrate-engine-core.ts diff --git a/package.json b/package.json index 29e8f2f..dd898b6 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "clean": "rm -rf packages/cli/dist dist", "docs": "./bun --filter @memory.build/docs-site dev", "docs:build": "./bun --filter @memory.build/docs-site build", + "engine-core:migrate": "./bun scripts/migrate-engine-core.ts", "web": "./bun --filter @memory.build/web dev", "web:build": "./bun --filter @memory.build/web build", "generate:master-key": "./bun scripts/generate-master-key.ts", diff --git a/packages/engine-core/migrate/idempotent/003_memory.sql b/packages/engine-core/migrate/idempotent/003_memory.sql index bdf99ed..1be5798 100644 --- a/packages/engine-core/migrate/idempotent/003_memory.sql +++ b/packages/engine-core/migrate/idempotent/003_memory.sql @@ -27,6 +27,41 @@ before update on {{schema}}.memory for each row execute function {{schema}}.memory_before_update(); +------------------------------------------------------------------------------- +-- get memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.get_memory +( _user_id uuid +, _id uuid default null +) +returns table +( id uuid +, tree ltree +, meta jsonb +, temporal tstzrange +, content text +, created_at timestamptz +, updated_at timestamptz +, has_embedding bool +) +as $func$ + select + m.id + , m.tree + , m.meta + , m.temporal + , m.content + , m.created_at + , m.updated_at + , m.embedding is not null + into _memory + from {{schema}}.memory m + where m.id = _id + and {{schema}}.has_tree_privilege(_user_id, m.tree, array['read']) +$func$ language sql stable security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + ------------------------------------------------------------------------------- -- create memory ------------------------------------------------------------------------------- @@ -82,17 +117,31 @@ create or replace function {{schema}}.update_memory returns bool as $func$ begin - if not {{schema}}.has_tree_privilege(_user_id, _tree, array['update']) then - raise exception 'user (%) must be a superuser or own or have update on the tree path %', _user_id, _tree - using errcode = 'insufficient_privilege'; - end if; - - update {{schema}}.memory set + with p as materialized + ( + select p.tree_path, p.actions + from {{schema}}.calc_tree_privileges(_user_id) p + ) + update {{schema}}.memory m set tree = _tree , meta = meta || _meta , temporal = _temporal , content = _content - where id = _id + where m.id = _id + and exists + ( + select 1 + from p + where p.tree_path @> m.tree + and p.actions @> array['update'] + ) + and (m.tree @> _tree or exists + ( + select 1 + from p + where p.tree_path @> _tree + and p.actions @> array['insert'] + )) ; return found; end; @@ -101,91 +150,68 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp ; ------------------------------------------------------------------------------- --- move memories +-- move tree ------------------------------------------------------------------------------- -create or replace function {{schema}}.move_memories +create or replace function {{schema}}.move_tree ( _user_id uuid -, _query lquery -, _to ltree +, _src ltree +, _dst ltree +, _dry_run bool default false ) -returns uuid[] +returns bigint as $func$ declare - _moved uuid[]; + _moved bigint; begin - -- must have create on target tree path - if not {{schema}}.has_tree_privilege(_user_id, _to, array['create']) then - raise exception 'user (%) must be a superuser or own or have create on the tree path %', _user_id, _to + -- must have create on _dst tree path + if not {{schema}}.has_tree_privilege(_user_id, _dst, array['create']) then + raise exception 'user (%) must be a superuser or own or have create on the tree path %', _user_id, _dst using errcode = 'insufficient_privilege'; end if; - with x as + with p as materialized ( - -- must have update on source tree paths - select - p.role_id - , p.tree_path + select p.tree_path from {{schema}}.calc_tree_privileges(_user_id) p - where p.actions @> array['update'] ) - , u as + , x as ( - update {{schema}}.memory m set tree = _to - from x - where m.tree ~ _query - and x.tree_path @> m.tree - returning id - ) - select array_agg(u.id) into strict _moved - from u - ; - - return _moved; -end; -$func$ language plpgsql volatile security definer -set search_path to pg_catalog, {{schema}}, public, pg_temp -; - -------------------------------------------------------------------------------- --- move memories -------------------------------------------------------------------------------- -create or replace function {{schema}}.move_memories -( _user_id uuid -, _query ltxtquery -, _to ltree -) -returns uuid[] -as $func$ -declare - _moved uuid[]; -begin - -- must have create on target tree path - if not {{schema}}.has_tree_privilege(_user_id, _to, array['create']) then - raise exception 'user (%) must be a superuser or own or have create on the tree path %', _user_id, _to - using errcode = 'insufficient_privilege'; - end if; + select m.id + from {{schema}}.memory m + where _src @> m.tree + and exists + ( + select 1 + from p + where p.tree_path @> m.tree + and p.actions @> array['read'] + ) + and + ( + m.tree @> _dst + and exists + ( + select 1 + from p.tree_path @> m.tree - with x as - ( - -- must have update on source tree paths - select - p.role_id - , p.tree_path - from {{schema}}.calc_tree_privileges(_user_id) p - where p.actions @> array['update'] + ) + ) ) , u as ( - update {{schema}}.memory m set tree = _to + update {{schema}}.memory m + set tree = + case + when nlevel(m.tree) = nlevel(_src) then _dst + else _dst || subpath(m.tree, nlevel(_src), nlevel(m.tree) - nlevel(_src)) + end from x - where m.tree @ _query - and x.tree_path @> m.tree - returning id + where m.id = x.id + and not _dry_run ) - select array_agg(u.id) into strict _moved - from u + select count(*) into strict _moved + from x ; - return _moved; end; $func$ language plpgsql volatile security definer @@ -193,92 +219,42 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp ; ------------------------------------------------------------------------------- --- copy memories +-- copy tree ------------------------------------------------------------------------------- -create or replace function {{schema}}.copy_memories +create or replace function {{schema}}.copy_tree ( _user_id uuid -, _query lquery -, _to ltree +, _src ltree +, _dst ltree +, _dry_run bool default false ) -returns uuid[] +returns bigint as $func$ declare - _copied uuid[]; + _copied bigint; begin - -- must have create on target tree path - if not {{schema}}.has_tree_privilege(_user_id, _to, array['create']) then - raise exception 'user (%) must be a superuser or own or have create on the tree path %', _user_id, _to + -- must have create on _dst tree path + if not {{schema}}.has_tree_privilege(_user_id, _dst, array['create']) then + raise exception 'user (%) must be a superuser or own or have create on the tree path %', _user_id, _dst using errcode = 'insufficient_privilege'; end if; - with x as + with p as materialized ( - -- must have read on source tree paths - select - p.role_id - , p.tree_path + select p.tree_path from {{schema}}.calc_tree_privileges(_user_id) p where p.actions @> array['read'] ) - , i as + , m as ( - insert into {{schema}}.memory - ( meta - , tree - , temporal - , content - , embedding - , embedding_version - ) - select - m.meta - , _to - , m.temporal - , m.content - , m.embedding - , m.embedding_version + select m.* from {{schema}}.memory m - inner join x on (x.tree_path @> m.tree) - where m.tree ~ _query - returning id - ) - select array_agg(i.id) into strict _copied - from i - ; - - return _copied; -end; -$func$ language plpgsql volatile security definer -set search_path to pg_catalog, {{schema}}, public, pg_temp -; - -------------------------------------------------------------------------------- --- copy memories -------------------------------------------------------------------------------- -create or replace function {{schema}}.copy_memories -( _user_id uuid -, _query ltxtquery -, _to ltree -) -returns uuid[] -as $func$ -declare - _copied uuid[]; -begin - -- must have create on target tree path - if not {{schema}}.has_tree_privilege(_user_id, _to, array['create']) then - raise exception 'user (%) must be a superuser or own or have create on the tree path %', _user_id, _to - using errcode = 'insufficient_privilege'; - end if; - - with x as - ( - -- must have read on source tree paths - select - p.role_id - , p.tree_path - from {{schema}}.calc_tree_privileges(_user_id) p - where p.actions @> array['read'] + where _src @> m.tree + and exists + ( + select 1 + from p + where p.tree_path @> m.tree + ) ) , i as ( @@ -292,18 +268,19 @@ begin ) select m.meta - , _to + , case + when nlevel(m.tree) = nlevel(_src) then _dst + else _dst || subpath(m.tree, nlevel(_src), nlevel(m.tree) - nlevel(_src)) + end as dst , m.temporal , m.content , m.embedding , m.embedding_version - from {{schema}}.memory m - inner join x on (x.tree_path @> m.tree) - where m.tree @ _query - returning id + from m + where not _dry_run ) - select array_agg(i.id) into strict _copied - from i + select count(*) into strict _copied + from m ; return _copied; @@ -348,63 +325,141 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp ; ------------------------------------------------------------------------------- --- delete memories +-- delete tree ------------------------------------------------------------------------------- -create or replace function {{schema}}.delete_memories +create or replace function {{schema}}.delete_tree ( _user_id uuid -, _query lquery +, _tree ltree +, _dry_run bool default false ) -returns uuid[] +returns bigint as $func$ - with x as + with p as materialized ( - select - p.role_id - , p.tree_path + select p.tree_path from {{schema}}.calc_tree_privileges(_user_id) p where p.actions @> array['delete'] ) + , m as + ( + select m.id + from {{schema}}.memory m + where _tree @> m.tree + and exists + ( + select 1 + from p + where p.tree @> m.tree + ) + ) , d as ( delete from {{schema}}.memory m using x - where m.tree ~ _query - and x.tree_path @> m.tree - returning id + where m.id = x.id + and not _dry_run ) - select array_agg(d.id) - from d + select count(*) + from x $func$ language sql volatile security definer set search_path to pg_catalog, {{schema}}, public, pg_temp ; ------------------------------------------------------------------------------- --- delete memories +-- count tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.count_tree +( _user_id uuid +, _query lquery +, _actions text[] +) +returns bigint +as $func$ + with x as materialized + ( + select p.tree_path + from {{schema}}.calc_tree_privileges(_user_id) p + where p.actions @> (coalesce(_actions, array['read'])) + ) + select count(*) + from {{schema}}.memory m + where m.tree ~ _query + and exists + ( + select 1 + from x + where x.tree_path @> m.tree + ) +$func$ language sql stable security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- count tree ------------------------------------------------------------------------------- -create or replace function {{schema}}.delete_memories +create or replace function {{schema}}.count_tree ( _user_id uuid , _query ltxtquery +, _actions text[] ) -returns uuid[] +returns bigint as $func$ - with x as + with x as materialized ( - select - p.role_id - , p.tree_path + select p.tree_path from {{schema}}.calc_tree_privileges(_user_id) p - where p.actions @> array['delete'] + where p.actions @> (coalesce(_actions, array['read'])) ) - , d as + select count(*) + from {{schema}}.memory m + where m.tree @ _query + and exists ( - delete from {{schema}}.memory m - using x - where m.tree @ _query - and x.tree_path @> m.tree - returning id + select 1 + from x + where x.tree_path @> m.tree ) - select array_agg(d.id) - from d -$func$ language sql volatile security definer +$func$ language sql stable security definer +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_tree +( _user_id uuid +, _query lquery +) +returns table +( tree ltree +, count bigint +) +as $func$ + with p as + ( + select p.tree_path + from {{schema}}.calc_tree_privileges(_user_id) p + where p.actions @> array['read'] + ) + , m as + ( + select distinct m.id, m.tree + from {{schema}}.memory m + where m.tree ~ _query + and exists + ( + select 1 + from p + where p.tree_path @> m.tree + ) + ) + select + subltree(m.tree, 0, i) as tree + , count(m.id) as count + from m + cross join lateral generate_series(1, nlevel(m.tree)) i + group by 1 + order by 1 +$func$ language sql stable security definer set search_path to pg_catalog, {{schema}}, public, pg_temp ; diff --git a/packages/engine-core/migrate/migrate.integration.test.ts b/packages/engine-core/migrate/migrate.integration.test.ts index 764a076..9b38a2f 100644 --- a/packages/engine-core/migrate/migrate.integration.test.ts +++ b/packages/engine-core/migrate/migrate.integration.test.ts @@ -329,4 +329,196 @@ describe("migrateEngine", () => { `; expect(exists).toBe(false); }, 30_000); + + test("lists tree breadcrumbs without double-counting overlapping privileges", async () => { + const slug = randomSlug(); + const schema = schemaFor(slug); + const db = getSql(); + + await migrateEngine(db, { slug, targetVersion: "0.1.0" }); + + const [{ id: userId }] = await db` + insert into ${db(schema)}."user" (name) + values (${`tree_user_${slug}`}) + returning id + `; + + await db` + insert into ${db(schema)}.tree_owner (tree_path, user_id) + values ('work'::ltree, ${userId}::uuid) + `; + await db` + insert into ${db(schema)}.tree_grant (user_id, tree_path, actions) + values (${userId}::uuid, 'work.api'::ltree, array['read']::text[]) + `; + await db` + insert into ${db(schema)}.memory (tree, content) + values + ('work.api.auth'::ltree, 'auth') + , ('work.api.search'::ltree, 'search') + , ('work.ui'::ltree, 'ui') + `; + + const rows = await db` + select tree::text, count::int + from ${db(schema)}.list_tree(${userId}::uuid, 'work.api.*{0,}'::lquery) + `; + + expect(rows).toEqual([ + { tree: "work", count: 2 }, + { tree: "work.api", count: 2 }, + { tree: "work.api.auth", count: 1 }, + { tree: "work.api.search", count: 1 }, + ]); + }); + + test("moves a tree by rewriting the source prefix", async () => { + const slug = randomSlug(); + const schema = schemaFor(slug); + const db = getSql(); + + await migrateEngine(db, { slug, targetVersion: "0.1.0" }); + + const [{ id: userId }] = await db` + insert into ${db(schema)}."user" (name) + values (${`move_user_${slug}`}) + returning id + `; + + await db` + insert into ${db(schema)}.tree_owner (tree_path, user_id) + values + ('work'::ltree, ${userId}::uuid) + , ('archive'::ltree, ${userId}::uuid) + `; + await db` + insert into ${db(schema)}.tree_grant (user_id, tree_path, actions) + values (${userId}::uuid, 'work.api'::ltree, array['update']::text[]) + `; + await db` + insert into ${db(schema)}.memory (tree, content) + values + ('work.api'::ltree, 'api') + , ('work.api.auth'::ltree, 'auth') + , ('work.ui'::ltree, 'ui') + `; + + const [{ count: dryRunCount }] = await db` + select ${db(schema)}.move_tree( + ${userId}::uuid, + 'work.api'::ltree, + 'archive.api'::ltree, + true + )::int as count + `; + expect(dryRunCount).toBe(2); + + const dryRunRows = await db` + select tree::text, content + from ${db(schema)}.memory + order by tree::text + `; + expect(dryRunRows).toEqual([ + { tree: "work.api", content: "api" }, + { tree: "work.api.auth", content: "auth" }, + { tree: "work.ui", content: "ui" }, + ]); + + const [{ count: moveCount }] = await db` + select ${db(schema)}.move_tree( + ${userId}::uuid, + 'work.api'::ltree, + 'archive.api'::ltree, + false + )::int as count + `; + expect(moveCount).toBe(2); + + const movedRows = await db` + select tree::text, content + from ${db(schema)}.memory + order by tree::text + `; + expect(movedRows).toEqual([ + { tree: "archive.api", content: "api" }, + { tree: "archive.api.auth", content: "auth" }, + { tree: "work.ui", content: "ui" }, + ]); + }); + + test("copies a tree by rewriting the source prefix", async () => { + const slug = randomSlug(); + const schema = schemaFor(slug); + const db = getSql(); + + await migrateEngine(db, { slug, targetVersion: "0.1.0" }); + + const [{ id: userId }] = await db` + insert into ${db(schema)}."user" (name) + values (${`copy_user_${slug}`}) + returning id + `; + + await db` + insert into ${db(schema)}.tree_owner (tree_path, user_id) + values + ('work'::ltree, ${userId}::uuid) + , ('archive'::ltree, ${userId}::uuid) + `; + await db` + insert into ${db(schema)}.tree_grant (user_id, tree_path, actions) + values (${userId}::uuid, 'work.api'::ltree, array['read']::text[]) + `; + await db` + insert into ${db(schema)}.memory (tree, content) + values + ('work.api'::ltree, 'api') + , ('work.api.auth'::ltree, 'auth') + , ('work.ui'::ltree, 'ui') + `; + + const [{ count: dryRunCount }] = await db` + select ${db(schema)}.copy_tree( + ${userId}::uuid, + 'work.api'::ltree, + 'archive.api'::ltree, + true + )::int as count + `; + expect(dryRunCount).toBe(2); + + const dryRunRows = await db` + select tree::text, content + from ${db(schema)}.memory + order by tree::text, content + `; + expect(dryRunRows).toEqual([ + { tree: "work.api", content: "api" }, + { tree: "work.api.auth", content: "auth" }, + { tree: "work.ui", content: "ui" }, + ]); + + const [{ count: copyCount }] = await db` + select ${db(schema)}.copy_tree( + ${userId}::uuid, + 'work.api'::ltree, + 'archive.api'::ltree, + false + )::int as count + `; + expect(copyCount).toBe(2); + + const copiedRows = await db` + select tree::text, content + from ${db(schema)}.memory + order by tree::text, content + `; + expect(copiedRows).toEqual([ + { tree: "archive.api", content: "api" }, + { tree: "archive.api.auth", content: "auth" }, + { tree: "work.api", content: "api" }, + { tree: "work.api.auth", content: "auth" }, + { tree: "work.ui", content: "ui" }, + ]); + }); }); diff --git a/scripts/migrate-engine-core.ts b/scripts/migrate-engine-core.ts new file mode 100644 index 0000000..2800a40 --- /dev/null +++ b/scripts/migrate-engine-core.ts @@ -0,0 +1,29 @@ +#!/usr/bin/env bun + +import { SQL } from "bun"; +import { bootstrapEngineDatabase } from "../packages/engine-core/migrate/bootstrap"; +import { migrateEngine } from "../packages/engine-core/migrate/migrate"; +import { slugToSchema } from "../packages/engine-core/slug"; + +const ENGINE_SLUG = "dev000000001"; +const TARGET_VERSION = process.env.TARGET_VERSION ?? "0.1.0"; +const DATABASE_URL = + process.env.DATABASE_URL ?? + process.env.ENGINE_DATABASE_URL ?? + "postgresql://postgres@localhost:5432/postgres"; + +const sql = new SQL(DATABASE_URL); + +try { + console.log( + `Bootstrapping engine database and migrating ${slugToSchema(ENGINE_SLUG)} to ${TARGET_VERSION}`, + ); + await bootstrapEngineDatabase(sql); + await migrateEngine(sql, { + slug: ENGINE_SLUG, + targetVersion: TARGET_VERSION, + }); + console.log(`Engine schema ${slugToSchema(ENGINE_SLUG)} is up to date.`); +} finally { + await sql.close(); +} From f31254ca037b3304c2d4928fd866b896ed12c474 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Wed, 20 May 2026 14:17:02 -0500 Subject: [PATCH 13/17] foo --- packages/engine-core/migrate/bootstrap.ts | 4 + .../idempotent/001_role_membership.sql | 23 +- ...ree_privileges.sql => 002_tree_access.sql} | 37 +- .../migrate/idempotent/003_memory.sql | 356 ++++++++++++------ .../idempotent/004_embedding_queue.sql | 6 +- .../migrate/idempotent/005_tree_ownership.sql | 4 +- .../migrate/idempotent/006_tree_grant.sql | 4 +- .../migrate/incremental/000_provision.sql | 17 + .../incremental/002_role_membership.sql | 2 +- ...004_tree_grant.sql => 003_tree_access.sql} | 8 +- .../incremental/003_tree_ownership.sql | 11 - .../{005_memory.sql => 004_memory.sql} | 0 packages/engine-core/migrate/migrate.ts | 43 +-- 13 files changed, 304 insertions(+), 211 deletions(-) rename packages/engine-core/migrate/idempotent/{002_tree_privileges.sql => 002_tree_access.sql} (55%) create mode 100644 packages/engine-core/migrate/incremental/000_provision.sql rename packages/engine-core/migrate/incremental/{004_tree_grant.sql => 003_tree_access.sql} (63%) delete mode 100644 packages/engine-core/migrate/incremental/003_tree_ownership.sql rename packages/engine-core/migrate/incremental/{005_memory.sql => 004_memory.sql} (100%) diff --git a/packages/engine-core/migrate/bootstrap.ts b/packages/engine-core/migrate/bootstrap.ts index 6fd0c4e..a5f747e 100644 --- a/packages/engine-core/migrate/bootstrap.ts +++ b/packages/engine-core/migrate/bootstrap.ts @@ -53,9 +53,11 @@ export async function bootstrapEngineDatabase( ensureExtension(tx, extension.name, extension.minVersion), }); } + /* TODO: remove await span("engine_core.bootstrap.ensure_roles", { callback: () => ensureRoles(tx), }); + */ }); info("Engine core bootstrap completed", attributes); } catch (error) { @@ -159,6 +161,7 @@ async function ensureExtension( } } +/* TODO: remove this async function ensureRoles(tx: SQL): Promise { await tx.unsafe(` do $block$ @@ -183,3 +186,4 @@ async function ensureRoles(tx: SQL): Promise { $block$; `); } +*/ diff --git a/packages/engine-core/migrate/idempotent/001_role_membership.sql b/packages/engine-core/migrate/idempotent/001_role_membership.sql index a4fbc9d..aed8f11 100644 --- a/packages/engine-core/migrate/idempotent/001_role_membership.sql +++ b/packages/engine-core/migrate/idempotent/001_role_membership.sql @@ -25,6 +25,7 @@ as $func$ where id = _member_id ) $func$ language sql stable security invoker parallel safe +set search_path to pg_catalog, {{schema}}, public, pg_temp ; ------------------------------------------------------------------------------- @@ -45,7 +46,8 @@ begin end if; return new; end; -$func$ language plpgsql volatile security definer +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp ; create or replace trigger role_membership_before_write_trg @@ -102,8 +104,7 @@ as $func$ from {{schema}}.calc_role_membership(_user_id) x where x.superuser ) -$func$ language sql stable security definer -set search_path to pg_catalog, {{schema}}, pg_temp +$func$ language sql stable security invoker ; ------------------------------------------------------------------------------- @@ -113,7 +114,7 @@ create or replace function {{schema}}.grant_role_membership ( _grantor_id uuid , _role_id uuid , _member_id uuid -, _with_admin_option bool default false +, _admin bool default false ) returns void as $func$ @@ -132,7 +133,7 @@ begin from {{schema}}.role_membership rm where rm.role_id = _role_id and rm.member_id = _grantor_id - and rm.with_admin_option + and rm.admin ) or {{schema}}.is_superuser(_grantor_id) -- or are they a superuser (even indirectly)? into strict _allowed @@ -147,18 +148,18 @@ begin insert into {{schema}}.role_membership ( role_id , member_id - , with_admin_option + , admin ) values ( _role_id , _member_id - , _with_admin_option + , _admin ) on conflict (member_id, role_id) - do update set with_admin_option = _with_admin_option + do update set admin = _admin ; end; -$func$ language plpgsql volatile security definer +$func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, pg_temp ; @@ -186,7 +187,7 @@ begin from {{schema}}.role_membership rm where rm.role_id = _role_id and rm.member_id = _revoker_id - and rm.with_admin_option + and rm.admin ) or {{schema}}.is_superuser(_revoker_id) -- or are they a superuser (even indirectly)? into strict _allowed @@ -202,6 +203,6 @@ begin and d.member_id = _member_id ; end; -$func$ language plpgsql volatile security definer +$func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, pg_temp ; diff --git a/packages/engine-core/migrate/idempotent/002_tree_privileges.sql b/packages/engine-core/migrate/idempotent/002_tree_access.sql similarity index 55% rename from packages/engine-core/migrate/idempotent/002_tree_privileges.sql rename to packages/engine-core/migrate/idempotent/002_tree_access.sql index 94049a4..6400ccd 100644 --- a/packages/engine-core/migrate/idempotent/002_tree_privileges.sql +++ b/packages/engine-core/migrate/idempotent/002_tree_access.sql @@ -1,12 +1,11 @@ ------------------------------------------------------------------------------- --- calc_tree_privileges +-- calc_tree_access ------------------------------------------------------------------------------- -create or replace function {{schema}}.calc_tree_privileges(_user_id uuid) +create or replace function {{schema}}.calc_tree_access(_user_id uuid) returns table ( role_id uuid , tree_path ltree -, actions text[] -, reason text +, access int2 ) as $func$ with r as @@ -21,47 +20,37 @@ as $func$ select r.role_id , ''::ltree as tree_path - , array['read', 'create', 'update', 'delete'] as actions - , 'superuser' as reason + , 3::int2 /* owner */ as access from r where r.superuser union all - -- ownership - select - r.role_id - , o.tree_path - , array['read', 'create', 'update', 'delete'] as actions - , 'owner' as reason - from r - inner join {{schema}}.tree_owner o on (r.role_id = o.user_id) - union all -- grants select r.role_id - , g.tree_path - , g.actions - , 'grant' as reason + , a.tree_path + , a.access::int2 from r - inner join {{schema}}.tree_grant g on (r.role_id = g.user_id) + inner join {{schema}}.tree_access a on (r.role_id = a.user_id) $func$ language sql stable security invoker ; ------------------------------------------------------------------------------- --- has_tree_privilege +-- has_tree_access ------------------------------------------------------------------------------- -create or replace function {{schema}}.has_tree_privilege +create or replace function {{schema}}.has_tree_access ( _user_id uuid , _tree_path ltree -, _actions text[] +, _access int4 ) returns bool as $func$ select exists ( select 1 - from {{schema}}.calc_tree_privileges(_user_id) x + from {{schema}}.calc_tree_access(_user_id) x where x.tree_path @> _tree_path - and x.actions @> _actions + and x.access >= _access + and _access in (1, 2, 3) ) $func$ language sql stable security invoker ; diff --git a/packages/engine-core/migrate/idempotent/003_memory.sql b/packages/engine-core/migrate/idempotent/003_memory.sql index 1be5798..6de72b3 100644 --- a/packages/engine-core/migrate/idempotent/003_memory.sql +++ b/packages/engine-core/migrate/idempotent/003_memory.sql @@ -18,7 +18,7 @@ begin return new; end; -$func$ language plpgsql volatile security definer +$func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp -- public required for pgvector's `is not distinct from` ; @@ -54,11 +54,10 @@ as $func$ , m.created_at , m.updated_at , m.embedding is not null - into _memory from {{schema}}.memory m where m.id = _id - and {{schema}}.has_tree_privilege(_user_id, m.tree, array['read']) -$func$ language sql stable security definer + and {{schema}}.has_tree_access(_user_id, m.tree, 1) +$func$ language sql stable security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; @@ -76,8 +75,8 @@ create or replace function {{schema}}.create_memory returns uuid as $func$ begin - if not {{schema}}.has_tree_privilege(_user_id, _tree, array['create']) then - raise exception 'user (%) must be a superuser or own or have create on the tree path %', _user_id, _tree + if not {{schema}}.has_tree_access(_user_id, _tree, 2) then + raise exception 'insufficient tree access' using errcode = 'insufficient_privilege'; end if; @@ -99,53 +98,100 @@ begin ; return _id; end; -$func$ language plpgsql volatile security definer +$func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; ------------------------------------------------------------------------------- --- update memory +-- patch memory ------------------------------------------------------------------------------- -create or replace function {{schema}}.update_memory +create or replace function {{schema}}.patch_memory ( _user_id uuid , _id uuid -, _tree ltree -, _content text -, _meta jsonb default '{}' -, _temporal tstzrange default null +, _patch jsonb ) returns bool as $func$ +declare + _src ltree; + _dst ltree; + _ok bool; begin - with p as materialized - ( - select p.tree_path, p.actions - from {{schema}}.calc_tree_privileges(_user_id) p - ) - update {{schema}}.memory m set - tree = _tree - , meta = meta || _meta - , temporal = _temporal - , content = _content + -- at least one valid field must be present + select count(*) filter (where k in ('meta', 'tree', 'temporal', 'content')) > 0 + into strict _ok + from jsonb_each(_patch) o(k, v) + ; + + if not _ok then + raise exception 'no valid patch fields found' + using errcode = 'invalid_parameter_value'; + end if; + + _dst = (_patch->>'tree')::ltree; + + -- cannot set tree to null + if _patch ? 'tree' and _dst is null then + raise exception 'tree cannot be set to null' + using errcode = 'invalid_parameter_value'; + end if; + + -- find the existing memory and get it's tree + select m.tree into _src + from {{schema}}.memory m where m.id = _id - and exists + for update -- don't let anyone "move" the memory while we're working on it + ; + + if not found then + return false; + end if; + + with a as materialized ( - select 1 - from p - where p.tree_path @> m.tree - and p.actions @> array['update'] + select a.tree_path, a.access + from {{schema}}.calc_tree_access(_user_id) a ) - and (m.tree @> _tree or exists - ( - select 1 - from p - where p.tree_path @> _tree - and p.actions @> array['insert'] - )) + select + exists + ( + select 1 + from a + where a.tree_path @> _src + and a.access >= 2 + ) + and + ( + _dst is null + or _src @> _dst + or exists + ( + select 1 + from a + where a.tree_path @> _dst + and a.access >= 2 + ) + ) + into strict _ok ; - return found; + + if not _ok then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + update {{schema}}.memory m set + tree = case when _patch ? 'tree' then (_patch->>'tree')::ltree else m.tree end + , meta = case when _patch ? 'meta' then _patch->'meta' else m.meta end + , temporal = case when _patch ? 'temporal' then (_patch->>'temporal')::tstzrange else m.temporal end + , content = case when _patch ? 'content' then _patch->>'content' else m.content end + where id = _id + returning id into _id + ; + + return _id is not null; end; -$func$ language plpgsql volatile security definer +$func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; @@ -161,41 +207,50 @@ create or replace function {{schema}}.move_tree returns bigint as $func$ declare + _has_src bool; + _has_dst bool; _moved bigint; begin - -- must have create on _dst tree path - if not {{schema}}.has_tree_privilege(_user_id, _dst, array['create']) then - raise exception 'user (%) must be a superuser or own or have create on the tree path %', _user_id, _dst - using errcode = 'insufficient_privilege'; - end if; - - with p as materialized + -- must have read/write on _src + -- must have read/write on _dst + with a as materialized ( - select p.tree_path - from {{schema}}.calc_tree_privileges(_user_id) p + select a.tree_path, a.access + from {{schema}}.calc_tree_access(_user_id) a ) - , x as - ( - select m.id - from {{schema}}.memory m - where _src @> m.tree - and exists + select + exists ( select 1 - from p - where p.tree_path @> m.tree - and p.actions @> array['read'] + from a + where a.tree_path @> _src + and a.access >= 2 ) - and + , exists ( - m.tree @> _dst - and exists - ( - select 1 - from p.tree_path @> m.tree - - ) + select 1 + from a + where a.tree_path @> _dst + and a.access >= 2 ) + into strict _has_src, _has_dst + ; + + if not _has_src then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + if not _has_dst then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + with x as + ( + select m.id + from {{schema}}.memory m + where _src @> m.tree ) , u as ( @@ -214,7 +269,7 @@ begin ; return _moved; end; -$func$ language plpgsql volatile security definer +$func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; @@ -230,31 +285,50 @@ create or replace function {{schema}}.copy_tree returns bigint as $func$ declare + _has_src bool; + _has_dst bool; _copied bigint; begin - -- must have create on _dst tree path - if not {{schema}}.has_tree_privilege(_user_id, _dst, array['create']) then - raise exception 'user (%) must be a superuser or own or have create on the tree path %', _user_id, _dst + -- must have read on _src + -- must have read/write on _dst + with a as materialized + ( + select a.tree_path, a.access + from {{schema}}.calc_tree_access(_user_id) a + ) + select + exists + ( + select 1 + from a + where a.tree_path @> _src + and a.access >= 1 + ) + , exists + ( + select 1 + from a + where a.tree_path @> _dst + and a.access >= 2 + ) + into strict _has_src, _has_dst + ; + + if not _has_src then + raise exception 'insufficient tree access' using errcode = 'insufficient_privilege'; end if; - with p as materialized - ( - select p.tree_path - from {{schema}}.calc_tree_privileges(_user_id) p - where p.actions @> array['read'] - ) - , m as + if not _has_dst then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + with m as ( select m.* from {{schema}}.memory m where _src @> m.tree - and exists - ( - select 1 - from p - where p.tree_path @> m.tree - ) ) , i as ( @@ -285,7 +359,7 @@ begin return _copied; end; -$func$ language plpgsql volatile security definer +$func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; @@ -304,14 +378,15 @@ begin select m.tree into _tree from {{schema}}.memory m where m.id = _id + for update ; if not found then return false; end if; - if not {{schema}}.has_tree_privilege(_user_id, _tree, array['delete']) then - raise exception 'user (%) must be a superuser or own or have delete on the tree path %', _user_id, _tree + if not {{schema}}.has_tree_access(_user_id, _tree, 2) then + raise exception 'insufficient tree access' using errcode = 'insufficient_privilege'; end if; @@ -320,7 +395,7 @@ begin ; return found; end; -$func$ language plpgsql volatile security definer +$func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; @@ -334,34 +409,75 @@ create or replace function {{schema}}.delete_tree ) returns bigint as $func$ - with p as materialized +declare + _has_access bool; + _deleted bigint; +begin + -- must have read/write on _tree + select exists ( - select p.tree_path - from {{schema}}.calc_tree_privileges(_user_id) p - where p.actions @> array['delete'] + select 1 + from {{schema}}.calc_tree_access(_user_id) a + where a.tree_path @> _tree + and a.access >= 2 ) - , m as - ( - select m.id + into strict _has_access + ; + + if not _has_access then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + if _dry_run then + select count(*) into strict _deleted from {{schema}}.memory m where _tree @> m.tree - and exists + ; + else + with d as ( - select 1 - from p - where p.tree @> m.tree + delete from {{schema}}.memory m + where _tree @> m.tree + returning id ) - ) - , d as + select count(*) into strict _deleted + from d + ; + end if; + + return _deleted; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- count tree +------------------------------------------------------------------------------- +create or replace function {{schema}}.count_tree +( _user_id uuid +, _tree ltree +, _access int4 +) +returns bigint +as $func$ + with x as materialized ( - delete from {{schema}}.memory m - using x - where m.id = x.id - and not _dry_run + select a.tree_path + from {{schema}}.calc_tree_access(_user_id) a + where a.access >= _access ) select count(*) - from x -$func$ language sql volatile security definer + from {{schema}}.memory m + where _tree @> m.tree + and exists + ( + select 1 + from x + where x.tree_path @> m.tree + ) +$func$ language sql stable security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; @@ -371,15 +487,15 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp create or replace function {{schema}}.count_tree ( _user_id uuid , _query lquery -, _actions text[] +, _access int4 ) returns bigint as $func$ with x as materialized ( - select p.tree_path - from {{schema}}.calc_tree_privileges(_user_id) p - where p.actions @> (coalesce(_actions, array['read'])) + select a.tree_path + from {{schema}}.calc_tree_access(_user_id) a + where a.access >= _access ) select count(*) from {{schema}}.memory m @@ -390,7 +506,7 @@ as $func$ from x where x.tree_path @> m.tree ) -$func$ language sql stable security definer +$func$ language sql stable security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; @@ -400,15 +516,15 @@ set search_path to pg_catalog, {{schema}}, public, pg_temp create or replace function {{schema}}.count_tree ( _user_id uuid , _query ltxtquery -, _actions text[] +, _access int4 ) returns bigint as $func$ with x as materialized ( - select p.tree_path - from {{schema}}.calc_tree_privileges(_user_id) p - where p.actions @> (coalesce(_actions, array['read'])) + select a.tree_path + from {{schema}}.calc_tree_access(_user_id) a + where a.access >= _access ) select count(*) from {{schema}}.memory m @@ -419,7 +535,7 @@ as $func$ from x where x.tree_path @> m.tree ) -$func$ language sql stable security definer +$func$ language sql stable security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; @@ -435,11 +551,11 @@ returns table , count bigint ) as $func$ - with p as + with a as materialized ( - select p.tree_path - from {{schema}}.calc_tree_privileges(_user_id) p - where p.actions @> array['read'] + select a.tree_path + from {{schema}}.calc_tree_access(_user_id) a + where a.access >= 1 ) , m as ( @@ -449,8 +565,8 @@ as $func$ and exists ( select 1 - from p - where p.tree_path @> m.tree + from a + where a.tree_path @> m.tree ) ) select @@ -460,6 +576,6 @@ as $func$ cross join lateral generate_series(1, nlevel(m.tree)) i group by 1 order by 1 -$func$ language sql stable security definer +$func$ language sql stable security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; diff --git a/packages/engine-core/migrate/idempotent/004_embedding_queue.sql b/packages/engine-core/migrate/idempotent/004_embedding_queue.sql index fa19348..52f9cb5 100644 --- a/packages/engine-core/migrate/idempotent/004_embedding_queue.sql +++ b/packages/engine-core/migrate/idempotent/004_embedding_queue.sql @@ -11,7 +11,7 @@ begin return new; end; $func$ -language plpgsql volatile security definer +language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, pg_temp ; @@ -131,7 +131,7 @@ begin end loop; end; $func$ -language plpgsql volatile security definer +language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, pg_temp ; @@ -158,6 +158,6 @@ begin return pruned; end; $func$ -language plpgsql volatile security definer +language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, pg_temp ; diff --git a/packages/engine-core/migrate/idempotent/005_tree_ownership.sql b/packages/engine-core/migrate/idempotent/005_tree_ownership.sql index 6a8d8b8..8c643de 100644 --- a/packages/engine-core/migrate/idempotent/005_tree_ownership.sql +++ b/packages/engine-core/migrate/idempotent/005_tree_ownership.sql @@ -55,7 +55,7 @@ begin set user_id = _owner_id ; end; -$func$ language plpgsql volatile security definer +$func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; @@ -92,6 +92,6 @@ begin and user_id = _owner_id ; end; -$func$ language plpgsql volatile security definer +$func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; diff --git a/packages/engine-core/migrate/idempotent/006_tree_grant.sql b/packages/engine-core/migrate/idempotent/006_tree_grant.sql index fc4b502..fe9c03a 100644 --- a/packages/engine-core/migrate/idempotent/006_tree_grant.sql +++ b/packages/engine-core/migrate/idempotent/006_tree_grant.sql @@ -40,7 +40,7 @@ begin ) ; end; -$func$ language plpgsql volatile security definer +$func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; @@ -102,6 +102,6 @@ begin ; end if; end; -$func$ language plpgsql volatile security definer +$func$ language plpgsql volatile security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; diff --git a/packages/engine-core/migrate/incremental/000_provision.sql b/packages/engine-core/migrate/incremental/000_provision.sql new file mode 100644 index 0000000..7a05de0 --- /dev/null +++ b/packages/engine-core/migrate/incremental/000_provision.sql @@ -0,0 +1,17 @@ +create schema {{schema}}; + +grant usage on schema {{schema}} to me_ro, me_rw, me_embed; + +create table {{schema}}.version +( version text not null +, at timestamptz not null default now() +); + +create unique index version_singleton_idx on {{schema}}.version ((true)); +insert into {{schema}}.version (version) values ('0.0.0'); + +create table {{schema}}.migration +( name text not null constraint migration_pkey primary key +, applied_at_version text not null +, applied_at timestamptz not null default pg_catalog.clock_timestamp() +); diff --git a/packages/engine-core/migrate/incremental/002_role_membership.sql b/packages/engine-core/migrate/incremental/002_role_membership.sql index 2257d6a..5968a99 100644 --- a/packages/engine-core/migrate/incremental/002_role_membership.sql +++ b/packages/engine-core/migrate/incremental/002_role_membership.sql @@ -4,7 +4,7 @@ create table {{schema}}.role_membership ( role_id uuid not null references {{schema}}."user"(id) on delete cascade , member_id uuid not null references {{schema}}."user"(id) on delete cascade -, with_admin_option boolean not null default false +, admin boolean not null default false , created_at timestamptz not null default now() , constraint pkey_role_membership primary key (member_id, role_id) , constraint no_self_membership check (role_id != member_id) diff --git a/packages/engine-core/migrate/incremental/004_tree_grant.sql b/packages/engine-core/migrate/incremental/003_tree_access.sql similarity index 63% rename from packages/engine-core/migrate/incremental/004_tree_grant.sql rename to packages/engine-core/migrate/incremental/003_tree_access.sql index cb6164b..86b268a 100644 --- a/packages/engine-core/migrate/incremental/004_tree_grant.sql +++ b/packages/engine-core/migrate/incremental/003_tree_access.sql @@ -1,12 +1,12 @@ ------------------------------------------------------------------------------- --- tree grant +-- tree access ------------------------------------------------------------------------------- -create table {{schema}}.tree_grant +create table {{schema}}.tree_access ( user_id uuid not null references {{schema}}."user"(id) on delete cascade , tree_path ltree not null -, actions text[] not null check (actions <@ '{read,create,update,delete}'::text[]) +, access int2 not null check (access in (1, 2, 3)) -- read, read/write, owner , created_at timestamptz not null default now() , constraint pkey_tree_grant primary key (user_id, tree_path) ); -create index idx_tree_grant_path on {{schema}}.tree_grant using gist (tree_path); +create index idx_tree_access_path on {{schema}}.tree_access using gist (tree_path); diff --git a/packages/engine-core/migrate/incremental/003_tree_ownership.sql b/packages/engine-core/migrate/incremental/003_tree_ownership.sql deleted file mode 100644 index 2eb4c17..0000000 --- a/packages/engine-core/migrate/incremental/003_tree_ownership.sql +++ /dev/null @@ -1,11 +0,0 @@ -------------------------------------------------------------------------------- --- tree ownership -------------------------------------------------------------------------------- -create table {{schema}}.tree_owner -( tree_path ltree not null primary key -, user_id uuid not null references {{schema}}."user" (id) on delete cascade -, created_at timestamptz not null default now() -); - -create index idx_tree_owner_user on {{schema}}.tree_owner (user_id); -create index idx_tree_owner_gist on {{schema}}.tree_owner using gist (tree_path); diff --git a/packages/engine-core/migrate/incremental/005_memory.sql b/packages/engine-core/migrate/incremental/004_memory.sql similarity index 100% rename from packages/engine-core/migrate/incremental/005_memory.sql rename to packages/engine-core/migrate/incremental/004_memory.sql diff --git a/packages/engine-core/migrate/migrate.ts b/packages/engine-core/migrate/migrate.ts index e548dbc..e4dc4f8 100644 --- a/packages/engine-core/migrate/migrate.ts +++ b/packages/engine-core/migrate/migrate.ts @@ -4,17 +4,17 @@ import type { SQL } from "bun"; import { semver } from "bun"; import { isValidSlug, slugToSchema } from "../slug"; -import incremental001 from "./incremental/001_user.sql" with { type: "text" }; -import incremental002 from "./incremental/002_role_membership.sql" with { +import provisionSql from "./incremental/000_provision.sql" with { type: "text", }; -import incremental003 from "./incremental/003_tree_ownership.sql" with { +import incremental001 from "./incremental/001_user.sql" with { type: "text" }; +import incremental002 from "./incremental/002_role_membership.sql" with { type: "text", }; -import incremental004 from "./incremental/004_tree_grant.sql" with { +import incremental003 from "./incremental/003_tree_access.sql" with { type: "text", }; -import incremental005 from "./incremental/005_memory.sql" with { type: "text" }; +import incremental004 from "./incremental/004_memory.sql" with { type: "text" }; import incremental006 from "./incremental/006_embedding_queue.sql" with { type: "text", }; @@ -27,16 +27,15 @@ interface Incremental { const incrementals: Incremental[] = [ { name: "001_user", sql: incremental001 }, { name: "002_role_membership", sql: incremental002 }, - { name: "003_tree_ownership", sql: incremental003 }, - { name: "004_tree_grant", sql: incremental004 }, - { name: "005_memory", sql: incremental005 }, + { name: "003_tree_access", sql: incremental003 }, + { name: "004_memory", sql: incremental004 }, { name: "006_embedding_queue", sql: incremental006 }, ]; import idempotent001 from "./idempotent/001_role_membership.sql" with { type: "text", }; -import idempotent002 from "./idempotent/002_tree_privileges.sql" with { +import idempotent002 from "./idempotent/002_tree_access.sql" with { type: "text", }; import idempotent003 from "./idempotent/003_memory.sql" with { type: "text" }; @@ -57,7 +56,7 @@ interface Idempotent { const idempotents: Idempotent[] = [ { name: "001_role_membership", sql: idempotent001 }, - { name: "002_tree_privileges", sql: idempotent002 }, + { name: "002_tree_access", sql: idempotent002 }, { name: "003_memory", sql: idempotent003 }, { name: "004_embedding_queue", sql: idempotent004 }, { name: "005_tree_ownership", sql: idempotent005 }, @@ -261,29 +260,7 @@ async function doesEngineExist(tx: SQL, schema: string): Promise { } async function provisionEngine(tx: SQL, schema: string): Promise { - await tx`create schema ${tx(schema)}`; - - // grant usage to all roles - await tx`grant usage on schema ${tx(schema)} to me_ro, me_rw, me_embed`; - - // version tracking table (single row) - await tx` - create table ${tx(schema)}.version - ( version text not null - , at timestamptz not null default now() - ) - `; - await tx`create unique index version_singleton_idx on ${tx(schema)}.version ((true))`; // only allow one row - await tx`insert into ${tx(schema)}.version (version) values ('0.0.0')`; - - // migration tracking table - await tx` - create table ${tx(schema)}.migration - ( name text not null constraint migration_pkey primary key - , applied_at_version text not null - , applied_at timestamptz not null default pg_catalog.clock_timestamp() - ) - `; + await tx.unsafe(template(provisionSql, { schema })); } async function runMigrations( From bf130948d0800787b40185aa08f14effcdf51de8 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Wed, 20 May 2026 17:23:00 -0500 Subject: [PATCH 14/17] foo --- .../migrate/idempotent/001_user.sql | 0 ...membership.sql => 002_role_membership.sql} | 34 ++++ .../migrate/idempotent/002_tree_access.sql | 56 ------ .../migrate/idempotent/003_tree_access.sql | 156 +++++++++++++++ .../{003_memory.sql => 004_memory.sql} | 0 .../migrate/idempotent/005_search.sql | 177 ++++++++++++++++++ .../migrate/idempotent/005_tree_ownership.sql | 97 ---------- ...ding_queue.sql => 006_embedding_queue.sql} | 0 .../migrate/idempotent/006_tree_grant.sql | 107 ----------- .../migrate/incremental/003_tree_access.sql | 2 +- packages/engine-core/migrate/migrate.ts | 28 ++- 11 files changed, 380 insertions(+), 277 deletions(-) create mode 100644 packages/engine-core/migrate/idempotent/001_user.sql rename packages/engine-core/migrate/idempotent/{001_role_membership.sql => 002_role_membership.sql} (88%) delete mode 100644 packages/engine-core/migrate/idempotent/002_tree_access.sql create mode 100644 packages/engine-core/migrate/idempotent/003_tree_access.sql rename packages/engine-core/migrate/idempotent/{003_memory.sql => 004_memory.sql} (100%) create mode 100644 packages/engine-core/migrate/idempotent/005_search.sql delete mode 100644 packages/engine-core/migrate/idempotent/005_tree_ownership.sql rename packages/engine-core/migrate/idempotent/{004_embedding_queue.sql => 006_embedding_queue.sql} (100%) delete mode 100644 packages/engine-core/migrate/idempotent/006_tree_grant.sql diff --git a/packages/engine-core/migrate/idempotent/001_user.sql b/packages/engine-core/migrate/idempotent/001_user.sql new file mode 100644 index 0000000..e69de29 diff --git a/packages/engine-core/migrate/idempotent/001_role_membership.sql b/packages/engine-core/migrate/idempotent/002_role_membership.sql similarity index 88% rename from packages/engine-core/migrate/idempotent/001_role_membership.sql rename to packages/engine-core/migrate/idempotent/002_role_membership.sql index aed8f11..e82b78f 100644 --- a/packages/engine-core/migrate/idempotent/001_role_membership.sql +++ b/packages/engine-core/migrate/idempotent/002_role_membership.sql @@ -107,6 +107,40 @@ as $func$ $func$ language sql stable security invoker ; +------------------------------------------------------------------------------- +-- list_direct_role_members +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_direct_role_members +( _requestor_id uuid +, _role_id uuid +) +returns table +( member_id uuid +, admin bool +) +as $func$ + select + r.member_id + , r.admin + from {{schema}}.role_membership r + where r.role_id = _role_id + and + ( + -- requestor must be an admin on role + -- or requestor must be a superuser + exists + ( + select 1 + from {{schema}}.role_membership m + where m.role_id = _role_id + and m.member_id = _requestor_id + and m.admin + ) + or {{schema}}.is_superuser(_requestor_id) + ) +$func$ language sql stable security invoker +; + ------------------------------------------------------------------------------- -- grant_role_membership ------------------------------------------------------------------------------- diff --git a/packages/engine-core/migrate/idempotent/002_tree_access.sql b/packages/engine-core/migrate/idempotent/002_tree_access.sql deleted file mode 100644 index 6400ccd..0000000 --- a/packages/engine-core/migrate/idempotent/002_tree_access.sql +++ /dev/null @@ -1,56 +0,0 @@ -------------------------------------------------------------------------------- --- calc_tree_access -------------------------------------------------------------------------------- -create or replace function {{schema}}.calc_tree_access(_user_id uuid) -returns table -( role_id uuid -, tree_path ltree -, access int2 -) -as $func$ - with r as - ( - -- the user and the roles they belong to - select - x.role_id - , x.superuser - from {{schema}}.calc_role_membership(_user_id) x - ) - -- superuser - select - r.role_id - , ''::ltree as tree_path - , 3::int2 /* owner */ as access - from r - where r.superuser - union all - -- grants - select - r.role_id - , a.tree_path - , a.access::int2 - from r - inner join {{schema}}.tree_access a on (r.role_id = a.user_id) -$func$ language sql stable security invoker -; - -------------------------------------------------------------------------------- --- has_tree_access -------------------------------------------------------------------------------- -create or replace function {{schema}}.has_tree_access -( _user_id uuid -, _tree_path ltree -, _access int4 -) -returns bool -as $func$ - select exists - ( - select 1 - from {{schema}}.calc_tree_access(_user_id) x - where x.tree_path @> _tree_path - and x.access >= _access - and _access in (1, 2, 3) - ) -$func$ language sql stable security invoker -; diff --git a/packages/engine-core/migrate/idempotent/003_tree_access.sql b/packages/engine-core/migrate/idempotent/003_tree_access.sql new file mode 100644 index 0000000..6180d82 --- /dev/null +++ b/packages/engine-core/migrate/idempotent/003_tree_access.sql @@ -0,0 +1,156 @@ +------------------------------------------------------------------------------- +-- calc_tree_access +------------------------------------------------------------------------------- +create or replace function {{schema}}.calc_tree_access(_user_id uuid) +returns table +( role_id uuid +, tree_path ltree +, access int2 +) +as $func$ + with r as + ( + -- the user and the roles they belong to + select + x.role_id + , x.superuser + from {{schema}}.calc_role_membership(_user_id) x + ) + -- superuser + select + r.role_id + , ''::ltree as tree_path + , 3::int2 /* owner */ as access + from r + where r.superuser + union all + -- grants + select + r.role_id + , a.tree_path + , a.access::int2 + from r + inner join {{schema}}.tree_access a on (r.role_id = a.user_id) +$func$ language sql stable security invoker +; + +------------------------------------------------------------------------------- +-- has_tree_access +------------------------------------------------------------------------------- +create or replace function {{schema}}.has_tree_access +( _user_id uuid +, _tree_path ltree +, _access int4 +) +returns bool +as $func$ + select exists + ( + select 1 + from {{schema}}.calc_tree_access(_user_id) x + where x.tree_path @> _tree_path + and x.access >= _access + and _access in (1, 2, 3) + ) +$func$ language sql stable security invoker +; + +------------------------------------------------------------------------------- +-- set_tree_access +------------------------------------------------------------------------------- +create or replace function {{schema}}.set_tree_access +( _grantor_id uuid +, _tree_path ltree +, _user_id uuid +, _access int4 +) +returns bool +as $func$ +begin + -- grantor must be superuser or owner of tree + if not {{schema}}.has_tree_access(_grantor_id, _tree_path, 3) then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + insert into {{schema}}.tree_access + ( user_id + , tree_path + , access + ) + values + ( _user_id + , _tree_path + , _access::int2 + ) + on conflict (user_id, tree_path) do update + set access = _access::int2 + ; + + return found; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- remove_all_tree_access +------------------------------------------------------------------------------- +create or replace function {{schema}}.remove_all_tree_access +( _grantor_id uuid +, _tree_path ltree +, _user_id uuid +) +returns bool +as $func$ +begin + -- grantor must be superuser or owner of tree + if not {{schema}}.has_tree_access(_grantor_id, _tree_path, 3) then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + delete from {{schema}}.tree_access + where user_id = _user_id + and _tree_path @> tree_path + ; + + return found; +end; +$func$ language plpgsql volatile security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; + +------------------------------------------------------------------------------- +-- list_tree_access +------------------------------------------------------------------------------- +create or replace function {{schema}}.list_tree_access +( _requestor_id uuid +, _tree_path ltree +) +returns table +( tree_path ltree +, user_id uuid +, access int2 +) +as $func$ +begin + -- grantor must be superuser or owner of tree + if not {{schema}}.has_tree_access(_requestor_id, _tree_path, 3) then + raise exception 'insufficient tree access' + using errcode = 'insufficient_privilege'; + end if; + + return query + select + a.tree_path + , a.user_id + , a.access + from {{schema}}.tree_access a + where _tree_path @> a.tree_path + order by a.tree_path, a.user_id + ; +end; +$func$ language plpgsql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/engine-core/migrate/idempotent/003_memory.sql b/packages/engine-core/migrate/idempotent/004_memory.sql similarity index 100% rename from packages/engine-core/migrate/idempotent/003_memory.sql rename to packages/engine-core/migrate/idempotent/004_memory.sql diff --git a/packages/engine-core/migrate/idempotent/005_search.sql b/packages/engine-core/migrate/idempotent/005_search.sql new file mode 100644 index 0000000..c322cbc --- /dev/null +++ b/packages/engine-core/migrate/idempotent/005_search.sql @@ -0,0 +1,177 @@ + +------------------------------------------------------------------------------- +-- search_memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.search_memory +( _user_id uuid +, _bm25 bm25query default null +, _ltree ltree default null +, _lquery lquery default null +, _ltxtquery ltxtquery default null +, _meta_contains jsonb default null +, _temporal_contains tstzrange default null +, _temporal_overlaps tstzrange default null +, _temporal_before timestamptz default null +, _temporal_after timestamptz default null +, _regexp text default null +, _limit bigint default 10 +) +returns table +( id uuid +, meta jsonb +, tree ltree +, temporal tstzrange +, content text +, has_embedding bool +, created_at timestamptz +, updated_at timestamptz +, score float8 +) +as $func$ +declare + _filter_count int = 0; + _score text; + _filters text[] = '{}'::text; + _order_by text; + _sql text; +begin + -- min 1, max 1000, default 10 + _limit = greatest(least(coalesce(_limit, 10), 1000), 1); + + -- score and order by + case when _bm25 is not null then + _filter_count = _filter_count + 1; + _score = format($sql$, (m.content <@> %L::bm25query) * -1 as score$sql$, _bm25); + _order_by = format($sql$order by m.content <@> %L::bm25query, m.id$sql$, _bm25); + else + _score = $sql$, -1 as score$sql$; + _order_by = $sql$order by m.id; + end case; + + -- ltree + if _ltree is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and %L::ltree @> m.tree$sql$, _ltree) + ); + end if; + + -- lquery + if _lquery is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and m.tree ~ %L::lquery$sql$, _lquery) + ); + end if; + + -- ltxtquery + if _ltxtquery is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and m.tree @ %L::ltxtquery$sql$, _ltxtquery) + ); + end if; + + -- meta_contains + if _meta_contains is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and m.meta @> %L::jsonb$sql$, _meta_contains) + ); + end if; + + -- temporal_contains + if _temporal_contains is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and %L::tstzrange @> m.temporal$sql$, _temporal_contains) + ); + end if; + + -- temporal_overlaps + if _temporal_overlaps is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and %L::tstzrange && m.temporal$sql$, _temporal_overlaps) + ); + end if; + + -- temporal_before + if _temporal_before is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and m.temporal << tstzrange(%L::timestamptz, %L::timestamptz, '[]')$sql$, _temporal_before, _temporal_before) + ); + end if; + + -- temporal_after + if _temporal_after is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and tstzrange(%L::timestamptz, %L::timestamptz, '[]') << m.temporal$sql$, _temporal_after, _temporal_after) + ); + end if; + + -- regexp + if _regexp is not null then + if _filter_count = 0 then + raise exception 'regexp must not be the only filter criteria' + using errcode = 'invalid_parameter_value'; + end if; + _filters = array_append + ( _filters + , format($sql$and m.content ~* %L::text$sql$, _regexp) + ); + end if; + + -- construct the query + _sql = format( + $sql$ + with x as + ( + select a.tree_path + from {{schema}}.calc_tree_access($1) a + where a.access >= 1 + ) + select + m.id + , m.meta + , m.tree + , m.temporal + , m.content + , m.embedding is not null + , m.created_at + , m.updated_at + %s + from {{schema}}.memory m + where exists + ( + select 1 + from x + where x.tree_path @> m.tree + ) + %s + %s + limit $2 + $sql$ + , _score + , ( + select string_agg(x, E'\n ') + from unnest(_filters) x + ) + , _order_by + ); + + return query execute _sql using _user_id, _limit; +end; +$func$ language plpgsql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/engine-core/migrate/idempotent/005_tree_ownership.sql b/packages/engine-core/migrate/idempotent/005_tree_ownership.sql deleted file mode 100644 index 8c643de..0000000 --- a/packages/engine-core/migrate/idempotent/005_tree_ownership.sql +++ /dev/null @@ -1,97 +0,0 @@ - -------------------------------------------------------------------------------- --- is_tree_owner -------------------------------------------------------------------------------- -create or replace function {{schema}}.is_tree_owner -( _user_id uuid -, _tree_path ltree -) -returns bool -as $func$ - with r as - ( - select * - from {{schema}}.calc_role_membership(_user_id) - ) - select - exists (select 1 from r where r.superuser) -- is user a superuser? - or exists - ( - -- does user own the path? - select 1 - from r - inner join {{schema}}.tree_owner o on (r.role_id = o.user_id) - where o.tree_path @> _tree_path - ) -$func$ language sql volatile security invoker parallel safe -; - -------------------------------------------------------------------------------- --- grant_tree_ownership -------------------------------------------------------------------------------- -create or replace function {{schema}}.grant_tree_ownership -( _grantor_id uuid -, _tree_path ltree -, _owner_id uuid -) -returns void -as $func$ -begin - -- is grantor allowed to do this? - if not {{schema}}.is_tree_owner(_grantor_id, _tree_path) then - raise exception 'grantor (%) must be a superuser or own the tree path %', _grantor_id, _tree_path - using errcode = 'insufficient_privilege'; - end if; - - insert into {{schema}}.tree_owner - ( tree_path - , user_id - ) - values - ( _tree_path - , _owner_id - ) - on conflict (tree_path) do update - set user_id = _owner_id - ; -end; -$func$ language plpgsql volatile security invoker -set search_path to pg_catalog, {{schema}}, public, pg_temp -; - -------------------------------------------------------------------------------- --- revoke_tree_ownership -------------------------------------------------------------------------------- -create or replace function {{schema}}.revoke_tree_ownership -( _revoker_id uuid -, _tree_path ltree -, _owner_id uuid -) -returns void -as $func$ -begin - -- checking permissions is expensive (relatively) - -- ensure this operation even makes sense first - perform 1 - from {{schema}}.tree_owner o - where o.tree_path = _tree_path - and o.user_id = _owner_id - ; - if not found then - return; - end if; - - -- is revoker allowed to do this? - if not {{schema}}.is_tree_owner(_revoker_id, _tree_path) then - raise exception 'revoker (%) must be a superuser or own the tree path %', _revoker_id, _tree_path - using errcode = 'insufficient_privilege'; - end if; - - delete from {{schema}}.tree_owner - where tree_path = _tree_path - and user_id = _owner_id - ; -end; -$func$ language plpgsql volatile security invoker -set search_path to pg_catalog, {{schema}}, public, pg_temp -; diff --git a/packages/engine-core/migrate/idempotent/004_embedding_queue.sql b/packages/engine-core/migrate/idempotent/006_embedding_queue.sql similarity index 100% rename from packages/engine-core/migrate/idempotent/004_embedding_queue.sql rename to packages/engine-core/migrate/idempotent/006_embedding_queue.sql diff --git a/packages/engine-core/migrate/idempotent/006_tree_grant.sql b/packages/engine-core/migrate/idempotent/006_tree_grant.sql deleted file mode 100644 index fe9c03a..0000000 --- a/packages/engine-core/migrate/idempotent/006_tree_grant.sql +++ /dev/null @@ -1,107 +0,0 @@ - -------------------------------------------------------------------------------- --- grant_tree_actions -------------------------------------------------------------------------------- -create or replace function {{schema}}.grant_tree_actions -( _grantor_id uuid -, _actions text[] -, _tree_path ltree -, _user_id uuid -) -returns void -as $func$ -begin - -- is grantor allowed to do this? - if not {{schema}}.is_tree_owner(_grantor_id, _tree_path) then - raise exception 'grantor (%) must be a superuser or own the tree path %', _grantor_id, _tree_path - using errcode = 'insufficient_privilege'; - end if; - - insert into {{schema}}.tree_grant as g - ( user_id - , tree_path - , actions - ) - values - ( _user_id - , _tree_path - , coalesce(array(select distinct a.action from unnest(_actions) a(action) order by a.action), '{}') - ) - on conflict (user_id, tree_path) do update - set actions = coalesce - ( - array - ( - select distinct a.action - from unnest(g.actions || excluded.actions) a(action) - order by a.action - ) - , '{}' - ) - ; -end; -$func$ language plpgsql volatile security invoker -set search_path to pg_catalog, {{schema}}, public, pg_temp -; - -------------------------------------------------------------------------------- --- revoke_tree_actions -------------------------------------------------------------------------------- -create or replace function {{schema}}.revoke_tree_actions -( _revoker_id uuid -, _actions text[] -, _tree_path ltree -, _user_id uuid -) -returns void -as $func$ -declare - _existing_actions text[]; - _remaining_actions text[]; - _remaining_action_count int8; -begin - -- checking permissions is expensive (relatively) - -- ensure this operation even makes sense first - select g.actions into _existing_actions - from {{schema}}.tree_grant g - where g.tree_path = _tree_path - and g.user_id = _user_id - and g.actions && _actions - ; - if not found then - return; - end if; - - -- is revoker allowed to do this? - if not {{schema}}.is_tree_owner(_revoker_id, _tree_path) then - raise exception 'revoker (%) must be a superuser or own the tree path %', _revoker_id, _tree_path - using errcode = 'insufficient_privilege'; - end if; - - -- calc remaining actions - select coalesce(array_agg(x.action order by x.action), '{}'), count(*) - into strict _remaining_actions, _remaining_action_count - from - ( - select unnest(_existing_actions) as action - except - select unnest(_actions) as action - ) x - ; - - if _remaining_action_count = 0 then - delete from {{schema}}.tree_grant g - where g.user_id = _user_id - and g.tree_path = _tree_path - ; - else - update {{schema}}.tree_grant g - set actions = _remaining_actions - where g.user_id = _user_id - and g.tree_path = _tree_path - ; - end if; -end; -$func$ language plpgsql volatile security invoker -set search_path to pg_catalog, {{schema}}, public, pg_temp -; diff --git a/packages/engine-core/migrate/incremental/003_tree_access.sql b/packages/engine-core/migrate/incremental/003_tree_access.sql index 86b268a..345c1d3 100644 --- a/packages/engine-core/migrate/incremental/003_tree_access.sql +++ b/packages/engine-core/migrate/incremental/003_tree_access.sql @@ -6,7 +6,7 @@ create table {{schema}}.tree_access , tree_path ltree not null , access int2 not null check (access in (1, 2, 3)) -- read, read/write, owner , created_at timestamptz not null default now() -, constraint pkey_tree_grant primary key (user_id, tree_path) +, constraint pkey_tree_access primary key (user_id, tree_path) ); create index idx_tree_access_path on {{schema}}.tree_access using gist (tree_path); diff --git a/packages/engine-core/migrate/migrate.ts b/packages/engine-core/migrate/migrate.ts index e4dc4f8..15443f5 100644 --- a/packages/engine-core/migrate/migrate.ts +++ b/packages/engine-core/migrate/migrate.ts @@ -32,20 +32,16 @@ const incrementals: Incremental[] = [ { name: "006_embedding_queue", sql: incremental006 }, ]; -import idempotent001 from "./idempotent/001_role_membership.sql" with { +import idempotent001 from "./idempotent/001_user.sql" with { type: "text" }; +import idempotent002 from "./idempotent/002_role_membership.sql" with { type: "text", }; -import idempotent002 from "./idempotent/002_tree_access.sql" with { +import idempotent003 from "./idempotent/003_tree_access.sql" with { type: "text", }; -import idempotent003 from "./idempotent/003_memory.sql" with { type: "text" }; -import idempotent004 from "./idempotent/004_embedding_queue.sql" with { - type: "text", -}; -import idempotent005 from "./idempotent/005_tree_ownership.sql" with { - type: "text", -}; -import idempotent006 from "./idempotent/006_tree_grant.sql" with { +import idempotent004 from "./idempotent/004_memory.sql" with { type: "text" }; +import idempotent005 from "./idempotent/005_search.sql" with { type: "text" }; +import idempotent006 from "./idempotent/006_embedding_queue.sql" with { type: "text", }; @@ -55,12 +51,12 @@ interface Idempotent { } const idempotents: Idempotent[] = [ - { name: "001_role_membership", sql: idempotent001 }, - { name: "002_tree_access", sql: idempotent002 }, - { name: "003_memory", sql: idempotent003 }, - { name: "004_embedding_queue", sql: idempotent004 }, - { name: "005_tree_ownership", sql: idempotent005 }, - { name: "006_tree_grant", sql: idempotent006 }, + { name: "001_user", sql: idempotent001 }, + { name: "002_role_membership", sql: idempotent002 }, + { name: "003_tree_access", sql: idempotent003 }, + { name: "004_memory", sql: idempotent004 }, + { name: "005_search", sql: idempotent005 }, + { name: "006_embedding_queue", sql: idempotent006 }, ]; export interface MigrateEngineOptions { From 75ef00811dce49d11d2619cab1d6f74d938caa06 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Wed, 20 May 2026 17:48:10 -0500 Subject: [PATCH 15/17] Foo --- .../migrate/idempotent/005_search.sql | 48 +++++++++++++++---- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/packages/engine-core/migrate/idempotent/005_search.sql b/packages/engine-core/migrate/idempotent/005_search.sql index c322cbc..613ca28 100644 --- a/packages/engine-core/migrate/idempotent/005_search.sql +++ b/packages/engine-core/migrate/idempotent/005_search.sql @@ -5,11 +5,13 @@ create or replace function {{schema}}.search_memory ( _user_id uuid , _bm25 bm25query default null +, _vec halfvec({{embedding_dimensions}}) default null +, _max_vec_dist float8 default null , _ltree ltree default null , _lquery lquery default null , _ltxtquery ltxtquery default null , _meta_contains jsonb default null -, _temporal_contains tstzrange default null +, _temporal_within tstzrange default null , _temporal_overlaps tstzrange default null , _temporal_before timestamptz default null , _temporal_after timestamptz default null @@ -35,14 +37,42 @@ declare _order_by text; _sql text; begin + -- _bm25 OR _vec but NOT BOTH + if _bm25 is not null and _vec is not null then + raise exception 'providing both _bm25 and _vec is not supported' + using errcode = 'invalid_parameter_value'; + end if; + + if _max_vec_dist is not null and _vec is null then + raise exception '_max_vec_dist provided but _vec was not provided' + using errcode = 'invalid_parameter_value'; + end if; + -- min 1, max 1000, default 10 _limit = greatest(least(coalesce(_limit, 10), 1000), 1); + -- bm25 or semantic -- score and order by - case when _bm25 is not null then + case + when _bm25 is not null then _filter_count = _filter_count + 1; _score = format($sql$, (m.content <@> %L::bm25query) * -1 as score$sql$, _bm25); _order_by = format($sql$order by m.content <@> %L::bm25query, m.id$sql$, _bm25); + when _vec is not null then + _filter_count = _filter_count + 1; + _score = format($sql$, (m.embedding <=> %L::halfvec({{embedding_dimensions}})) * -1 as score$sql$, _vec); + _order_by = format($sql$order by m.embedding <=> %L::halfvec({{embedding_dimensions}}), m.id$sql$, _vec); + _filters = array_append + ( _filters + , $sql$and m.embedding is not null$sql$ + ); + if _max_vec_dist is not null then + _filter_count = _filter_count + 1; + _filters = array_append + ( _filters + , format($sql$and (m.embedding <=> %L::halfvec({{embedding_dimensions}})) <= %L::float8$sql$, _vec, _max_vec_dist) + ); + end if; else _score = $sql$, -1 as score$sql$; _order_by = $sql$order by m.id; @@ -84,12 +114,12 @@ begin ); end if; - -- temporal_contains - if _temporal_contains is not null then + -- temporal_within + if _temporal_within is not null then _filter_count = _filter_count + 1; _filters = array_append ( _filters - , format($sql$and %L::tstzrange @> m.temporal$sql$, _temporal_contains) + , format($sql$and %L::tstzrange @> m.temporal$sql$, _temporal_within) ); end if; @@ -107,7 +137,7 @@ begin _filter_count = _filter_count + 1; _filters = array_append ( _filters - , format($sql$and m.temporal << tstzrange(%L::timestamptz, %L::timestamptz, '[]')$sql$, _temporal_before, _temporal_before) + , format($sql$and tstzrange('-infinity'::timestamptz, %L::timestamptz, '[]') @> m.temporal$sql$, _temporal_before) ); end if; @@ -116,7 +146,7 @@ begin _filter_count = _filter_count + 1; _filters = array_append ( _filters - , format($sql$and tstzrange(%L::timestamptz, %L::timestamptz, '[]') << m.temporal$sql$, _temporal_after, _temporal_after) + , format($sql$and tstzrange(%L::timestamptz, 'infinity'::timestamptz, '[]') @> m.temporal$sql$, _temporal_after) ); end if; @@ -132,10 +162,12 @@ begin ); end if; + assert array_length(_filters, 1) > 0; + -- construct the query _sql = format( $sql$ - with x as + with x as materialized ( select a.tree_path from {{schema}}.calc_tree_access($1) a From 628e337c32458ad7cf1a9087bbf5e9194ee786a8 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Wed, 20 May 2026 20:45:41 -0500 Subject: [PATCH 16/17] foo --- .../migrate/idempotent/005_search.sql | 130 +++++++++++++++++- packages/engine-core/migrate/migrate.ts | 2 + 2 files changed, 127 insertions(+), 5 deletions(-) diff --git a/packages/engine-core/migrate/idempotent/005_search.sql b/packages/engine-core/migrate/idempotent/005_search.sql index 613ca28..2790f70 100644 --- a/packages/engine-core/migrate/idempotent/005_search.sql +++ b/packages/engine-core/migrate/idempotent/005_search.sql @@ -1,4 +1,3 @@ - ------------------------------------------------------------------------------- -- search_memory ------------------------------------------------------------------------------- @@ -56,10 +55,14 @@ begin case when _bm25 is not null then _filter_count = _filter_count + 1; + -- <@> is negative bm25 score. smaller values means better match. order by this for index scans + -- negative score * -1 = score. higher score means better match _score = format($sql$, (m.content <@> %L::bm25query) * -1 as score$sql$, _bm25); _order_by = format($sql$order by m.content <@> %L::bm25query, m.id$sql$, _bm25); when _vec is not null then _filter_count = _filter_count + 1; + -- <=> is cosine distance. smaller distance means better match. order by this for index scans + -- distance * -1 = "score". higher score means better match _score = format($sql$, (m.embedding <=> %L::halfvec({{embedding_dimensions}})) * -1 as score$sql$, _vec); _order_by = format($sql$order by m.embedding <=> %L::halfvec({{embedding_dimensions}}), m.id$sql$, _vec); _filters = array_append @@ -162,8 +165,6 @@ begin ); end if; - assert array_length(_filters, 1) > 0; - -- construct the query _sql = format( $sql$ @@ -195,10 +196,11 @@ begin limit $2 $sql$ , _score - , ( + , coalesce + (( select string_agg(x, E'\n ') from unnest(_filters) x - ) + ), '') , _order_by ); @@ -207,3 +209,121 @@ end; $func$ language plpgsql stable security invoker set search_path to pg_catalog, {{schema}}, public, pg_temp ; + +------------------------------------------------------------------------------- +-- hybrid_search_memory +------------------------------------------------------------------------------- +create or replace function {{schema}}.hybrid_search_memory +( _user_id uuid +, _bm25 bm25query +, _vec halfvec({{embedding_dimensions}}) +, _max_vec_dist float8 default null +, _ltree ltree default null +, _lquery lquery default null +, _ltxtquery ltxtquery default null +, _meta_contains jsonb default null +, _temporal_within tstzrange default null +, _temporal_overlaps tstzrange default null +, _temporal_before timestamptz default null +, _temporal_after timestamptz default null +, _regexp text default null +, _k float8 default 60.0 +, _candidate_limit bigint default 30 +, _fulltext_weight float8 default 1.0 +, _semantic_weight float8 default 1.0 +, _limit bigint default 10 +) +returns table +( id uuid +, meta jsonb +, tree ltree +, temporal tstzrange +, content text +, has_embedding bool +, created_at timestamptz +, updated_at timestamptz +, score float8 +) +as $func$ +declare +begin + if _bm25 is null then + raise exception '_bm25 must not be null' + using errcode = 'invalid_parameter_value'; + end if; + + if _vec is null then + raise exception '_vec must not be null' + using errcode = 'invalid_parameter_value'; + end if; + + _k = greatest(coalesce(_k, 60.0), 0.0); + _limit = greatest(least(coalesce(_limit, 10), 1000), 1); + _candidate_limit = greatest + ( least(coalesce(_candidate_limit, 30), 1000) + , _limit + ); + _fulltext_weight = greatest(least(coalesce(_fulltext_weight, 1.0), 1.0), 0.0); + _semantic_weight = greatest(least(coalesce(_semantic_weight, 1.0), 1.0), 0.0); + + -- reciprocal rank fusion + return query + select + coalesce(x1.id, x2.id) as id + , coalesce(x1.meta, x2.meta) as meta + , coalesce(x1.tree, x2.tree) as tree + , coalesce(x1.temporal, x2.temporal) as temporal + , coalesce(x1.content, x2.content) as content + , coalesce(x1.has_embedding, x2.has_embedding) as has_embedding + , coalesce(x1.created_at, x2.created_at) as created_at + , coalesce(x1.updated_at, x2.updated_at) as updated_at + , coalesce(_fulltext_weight / (_k + x1.rank), 0.0) + + coalesce(_semantic_weight / (_k + x2.rank), 0.0) as score + from + ( + select + row_number() over (order by m.score desc, m.id) as rank + , m.* + from {{schema}}.search_memory + ( _user_id => _user_id + , _bm25 => _bm25 + , _ltree => _ltree + , _lquery => _lquery + , _ltxtquery => _ltxtquery + , _meta_contains => _meta_contains + , _temporal_within => _temporal_within + , _temporal_overlaps => _temporal_overlaps + , _temporal_before => _temporal_before + , _temporal_after => _temporal_after + , _regexp => _regexp + , _limit => _candidate_limit + ) m + ) x1 + full outer join + ( + select + row_number() over (order by m.score desc, m.id) as rank + , m.* + from {{schema}}.search_memory + ( _user_id => _user_id + , _vec => _vec + , _max_vec_dist => _max_vec_dist + , _ltree => _ltree + , _lquery => _lquery + , _ltxtquery => _ltxtquery + , _meta_contains => _meta_contains + , _temporal_within => _temporal_within + , _temporal_overlaps => _temporal_overlaps + , _temporal_before => _temporal_before + , _temporal_after => _temporal_after + , _regexp => _regexp + , _limit => _candidate_limit + ) m + ) x2 on (x1.id = x2.id) + order by score desc, id + limit _limit + ; +end; +$func$ language plpgsql stable security invoker +set search_path to pg_catalog, {{schema}}, public, pg_temp +; diff --git a/packages/engine-core/migrate/migrate.ts b/packages/engine-core/migrate/migrate.ts index 15443f5..3e362ed 100644 --- a/packages/engine-core/migrate/migrate.ts +++ b/packages/engine-core/migrate/migrate.ts @@ -15,6 +15,7 @@ import incremental003 from "./incremental/003_tree_access.sql" with { type: "text", }; import incremental004 from "./incremental/004_memory.sql" with { type: "text" }; +import incremental005 from "./incremental/005_search.sql" with { type: "text" }; import incremental006 from "./incremental/006_embedding_queue.sql" with { type: "text", }; @@ -29,6 +30,7 @@ const incrementals: Incremental[] = [ { name: "002_role_membership", sql: incremental002 }, { name: "003_tree_access", sql: incremental003 }, { name: "004_memory", sql: incremental004 }, + { name: "005_search", sql: incremental005 }, { name: "006_embedding_queue", sql: incremental006 }, ]; From 0bd608ae83e7c900f30eab8e594faea2c2f2e5d6 Mon Sep 17 00:00:00 2001 From: John Pruitt Date: Thu, 21 May 2026 12:39:52 -0500 Subject: [PATCH 17/17] foo --- packages/engine-core/migrate/incremental/000_provision.sql | 4 +--- packages/engine-core/migrate/incremental/001_user.sql | 5 ++--- .../{006_embedding_queue.sql => 005_embedding_queue.sql} | 0 packages/engine-core/migrate/migrate.ts | 6 ++---- 4 files changed, 5 insertions(+), 10 deletions(-) rename packages/engine-core/migrate/incremental/{006_embedding_queue.sql => 005_embedding_queue.sql} (100%) diff --git a/packages/engine-core/migrate/incremental/000_provision.sql b/packages/engine-core/migrate/incremental/000_provision.sql index 7a05de0..e98b9d9 100644 --- a/packages/engine-core/migrate/incremental/000_provision.sql +++ b/packages/engine-core/migrate/incremental/000_provision.sql @@ -1,13 +1,11 @@ create schema {{schema}}; -grant usage on schema {{schema}} to me_ro, me_rw, me_embed; - create table {{schema}}.version ( version text not null , at timestamptz not null default now() ); -create unique index version_singleton_idx on {{schema}}.version ((true)); +create unique index version_singleton_idx on {{schema}}.version ((true)); -- only ONE row allowed insert into {{schema}}.version (version) values ('0.0.0'); create table {{schema}}.migration diff --git a/packages/engine-core/migrate/incremental/001_user.sql b/packages/engine-core/migrate/incremental/001_user.sql index ecdba44..f93567d 100644 --- a/packages/engine-core/migrate/incremental/001_user.sql +++ b/packages/engine-core/migrate/incremental/001_user.sql @@ -2,12 +2,11 @@ ------------------------------------------------------------------------------- -- users ------------------------------------------------------------------------------- --- User: thing that accesses memories, or a role (can_login = false) --- identity_id is a soft FK to accounts.identity (nullable for service users) --- Note: "user" is a reserved word, must be quoted +-- note: "user" is a reserved word, must be quoted create table {{schema}}."user" ( id uuid primary key default uuidv7() check (uuid_extract_version(id) = 7) , name citext not null unique , superuser boolean not null default false +--, type text not null check (type in ('user', 'role', 'agent')) , created_at timestamptz not null default now() ); diff --git a/packages/engine-core/migrate/incremental/006_embedding_queue.sql b/packages/engine-core/migrate/incremental/005_embedding_queue.sql similarity index 100% rename from packages/engine-core/migrate/incremental/006_embedding_queue.sql rename to packages/engine-core/migrate/incremental/005_embedding_queue.sql diff --git a/packages/engine-core/migrate/migrate.ts b/packages/engine-core/migrate/migrate.ts index 3e362ed..83911df 100644 --- a/packages/engine-core/migrate/migrate.ts +++ b/packages/engine-core/migrate/migrate.ts @@ -15,8 +15,7 @@ import incremental003 from "./incremental/003_tree_access.sql" with { type: "text", }; import incremental004 from "./incremental/004_memory.sql" with { type: "text" }; -import incremental005 from "./incremental/005_search.sql" with { type: "text" }; -import incremental006 from "./incremental/006_embedding_queue.sql" with { +import incremental005 from "./incremental/005_embedding_queue.sql" with { type: "text", }; @@ -30,8 +29,7 @@ const incrementals: Incremental[] = [ { name: "002_role_membership", sql: incremental002 }, { name: "003_tree_access", sql: incremental003 }, { name: "004_memory", sql: incremental004 }, - { name: "005_search", sql: incremental005 }, - { name: "006_embedding_queue", sql: incremental006 }, + { name: "005_embedding_queue", sql: incremental005 }, ]; import idempotent001 from "./idempotent/001_user.sql" with { type: "text" };