diff --git a/docs/collections.md b/docs/collections.md index 89ba4f4..da40f22 100644 --- a/docs/collections.md +++ b/docs/collections.md @@ -91,8 +91,9 @@ Explicitly creates a new empty collection. Collections are also created automati CREATE COLLECTION CREATE COLLECTION HYBRID CREATE COLLECTION USING MODEL '' +CREATE COLLECTION USING VECTOR '' CREATE COLLECTION USING HYBRID -CREATE COLLECTION USING HYBRID DENSE MODEL '' +CREATE COLLECTION USING HYBRID [DENSE MODEL ''] [DENSE VECTOR ''] [SPARSE VECTOR ''] CREATE COLLECTION WITH VECTORS { on_disk: } CREATE COLLECTION WITH HNSW { m, ef_construct, full_scan_threshold, max_indexing_threads, on_disk, payload_m, inline_storage } CREATE COLLECTION WITH OPTIMIZERS { deleted_threshold, vacuum_min_vector_number, default_segment_number, max_segment_size, memmap_threshold, indexing_threshold, flush_interval_sec, max_optimization_threads, prevent_unoptimized } @@ -117,6 +118,11 @@ Dense-only collection (standard, uses default model dimensions): CREATE COLLECTION research_papers ``` +QQL-created dense collections use the configured dense vector name (`dense` by default). You can choose a different name explicitly: +```sql +CREATE COLLECTION research_papers USING VECTOR 'body' +``` + Dense-only collection pinned to a specific model (768-dimensional): ```sql CREATE COLLECTION research_papers USING MODEL 'BAAI/bge-base-en-v1.5' @@ -127,6 +133,11 @@ Hybrid collection (dense + sparse BM25, default models): CREATE COLLECTION research_papers HYBRID ``` +Hybrid collection with explicit vector names: +```sql +CREATE COLLECTION research_papers USING HYBRID DENSE VECTOR 'emb' SPARSE VECTOR 'lex' +``` + Hybrid collection with a custom dense model: ```sql CREATE COLLECTION research_papers USING HYBRID DENSE MODEL 'BAAI/bge-base-en-v1.5' @@ -150,6 +161,8 @@ QQL supports the same config blocks on both `CREATE COLLECTION` and `ALTER COLLE - `WITH PARAMS { replication_factor, write_consistency_factor, read_fan_out_factor, read_fan_out_delay_ms, on_disk_payload }` on alter - `ALTER COLLECTION ... QUANTIZE ...` supports the same quantization forms as create, plus `QUANTIZE DISABLED` +`ALTER COLLECTION ... WITH VECTORS { ... }` can update unnamed collections or named collections with one dense vector. Collections with multiple dense vectors are rejected because this syntax has no vector-name target. + Example: ```sql @@ -380,6 +393,7 @@ Replaces the stored dense vector for a **single point** identified by its ID. Th ``` UPDATE SET VECTOR WHERE id = '' [] UPDATE SET VECTOR WHERE id = [] +UPDATE SET VECTOR '' WHERE id = '' [] ``` The vector is provided as a JSON-style float array `[v1, v2, ..., vN]`. The array length must match the collection's configured vector dimensions. @@ -392,13 +406,17 @@ UPDATE articles SET VECTOR WHERE id = '3f2e1a4b-8c91-4d0e-b123-abc123def456' [0. -- Replace vector by integer ID UPDATE articles SET VECTOR WHERE id = 42 [0.1, 0.2, 0.3, 0.4] + +-- Replace a specific named vector +UPDATE articles SET VECTOR 'body' WHERE id = '3f2e1a4b-8c91-4d0e-b123-abc123def456' [0.1, 0.2, 0.3, 0.4] ``` **Notes:** - Only single-point updates are supported (by ID). Bulk or filter-based vector updates are not supported. - The point must already exist; this operation does not create new points. - The collection must exist; updating from a non-existent collection raises an error. -- For hybrid collections, the dense vector named `"dense"` is updated. Sparse vectors are managed separately. +- For named-vector collections, QQL updates the only dense vector when the target is unambiguous. Use `SET VECTOR ''` when a collection has multiple dense vectors. +- Sparse vectors are managed separately. --- diff --git a/docs/insert.md b/docs/insert.md index 4f33e07..a3f092a 100644 --- a/docs/insert.md +++ b/docs/insert.md @@ -16,8 +16,9 @@ If you include an `id` field in `VALUES`, QQL uses it as the Qdrant point ID. Su ``` INSERT INTO COLLECTION VALUES {} INSERT INTO COLLECTION VALUES {} USING MODEL '' +INSERT INTO COLLECTION VALUES {} USING VECTOR '' INSERT INTO COLLECTION VALUES {} USING HYBRID -INSERT INTO COLLECTION VALUES {} USING HYBRID DENSE MODEL '' SPARSE MODEL '' +INSERT INTO COLLECTION VALUES {} USING HYBRID [DENSE MODEL ''] [DENSE VECTOR ''] [SPARSE MODEL ''] [SPARSE VECTOR ''] ``` **Examples:** @@ -49,6 +50,17 @@ Insert into a hybrid collection (dense + sparse BM25 vectors): INSERT INTO COLLECTION articles VALUES {'text': 'Attention is all you need'} USING HYBRID ``` +Insert into a specific named dense vector: +```sql +INSERT INTO COLLECTION articles VALUES {'text': 'hello world'} USING VECTOR 'body' +``` + +Insert into a hybrid collection with external vector names: +```sql +INSERT INTO COLLECTION articles VALUES {'text': 'hello world'} + USING HYBRID DENSE VECTOR 'emb' SPARSE VECTOR 'lex' +``` + Insert with custom models for both dense and sparse: ```sql INSERT INTO COLLECTION articles VALUES {'text': 'hello world'} @@ -67,6 +79,7 @@ INSERT INTO COLLECTION articles VALUES {'text': 'hello world'} - `id`, when provided, must be an unsigned integer or UUID string. - If the collection already exists with a different vector size (from a different model), an error is raised with a clear message. - Hybrid inserts require a hybrid collection (created with `CREATE COLLECTION ... HYBRID`, auto-created on the first `USING HYBRID` insert, or **auto-detected** — if you omit `USING HYBRID` but the target collection is already a hybrid collection, QQL detects this and uses the hybrid insert path automatically). +- If a collection has multiple dense or sparse vectors, specify the target vector names explicitly. --- @@ -82,8 +95,9 @@ Each record may optionally include an `id` field. This is the preferred way to k ``` INSERT BULK INTO COLLECTION VALUES [, , ...] INSERT BULK INTO COLLECTION VALUES [, ...] USING MODEL '' +INSERT BULK INTO COLLECTION VALUES [, ...] USING VECTOR '' INSERT BULK INTO COLLECTION VALUES [, ...] USING HYBRID -INSERT BULK INTO COLLECTION VALUES [, ...] USING HYBRID DENSE MODEL '' SPARSE MODEL '' +INSERT BULK INTO COLLECTION VALUES [, ...] USING HYBRID [DENSE MODEL ''] [DENSE VECTOR ''] [SPARSE MODEL ''] [SPARSE VECTOR ''] ``` **Examples:** diff --git a/docs/programmatic.md b/docs/programmatic.md index 1d0e6b6..6e86021 100644 --- a/docs/programmatic.md +++ b/docs/programmatic.md @@ -139,7 +139,7 @@ with Connection("http://localhost:6333") as conn: # Inspect collection diagnostics result = conn.run_query("SHOW COLLECTION notes") print(result.data["topology"]) # "dense" or "hybrid" - print(result.data["vectors"]) # {"": {...}} or {"dense": {...}} + print(result.data["vectors"]) # named vectors, or {"": {...}} for unnamed external collections print(result.data["payload_schema"]) # field index info, or None ``` @@ -150,6 +150,8 @@ with Connection("http://localhost:6333") as conn: | `url` | `str` | `"http://localhost:6333"` | Qdrant instance URL | | `secret` | `str \| None` | `None` | API key; `None` for unauthenticated | | `default_model` | `str \| None` | `None` → `sentence-transformers/all-MiniLM-L6-v2` | Dense embedding model used when no `USING MODEL` clause is given | +| `default_dense_vector_name` | `str` | `"dense"` | Dense vector name used when QQL creates a collection and no explicit `USING VECTOR` name is given | +| `default_sparse_vector_name` | `str` | `"sparse"` | Sparse vector name used when QQL creates a hybrid collection and no explicit sparse vector name is given | ### Power-user: `executor` property diff --git a/docs/reference.md b/docs/reference.md index 1d7377a..f006077 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -33,6 +33,10 @@ Qdrant/bm25 INSERT INTO docs VALUES {'text': 'hello'} USING MODEL 'BAAI/bge-small-en-v1.5' SEARCH docs SIMILAR TO 'hello' LIMIT 5 USING MODEL 'BAAI/bge-small-en-v1.5' +-- Explicit vector names +INSERT INTO docs VALUES {'text': 'hello'} USING VECTOR 'body' +SEARCH docs SIMILAR TO 'hello' LIMIT 5 USING VECTOR 'body' + -- Hybrid with custom dense model SEARCH docs SIMILAR TO 'hello' LIMIT 5 USING HYBRID DENSE MODEL 'BAAI/bge-base-en-v1.5' @@ -42,6 +46,10 @@ SEARCH docs SIMILAR TO 'hello' LIMIT 5 USING HYBRID FUSION 'dbsf' -- Hybrid with both custom SEARCH docs SIMILAR TO 'hello' LIMIT 5 USING HYBRID DENSE MODEL 'BAAI/bge-base-en-v1.5' SPARSE MODEL 'prithivida/Splade_PP_en_v1' + +-- Hybrid with external vector names +SEARCH docs SIMILAR TO 'hello' LIMIT 5 + USING HYBRID DENSE VECTOR 'emb' SPARSE VECTOR 'lex' ``` ### Commonly available dense models (Fastembed) diff --git a/docs/scripts.md b/docs/scripts.md index 319707e..b95c18c 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -120,7 +120,7 @@ Done. 41 point(s) written. -- configured model (see: qql connect). -- ============================================================ -CREATE COLLECTION medical_records HYBRID +CREATE COLLECTION medical_records USING HYBRID DENSE VECTOR 'dense' SPARSE VECTOR 'sparse' -- Batch 1 / 1 (records 1–41) INSERT BULK INTO COLLECTION medical_records VALUES [ @@ -132,7 +132,7 @@ INSERT BULK INTO COLLECTION medical_records VALUES [ 'peer_reviewed': true }, ... -] USING HYBRID +] USING HYBRID DENSE VECTOR 'dense' SPARSE VECTOR 'sparse' -- ============================================================ -- End of dump @@ -155,8 +155,8 @@ qql execute backup.qql **Rules and notes:** - Points without a `'text'` payload field are **skipped** (counted in the footer comment). -- Hybrid collections produce `CREATE COLLECTION HYBRID` and `INSERT BULK ... USING HYBRID` statements. -- Dense collections produce plain `CREATE COLLECTION ` and `INSERT BULK` statements. +- Hybrid collections produce `CREATE COLLECTION USING HYBRID ...` and matching `INSERT BULK ... USING HYBRID ...` statements, including vector names when the source collection uses named vectors. +- Dense collections produce `CREATE COLLECTION USING VECTOR ''` for named vectors, or plain `CREATE COLLECTION ` for unnamed external collections. - All payload value types are preserved: strings, integers, floats, booleans (`true`/`false`), `null`, lists, and nested dicts. - Re-importing re-embeds all text using your currently configured model — use the same model as the original collection to preserve semantic accuracy. - Parent directories of the output path are created automatically. diff --git a/docs/search.md b/docs/search.md index 6a05b63..3083409 100644 --- a/docs/search.md +++ b/docs/search.md @@ -12,10 +12,11 @@ An optional `WHERE` clause filters the candidate set **before** similarity ranki ``` SEARCH SIMILAR TO '' LIMIT SEARCH SIMILAR TO '' LIMIT USING MODEL '' +SEARCH SIMILAR TO '' LIMIT USING VECTOR '' SEARCH SIMILAR TO '' LIMIT [USING MODEL ''] WHERE SEARCH SIMILAR TO '' LIMIT USING HYBRID -SEARCH SIMILAR TO '' LIMIT USING HYBRID [FUSION 'rrf|dbsf'] [DENSE MODEL ''] [SPARSE MODEL ''] [WHERE ] -SEARCH SIMILAR TO '' LIMIT USING SPARSE [MODEL ''] +SEARCH SIMILAR TO '' LIMIT USING HYBRID [FUSION 'rrf|dbsf'] [DENSE MODEL ''] [DENSE VECTOR ''] [SPARSE MODEL ''] [SPARSE VECTOR ''] [WHERE ] +SEARCH SIMILAR TO '' LIMIT USING SPARSE [MODEL ''] [VECTOR ''] SEARCH SIMILAR TO '' LIMIT EXACT SEARCH SIMILAR TO '' LIMIT [USING ...] [WHERE ] [RERANK] WITH { hnsw_ef: , exact: true|false, acorn: true|false, indexed_only: true|false, quantization: { ignore: true|false, rescore: true|false, oversampling: }, mmr_diversity: <0..1>, mmr_candidates: } SEARCH SIMILAR TO '' LIMIT [USING ...] [WHERE ] RERANK [MODEL ''] @@ -38,7 +39,17 @@ Hybrid search (combines dense semantic + sparse BM25 keyword retrieval via RRF b SEARCH articles SIMILAR TO 'attention mechanism' LIMIT 10 USING HYBRID ``` -Sparse-only search (queries only the `sparse` named vector — useful for pure keyword retrieval): +Search a specific named dense vector: +```sql +SEARCH articles SIMILAR TO 'attention mechanism' LIMIT 10 USING VECTOR 'body' +``` + +Hybrid search against external vector names: +```sql +SEARCH articles SIMILAR TO 'attention mechanism' LIMIT 10 USING HYBRID DENSE VECTOR 'emb' SPARSE VECTOR 'lex' +``` + +Sparse-only search (queries a sparse vector — useful for pure keyword retrieval): ```sql SEARCH medical_knowledge SIMILAR TO 'beta blocker contraindications' LIMIT 5 USING SPARSE ``` diff --git a/src/qql/__init__.py b/src/qql/__init__.py index c71f152..deeb737 100644 --- a/src/qql/__init__.py +++ b/src/qql/__init__.py @@ -5,7 +5,13 @@ except PackageNotFoundError: __version__ = "0.0.0+unknown" -from .config import DEFAULT_MODEL, QQLConfig, load_config +from .config import ( + DEFAULT_DENSE_VECTOR_NAME, + DEFAULT_MODEL, + DEFAULT_SPARSE_VECTOR_NAME, + QQLConfig, + load_config, +) from .connection import Connection from .exceptions import QQLError, QQLRuntimeError, QQLSyntaxError from .executor import ExecutionResult, Executor @@ -15,6 +21,9 @@ __all__ = [ "__version__", "Connection", + "DEFAULT_DENSE_VECTOR_NAME", + "DEFAULT_MODEL", + "DEFAULT_SPARSE_VECTOR_NAME", "QQLConfig", "QQLError", "QQLRuntimeError", diff --git a/src/qql/ast_nodes.py b/src/qql/ast_nodes.py index dbd8fd4..2e4985f 100644 --- a/src/qql/ast_nodes.py +++ b/src/qql/ast_nodes.py @@ -207,6 +207,8 @@ class InsertStmt: model: str | None # dense model; None → use config default hybrid: bool = False # if True, also embed + store sparse BM25 vector sparse_model: str | None = None # sparse model; None → SparseEmbedder.DEFAULT_MODEL + dense_vector: str | None = None + sparse_vector: str | None = None @dataclass(frozen=True) @@ -216,6 +218,8 @@ class InsertBulkStmt: model: str | None # dense model; None → use config default hybrid: bool = False sparse_model: str | None = None + dense_vector: str | None = None + sparse_vector: str | None = None @dataclass(frozen=True) @@ -225,6 +229,8 @@ class CreateCollectionStmt: model: str | None = None # dense model; None → use config default quantization: QuantizationConfig | None = None # optional QUANTIZE clause config: CollectionConfig | None = None + dense_vector: str | None = None + sparse_vector: str | None = None @dataclass(frozen=True) @@ -287,6 +293,8 @@ class SearchStmt: with_clause: SearchWith | None = None group_by: str | None = None # GROUP BY field name; None → normal flat search group_size: int = 3 # max points per group (ignored when group_by is None) + dense_vector: str | None = None + sparse_vector: str | None = None @dataclass(frozen=True) @@ -317,6 +325,7 @@ class UpdateVectorStmt: collection: str point_id: str | int vector: tuple[float, ...] # dense vector as immutable tuple (frozen=True compatible) + vector_name: str | None = None @dataclass(frozen=True) diff --git a/src/qql/cli.py b/src/qql/cli.py index 1b072ba..2b4ab16 100644 --- a/src/qql/cli.py +++ b/src/qql/cli.py @@ -27,7 +27,8 @@ Insert a point. 'text' is required and auto-vectorized. Optional: include [yellow]'id'[/yellow] in VALUES as an integer or UUID Optional: [yellow]USING MODEL[/yellow] '' - Optional: [yellow]USING HYBRID[/yellow] [DENSE MODEL ''] [SPARSE MODEL ''] + Optional: [yellow]USING VECTOR[/yellow] '' + Optional: [yellow]USING HYBRID[/yellow] [DENSE MODEL ''] [DENSE VECTOR ''] [SPARSE MODEL ''] [SPARSE VECTOR ''] [yellow]INSERT BULK INTO COLLECTION[/yellow] [yellow]VALUES[/yellow] [{[yellow]'text'[/yellow]: '...', ...}, ...] Batch insert multiple points in a single call. Each dict must contain 'text'. @@ -37,7 +38,8 @@ [yellow]CREATE COLLECTION[/yellow] [[yellow]HYBRID[/yellow]] Create a new collection. Add HYBRID for dense+sparse BM25 vectors. Optional: [yellow]USING MODEL[/yellow] '' - Optional: [yellow]USING HYBRID[/yellow] [DENSE MODEL ''] + Optional: [yellow]USING VECTOR[/yellow] '' + Optional: [yellow]USING HYBRID[/yellow] [DENSE MODEL ''] [DENSE VECTOR ''] [SPARSE VECTOR ''] Optional: [yellow]WITH VECTORS[/yellow] { on_disk: } Optional: [yellow]WITH HNSW[/yellow] { m, ef_construct, full_scan_threshold, max_indexing_threads, on_disk, payload_m, inline_storage } Optional: [yellow]WITH OPTIMIZERS[/yellow] { deleted_threshold, vacuum_min_vector_number, default_segment_number, max_segment_size, memmap_threshold, indexing_threshold, flush_interval_sec, max_optimization_threads, prevent_unoptimized } @@ -87,8 +89,9 @@ [yellow]SEARCH[/yellow] [yellow]SIMILAR TO[/yellow] '' [yellow]LIMIT[/yellow] Semantic search by vector similarity. Optional: [yellow]USING MODEL[/yellow] '' - Optional: [yellow]USING HYBRID[/yellow] [FUSION 'rrf|dbsf'] [DENSE MODEL ''] [SPARSE MODEL ''] - Optional: [yellow]USING SPARSE[/yellow] [MODEL ''] sparse-vector-only search + Optional: [yellow]USING VECTOR[/yellow] '' + Optional: [yellow]USING HYBRID[/yellow] [FUSION 'rrf|dbsf'] [DENSE MODEL ''] [DENSE VECTOR ''] [SPARSE MODEL ''] [SPARSE VECTOR ''] + Optional: [yellow]USING SPARSE[/yellow] [MODEL ''] [VECTOR ''] sparse-vector-only search Optional: [yellow]WHERE[/yellow] (e.g. WHERE year > 2020 AND status = 'ok') Optional: [yellow]RERANK[/yellow] [MODEL ''] rerank results with a cross-encoder Optional: [yellow]EXACT[/yellow] bypass HNSW and perform exact search @@ -107,7 +110,8 @@ [yellow]DELETE FROM[/yellow] [yellow]WHERE id =[/yellow] '' Delete a point by its ID. - [yellow]UPDATE[/yellow] [yellow]SET VECTOR WHERE id =[/yellow] ''| [] + [yellow]UPDATE[/yellow] [yellow]SET VECTOR[/yellow] [yellow]WHERE id =[/yellow] ''| [] + [yellow]UPDATE[/yellow] [yellow]SET VECTOR[/yellow] '' [yellow]WHERE id =[/yellow] ''| [] Replace the dense vector for a single point by ID. The point must already exist. Vector is a float array: [0.1, 0.2, ..., 0.N] diff --git a/src/qql/config.py b/src/qql/config.py index b679bfd..8c41074 100644 --- a/src/qql/config.py +++ b/src/qql/config.py @@ -8,6 +8,8 @@ CONFIG_PATH = CONFIG_DIR / "config.json" DEFAULT_MODEL = "sentence-transformers/all-MiniLM-L6-v2" +DEFAULT_DENSE_VECTOR_NAME = "dense" +DEFAULT_SPARSE_VECTOR_NAME = "sparse" @dataclass @@ -15,6 +17,8 @@ class QQLConfig: url: str secret: str | None = None default_model: str = DEFAULT_MODEL + default_dense_vector_name: str = DEFAULT_DENSE_VECTOR_NAME + default_sparse_vector_name: str = DEFAULT_SPARSE_VECTOR_NAME def save_config(cfg: QQLConfig) -> None: @@ -33,6 +37,12 @@ def load_config() -> QQLConfig | None: url=data["url"], secret=data.get("secret"), default_model=data.get("default_model", DEFAULT_MODEL), + default_dense_vector_name=data.get( + "default_dense_vector_name", DEFAULT_DENSE_VECTOR_NAME + ), + default_sparse_vector_name=data.get( + "default_sparse_vector_name", DEFAULT_SPARSE_VECTOR_NAME + ), ) diff --git a/src/qql/dumper.py b/src/qql/dumper.py index 9c0da2e..8bef854 100644 --- a/src/qql/dumper.py +++ b/src/qql/dumper.py @@ -77,6 +77,48 @@ def _is_hybrid(collection: str, client: QdrantClient) -> bool: return isinstance(sparse_vectors, dict) and bool(sparse_vectors) +def _single_named_vector_name(names: list[str], kind: str) -> str: + if len(names) == 1: + return names[0] + if not names: + raise ValueError(f"Collection has no {kind} vector to dump") + raise ValueError( + f"Collection has multiple {kind} vectors; QQL dump requires explicit single-vector topology" + ) + + +def _dense_vector_config(info: Any) -> Any: + vectors = info.config.params.vectors # type: ignore[union-attr] + if isinstance(vectors, dict): + return vectors[_single_named_vector_name(list(vectors.keys()), "dense")] + return vectors + + +def _create_using_clause(info: Any, hybrid: bool) -> str: + params = info.config.params + vectors = params.vectors # type: ignore[union-attr] + sparse_vectors = params.sparse_vectors or {} + + dense_name: str | None = None + if isinstance(vectors, dict): + dense_name = _single_named_vector_name(list(vectors.keys()), "dense") + + sparse_name: str | None = None + if isinstance(sparse_vectors, dict) and sparse_vectors: + sparse_name = _single_named_vector_name(list(sparse_vectors.keys()), "sparse") + + if hybrid: + parts = [" USING HYBRID"] + if dense_name is not None: + parts.append(f" DENSE VECTOR {_serialize_value(dense_name)}") + if sparse_name is not None: + parts.append(f" SPARSE VECTOR {_serialize_value(sparse_name)}") + return "".join(parts) + if dense_name is not None: + return f" USING VECTOR {_serialize_value(dense_name)}" + return "" + + def _quantization_clause(info: Any) -> str: quant = info.config.quantization_config if quant is None: @@ -118,8 +160,7 @@ def _quantization_clause(info: Any) -> str: def _config_clauses(info: Any) -> str: clauses: list[str] = [] params = info.config.params - vectors = params.vectors # type: ignore[union-attr] - dense_vectors = vectors.get("dense") if isinstance(vectors, dict) else vectors + dense_vectors = _dense_vector_config(info) if dense_vectors is not None and getattr(dense_vectors, "on_disk", None) is not None: clauses.append(f"WITH VECTORS {{ on_disk: {'true' if dense_vectors.on_disk else 'false'} }}") @@ -211,7 +252,7 @@ def dump_collection( sparse_vectors = info.config.params.sparse_vectors hybrid = isinstance(sparse_vectors, dict) and bool(sparse_vectors) col_type = "hybrid (dense + sparse)" if hybrid else "dense" - using_clause = " USING HYBRID" if hybrid else "" + using_clause = _create_using_clause(info, hybrid) # ── First pass: count total points for the header ───────────────────── count_info = client.count(collection_name=collection, exact=True) @@ -248,11 +289,10 @@ def dump_collection( ) # ── CREATE statement ────────────────────────────────────────────── - hybrid_suffix = " HYBRID" if hybrid else "" config_suffix = _config_clauses(info) quantization_suffix = _quantization_clause(info) f.write( - f"CREATE COLLECTION {collection}{hybrid_suffix}{config_suffix}{quantization_suffix}\n\n" + f"CREATE COLLECTION {collection}{using_clause}{config_suffix}{quantization_suffix}\n\n" ) # ── Paginate and write INSERT BULK batches ──────────────────────── diff --git a/src/qql/executor.py b/src/qql/executor.py index 946ca7e..46343c3 100644 --- a/src/qql/executor.py +++ b/src/qql/executor.py @@ -74,7 +74,6 @@ AlterCollectionStmt, AndExpr, BetweenExpr, - CollectionParamsConfig, CollectionConfig, CompareExpr, CreateCollectionStmt, @@ -94,7 +93,6 @@ MatchTextExpr, NotExpr, NotInExpr, - OptimizersRuntimeConfig, OrExpr, QuantizationUpdate, QuantizationConfig, @@ -108,8 +106,6 @@ ShowCollectionsStmt, UpdateVectorStmt, UpdatePayloadStmt, - VectorsConfig, - HnswRuntimeConfig, ) from .config import QQLConfig from .embedder import CrossEncoderEmbedder, Embedder, SparseEmbedder @@ -128,6 +124,70 @@ class ExecutionResult: data: Any = None +@dataclass(frozen=True) +class CollectionTopology: + exists: bool + is_named_dense: bool + has_unnamed_dense: bool = False + dense_names: tuple[str, ...] = () + sparse_names: tuple[str, ...] = () + + @property + def has_dense(self) -> bool: + return self.has_unnamed_dense or bool(self.dense_names) + + @property + def has_sparse(self) -> bool: + return bool(self.sparse_names) + + @property + def is_hybrid(self) -> bool: + return self.has_dense and self.has_sparse + + def dense_using(self, explicit: str | None = None) -> str | None: + if explicit is not None: + if self.exists and self.has_unnamed_dense: + raise QQLRuntimeError( + "Collection uses an unnamed dense vector; omit USING VECTOR" + ) + if self.exists and explicit not in self.dense_names: + raise QQLRuntimeError( + f"Collection has no dense vector named '{explicit}'" + ) + return explicit + if self.has_unnamed_dense: + return None + if len(self.dense_names) == 1: + return self.dense_names[0] + if not self.dense_names: + raise QQLRuntimeError("Collection has no dense vector") + raise QQLRuntimeError( + "Collection has multiple dense vectors; specify one with USING VECTOR ''" + ) + + def dense_payload_name(self, explicit: str | None = None) -> str | None: + return self.dense_using(explicit) + + def dense_config_key(self, explicit: str | None = None) -> str: + name = self.dense_using(explicit) + return "" if name is None else name + + def sparse_using(self, explicit: str | None = None) -> str: + if explicit is not None: + if self.exists and explicit not in self.sparse_names: + raise QQLRuntimeError( + f"Collection has no sparse vector named '{explicit}'" + ) + return explicit + if len(self.sparse_names) == 1: + return self.sparse_names[0] + if not self.sparse_names: + raise QQLRuntimeError("Collection has no sparse vector") + raise QQLRuntimeError( + "Collection has multiple sparse vectors; specify one with USING SPARSE VECTOR ''" + ) + + class Executor: def __init__(self, client: QdrantClient, config: QQLConfig) -> None: self._client = client @@ -168,13 +228,51 @@ def execute(self, node: ASTNode) -> ExecutionResult: # ── Statement executors ─────────────────────────────────────────────── + def _resolve_topology(self, name: str) -> CollectionTopology: + if not self._client.collection_exists(name): + return CollectionTopology(exists=False, is_named_dense=False) + + info = self._client.get_collection(name) + params = info.config.params + vectors = params.vectors # type: ignore[union-attr] + sparse_vectors = params.sparse_vectors or {} + + if isinstance(vectors, dict): + dense_names = tuple(vectors.keys()) + has_unnamed_dense = False + is_named_dense = True + elif vectors is None: + dense_names = () + has_unnamed_dense = False + is_named_dense = False + else: + dense_names = () + has_unnamed_dense = True + is_named_dense = False + + sparse_names = ( + tuple(sparse_vectors.keys()) if isinstance(sparse_vectors, dict) else () + ) + return CollectionTopology( + exists=True, + is_named_dense=is_named_dense, + has_unnamed_dense=has_unnamed_dense, + dense_names=dense_names, + sparse_names=sparse_names, + ) + + def _default_dense_vector_name(self) -> str: + return self._config.default_dense_vector_name + + def _default_sparse_vector_name(self) -> str: + return self._config.default_sparse_vector_name + def _execute_insert(self, node: InsertStmt) -> ExecutionResult: if "text" not in node.values: raise QQLRuntimeError("INSERT requires a 'text' field in VALUES") - # Auto-detect hybrid when the user omitted USING HYBRID but the - # collection already exists as a hybrid (named-vector) collection. - use_hybrid = node.hybrid or self._collection_is_hybrid(node.collection) + topology = self._resolve_topology(node.collection) + use_hybrid = node.hybrid or (topology.exists and topology.is_hybrid) # ── Hybrid INSERT: dense + sparse vectors ────────────────────────── if use_hybrid: @@ -190,17 +288,27 @@ def _execute_insert(self, node: InsertStmt) -> ExecutionResult: values=sparse_obj["values"], ) - # Auto-create hybrid collection if it doesn't exist yet - if not self._client.collection_exists(node.collection): + dense_name = node.dense_vector or self._default_dense_vector_name() + sparse_name = node.sparse_vector or self._default_sparse_vector_name() + + if topology.exists: + resolved_dense = topology.dense_using(node.dense_vector) + if resolved_dense is None: + raise QQLRuntimeError( + "Hybrid collections must use named dense vectors" + ) + dense_name = resolved_dense + sparse_name = topology.sparse_using(node.sparse_vector) + else: self._create_collection_and_wait( collection_name=node.collection, vectors_config={ - "dense": VectorParams( + dense_name: VectorParams( size=len(dense_vector), distance=Distance.COSINE ) }, sparse_vectors_config={ - "sparse": SparseVectorParams(modifier=Modifier.IDF) + sparse_name: SparseVectorParams(modifier=Modifier.IDF) }, ) @@ -212,7 +320,7 @@ def _execute_insert(self, node: InsertStmt) -> ExecutionResult: points=[ PointStruct( id=point_id, - vector={"dense": dense_vector, "sparse": sparse_vector}, + vector={dense_name: dense_vector, sparse_name: sparse_vector}, payload=payload, ) ], @@ -231,8 +339,10 @@ def _execute_insert(self, node: InsertStmt) -> ExecutionResult: embedder = Embedder(model_name) vector = embedder.embed(node.values["text"]) - self._ensure_collection(node.collection, len(vector)) - point_vector = self._build_dense_point_vector(node.collection, vector) + self._ensure_collection( + node.collection, len(vector), topology, node.dense_vector + ) + point_vector = self._build_dense_point_vector(topology, vector, node.dense_vector) point_id, payload = self._extract_point_id_and_payload(node.values) @@ -260,9 +370,8 @@ def _execute_insert_bulk(self, node: InsertBulkStmt) -> ExecutionResult: f"INSERT BULK: item at index {i} is missing required 'text' field" ) - # Auto-detect hybrid when the user omitted USING HYBRID but the - # collection already exists as a hybrid (named-vector) collection. - use_hybrid = node.hybrid or self._collection_is_hybrid(node.collection) + topology = self._resolve_topology(node.collection) + use_hybrid = node.hybrid or (topology.exists and topology.is_hybrid) # ── Hybrid bulk INSERT: dense + sparse vectors ───────────────────── if use_hybrid: @@ -270,11 +379,24 @@ def _execute_insert_bulk(self, node: InsertBulkStmt) -> ExecutionResult: sparse_model_name = node.sparse_model or SparseEmbedder.DEFAULT_MODEL dense_embedder = Embedder(dense_model) sparse_embedder = SparseEmbedder(sparse_model_name) + dense_name = node.dense_vector or self._default_dense_vector_name() + sparse_name = node.sparse_vector or self._default_sparse_vector_name() + if topology.exists: + resolved_dense = topology.dense_using(node.dense_vector) + if resolved_dense is None: + raise QQLRuntimeError( + "Hybrid collections must use named dense vectors" + ) + dense_name = resolved_dense + sparse_name = topology.sparse_using(node.sparse_vector) + first_dense_vector: list[float] | None = None points: list[PointStruct] = [] for vals in node.values_list: point_id, payload = self._extract_point_id_and_payload(vals) dense_vector = dense_embedder.embed(vals["text"]) + if first_dense_vector is None: + first_dense_vector = dense_vector sparse_obj = sparse_embedder.embed(vals["text"]) sparse_vector = SparseVector( indices=sparse_obj["indices"], values=sparse_obj["values"] @@ -282,20 +404,20 @@ def _execute_insert_bulk(self, node: InsertBulkStmt) -> ExecutionResult: points.append( PointStruct( id=point_id, - vector={"dense": dense_vector, "sparse": sparse_vector}, + vector={dense_name: dense_vector, sparse_name: sparse_vector}, payload=payload, ) ) - if not self._client.collection_exists(node.collection): - first_dense = dense_embedder.embed(node.values_list[0]["text"]) + if not topology.exists: + assert first_dense_vector is not None self._create_collection_and_wait( collection_name=node.collection, vectors_config={ - "dense": VectorParams(size=len(first_dense), distance=Distance.COSINE) + dense_name: VectorParams(size=len(first_dense_vector), distance=Distance.COSINE) }, sparse_vectors_config={ - "sparse": SparseVectorParams(modifier=Modifier.IDF) + sparse_name: SparseVectorParams(modifier=Modifier.IDF) }, ) @@ -317,16 +439,24 @@ def _execute_insert_bulk(self, node: InsertBulkStmt) -> ExecutionResult: model_name = node.model or self._config.default_model embedder = Embedder(model_name) + first_vector: list[float] | None = None points = [] for vals in node.values_list: vector = embedder.embed(vals["text"]) + if first_vector is None: + first_vector = vector point_id, payload = self._extract_point_id_and_payload(vals) - point_vector = self._build_dense_point_vector(node.collection, vector) + point_vector = self._build_dense_point_vector( + topology, vector, node.dense_vector + ) points.append( PointStruct(id=point_id, vector=point_vector, payload=payload) ) - self._ensure_collection(node.collection, len(vector)) + assert first_vector is not None + self._ensure_collection( + node.collection, len(first_vector), topology, node.dense_vector + ) try: self._client.upsert( @@ -376,17 +506,19 @@ def _execute_create(self, node: CreateCollectionStmt) -> ExecutionResult: if node.hybrid: embedder = Embedder(dense_model_name) dims = embedder.dimensions + dense_name = node.dense_vector or self._default_dense_vector_name() + sparse_name = node.sparse_vector or self._default_sparse_vector_name() create_kwargs: dict[str, Any] = { "collection_name": node.collection, "vectors_config": { - "dense": VectorParams( + dense_name: VectorParams( size=dims, distance=Distance.COSINE, on_disk=vector_on_disk, ) }, "sparse_vectors_config": { - "sparse": SparseVectorParams(modifier=Modifier.IDF) + sparse_name: SparseVectorParams(modifier=Modifier.IDF) }, } if quant_config is not None: @@ -408,13 +540,16 @@ def _execute_create(self, node: CreateCollectionStmt) -> ExecutionResult: # ── Standard dense-only collection ───────────────────────────────── embedder = Embedder(dense_model_name) dims = embedder.dimensions + dense_name = node.dense_vector or self._default_dense_vector_name() create_kwargs = { "collection_name": node.collection, - "vectors_config": VectorParams( - size=dims, - distance=Distance.COSINE, - on_disk=vector_on_disk, - ), + "vectors_config": { + dense_name: VectorParams( + size=dims, + distance=Distance.COSINE, + on_disk=vector_on_disk, + ) + }, } if quant_config is not None: create_kwargs["quantization_config"] = quant_config @@ -432,9 +567,10 @@ def _execute_create(self, node: CreateCollectionStmt) -> ExecutionResult: def _execute_alter_collection(self, node: AlterCollectionStmt) -> ExecutionResult: if not self._client.collection_exists(node.collection): raise QQLRuntimeError(f"Collection '{node.collection}' does not exist") + topology = self._resolve_topology(node.collection) update_kwargs: dict[str, Any] = {"collection_name": node.collection} - vectors_config = self._build_vectors_config_diff(node.collection, node.config) + vectors_config = self._build_vectors_config_diff(topology, node.config) hnsw_config = self._build_hnsw_config(node.config) optimizers_config = self._build_optimizers_config(node.config) collection_params = self._build_collection_params_diff(node.config) @@ -694,6 +830,7 @@ def _execute_select(self, node: SelectStmt) -> ExecutionResult: def _execute_search(self, node: SearchStmt) -> ExecutionResult: if not self._client.collection_exists(node.collection): raise QQLRuntimeError(f"Collection '{node.collection}' does not exist") + topology = self._resolve_topology(node.collection) # Build WHERE filter (shared by both hybrid and dense-only paths) qdrant_filter: Filter | None = None @@ -711,7 +848,9 @@ def _execute_search(self, node: SearchStmt) -> ExecutionResult: # ── GROUP BY SEARCH: delegate to query_points_groups() ───────────── if node.group_by is not None: - return self._execute_search_groups(node, qdrant_filter, search_params) + return self._execute_search_groups( + node, qdrant_filter, search_params, topology + ) # ── Hybrid SEARCH: prefetch dense+sparse, fuse with the requested strategy ── if node.hybrid: @@ -727,13 +866,13 @@ def _execute_search(self, node: SearchStmt) -> ExecutionResult: prefetch=[ Prefetch( query=dense_vector, - using="dense", + using=topology.dense_using(node.dense_vector), limit=node.limit * _HYBRID_PREFETCH_MULTIPLIER, params=search_params, ), Prefetch( query=sparse_vector, - using="sparse", + using=topology.sparse_using(node.sparse_vector), limit=node.limit * _HYBRID_PREFETCH_MULTIPLIER, params=search_params, ), @@ -764,7 +903,7 @@ def _execute_search(self, node: SearchStmt) -> ExecutionResult: data=results, ) - # ── Sparse-only SEARCH: query the "sparse" named vector directly ───── + # ── Sparse-only SEARCH: query the selected sparse vector directly ─── if node.sparse_only: sparse_model_name = node.sparse_model or SparseEmbedder.DEFAULT_MODEL sparse_embedder = SparseEmbedder(sparse_model_name) @@ -778,7 +917,7 @@ def _execute_search(self, node: SearchStmt) -> ExecutionResult: response = self._client.query_points( collection_name=node.collection, query=sparse_vector, - using="sparse", + using=topology.sparse_using(node.sparse_vector), limit=fetch_limit, query_filter=qdrant_filter, search_params=search_params, @@ -811,7 +950,7 @@ def _execute_search(self, node: SearchStmt) -> ExecutionResult: vector = embedder.embed(node.query_text) try: - query_using = self._get_dense_vector_name(node.collection) + query_using = topology.dense_using(node.dense_vector) response = self._client.query_points( collection_name=node.collection, query=self._build_dense_query(vector, node.with_clause), @@ -1202,14 +1341,19 @@ def _build_collection_params_diff( def _build_vectors_config_diff( self, - collection_name: str, + topology: CollectionTopology, config: CollectionConfig | None, ) -> dict[str, VectorParamsDiff] | None: if config is None or config.vectors is None: return None - vector_name = self._get_dense_vector_name(collection_name) - if vector_name is None: - vector_name = "" + try: + vector_name = topology.dense_config_key() + except QQLRuntimeError as e: + if "multiple dense vectors" in str(e): + raise QQLRuntimeError( + "ALTER COLLECTION WITH VECTORS requires a collection with one dense vector" + ) from e + raise return { vector_name: VectorParamsDiff(on_disk=config.vectors.on_disk), } @@ -1382,26 +1526,15 @@ def _extract_point_id_and_payload( "INSERT id must be an unsigned integer or UUID string when provided" ) - def _get_dense_vector_name(self, collection_name: str) -> str | None: - """Return the dense vector name for named-vector collections. - - Dense-only QQL searches should keep working against hybrid collections, - which store vectors under the explicit ``dense`` name. - """ - info = self._client.get_collection(collection_name) - vectors = info.config.params.vectors # type: ignore[union-attr] - if isinstance(vectors, dict): - return "dense" - return None - def _build_dense_point_vector( self, - collection_name: str, + topology: CollectionTopology, vector: list[float], + explicit_vector: str | None, ) -> list[float] | dict[str, list[float]]: - if not self._client.collection_exists(collection_name): - return vector - vector_name = self._get_dense_vector_name(collection_name) + if not topology.exists: + return {explicit_vector or self._default_dense_vector_name(): vector} + vector_name = topology.dense_payload_name(explicit_vector) if vector_name is None: return vector return {vector_name: vector} @@ -1463,6 +1596,7 @@ def _execute_search_groups( node: SearchStmt, qdrant_filter: Filter | None, search_params: SearchParams | None, + topology: CollectionTopology, ) -> ExecutionResult: """Execute SEARCH ... GROUP BY using query_points_groups().""" try: @@ -1478,13 +1612,13 @@ def _execute_search_groups( prefetch=[ Prefetch( query=dense_vector, - using="dense", + using=topology.dense_using(node.dense_vector), limit=node.limit * _HYBRID_PREFETCH_MULTIPLIER, params=search_params, ), Prefetch( query=sparse_vector, - using="sparse", + using=topology.sparse_using(node.sparse_vector), limit=node.limit * _HYBRID_PREFETCH_MULTIPLIER, params=search_params, ), @@ -1506,16 +1640,17 @@ def _execute_search_groups( collection_name=node.collection, group_by=node.group_by, query=sparse_vector, - using="sparse", + using=topology.sparse_using(node.sparse_vector), limit=node.limit, group_size=node.group_size, query_filter=qdrant_filter, + search_params=search_params, ) label = "sparse, grouped" else: model_name = node.model or self._config.default_model vector = Embedder(model_name).embed(node.query_text) - query_using = self._get_dense_vector_name(node.collection) + query_using = topology.dense_using(node.dense_vector) response = self._client.query_points_groups( collection_name=node.collection, group_by=node.group_by, @@ -1550,8 +1685,8 @@ def _execute_update_vector(self, node: UpdateVectorStmt) -> ExecutionResult: """Execute UPDATE ... SET VECTOR using update_vectors().""" if not self._client.collection_exists(node.collection): raise QQLRuntimeError(f"Collection '{node.collection}' does not exist") - # Named-vector collections (hybrid) use "dense"; unnamed use plain list. - vector_name = self._get_dense_vector_name(node.collection) + topology = self._resolve_topology(node.collection) + vector_name = topology.dense_payload_name(node.vector_name) vector_struct: Any = ( {vector_name: list(node.vector)} if vector_name else list(node.vector) ) @@ -1745,28 +1880,35 @@ def _build_quantization_config( ) raise QQLRuntimeError(f"Unknown quantization type: {qc.type}") - def _collection_is_hybrid(self, name: str) -> bool: - """Return True if *name* exists and uses sparse vectors (hybrid collection).""" - if not self._client.collection_exists(name): - return False - info = self._client.get_collection(name) - sparse_vectors = info.config.params.sparse_vectors - return isinstance(sparse_vectors, dict) and bool(sparse_vectors) - - def _ensure_collection(self, name: str, vector_size: int) -> None: + def _ensure_collection( + self, + name: str, + vector_size: int, + topology: CollectionTopology, + explicit_vector: str | None, + ) -> None: """Create the collection if it doesn't exist. Raises on dimension mismatch. - For named-vector (hybrid) collections the validation is skipped — those - collections are managed directly by the hybrid insert/create paths. + QQL-created dense collections use the configured dense vector name. + Externally created unnamed collections still accept plain dense vectors. """ - if self._client.collection_exists(name): + if topology.exists: info = self._client.get_collection(name) vectors = info.config.params.vectors # type: ignore[union-attr] if isinstance(vectors, dict): - # Named-vector (hybrid) collection — skip validation here; - # the hybrid insert path manages its own collection creation. - pass - else: + vector_name = topology.dense_using(explicit_vector) + if vector_name is None: + raise QQLRuntimeError("Collection has no dense vector") + vector_config = vectors[vector_name] + expected_size = getattr(vector_config, "size", None) + if expected_size is not None and expected_size != vector_size: + raise QQLRuntimeError( + f"Vector dimension mismatch: collection '{name}' vector " + f"'{vector_name}' expects {expected_size} dims, but " + f"model produces {vector_size} dims. Specify a compatible " + "model with USING MODEL ''." + ) + elif vectors is not None: # Unnamed single-vector collection: validate dimensions if vectors.size != vector_size: raise QQLRuntimeError( @@ -1774,10 +1916,16 @@ def _ensure_collection(self, name: str, vector_size: int) -> None: f"{vectors.size} dims, but model produces {vector_size} dims. " f"Specify a compatible model with USING MODEL ''." ) + else: + raise QQLRuntimeError("Collection has no dense vector") else: self._create_collection_and_wait( collection_name=name, - vectors_config=VectorParams(size=vector_size, distance=Distance.COSINE), + vectors_config={ + explicit_vector or self._default_dense_vector_name(): VectorParams( + size=vector_size, distance=Distance.COSINE + ) + }, ) def _create_collection_and_wait(self, **kwargs: Any) -> None: diff --git a/src/qql/parser.py b/src/qql/parser.py index 37d25e5..17fdafa 100644 --- a/src/qql/parser.py +++ b/src/qql/parser.py @@ -114,26 +114,45 @@ def _parse_insert(self) -> InsertStmt | InsertBulkStmt: model: str | None = None hybrid: bool = False sparse_model: str | None = None + dense_vector: str | None = None + sparse_vector: str | None = None if self._peek().kind == TokenKind.USING: self._advance() # consume USING if self._peek().kind == TokenKind.HYBRID: self._advance() # consume HYBRID hybrid = True - # Optional DENSE MODEL and/or SPARSE MODEL sub-clauses, any order + # Optional DENSE/SPARSE MODEL or VECTOR sub-clauses, any order while self._peek().kind in (TokenKind.DENSE, TokenKind.SPARSE): sub = self._advance() - self._expect(TokenKind.MODEL) - m = self._expect(TokenKind.STRING).value - if sub.kind == TokenKind.DENSE: - model = m + if self._peek().kind == TokenKind.MODEL: + self._advance() + m = self._expect(TokenKind.STRING).value + if sub.kind == TokenKind.DENSE: + model = m + else: + sparse_model = m + elif self._peek().kind == TokenKind.VECTOR: + self._advance() + name = self._expect(TokenKind.STRING).value + if sub.kind == TokenKind.DENSE: + dense_vector = name + else: + sparse_vector = name else: - sparse_model = m + raise QQLSyntaxError( + "Expected MODEL or VECTOR after DENSE/SPARSE in USING HYBRID", + self._peek().pos, + ) + elif self._peek().kind == TokenKind.VECTOR: + self._advance() + dense_vector = self._expect(TokenKind.STRING).value else: self._expect(TokenKind.MODEL) model = self._expect(TokenKind.STRING).value return InsertStmt( collection=collection, values=values, model=model, hybrid=hybrid, sparse_model=sparse_model, + dense_vector=dense_vector, sparse_vector=sparse_vector, ) def _parse_insert_bulk_body(self) -> InsertBulkStmt: @@ -153,6 +172,8 @@ def _parse_insert_bulk_body(self) -> InsertBulkStmt: model: str | None = None hybrid: bool = False sparse_model: str | None = None + dense_vector: str | None = None + sparse_vector: str | None = None if self._peek().kind == TokenKind.USING: self._advance() # consume USING if self._peek().kind == TokenKind.HYBRID: @@ -160,18 +181,35 @@ def _parse_insert_bulk_body(self) -> InsertBulkStmt: hybrid = True while self._peek().kind in (TokenKind.DENSE, TokenKind.SPARSE): sub = self._advance() - self._expect(TokenKind.MODEL) - m = self._expect(TokenKind.STRING).value - if sub.kind == TokenKind.DENSE: - model = m + if self._peek().kind == TokenKind.MODEL: + self._advance() + m = self._expect(TokenKind.STRING).value + if sub.kind == TokenKind.DENSE: + model = m + else: + sparse_model = m + elif self._peek().kind == TokenKind.VECTOR: + self._advance() + name = self._expect(TokenKind.STRING).value + if sub.kind == TokenKind.DENSE: + dense_vector = name + else: + sparse_vector = name else: - sparse_model = m + raise QQLSyntaxError( + "Expected MODEL or VECTOR after DENSE/SPARSE in USING HYBRID", + self._peek().pos, + ) + elif self._peek().kind == TokenKind.VECTOR: + self._advance() + dense_vector = self._expect(TokenKind.STRING).value else: self._expect(TokenKind.MODEL) model = self._expect(TokenKind.STRING).value return InsertBulkStmt( collection=collection, values_list=values_list, model=model, hybrid=hybrid, sparse_model=sparse_model, + dense_vector=dense_vector, sparse_vector=sparse_vector, ) def _parse_create(self) -> CreateCollectionStmt | CreateIndexStmt: @@ -181,9 +219,10 @@ def _parse_create(self) -> CreateCollectionStmt | CreateIndexStmt: collection = self._parse_identifier() hybrid: bool = False model: str | None = None + dense_vector: str | None = None + sparse_vector: str | None = None if self._peek().kind == TokenKind.HYBRID: - # Bare HYBRID shorthand — backward compat self._advance() hybrid = True elif self._peek().kind == TokenKind.USING: @@ -191,11 +230,31 @@ def _parse_create(self) -> CreateCollectionStmt | CreateIndexStmt: if self._peek().kind == TokenKind.HYBRID: self._advance() # consume HYBRID hybrid = True - # Optional DENSE MODEL sub-clause - if self._peek().kind == TokenKind.DENSE: - self._advance() # consume DENSE - self._expect(TokenKind.MODEL) - model = self._expect(TokenKind.STRING).value + while self._peek().kind in (TokenKind.DENSE, TokenKind.SPARSE): + sub = self._advance() + if self._peek().kind == TokenKind.MODEL: + self._advance() + if sub.kind != TokenKind.DENSE: + raise QQLSyntaxError( + "CREATE COLLECTION supports MODEL only for DENSE vectors", + self._peek().pos, + ) + model = self._expect(TokenKind.STRING).value + elif self._peek().kind == TokenKind.VECTOR: + self._advance() + name = self._expect(TokenKind.STRING).value + if sub.kind == TokenKind.DENSE: + dense_vector = name + else: + sparse_vector = name + else: + raise QQLSyntaxError( + "Expected MODEL or VECTOR after DENSE/SPARSE in USING HYBRID", + self._peek().pos, + ) + elif self._peek().kind == TokenKind.VECTOR: + self._advance() + dense_vector = self._expect(TokenKind.STRING).value else: self._expect(TokenKind.MODEL) model = self._expect(TokenKind.STRING).value @@ -209,6 +268,8 @@ def _parse_create(self) -> CreateCollectionStmt | CreateIndexStmt: model=model, quantization=quantization, config=config, + dense_vector=dense_vector, + sparse_vector=sparse_vector, ) self._expect(TokenKind.INDEX) @@ -631,12 +692,14 @@ def _parse_search(self) -> SearchStmt: fusion: str | None = None sparse_only: bool = False sparse_model: str | None = None + dense_vector: str | None = None + sparse_vector: str | None = None if self._peek().kind == TokenKind.USING: self._advance() # consume USING if self._peek().kind == TokenKind.HYBRID: self._advance() # consume HYBRID hybrid = True - # Optional FUSION / DENSE MODEL / SPARSE MODEL sub-clauses, any order. + # Optional FUSION / DENSE|SPARSE MODEL|VECTOR sub-clauses, any order. while self._peek().kind in (TokenKind.FUSION, TokenKind.DENSE, TokenKind.SPARSE): sub = self._advance() if sub.kind == TokenKind.FUSION: @@ -648,18 +711,37 @@ def _parse_search(self) -> SearchStmt: value_tok.pos, ) continue - self._expect(TokenKind.MODEL) - m = self._expect(TokenKind.STRING).value - if sub.kind == TokenKind.DENSE: - model = m + if self._peek().kind == TokenKind.MODEL: + self._advance() + m = self._expect(TokenKind.STRING).value + if sub.kind == TokenKind.DENSE: + model = m + else: + sparse_model = m + elif self._peek().kind == TokenKind.VECTOR: + self._advance() + name = self._expect(TokenKind.STRING).value + if sub.kind == TokenKind.DENSE: + dense_vector = name + else: + sparse_vector = name else: - sparse_model = m + raise QQLSyntaxError( + "Expected MODEL or VECTOR after DENSE/SPARSE in USING HYBRID", + self._peek().pos, + ) elif self._peek().kind == TokenKind.SPARSE: self._advance() # consume SPARSE sparse_only = True - if self._peek().kind == TokenKind.MODEL: - self._advance() # consume MODEL - sparse_model = self._expect(TokenKind.STRING).value + while self._peek().kind in (TokenKind.MODEL, TokenKind.VECTOR): + sub = self._advance() + if sub.kind == TokenKind.MODEL: + sparse_model = self._expect(TokenKind.STRING).value + else: + sparse_vector = self._expect(TokenKind.STRING).value + elif self._peek().kind == TokenKind.VECTOR: + self._advance() + dense_vector = self._expect(TokenKind.STRING).value else: self._expect(TokenKind.MODEL) model = self._expect(TokenKind.STRING).value @@ -743,6 +825,8 @@ def _parse_search(self) -> SearchStmt: with_clause=with_clause, group_by=group_by, group_size=group_size, + dense_vector=dense_vector, + sparse_vector=sparse_vector, ) def _parse_recommend(self) -> RecommendStmt: @@ -844,6 +928,9 @@ def _parse_update(self) -> UpdateVectorStmt | UpdatePayloadStmt: if self._peek().kind == TokenKind.VECTOR: self._advance() # consume VECTOR + vector_name: str | None = None + if self._peek().kind == TokenKind.STRING: + vector_name = self._advance().value self._expect(TokenKind.WHERE) self._expect(TokenKind.ID) self._expect(TokenKind.EQUALS) @@ -872,6 +959,7 @@ def _parse_update(self) -> UpdateVectorStmt | UpdatePayloadStmt: collection=collection, point_id=point_id, vector=coerced, + vector_name=vector_name, ) if self._peek().kind == TokenKind.PAYLOAD: diff --git a/tests/test_dumper.py b/tests/test_dumper.py index 32ee430..a87be80 100644 --- a/tests/test_dumper.py +++ b/tests/test_dumper.py @@ -171,7 +171,7 @@ def test_writes_create_statement_hybrid(self, tmp_path, mocker): client = _make_client(mocker, hybrid=True, points=[{"text": "hello"}]) dump_collection("col", out, client, null_console(), null_console()) content = (tmp_path / "dump.qql").read_text() - assert "CREATE COLLECTION col HYBRID" in content + assert "CREATE COLLECTION col USING HYBRID DENSE VECTOR 'dense' SPARSE VECTOR 'sparse'" in content def test_writes_collection_config_and_quantization(self, tmp_path, mocker): from qdrant_client.models import ( @@ -213,7 +213,7 @@ def test_hybrid_insert_bulk_has_using_hybrid(self, tmp_path, mocker): client = _make_client(mocker, hybrid=True, points=[{"text": "hello"}]) dump_collection("col", out, client, null_console(), null_console()) content = (tmp_path / "dump.qql").read_text() - assert "] USING HYBRID" in content + assert "] USING HYBRID DENSE VECTOR 'dense' SPARSE VECTOR 'sparse'" in content def test_dense_insert_bulk_has_no_using_clause(self, tmp_path, mocker): out = str(tmp_path / "dump.qql") diff --git a/tests/test_executor.py b/tests/test_executor.py index 41404d7..3f1a366 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -293,7 +293,7 @@ def test_create_collection_passes_all_new_config_blocks(self, executor, mock_cli ) executor.execute(node) kw = mock_client.create_collection.call_args.kwargs - assert kw["vectors_config"].on_disk is True + assert kw["vectors_config"]["dense"].on_disk is True assert kw["hnsw_config"].m == 32 assert kw["hnsw_config"].ef_construct == 200 assert kw["hnsw_config"].full_scan_threshold == 5000 @@ -358,6 +358,24 @@ def test_alter_collection_named_vectors_use_dense_key(self, executor, mock_clien kw = mock_client.update_collection.call_args.kwargs assert kw["vectors_config"]["dense"].on_disk is True + def test_alter_collection_vectors_rejects_multiple_dense_vectors( + self, executor, mock_client + ): + from qdrant_client.models import Distance, VectorParams + + mock_client.collection_exists.return_value = True + mock_client.get_collection.return_value.config.params.vectors = { + "title": VectorParams(size=384, distance=Distance.COSINE), + "body": VectorParams(size=384, distance=Distance.COSINE), + } + mock_client.get_collection.return_value.config.params.sparse_vectors = None + node = AlterCollectionStmt( + collection="named_col", + config=CollectionConfig(vectors=VectorsConfig(on_disk=True)), + ) + with pytest.raises(QQLRuntimeError, match="one dense vector"): + executor.execute(node) + def test_alter_collection_can_disable_quantization(self, executor, mock_client): mock_client.collection_exists.return_value = True node = AlterCollectionStmt( @@ -368,11 +386,11 @@ def test_alter_collection_can_disable_quantization(self, executor, mock_client): kw = mock_client.update_collection.call_args.kwargs assert kw["quantization_config"].value == "Disabled" - def test_collection_is_hybrid_depends_on_sparse_vectors(self, executor, mock_client, mocker): + def test_resolved_topology_depends_on_sparse_vectors(self, executor, mock_client, mocker): mock_client.collection_exists.return_value = True mock_client.get_collection.return_value.config.params.vectors = {"dense": object()} mock_client.get_collection.return_value.config.params.sparse_vectors = None - assert executor._collection_is_hybrid("named_dense") is False + assert executor._resolve_topology("named_dense").is_hybrid is False def test_insert_named_dense_collection_uses_named_vector_payload(self, executor, mock_client): mock_client.collection_exists.return_value = True @@ -519,7 +537,8 @@ def test_create_dense_with_model_uses_scalar_vectors(self, mock_client, cfg, moc node = CreateCollectionStmt(collection="col", model="BAAI/bge-base-en-v1.5") executor.execute(node) kw = mock_client.create_collection.call_args.kwargs - assert isinstance(kw["vectors_config"], VectorParams) + assert isinstance(kw["vectors_config"], dict) + assert isinstance(kw["vectors_config"]["dense"], VectorParams) assert "sparse_vectors_config" not in kw def test_create_existing_noop_with_model(self, executor, mock_client): @@ -1034,7 +1053,7 @@ def test_search_with_indexed_only_and_quantization_forwards_search_params( assert search_params.quantization.oversampling == pytest.approx(2.5) def test_sparse_search_forwards_search_params(self, executor, mock_client, mocker): - mock_client.collection_exists.return_value = True + _mock_hybrid_collection(mock_client) mock_response = mocker.MagicMock() mock_response.points = [] mock_client.query_points.return_value = mock_response @@ -1616,6 +1635,45 @@ def mock_sparse_embedder(mocker): return mock +def _mock_named_dense_collection(mock_client, name: str = "dense", size: int = 384): + from qdrant_client.models import Distance, VectorParams + + mock_client.collection_exists.return_value = True + mock_client.get_collection.return_value.config.params.vectors = { + name: VectorParams(size=size, distance=Distance.COSINE) + } + mock_client.get_collection.return_value.config.params.sparse_vectors = None + + +def _mock_hybrid_collection( + mock_client, + dense_name: str = "dense", + sparse_name: str = "sparse", + size: int = 384, +): + from qdrant_client.models import Distance, SparseVectorParams, VectorParams + + mock_client.collection_exists.return_value = True + mock_client.get_collection.return_value.config.params.vectors = { + dense_name: VectorParams(size=size, distance=Distance.COSINE) + } + mock_client.get_collection.return_value.config.params.sparse_vectors = { + sparse_name: SparseVectorParams() + } + + +def _mock_unnamed_hybrid_collection(mock_client, sparse_name: str = "sparse"): + from qdrant_client.models import Distance, SparseVectorParams, VectorParams + + mock_client.collection_exists.return_value = True + mock_client.get_collection.return_value.config.params.vectors = VectorParams( + size=384, distance=Distance.COSINE + ) + mock_client.get_collection.return_value.config.params.sparse_vectors = { + sparse_name: SparseVectorParams() + } + + class TestHybridCreate: def test_create_hybrid_uses_named_vector_config(self, executor, mock_client): node = CreateCollectionStmt(collection="articles", hybrid=True) @@ -1641,15 +1699,28 @@ def test_create_non_hybrid_unchanged(self, executor, mock_client): node = CreateCollectionStmt(collection="col", hybrid=False) executor.execute(node) kw = mock_client.create_collection.call_args.kwargs - assert isinstance(kw["vectors_config"], VectorParams) + assert isinstance(kw["vectors_config"], dict) + assert isinstance(kw["vectors_config"]["dense"], VectorParams) assert "sparse_vectors_config" not in kw + def test_create_uses_explicit_vector_names(self, executor, mock_client): + node = CreateCollectionStmt( + collection="col", + hybrid=True, + dense_vector="emb", + sparse_vector="lex", + ) + executor.execute(node) + kw = mock_client.create_collection.call_args.kwargs + assert "emb" in kw["vectors_config"] + assert "lex" in kw["sparse_vectors_config"] + class TestHybridInsert: def test_hybrid_insert_upsert_has_named_vectors( self, executor, mock_client, mock_sparse_embedder ): - mock_client.collection_exists.return_value = True + _mock_hybrid_collection(mock_client) node = InsertStmt( collection="col", values={"text": "hello"}, model=None, hybrid=True ) @@ -1666,7 +1737,7 @@ def test_hybrid_insert_sparse_is_SparseVector( ): from qdrant_client.models import SparseVector - mock_client.collection_exists.return_value = True + _mock_hybrid_collection(mock_client) node = InsertStmt( collection="col", values={"text": "hello"}, model=None, hybrid=True ) @@ -1689,7 +1760,7 @@ def test_hybrid_insert_auto_creates_hybrid_collection( def test_hybrid_insert_skips_create_when_exists( self, executor, mock_client, mock_sparse_embedder ): - mock_client.collection_exists.return_value = True + _mock_hybrid_collection(mock_client) node = InsertStmt( collection="col", values={"text": "hello"}, model=None, hybrid=True ) @@ -1699,7 +1770,7 @@ def test_hybrid_insert_skips_create_when_exists( def test_hybrid_insert_uses_custom_dense_model( self, executor, mock_client, mock_sparse_embedder, mocker ): - mock_client.collection_exists.return_value = True + _mock_hybrid_collection(mock_client) node = InsertStmt( collection="col", values={"text": "hi"}, model="BAAI/bge-small-en-v1.5", hybrid=True, @@ -1713,7 +1784,7 @@ def test_hybrid_insert_uses_custom_dense_model( def test_hybrid_insert_uses_custom_sparse_model( self, executor, mock_client, mocker ): - mock_client.collection_exists.return_value = True + _mock_hybrid_collection(mock_client) mock_sparse = mocker.MagicMock() mock_sparse.embed.return_value = FAKE_SPARSE sparse_cls = mocker.patch("qql.executor.SparseEmbedder", return_value=mock_sparse) @@ -1724,13 +1795,24 @@ def test_hybrid_insert_uses_custom_sparse_model( executor.execute(node) sparse_cls.assert_called_once_with("prithivida/Splade_PP_en_v1") + def test_hybrid_insert_uses_external_vector_names( + self, executor, mock_client, mock_sparse_embedder + ): + _mock_hybrid_collection(mock_client, dense_name="emb", sparse_name="lex") + node = InsertStmt( + collection="col", values={"text": "hello"}, model=None, hybrid=True + ) + executor.execute(node) + point = mock_client.upsert.call_args.kwargs["points"][0] + assert set(point.vector) == {"emb", "lex"} + def test_non_hybrid_insert_uses_flat_vector(self, executor, mock_client): node = InsertStmt( collection="col", values={"text": "hello"}, model=None, hybrid=False ) executor.execute(node) point = mock_client.upsert.call_args.kwargs["points"][0] - assert isinstance(point.vector, list) + assert point.vector == {"dense": FAKE_VECTOR} def test_hybrid_insert_missing_text_raises( self, executor, mock_client, mock_sparse_embedder @@ -1741,12 +1823,35 @@ def test_hybrid_insert_missing_text_raises( with pytest.raises(QQLRuntimeError, match="'text' field"): executor.execute(node) + def test_hybrid_insert_rejects_unnamed_dense_collection( + self, executor, mock_client, mock_sparse_embedder + ): + _mock_unnamed_hybrid_collection(mock_client) + node = InsertStmt( + collection="col", values={"text": "hello"}, model=None, hybrid=True + ) + with pytest.raises(QQLRuntimeError, match="named dense vectors"): + executor.execute(node) + + def test_hybrid_bulk_insert_rejects_unnamed_dense_collection( + self, executor, mock_client, mock_sparse_embedder + ): + _mock_unnamed_hybrid_collection(mock_client) + node = InsertBulkStmt( + collection="col", values_list=({"text": "hello"},), model=None, hybrid=True + ) + with pytest.raises(QQLRuntimeError, match="named dense vectors"): + executor.execute(node) + class TestHybridSearch: + @pytest.fixture(autouse=True) + def _collection(self, mock_client): + _mock_hybrid_collection(mock_client) + def test_hybrid_search_uses_prefetch( self, executor, mock_client, mock_sparse_embedder, mocker ): - mock_client.collection_exists.return_value = True mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -1767,7 +1872,6 @@ def test_hybrid_search_uses_rrf_fusion( ): from qdrant_client.models import Fusion, FusionQuery - mock_client.collection_exists.return_value = True mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -1785,7 +1889,6 @@ def test_hybrid_search_uses_dbsf_fusion( ): from qdrant_client.models import Fusion, FusionQuery - mock_client.collection_exists.return_value = True mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -1806,7 +1909,6 @@ def test_hybrid_search_uses_dbsf_fusion( def test_hybrid_search_prefetch_limit_is_4x( self, executor, mock_client, mock_sparse_embedder, mocker ): - mock_client.collection_exists.return_value = True mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -1821,7 +1923,6 @@ def test_hybrid_search_prefetch_limit_is_4x( def test_hybrid_search_prefetch_using_fields( self, executor, mock_client, mock_sparse_embedder, mocker ): - mock_client.collection_exists.return_value = True mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -1834,10 +1935,82 @@ def test_hybrid_search_prefetch_using_fields( usings = {p.using for p in prefetches} assert usings == {"dense", "sparse"} - def test_hybrid_search_forwards_search_params_to_prefetch( + def test_hybrid_search_uses_external_vector_names( self, executor, mock_client, mock_sparse_embedder, mocker ): + _mock_hybrid_collection(mock_client, dense_name="emb", sparse_name="lex") + mock_resp = mocker.MagicMock() + mock_resp.points = [] + mock_client.query_points.return_value = mock_resp + + node = SearchStmt( + collection="col", query_text="q", limit=5, model=None, hybrid=True + ) + executor.execute(node) + prefetches = mock_client.query_points.call_args.kwargs["prefetch"] + assert {p.using for p in prefetches} == {"emb", "lex"} + + def test_dense_search_requires_explicit_name_for_ambiguous_collection( + self, executor, mock_client, mocker + ): + from qdrant_client.models import Distance, VectorParams + mock_client.collection_exists.return_value = True + mock_client.get_collection.return_value.config.params.vectors = { + "title": VectorParams(size=384, distance=Distance.COSINE), + "body": VectorParams(size=384, distance=Distance.COSINE), + } + mock_client.get_collection.return_value.config.params.sparse_vectors = None + node = SearchStmt(collection="col", query_text="q", limit=5, model=None) + with pytest.raises(QQLRuntimeError, match="multiple dense vectors"): + executor.execute(node) + + def test_dense_search_explicit_vector_resolves_ambiguous_collection( + self, executor, mock_client, mocker + ): + from qdrant_client.models import Distance, VectorParams + + mock_client.collection_exists.return_value = True + mock_client.get_collection.return_value.config.params.vectors = { + "title": VectorParams(size=384, distance=Distance.COSINE), + "body": VectorParams(size=384, distance=Distance.COSINE), + } + mock_client.get_collection.return_value.config.params.sparse_vectors = None + mock_resp = mocker.MagicMock() + mock_resp.points = [] + mock_client.query_points.return_value = mock_resp + + node = SearchStmt( + collection="col", query_text="q", limit=5, model=None, dense_vector="body" + ) + executor.execute(node) + assert mock_client.query_points.call_args.kwargs["using"] == "body" + + def test_dense_search_explicit_vector_unknown_name_raises( + self, executor, mock_client + ): + from qdrant_client.models import Distance, VectorParams + + mock_client.collection_exists.return_value = True + mock_client.get_collection.return_value.config.params.vectors = { + "title": VectorParams(size=384, distance=Distance.COSINE), + "body": VectorParams(size=384, distance=Distance.COSINE), + } + mock_client.get_collection.return_value.config.params.sparse_vectors = None + + node = SearchStmt( + collection="col", + query_text="q", + limit=5, + model=None, + dense_vector="missing_name", + ) + with pytest.raises(QQLRuntimeError, match="no dense vector named"): + executor.execute(node) + + def test_hybrid_search_forwards_search_params_to_prefetch( + self, executor, mock_client, mock_sparse_embedder, mocker + ): mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -1863,7 +2036,6 @@ def test_hybrid_search_with_where_filter( from qql.ast_nodes import CompareExpr from qdrant_client.models import Filter - mock_client.collection_exists.return_value = True mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -1888,7 +2060,6 @@ def test_hybrid_search_nonexistent_collection_raises( executor.execute(node) def test_non_hybrid_search_unchanged(self, executor, mock_client, mocker): - mock_client.collection_exists.return_value = True mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -1903,7 +2074,6 @@ def test_non_hybrid_search_unchanged(self, executor, mock_client, mocker): def test_hybrid_search_uses_custom_sparse_model( self, executor, mock_client, mocker ): - mock_client.collection_exists.return_value = True mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -1920,17 +2090,17 @@ def test_hybrid_search_uses_custom_sparse_model( sparse_cls.assert_called_once_with("prithivida/Splade_PP_en_v1") -class TestEnsureCollectionHybridCompat: - def test_named_vector_collection_skips_validation(self, executor, mock_client): +class TestEnsureCollectionValidation: + def test_named_vector_collection_validates_selected_vector(self, executor, mock_client): from qdrant_client.models import VectorParams mock_client.collection_exists.return_value = True - # Simulate a named-vector (hybrid) collection: vectors is a dict mock_client.get_collection.return_value.config.params.vectors = { "dense": VectorParams(size=384, distance="Cosine") } - # Should not raise even with a different size argument - executor._ensure_collection("hybrid_col", 384) + mock_client.get_collection.return_value.config.params.sparse_vectors = None + topology = executor._resolve_topology("hybrid_col") + executor._ensure_collection("hybrid_col", 384, topology, None) mock_client.create_collection.assert_not_called() def test_unnamed_vector_mismatch_still_raises(self, executor, mock_client): @@ -1940,8 +2110,10 @@ def test_unnamed_vector_mismatch_still_raises(self, executor, mock_client): mock_client.get_collection.return_value.config.params.vectors = VectorParams( size=768, distance="Cosine" ) + mock_client.get_collection.return_value.config.params.sparse_vectors = None + topology = executor._resolve_topology("col") with pytest.raises(QQLRuntimeError, match="dimension mismatch"): - executor._ensure_collection("col", 384) + executor._ensure_collection("col", 384, topology, None) FAKE_SPARSE = {"indices": [1, 42, 100], "values": [0.22, 0.8, 0.3]} @@ -2102,7 +2274,7 @@ def test_rerank_custom_model_forwarded( def test_rerank_hybrid_search_message( self, executor, mock_client, mock_cross_encoder, mocker ): - mock_client.collection_exists.return_value = True + _mock_hybrid_collection(mock_client) mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -2122,6 +2294,10 @@ def test_rerank_hybrid_search_message( class TestSparseOnlySearch: + @pytest.fixture(autouse=True) + def _collection(self, mock_client): + _mock_hybrid_collection(mock_client) + @pytest.fixture def mock_sparse(self, mocker): mock = mocker.MagicMock() @@ -2132,7 +2308,6 @@ def mock_sparse(self, mocker): def test_sparse_only_calls_query_embed( self, executor, mock_client, mock_sparse, mocker ): - mock_client.collection_exists.return_value = True mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -2147,7 +2322,6 @@ def test_sparse_only_calls_query_embed( def test_sparse_only_queries_sparse_vector_name( self, executor, mock_client, mock_sparse, mocker ): - mock_client.collection_exists.return_value = True mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -2163,7 +2337,6 @@ def test_sparse_only_queries_sparse_vector_name( def test_sparse_only_message_contains_sparse( self, executor, mock_client, mock_sparse, mocker ): - mock_client.collection_exists.return_value = True mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -2178,7 +2351,6 @@ def test_sparse_only_message_contains_sparse( def test_sparse_only_uses_custom_model( self, executor, mock_client, mocker ): - mock_client.collection_exists.return_value = True mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -2197,7 +2369,6 @@ def test_sparse_only_uses_custom_model( def test_sparse_only_uses_default_model_when_none( self, executor, mock_client, mocker ): - mock_client.collection_exists.return_value = True mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -2219,7 +2390,6 @@ def test_sparse_only_uses_default_model_when_none( def test_sparse_only_with_rerank_message( self, executor, mock_client, mock_sparse, mocker ): - mock_client.collection_exists.return_value = True mock_resp = mocker.MagicMock() mock_resp.points = [] mock_client.query_points.return_value = mock_resp @@ -2574,7 +2744,7 @@ def test_group_by_passes_group_size_to_qdrant(self, executor, mock_client, mocke assert kwargs["group_by"] == "tag" def test_group_by_hybrid_uses_query_points_groups(self, executor, mock_client, mocker): - mock_client.collection_exists.return_value = True + _mock_hybrid_collection(mock_client) mock_response = mocker.MagicMock() mock_response.groups = [] mock_client.query_points_groups.return_value = mock_response @@ -2630,8 +2800,12 @@ def test_update_vector_calls_update_vectors(self, executor, mock_client): def test_update_vector_passes_correct_point_id(self, executor, mock_client): from qql.ast_nodes import UpdateVectorStmt + from qdrant_client.models import VectorParams mock_client.collection_exists.return_value = True - mock_client.get_collection.return_value.config.params.vectors = {} # non-dict → unnamed + mock_client.get_collection.return_value.config.params.vectors = VectorParams( + size=2, distance="Cosine" + ) + mock_client.get_collection.return_value.config.params.sparse_vectors = None node = UpdateVectorStmt( collection="notes", point_id=42, vector=(0.5, 0.6) ) @@ -2733,8 +2907,11 @@ def test_update_payload_passes_wait_true(self, executor, mock_client): class TestSearchGroupBySparse: """Gap 1 & 6 — sparse-only grouped search must use the sparse path.""" + @pytest.fixture(autouse=True) + def _collection(self, mock_client): + _mock_hybrid_collection(mock_client) + def test_sparse_only_grouped_calls_query_points_groups(self, executor, mock_client, mocker): - mock_client.collection_exists.return_value = True mock_response = mocker.MagicMock() mock_response.groups = [] mock_client.query_points_groups.return_value = mock_response @@ -2757,7 +2934,6 @@ def test_sparse_only_grouped_calls_query_points_groups(self, executor, mock_clie mock_client.query_points.assert_not_called() def test_sparse_only_grouped_label_in_message(self, executor, mock_client, mocker): - mock_client.collection_exists.return_value = True mock_response = mocker.MagicMock() mock_response.groups = [] mock_client.query_points_groups.return_value = mock_response @@ -2780,7 +2956,7 @@ class TestSearchGroupByAdvanced: def test_grouped_hybrid_fusion_dbsf(self, executor, mock_client, mocker): from qdrant_client.models import Fusion - mock_client.collection_exists.return_value = True + _mock_hybrid_collection(mock_client) mock_response = mocker.MagicMock() mock_response.groups = [] mock_client.query_points_groups.return_value = mock_response @@ -2818,11 +2994,12 @@ class TestUpdateVectorVectorShape: """Gaps 12 & 13 — verify exact vector shape sent to Qdrant for named/unnamed collections.""" def test_update_vector_unnamed_collection_sends_plain_list(self, executor, mock_client): + from qdrant_client.models import Distance, VectorParams from qql.ast_nodes import UpdateVectorStmt + mock_client.collection_exists.return_value = True - # Unnamed collection: get_collection returns non-dict vectors info = mock_client.get_collection.return_value - info.config.params.vectors = [None] # list → not a dict → unnamed + info.config.params.vectors = VectorParams(size=3, distance=Distance.COSINE) node = UpdateVectorStmt(collection="articles", point_id=1, vector=(0.1, 0.2, 0.3)) executor.execute(node) @@ -2834,9 +3011,9 @@ def test_update_vector_unnamed_collection_sends_plain_list(self, executor, mock_ def test_update_vector_named_collection_sends_dict(self, executor, mock_client): from qql.ast_nodes import UpdateVectorStmt mock_client.collection_exists.return_value = True - # Named collection: get_collection returns dict vectors info = mock_client.get_collection.return_value - info.config.params.vectors = {"dense": object(), "sparse": object()} # dict → named + info.config.params.vectors = {"dense": object()} + info.config.params.sparse_vectors = {"sparse": object()} node = UpdateVectorStmt(collection="articles", point_id="id-1", vector=(0.5, 0.6)) executor.execute(node) @@ -2904,11 +3081,16 @@ class TestGroupedCustomModelForwarding: def test_grouped_dense_custom_model_forwarded(self, executor, mock_client, mocker): """Dense grouped search with USING MODEL must instantiate Embedder with that model.""" + from qdrant_client.models import VectorParams + mock_client.collection_exists.return_value = True mock_response = mocker.MagicMock() mock_response.groups = [] mock_client.query_points_groups.return_value = mock_response - mock_client.get_collection.return_value.config.params.vectors = None # unnamed + mock_client.get_collection.return_value.config.params.vectors = VectorParams( + size=384, distance="Cosine" + ) + mock_client.get_collection.return_value.config.params.sparse_vectors = None embedder_cls = mocker.patch("qql.executor.Embedder") embedder_instance = mocker.MagicMock() @@ -2926,7 +3108,7 @@ def test_grouped_dense_custom_model_forwarded(self, executor, mock_client, mocke def test_grouped_hybrid_custom_dense_model_forwarded(self, executor, mock_client, mocker): """Hybrid grouped search must instantiate Embedder with the custom dense model.""" - mock_client.collection_exists.return_value = True + _mock_hybrid_collection(mock_client) mock_response = mocker.MagicMock() mock_response.groups = [] mock_client.query_points_groups.return_value = mock_response @@ -2947,7 +3129,7 @@ def test_grouped_hybrid_custom_dense_model_forwarded(self, executor, mock_client def test_grouped_hybrid_custom_sparse_model_forwarded(self, executor, mock_client, mocker): """Hybrid grouped search must instantiate SparseEmbedder with the custom sparse model.""" - mock_client.collection_exists.return_value = True + _mock_hybrid_collection(mock_client) mock_response = mocker.MagicMock() mock_response.groups = [] mock_client.query_points_groups.return_value = mock_response @@ -2972,11 +3154,16 @@ class TestGroupedSearchParamsDepth: def test_grouped_search_hnsw_ef_value_forwarded(self, executor, mock_client, mocker): """search_params for dense grouped search must carry the hnsw_ef value.""" + from qdrant_client.models import VectorParams + mock_client.collection_exists.return_value = True mock_response = mocker.MagicMock() mock_response.groups = [] mock_client.query_points_groups.return_value = mock_response - mock_client.get_collection.return_value.config.params.vectors = None # unnamed + mock_client.get_collection.return_value.config.params.vectors = VectorParams( + size=384, distance="Cosine" + ) + mock_client.get_collection.return_value.config.params.sparse_vectors = None embedder_cls = mocker.patch("qql.executor.Embedder") embedder_cls.return_value.embed.return_value = [0.1] * 384 @@ -2994,11 +3181,16 @@ def test_grouped_search_hnsw_ef_value_forwarded(self, executor, mock_client, moc def test_grouped_search_exact_value_forwarded(self, executor, mock_client, mocker): """search_params for dense grouped search must carry exact=True when set.""" + from qdrant_client.models import VectorParams + mock_client.collection_exists.return_value = True mock_response = mocker.MagicMock() mock_response.groups = [] mock_client.query_points_groups.return_value = mock_response - mock_client.get_collection.return_value.config.params.vectors = None # unnamed + mock_client.get_collection.return_value.config.params.vectors = VectorParams( + size=384, distance="Cosine" + ) + mock_client.get_collection.return_value.config.params.sparse_vectors = None embedder_cls = mocker.patch("qql.executor.Embedder") embedder_cls.return_value.embed.return_value = [0.1] * 384 @@ -3016,7 +3208,7 @@ def test_grouped_search_exact_value_forwarded(self, executor, mock_client, mocke def test_grouped_hybrid_prefetch_params_forwarded(self, executor, mock_client, mocker): """Hybrid grouped search must forward search_params into each Prefetch.params.""" - mock_client.collection_exists.return_value = True + _mock_hybrid_collection(mock_client) mock_response = mocker.MagicMock() mock_response.groups = [] mock_client.query_points_groups.return_value = mock_response @@ -3066,7 +3258,7 @@ def test_flat_hybrid_search_uses_build_hybrid_vectors(self, executor, mock_clien """The flat hybrid path must call _build_hybrid_vectors (not inline Embedder calls).""" from qdrant_client.models import SparseVector - mock_client.collection_exists.return_value = True + _mock_hybrid_collection(mock_client) mock_response = mocker.MagicMock() mock_response.points = [] mock_client.query_points.return_value = mock_response @@ -3088,7 +3280,7 @@ def test_grouped_hybrid_search_uses_build_hybrid_vectors(self, executor, mock_cl """The grouped hybrid path must call _build_hybrid_vectors (not inline Embedder calls).""" from qdrant_client.models import SparseVector - mock_client.collection_exists.return_value = True + _mock_hybrid_collection(mock_client) mock_response = mocker.MagicMock() mock_response.groups = [] mock_client.query_points_groups.return_value = mock_response diff --git a/tests/test_parser.py b/tests/test_parser.py index efa30cd..3f95f89 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -27,7 +27,6 @@ OrExpr, QuantizationUpdate, QuantizationType, - QuantizationSearchWith, RecommendStmt, SelectStmt, ScrollStmt, @@ -779,6 +778,11 @@ def test_insert_using_model_still_works(self): assert node.model == "my-model" assert node.sparse_model is None + def test_insert_using_vector_sets_dense_vector_name(self): + node = parse("INSERT INTO COLLECTION col VALUES {'text': 'hi'} USING VECTOR 'emb'") + assert node.hybrid is False + assert node.dense_vector == "emb" + def test_insert_hybrid_dense_model(self): node = parse( "INSERT INTO COLLECTION col VALUES {'text': 'hi'} " @@ -815,6 +819,25 @@ def test_insert_hybrid_both_models_reversed_order(self): assert node.model == "BAAI/bge-base-en-v1.5" assert node.sparse_model == "Qdrant/bm25" + def test_insert_hybrid_vector_names(self): + node = parse( + "INSERT INTO COLLECTION col VALUES {'text': 'hi'} " + "USING HYBRID DENSE VECTOR 'emb' SPARSE VECTOR 'lex'" + ) + assert node.hybrid is True + assert node.dense_vector == "emb" + assert node.sparse_vector == "lex" + + def test_insert_bulk_hybrid_vector_names(self): + node = parse( + "INSERT BULK INTO COLLECTION col VALUES [{'text': 'hi'}] " + "USING HYBRID DENSE VECTOR 'emb' SPARSE VECTOR 'lex'" + ) + assert isinstance(node, InsertBulkStmt) + assert node.hybrid is True + assert node.dense_vector == "emb" + assert node.sparse_vector == "lex" + class TestHybridSearch: def test_search_using_hybrid_sets_flag(self): @@ -835,6 +858,11 @@ def test_search_using_model_still_works(self): assert node.model == "my-model" assert node.sparse_model is None + def test_search_using_vector_sets_dense_vector_name(self): + node = parse("SEARCH articles SIMILAR TO 'ml' LIMIT 5 USING VECTOR 'emb'") + assert node.hybrid is False + assert node.dense_vector == "emb" + def test_search_hybrid_dense_model(self): node = parse( "SEARCH articles SIMILAR TO 'ml' LIMIT 10 " @@ -871,6 +899,20 @@ def test_search_hybrid_both_models_reversed_order(self): assert node.model == "BAAI/bge-base-en-v1.5" assert node.sparse_model == "Qdrant/bm25" + def test_search_hybrid_vector_names(self): + node = parse( + "SEARCH articles SIMILAR TO 'ml' LIMIT 10 " + "USING HYBRID DENSE VECTOR 'emb' SPARSE VECTOR 'lex'" + ) + assert node.hybrid is True + assert node.dense_vector == "emb" + assert node.sparse_vector == "lex" + + def test_search_sparse_vector_name(self): + node = parse("SEARCH articles SIMILAR TO 'ml' LIMIT 10 USING SPARSE VECTOR 'lex'") + assert node.sparse_only is True + assert node.sparse_vector == "lex" + def test_search_hybrid_with_where(self): node = parse( "SEARCH articles SIMILAR TO 'ml' LIMIT 10 USING HYBRID WHERE year > 2020" @@ -1089,14 +1131,6 @@ def test_with_trailing_comma(self): node = parse("SEARCH col SIMILAR TO 'q' LIMIT 5 WITH { hnsw_ef: 256, }") assert node.with_clause.hnsw_ef == 256 - def test_with_quantization_unknown_key_raises(self): - with pytest.raises(QQLSyntaxError): - parse( - "SEARCH col SIMILAR TO 'q' LIMIT 5 " - "WITH { quantization: { unknown: true } }" - ) - - class TestSparseOnlySearch: def test_using_sparse_sets_flag(self): node = parse("SEARCH col SIMILAR TO 'q' LIMIT 5 USING SPARSE") @@ -1336,6 +1370,20 @@ def test_turbo_with_model(self): assert node.quantization.type == QuantizationType.TURBO assert node.quantization.turbo_bits == 1.5 + def test_create_using_vector_sets_dense_vector_name(self): + node = parse("CREATE COLLECTION articles USING VECTOR 'emb'") + assert node.dense_vector == "emb" + assert node.hybrid is False + + def test_create_hybrid_vector_names(self): + node = parse( + "CREATE COLLECTION articles USING HYBRID " + "DENSE VECTOR 'emb' SPARSE VECTOR 'lex'" + ) + assert node.hybrid is True + assert node.dense_vector == "emb" + assert node.sparse_vector == "lex" + def test_turbo_with_hybrid_dense_model(self): node = parse("CREATE COLLECTION articles USING HYBRID DENSE MODEL 'x' QUANTIZE TURBO BITS 1 ALWAYS RAM") assert node.hybrid is True @@ -1428,6 +1476,13 @@ def test_update_vector_by_string_id(self): assert node.point_id == "abc-123" assert node.vector == (0.1, 0.2, 0.3) + def test_update_vector_with_vector_name(self): + from qql.ast_nodes import UpdateVectorStmt + node = parse("UPDATE articles SET VECTOR 'emb' WHERE id = 'abc-123' [0.1, 0.2]") + assert isinstance(node, UpdateVectorStmt) + assert node.vector_name == "emb" + assert node.vector == (0.1, 0.2) + def test_update_vector_by_integer_id(self): from qql.ast_nodes import UpdateVectorStmt node = parse("UPDATE articles SET VECTOR WHERE id = 42 [0.1, 0.2]")