From 26798ee24aa5d35e31e5ca4317d7ba98e4f2ee6a Mon Sep 17 00:00:00 2001 From: Srimon Date: Mon, 18 May 2026 17:03:01 +0530 Subject: [PATCH 1/2] feat: Add ALTER statement support and enhance collection configuration - Introduced ALTER statement handling in the lexer and parser, allowing modifications to existing collections. - Expanded the TokenKind enum to include new keywords: VECTORS, OPTIMIZERS, PARAMS, and DISABLED. - Enhanced collection configuration parsing to support new configuration blocks for VECTORS, OPTIMIZERS, and PARAMS. - Updated the CreateCollectionStmt and AlterCollectionStmt to accept and process new configuration parameters. - Improved error handling for duplicate configuration blocks and invalid parameter usage. - Enhanced tests to cover new functionality, including parsing and execution of ALTER statements and collection configurations. - Updated comment stripping logic to preserve double dashes within string literals. --- README.md | 4 +- docs/collections.md | 36 +++-- docs/scripts.md | 2 +- src/qql/ast_nodes.py | 62 ++++++++- src/qql/cli.py | 39 +++++- src/qql/dumper.py | 127 ++++++++++++++++- src/qql/executor.py | 275 +++++++++++++++++++++++++++++++++++-- src/qql/lexer.py | 10 ++ src/qql/parser.py | 301 +++++++++++++++++++++++++++++++++++------ src/qql/script.py | 50 +++++-- tests/test_dumper.py | 62 +++++++++ tests/test_executor.py | 175 +++++++++++++++++++++++- tests/test_lexer.py | 20 +++ tests/test_parser.py | 94 +++++++++++++ tests/test_script.py | 18 +++ 15 files changed, 1187 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 4dc5663..372a8b4 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,9 @@ SELECT * FROM articles WHERE id = '3f2e1a4b-...' -- Collections CREATE COLLECTION articles CREATE COLLECTION articles HYBRID -CREATE COLLECTION articles HNSW { payload_m: 16 } +CREATE COLLECTION articles WITH HNSW { payload_m: 16 } +CREATE COLLECTION articles WITH VECTORS { on_disk: true } WITH HNSW { full_scan_threshold: 10000 } +ALTER COLLECTION articles WITH OPTIMIZERS { indexing_threshold: 10000 } CREATE COLLECTION articles QUANTIZE SCALAR CREATE COLLECTION articles QUANTIZE TURBO CREATE COLLECTION articles QUANTIZE TURBO BITS 2 diff --git a/docs/collections.md b/docs/collections.md index ebb0862..89ba4f4 100644 --- a/docs/collections.md +++ b/docs/collections.md @@ -93,10 +93,22 @@ CREATE COLLECTION HYBRID CREATE COLLECTION USING MODEL '' CREATE COLLECTION USING HYBRID CREATE COLLECTION USING HYBRID DENSE MODEL '' -CREATE COLLECTION HNSW { payload_m: } -``` - -Any of the above forms can be followed by an optional `QUANTIZE` clause and/or `HNSW { payload_m: }`. +CREATE COLLECTION WITH VECTORS { on_disk: } +CREATE COLLECTION WITH HNSW { m, ef_construct, full_scan_threshold, max_indexing_threads, on_disk, payload_m, inline_storage } +CREATE COLLECTION WITH OPTIMIZERS { deleted_threshold, vacuum_min_vector_number, default_segment_number, max_segment_size, memmap_threshold, indexing_threshold, flush_interval_sec, max_optimization_threads, prevent_unoptimized } +CREATE COLLECTION WITH PARAMS { replication_factor, write_consistency_factor, on_disk_payload } +ALTER COLLECTION WITH HNSW { ... } +ALTER COLLECTION WITH VECTORS { ... } +ALTER COLLECTION WITH OPTIMIZERS { ... } +ALTER COLLECTION WITH PARAMS { ... } +ALTER COLLECTION QUANTIZE SCALAR [QUANTILE <0.0–1.0>] [ALWAYS RAM] +ALTER COLLECTION QUANTIZE BINARY [ALWAYS RAM] +ALTER COLLECTION QUANTIZE PRODUCT [ALWAYS RAM] +ALTER COLLECTION QUANTIZE TURBO [BITS <1|1.5|2|4>] [ALWAYS RAM] +ALTER COLLECTION QUANTIZE DISABLED +``` + +Any `CREATE COLLECTION` form can be followed by an optional `QUANTIZE` clause and one or more `WITH ... { ... }` config blocks. **Examples:** @@ -122,21 +134,27 @@ CREATE COLLECTION research_papers USING HYBRID DENSE MODEL 'BAAI/bge-base-en-v1. Dense collection with payload-aware HNSW links: ```sql -CREATE COLLECTION research_papers HNSW {payload_m: 16} +CREATE COLLECTION research_papers WITH HNSW { payload_m: 16 } ``` When `USING MODEL` is omitted, the collection uses the **default embedding model's dimensions** (384 for `all-MiniLM-L6-v2`). If the collection already exists, the command succeeds with a message and does nothing. -### HNSW clause +### Collection config blocks -QQL currently supports one explicit HNSW knob during collection creation: +QQL supports the same config blocks on both `CREATE COLLECTION` and `ALTER COLLECTION`: -- `payload_m` — enables payload-aware HNSW connectivity used by Qdrant for filtered / tenant-aware workloads +- `WITH VECTORS { on_disk }` +- `WITH HNSW { m, ef_construct, full_scan_threshold, max_indexing_threads, on_disk, payload_m, inline_storage }` +- `WITH OPTIMIZERS { deleted_threshold, vacuum_min_vector_number, default_segment_number, max_segment_size, memmap_threshold, indexing_threshold, flush_interval_sec, max_optimization_threads, prevent_unoptimized }` +- `WITH PARAMS { replication_factor, write_consistency_factor, on_disk_payload }` on create +- `WITH PARAMS { replication_factor, write_consistency_factor, read_fan_out_factor, read_fan_out_delay_ms, on_disk_payload }` on alter +- `ALTER COLLECTION ... QUANTIZE ...` supports the same quantization forms as create, plus `QUANTIZE DISABLED` Example: ```sql -CREATE COLLECTION tenant_docs USING HYBRID HNSW {payload_m: 16} +CREATE COLLECTION tenant_docs USING HYBRID WITH HNSW { payload_m: 16, m: 32 } +ALTER COLLECTION tenant_docs WITH OPTIMIZERS { indexing_threshold: 10000 } ``` --- diff --git a/docs/scripts.md b/docs/scripts.md index e009efa..319707e 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -74,7 +74,7 @@ Done. 3/3 statement(s) succeeded. Export every point in a collection to a `.qql` script file. The generated file is valid QQL that re-creates the collection and re-inserts all payload data. Points are written in batches of 50 as `INSERT BULK` statements. -> **Scope of a dump:** The generated script preserves collection topology (dense vs hybrid) and all point payloads. It does **not** preserve quantization config, pinned model / vector dimensions, or payload indexes — those must be re-applied manually after import if needed. +> **Scope of a dump:** The generated script preserves collection topology (dense vs hybrid), collection config blocks, quantization config, and all point payloads. It does **not** preserve pinned model / vector dimensions or payload indexes — those must be re-applied manually after import if needed. **CLI usage:** ```bash diff --git a/src/qql/ast_nodes.py b/src/qql/ast_nodes.py index 3333230..dbd8fd4 100644 --- a/src/qql/ast_nodes.py +++ b/src/qql/ast_nodes.py @@ -40,6 +40,58 @@ class QuantizationSearchWith: oversampling: float | None = None +@dataclass(frozen=True) +class VectorsConfig: + on_disk: bool | None = None + + +@dataclass(frozen=True) +class HnswRuntimeConfig: + m: int | None = None + ef_construct: int | None = None + full_scan_threshold: int | None = None + max_indexing_threads: int | None = None + on_disk: bool | None = None + payload_m: int | None = None + inline_storage: bool | None = None + + +@dataclass(frozen=True) +class OptimizersRuntimeConfig: + deleted_threshold: float | None = None + vacuum_min_vector_number: int | None = None + default_segment_number: int | None = None + max_segment_size: int | None = None + memmap_threshold: int | None = None + indexing_threshold: int | None = None + flush_interval_sec: int | None = None + max_optimization_threads: int | str | None = None + prevent_unoptimized: bool | None = None + + +@dataclass(frozen=True) +class CollectionParamsConfig: + replication_factor: int | None = None + write_consistency_factor: int | None = None + read_fan_out_factor: int | None = None + read_fan_out_delay_ms: int | None = None + on_disk_payload: bool | None = None + + +@dataclass(frozen=True) +class CollectionConfig: + vectors: VectorsConfig | None = None + hnsw: HnswRuntimeConfig | None = None + optimizers: OptimizersRuntimeConfig | None = None + params: CollectionParamsConfig | None = None + + +@dataclass(frozen=True) +class QuantizationUpdate: + config: QuantizationConfig | None = None + disabled: bool = False + + # ── Filter expression leaf nodes ────────────────────────────────────────────── @dataclass(frozen=True) @@ -172,7 +224,14 @@ class CreateCollectionStmt: hybrid: bool = False # if True, create with dense + sparse named vectors model: str | None = None # dense model; None → use config default quantization: QuantizationConfig | None = None # optional QUANTIZE clause - payload_m: int | None = None # optional HNSW { payload_m: N } clause + config: CollectionConfig | None = None + + +@dataclass(frozen=True) +class AlterCollectionStmt: + collection: str + config: CollectionConfig | None = None + quantization: QuantizationUpdate | None = None @dataclass(frozen=True) @@ -274,6 +333,7 @@ class UpdatePayloadStmt: InsertStmt | InsertBulkStmt | CreateCollectionStmt + | AlterCollectionStmt | CreateIndexStmt | DropCollectionStmt | ShowCollectionsStmt diff --git a/src/qql/cli.py b/src/qql/cli.py index 78b5200..1b072ba 100644 --- a/src/qql/cli.py +++ b/src/qql/cli.py @@ -38,12 +38,28 @@ Create a new collection. Add HYBRID for dense+sparse BM25 vectors. Optional: [yellow]USING MODEL[/yellow] '' Optional: [yellow]USING HYBRID[/yellow] [DENSE MODEL ''] - Optional: [yellow]HNSW[/yellow] { payload_m: } + Optional: [yellow]WITH VECTORS[/yellow] { on_disk: } + Optional: [yellow]WITH HNSW[/yellow] { m, ef_construct, full_scan_threshold, max_indexing_threads, on_disk, payload_m, inline_storage } + Optional: [yellow]WITH OPTIMIZERS[/yellow] { deleted_threshold, vacuum_min_vector_number, default_segment_number, max_segment_size, memmap_threshold, indexing_threshold, flush_interval_sec, max_optimization_threads, prevent_unoptimized } + Optional: [yellow]WITH PARAMS[/yellow] { replication_factor, write_consistency_factor, on_disk_payload } Optional: [yellow]QUANTIZE SCALAR[/yellow] [QUANTILE <0.0–1.0>] [ALWAYS RAM] Optional: [yellow]QUANTIZE BINARY[/yellow] [ALWAYS RAM] Optional: [yellow]QUANTIZE PRODUCT[/yellow] [ALWAYS RAM] (4× compression) + Optional: [yellow]QUANTIZE TURBO[/yellow] [BITS <1|1.5|2|4>] [ALWAYS RAM] QUANTIZE may be combined with any HYBRID or MODEL clause. + [yellow]ALTER COLLECTION[/yellow] + Update runtime collection config without recreating the collection. + Optional: [yellow]WITH VECTORS[/yellow] { on_disk: } + Optional: [yellow]WITH HNSW[/yellow] { m, ef_construct, full_scan_threshold, max_indexing_threads, on_disk, payload_m, inline_storage } + Optional: [yellow]WITH OPTIMIZERS[/yellow] { deleted_threshold, vacuum_min_vector_number, default_segment_number, max_segment_size, memmap_threshold, indexing_threshold, flush_interval_sec, max_optimization_threads, prevent_unoptimized } + Optional: [yellow]WITH PARAMS[/yellow] { replication_factor, write_consistency_factor, read_fan_out_factor, read_fan_out_delay_ms, on_disk_payload } + Optional: [yellow]QUANTIZE SCALAR[/yellow] [QUANTILE <0.0–1.0>] [ALWAYS RAM] + Optional: [yellow]QUANTIZE BINARY[/yellow] [ALWAYS RAM] + Optional: [yellow]QUANTIZE PRODUCT[/yellow] [ALWAYS RAM] + Optional: [yellow]QUANTIZE TURBO[/yellow] [BITS <1|1.5|2|4>] [ALWAYS RAM] + Optional: [yellow]QUANTIZE DISABLED[/yellow] + [yellow]DROP COLLECTION[/yellow] Delete a collection and all its points. @@ -400,7 +416,10 @@ def _format_collection_diagnostics(data: dict) -> str: vectors = data["vectors"] for vname, vconf in vectors.items(): label = f" Vector '{vname}'" if vname else " Vector" - lines.append(f"{label} : {vconf['size']} dims, {vconf['distance']} distance") + suffix = "" + if vconf.get("on_disk") is not None: + suffix = f", on_disk={vconf['on_disk']}" + lines.append(f"{label} : {vconf['size']} dims, {vconf['distance']} distance{suffix}") # Sparse vectors if data["sparse_vectors"]: @@ -421,6 +440,8 @@ def _format_collection_diagnostics(data: dict) -> str: lines.append(f" HNSW on_disk : {hnsw['on_disk']}") if hnsw.get("payload_m") is not None: lines.append(f" HNSW payload_m : {hnsw['payload_m']}") + if hnsw.get("inline_storage") is not None: + lines.append(f" HNSW inline_storage : {hnsw['inline_storage']}") # Payload schema schema = data["payload_schema"] @@ -439,11 +460,21 @@ def _format_collection_diagnostics(data: dict) -> str: else: lines.append(" Payload indexes : none") + sharding = data["sharding"] + if sharding.get("replication_factor") is not None: + lines.append(f" Replication factor : {sharding['replication_factor']}") + if sharding.get("write_consistency_factor") is not None: + lines.append(f" Write consistency : {sharding['write_consistency_factor']}") + if sharding.get("read_fan_out_factor") is not None: + lines.append(f" Read fan-out : {sharding['read_fan_out_factor']}") + if sharding.get("read_fan_out_delay_ms") is not None: + lines.append(f" Read fan-out delay : {sharding['read_fan_out_delay_ms']} ms") + if sharding.get("on_disk_payload") is not None: + lines.append(f" Payload on_disk : {sharding['on_disk_payload']}") + # Sharding sh = data["sharding"] lines.append(f" Shards : {sh['shard_number']}") - lines.append(f" Replicas : {sh['replication_factor']}") - lines.append(f" Write consistency : {sh['write_consistency_factor']}") return "\n".join(lines) diff --git a/src/qql/dumper.py b/src/qql/dumper.py index 4fcf9d7..9c0da2e 100644 --- a/src/qql/dumper.py +++ b/src/qql/dumper.py @@ -66,11 +66,120 @@ def _serialize_dict(d: dict[str, Any], indent: int = 4) -> str: # ── Collection type detection ───────────────────────────────────────────────── +def _collection_info(collection: str, client: QdrantClient) -> Any: + return client.get_collection(collection) + + def _is_hybrid(collection: str, client: QdrantClient) -> bool: - """Return True if the collection uses named vectors (dense + sparse).""" - info = client.get_collection(collection) - vectors = info.config.params.vectors # type: ignore[union-attr] - return isinstance(vectors, dict) + """Return True only when sparse vectors are configured.""" + info = _collection_info(collection, client) + sparse_vectors = info.config.params.sparse_vectors + return isinstance(sparse_vectors, dict) and bool(sparse_vectors) + + +def _quantization_clause(info: Any) -> str: + quant = info.config.quantization_config + if quant is None: + return "" + if hasattr(quant, "scalar"): + clause = " QUANTIZE SCALAR" + if quant.scalar.quantile is not None: + clause += f" QUANTILE {quant.scalar.quantile}" + if quant.scalar.always_ram: + clause += " ALWAYS RAM" + return clause + if hasattr(quant, "binary"): + clause = " QUANTIZE BINARY" + if quant.binary.always_ram: + clause += " ALWAYS RAM" + return clause + if hasattr(quant, "product"): + clause = " QUANTIZE PRODUCT" + if quant.product.always_ram: + clause += " ALWAYS RAM" + return clause + if hasattr(quant, "turbo"): + clause = " QUANTIZE TURBO" + bits = quant.turbo.bits + if bits is not None: + bit_map = { + "BITS4": "4", + "BITS2": "2", + "BITS1_5": "1.5", + "BITS1": "1", + } + clause += f" BITS {bit_map.get(getattr(bits, 'name', ''), str(bits))}" + if quant.turbo.always_ram: + clause += " ALWAYS RAM" + return clause + return "" + + +def _config_clauses(info: Any) -> str: + clauses: list[str] = [] + params = info.config.params + vectors = params.vectors # type: ignore[union-attr] + dense_vectors = vectors.get("dense") if isinstance(vectors, dict) else vectors + if dense_vectors is not None and getattr(dense_vectors, "on_disk", None) is not None: + clauses.append(f"WITH VECTORS {{ on_disk: {'true' if dense_vectors.on_disk else 'false'} }}") + + hnsw = info.config.hnsw_config + hnsw_items: list[str] = [] + for key in ( + "m", + "ef_construct", + "full_scan_threshold", + "max_indexing_threads", + "payload_m", + ): + value = getattr(hnsw, key, None) + if value is not None: + hnsw_items.append(f"{key}: {value}") + for key in ("on_disk", "inline_storage"): + value = getattr(hnsw, key, None) + if value is not None: + hnsw_items.append(f"{key}: {'true' if value else 'false'}") + if hnsw_items: + clauses.append(f"WITH HNSW {{ {', '.join(hnsw_items)} }}") + + optimizers = getattr(info.config, "optimizer_config", None) or getattr(info.config, "optimizers_config", None) + optimizer_items: list[str] = [] + if optimizers is not None: + for key in ( + "deleted_threshold", + "vacuum_min_vector_number", + "default_segment_number", + "max_segment_size", + "memmap_threshold", + "indexing_threshold", + "flush_interval_sec", + ): + value = getattr(optimizers, key, None) + if value is not None: + optimizer_items.append(f"{key}: {value}") + max_opt_threads = getattr(optimizers, "max_optimization_threads", None) + if max_opt_threads is not None: + value = getattr(max_opt_threads, "value", max_opt_threads) + optimizer_items.append(f"max_optimization_threads: {value}") + prevent_unoptimized = getattr(optimizers, "prevent_unoptimized", None) + if prevent_unoptimized is not None: + optimizer_items.append( + f"prevent_unoptimized: {'true' if prevent_unoptimized else 'false'}" + ) + if optimizer_items: + clauses.append(f"WITH OPTIMIZERS {{ {', '.join(optimizer_items)} }}") + + param_items: list[str] = [] + for key in ("replication_factor", "write_consistency_factor", "on_disk_payload"): + value = getattr(params, key, None) + if value is not None: + if isinstance(value, bool): + param_items.append(f"{key}: {'true' if value else 'false'}") + else: + param_items.append(f"{key}: {value}") + if param_items: + clauses.append(f"WITH PARAMS {{ {', '.join(param_items)} }}") + return (" " + " ".join(clauses)) if clauses else "" # ── Main entry point ────────────────────────────────────────────────────────── @@ -98,7 +207,9 @@ def dump_collection( ) return 0, 0 - hybrid = _is_hybrid(collection, client) + info = _collection_info(collection, client) + sparse_vectors = info.config.params.sparse_vectors + hybrid = isinstance(sparse_vectors, dict) and bool(sparse_vectors) col_type = "hybrid (dense + sparse)" if hybrid else "dense" using_clause = " USING HYBRID" if hybrid else "" @@ -138,7 +249,11 @@ def dump_collection( # ── CREATE statement ────────────────────────────────────────────── hybrid_suffix = " HYBRID" if hybrid else "" - f.write(f"CREATE COLLECTION {collection}{hybrid_suffix}\n\n") + config_suffix = _config_clauses(info) + quantization_suffix = _quantization_clause(info) + f.write( + f"CREATE COLLECTION {collection}{hybrid_suffix}{config_suffix}{quantization_suffix}\n\n" + ) # ── Paginate and write INSERT BULK batches ──────────────────────── offset = None diff --git a/src/qql/executor.py b/src/qql/executor.py index 450099b..946ca7e 100644 --- a/src/qql/executor.py +++ b/src/qql/executor.py @@ -11,8 +11,10 @@ AcornSearchParams, BinaryQuantization, BinaryQuantizationConfig, + CollectionParamsDiff, CompressionRatio, Distance, + Disabled, FieldCondition, Filter, Fusion, @@ -34,6 +36,7 @@ Mmr, Modifier, NearestQuery, + OptimizersConfigDiff, PayloadField, PayloadSchemaType, PointStruct, @@ -62,12 +65,17 @@ UuidIndexParams, UuidIndexType, VectorParams, + VectorParamsDiff, + MaxOptimizationThreadsSetting, ) from .ast_nodes import ( ASTNode, + AlterCollectionStmt, AndExpr, BetweenExpr, + CollectionParamsConfig, + CollectionConfig, CompareExpr, CreateCollectionStmt, CreateIndexStmt, @@ -86,7 +94,9 @@ MatchTextExpr, NotExpr, NotInExpr, + OptimizersRuntimeConfig, OrExpr, + QuantizationUpdate, QuantizationConfig, QuantizationType, RecommendStmt, @@ -98,6 +108,8 @@ ShowCollectionsStmt, UpdateVectorStmt, UpdatePayloadStmt, + VectorsConfig, + HnswRuntimeConfig, ) from .config import QQLConfig from .embedder import CrossEncoderEmbedder, Embedder, SparseEmbedder @@ -128,6 +140,8 @@ def execute(self, node: ASTNode) -> ExecutionResult: return self._execute_insert(node) if isinstance(node, CreateCollectionStmt): return self._execute_create(node) + if isinstance(node, AlterCollectionStmt): + return self._execute_alter_collection(node) if isinstance(node, CreateIndexStmt): return self._execute_create_index(node) if isinstance(node, DropCollectionStmt): @@ -218,6 +232,7 @@ def _execute_insert(self, node: InsertStmt) -> ExecutionResult: vector = embedder.embed(node.values["text"]) self._ensure_collection(node.collection, len(vector)) + point_vector = self._build_dense_point_vector(node.collection, vector) point_id, payload = self._extract_point_id_and_payload(node.values) @@ -225,7 +240,7 @@ def _execute_insert(self, node: InsertStmt) -> ExecutionResult: self._client.upsert( collection_name=node.collection, wait=True, - points=[PointStruct(id=point_id, vector=vector, payload=payload)], + points=[PointStruct(id=point_id, vector=point_vector, payload=payload)], ) except UnexpectedResponse as e: raise QQLRuntimeError(f"Qdrant error during INSERT: {e}") from e @@ -306,11 +321,12 @@ def _execute_insert_bulk(self, node: InsertBulkStmt) -> ExecutionResult: for vals in node.values_list: vector = embedder.embed(vals["text"]) point_id, payload = self._extract_point_id_and_payload(vals) + point_vector = self._build_dense_point_vector(node.collection, vector) points.append( - PointStruct(id=point_id, vector=vector, payload=payload) + PointStruct(id=point_id, vector=point_vector, payload=payload) ) - self._ensure_collection(node.collection, len(points[0].vector)) + self._ensure_collection(node.collection, len(vector)) try: self._client.upsert( @@ -346,12 +362,15 @@ def _execute_create(self, node: CreateCollectionStmt) -> ExecutionResult: if node.quantization is not None else "" ) - hnsw_config = ( - HnswConfigDiff(payload_m=node.payload_m) - if node.payload_m is not None + hnsw_config = self._build_hnsw_config(node.config) + optimizers_config = self._build_optimizers_config(node.config) + params_config = self._build_collection_params_create_kwargs(node.config) + config_label = self._describe_collection_config(node.config) + vector_on_disk = ( + node.config.vectors.on_disk + if node.config is not None and node.config.vectors is not None else None ) - hnsw_label = f", payload_m={node.payload_m}" if node.payload_m is not None else "" # ── Hybrid collection: named dense + sparse vectors ──────────────── if node.hybrid: @@ -360,7 +379,11 @@ def _execute_create(self, node: CreateCollectionStmt) -> ExecutionResult: create_kwargs: dict[str, Any] = { "collection_name": node.collection, "vectors_config": { - "dense": VectorParams(size=dims, distance=Distance.COSINE) + "dense": VectorParams( + size=dims, + distance=Distance.COSINE, + on_disk=vector_on_disk, + ) }, "sparse_vectors_config": { "sparse": SparseVectorParams(modifier=Modifier.IDF) @@ -370,12 +393,15 @@ def _execute_create(self, node: CreateCollectionStmt) -> ExecutionResult: create_kwargs["quantization_config"] = quant_config if hnsw_config is not None: create_kwargs["hnsw_config"] = hnsw_config + if optimizers_config is not None: + create_kwargs["optimizers_config"] = optimizers_config + create_kwargs.update(params_config) self._create_collection_and_wait(**create_kwargs) return ExecutionResult( success=True, message=( f"Collection '{node.collection}' created " - f"(hybrid: {dims}-dim dense + BM25 sparse, cosine distance{quant_label}{hnsw_label})" + f"(hybrid: {dims}-dim dense + BM25 sparse, cosine distance{quant_label}{config_label})" ), ) @@ -384,16 +410,59 @@ def _execute_create(self, node: CreateCollectionStmt) -> ExecutionResult: dims = embedder.dimensions create_kwargs = { "collection_name": node.collection, - "vectors_config": VectorParams(size=dims, distance=Distance.COSINE), + "vectors_config": VectorParams( + size=dims, + distance=Distance.COSINE, + on_disk=vector_on_disk, + ), } if quant_config is not None: create_kwargs["quantization_config"] = quant_config if hnsw_config is not None: create_kwargs["hnsw_config"] = hnsw_config + if optimizers_config is not None: + create_kwargs["optimizers_config"] = optimizers_config + create_kwargs.update(params_config) self._create_collection_and_wait(**create_kwargs) return ExecutionResult( success=True, - message=f"Collection '{node.collection}' created ({dims}-dimensional vectors, cosine distance{quant_label}{hnsw_label})", + message=f"Collection '{node.collection}' created ({dims}-dimensional vectors, cosine distance{quant_label}{config_label})", + ) + + def _execute_alter_collection(self, node: AlterCollectionStmt) -> ExecutionResult: + if not self._client.collection_exists(node.collection): + raise QQLRuntimeError(f"Collection '{node.collection}' does not exist") + + update_kwargs: dict[str, Any] = {"collection_name": node.collection} + vectors_config = self._build_vectors_config_diff(node.collection, node.config) + hnsw_config = self._build_hnsw_config(node.config) + optimizers_config = self._build_optimizers_config(node.config) + collection_params = self._build_collection_params_diff(node.config) + quantization_config = self._build_alter_quantization_config(node.quantization) + + if vectors_config is not None: + update_kwargs["vectors_config"] = vectors_config + if hnsw_config is not None: + update_kwargs["hnsw_config"] = hnsw_config + if optimizers_config is not None: + update_kwargs["optimizers_config"] = optimizers_config + if collection_params is not None: + update_kwargs["collection_params"] = collection_params + if quantization_config is not None: + update_kwargs["quantization_config"] = quantization_config + + try: + self._client.update_collection(**update_kwargs) + except UnexpectedResponse as e: + raise QQLRuntimeError(f"Qdrant error during ALTER COLLECTION: {e}") from e + + return ExecutionResult( + success=True, + message=( + f"Collection '{node.collection}' altered" + f"{self._describe_collection_config(node.config)}" + f"{self._describe_quantization_update(node.quantization)}" + ), ) def _execute_create_index(self, node: CreateIndexStmt) -> ExecutionResult: @@ -471,6 +540,7 @@ def _execute_show_collection(self, node: ShowCollectionStmt) -> ExecutionResult: vector_details[vname] = { "size": vconfig.size, "distance": str(vconfig.distance) if vconfig.distance else None, + "on_disk": vconfig.on_disk, } elif vectors is None: raise QQLRuntimeError( @@ -481,6 +551,7 @@ def _execute_show_collection(self, node: ShowCollectionStmt) -> ExecutionResult: "": { "size": vectors.size, "distance": str(vectors.distance) if vectors.distance else None, + "on_disk": vectors.on_disk, } } topology = "hybrid" if sparse_vector_params else "dense" @@ -522,6 +593,8 @@ def _execute_show_collection(self, node: ShowCollectionStmt) -> ExecutionResult: hnsw["on_disk"] = config.hnsw_config.on_disk if config.hnsw_config.payload_m is not None: hnsw["payload_m"] = config.hnsw_config.payload_m + if config.hnsw_config.inline_storage is not None: + hnsw["inline_storage"] = config.hnsw_config.inline_storage # ── Payload schema / indexes ─────────────────────────────────────── payload_indexes = {} @@ -533,6 +606,9 @@ def _execute_show_collection(self, node: ShowCollectionStmt) -> ExecutionResult: "shard_number": params.shard_number, "replication_factor": params.replication_factor, "write_consistency_factor": params.write_consistency_factor, + "read_fan_out_factor": params.read_fan_out_factor, + "read_fan_out_delay_ms": params.read_fan_out_delay_ms, + "on_disk_payload": params.on_disk_payload, } data = { @@ -1057,6 +1133,165 @@ def _build_search_params(self, with_clause: SearchWith | None) -> SearchParams | acorn=AcornSearchParams(enable=True) if with_clause.acorn else None, ) + def _build_hnsw_config(self, config: CollectionConfig | None) -> HnswConfigDiff | None: + if config is None or config.hnsw is None: + return None + hnsw = config.hnsw + return HnswConfigDiff( + m=hnsw.m, + ef_construct=hnsw.ef_construct, + full_scan_threshold=hnsw.full_scan_threshold, + max_indexing_threads=hnsw.max_indexing_threads, + on_disk=hnsw.on_disk, + payload_m=hnsw.payload_m, + inline_storage=hnsw.inline_storage, + ) + + def _build_optimizers_config( + self, + config: CollectionConfig | None, + ) -> OptimizersConfigDiff | None: + if config is None or config.optimizers is None: + return None + optimizers = config.optimizers + max_optimization_threads = optimizers.max_optimization_threads + if max_optimization_threads == "auto": + max_optimization_threads = MaxOptimizationThreadsSetting.AUTO + return OptimizersConfigDiff( + deleted_threshold=optimizers.deleted_threshold, + vacuum_min_vector_number=optimizers.vacuum_min_vector_number, + default_segment_number=optimizers.default_segment_number, + max_segment_size=optimizers.max_segment_size, + memmap_threshold=optimizers.memmap_threshold, + indexing_threshold=optimizers.indexing_threshold, + flush_interval_sec=optimizers.flush_interval_sec, + max_optimization_threads=max_optimization_threads, + prevent_unoptimized=optimizers.prevent_unoptimized, + ) + + def _build_collection_params_create_kwargs( + self, + config: CollectionConfig | None, + ) -> dict[str, Any]: + if config is None or config.params is None: + return {} + params = config.params + create_kwargs: dict[str, Any] = {} + if params.replication_factor is not None: + create_kwargs["replication_factor"] = params.replication_factor + if params.write_consistency_factor is not None: + create_kwargs["write_consistency_factor"] = params.write_consistency_factor + if params.on_disk_payload is not None: + create_kwargs["on_disk_payload"] = params.on_disk_payload + return create_kwargs + + def _build_collection_params_diff( + self, + config: CollectionConfig | None, + ) -> CollectionParamsDiff | None: + if config is None or config.params is None: + return None + params = config.params + return CollectionParamsDiff( + replication_factor=params.replication_factor, + write_consistency_factor=params.write_consistency_factor, + read_fan_out_factor=params.read_fan_out_factor, + read_fan_out_delay_ms=params.read_fan_out_delay_ms, + on_disk_payload=params.on_disk_payload, + ) + + def _build_vectors_config_diff( + self, + collection_name: str, + config: CollectionConfig | None, + ) -> dict[str, VectorParamsDiff] | None: + if config is None or config.vectors is None: + return None + vector_name = self._get_dense_vector_name(collection_name) + if vector_name is None: + vector_name = "" + return { + vector_name: VectorParamsDiff(on_disk=config.vectors.on_disk), + } + + def _build_alter_quantization_config( + self, + quantization: QuantizationUpdate | None, + ) -> ( + ScalarQuantization | BinaryQuantization | ProductQuantization | TurboQuantization | Disabled | None + ): + if quantization is None: + return None + if quantization.disabled: + return Disabled.DISABLED + if quantization.config is None: + return None + return self._build_quantization_config(quantization.config) + + def _describe_collection_config(self, config: CollectionConfig | None) -> str: + if config is None: + return "" + labels: list[str] = [] + if config.vectors is not None and config.vectors.on_disk is not None: + labels.append(f"vectors.on_disk={config.vectors.on_disk}") + if config.hnsw is not None: + hnsw = config.hnsw + if hnsw.m is not None: + labels.append(f"hnsw.m={hnsw.m}") + if hnsw.ef_construct is not None: + labels.append(f"hnsw.ef_construct={hnsw.ef_construct}") + if hnsw.full_scan_threshold is not None: + labels.append(f"hnsw.full_scan_threshold={hnsw.full_scan_threshold}") + if hnsw.max_indexing_threads is not None: + labels.append(f"hnsw.max_indexing_threads={hnsw.max_indexing_threads}") + if hnsw.on_disk is not None: + labels.append(f"hnsw.on_disk={hnsw.on_disk}") + if hnsw.payload_m is not None: + labels.append(f"hnsw.payload_m={hnsw.payload_m}") + if hnsw.inline_storage is not None: + labels.append(f"hnsw.inline_storage={hnsw.inline_storage}") + if config.optimizers is not None: + optimizers = config.optimizers + for key in ( + "deleted_threshold", + "vacuum_min_vector_number", + "default_segment_number", + "max_segment_size", + "memmap_threshold", + "indexing_threshold", + "flush_interval_sec", + "max_optimization_threads", + "prevent_unoptimized", + ): + value = getattr(optimizers, key) + if value is not None: + labels.append(f"optimizers.{key}={value}") + if config.params is not None: + params = config.params + for key in ( + "replication_factor", + "write_consistency_factor", + "read_fan_out_factor", + "read_fan_out_delay_ms", + "on_disk_payload", + ): + value = getattr(params, key) + if value is not None: + labels.append(f"params.{key}={value}") + return f", {', '.join(labels)}" if labels else "" + + def _describe_quantization_update( + self, + quantization: QuantizationUpdate | None, + ) -> str: + if quantization is None: + return "" + if quantization.disabled: + return ", quantization=disabled" + if quantization.config is not None: + return f", quantization={quantization.config.type.value}" + return "" + def _has_mmr(self, with_clause: SearchWith | None) -> bool: return with_clause is not None and ( with_clause.mmr_diversity is not None or with_clause.mmr_candidates is not None @@ -1159,6 +1394,18 @@ def _get_dense_vector_name(self, collection_name: str) -> str | None: return "dense" return None + def _build_dense_point_vector( + self, + collection_name: str, + vector: list[float], + ) -> list[float] | dict[str, list[float]]: + if not self._client.collection_exists(collection_name): + return vector + vector_name = self._get_dense_vector_name(collection_name) + if vector_name is None: + return vector + return {vector_name: vector} + def _apply_reranking( self, query: str, @@ -1499,12 +1746,12 @@ def _build_quantization_config( raise QQLRuntimeError(f"Unknown quantization type: {qc.type}") def _collection_is_hybrid(self, name: str) -> bool: - """Return True if *name* exists and uses named vectors (hybrid collection).""" + """Return True if *name* exists and uses sparse vectors (hybrid collection).""" if not self._client.collection_exists(name): return False info = self._client.get_collection(name) - vectors = info.config.params.vectors # type: ignore[union-attr] - return isinstance(vectors, dict) + sparse_vectors = info.config.params.sparse_vectors + return isinstance(sparse_vectors, dict) and bool(sparse_vectors) def _ensure_collection(self, name: str, vector_size: int) -> None: """Create the collection if it doesn't exist. Raises on dimension mismatch. diff --git a/src/qql/lexer.py b/src/qql/lexer.py index 72b09c0..39babee 100644 --- a/src/qql/lexer.py +++ b/src/qql/lexer.py @@ -31,7 +31,12 @@ class TokenKind(Enum): TURBO = auto() BITS = auto() HNSW = auto() + VECTORS = auto() + OPTIMIZERS = auto() + PARAMS = auto() + DISABLED = auto() CREATE = auto() + ALTER = auto() INDEX = auto() ON = auto() DROP = auto() @@ -131,7 +136,12 @@ class TokenKind(Enum): "TURBO": TokenKind.TURBO, "BITS": TokenKind.BITS, "HNSW": TokenKind.HNSW, + "VECTORS": TokenKind.VECTORS, + "OPTIMIZERS": TokenKind.OPTIMIZERS, + "PARAMS": TokenKind.PARAMS, + "DISABLED": TokenKind.DISABLED, "CREATE": TokenKind.CREATE, + "ALTER": TokenKind.ALTER, "INDEX": TokenKind.INDEX, "ON": TokenKind.ON, "DROP": TokenKind.DROP, diff --git a/src/qql/parser.py b/src/qql/parser.py index 0f8851b..37d25e5 100644 --- a/src/qql/parser.py +++ b/src/qql/parser.py @@ -2,8 +2,11 @@ from .ast_nodes import ( ASTNode, + AlterCollectionStmt, AndExpr, BetweenExpr, + CollectionParamsConfig, + CollectionConfig, CompareExpr, CreateCollectionStmt, CreateIndexStmt, @@ -22,7 +25,9 @@ MatchTextExpr, NotExpr, NotInExpr, + OptimizersRuntimeConfig, OrExpr, + QuantizationUpdate, QuantizationConfig, QuantizationType, QuantizationSearchWith, @@ -35,6 +40,8 @@ ShowCollectionsStmt, UpdateVectorStmt, UpdatePayloadStmt, + VectorsConfig, + HnswRuntimeConfig, ) from .exceptions import QQLSyntaxError from .lexer import Token, TokenKind @@ -65,6 +72,8 @@ def parse(self) -> ASTNode: node = self._parse_insert() elif tok.kind == TokenKind.CREATE: node = self._parse_create() + elif tok.kind == TokenKind.ALTER: + node = self._parse_alter() elif tok.kind == TokenKind.DROP: node = self._parse_drop() elif tok.kind == TokenKind.SHOW: @@ -191,38 +200,15 @@ def _parse_create(self) -> CreateCollectionStmt | CreateIndexStmt: self._expect(TokenKind.MODEL) model = self._expect(TokenKind.STRING).value - # ── Optional QUANTIZE clause ────────────────────────────────── - quantization: QuantizationConfig | None = None - payload_m: int | None = None - seen_quantize = False - seen_hnsw = False - while self._peek().kind in (TokenKind.QUANTIZE, TokenKind.HNSW): - if self._peek().kind == TokenKind.QUANTIZE: - if seen_quantize: - raise QQLSyntaxError( - "QUANTIZE clause may only appear once", - self._peek().pos, - ) - self._advance() # consume QUANTIZE - quantization = self._parse_quantize_clause() - seen_quantize = True - continue - - if seen_hnsw: - raise QQLSyntaxError( - "HNSW clause may only appear once", - self._peek().pos, - ) - self._advance() # consume HNSW - payload_m = self._parse_collection_hnsw_clause() - seen_hnsw = True + config = self._parse_collection_config_blocks(for_alter=False) + quantization = self._parse_optional_create_quantization() return CreateCollectionStmt( collection=collection, hybrid=hybrid, model=model, quantization=quantization, - payload_m=payload_m, + config=config, ) self._expect(TokenKind.INDEX) @@ -244,21 +230,258 @@ def _parse_create(self) -> CreateCollectionStmt | CreateIndexStmt: options=options, ) - def _parse_collection_hnsw_clause(self) -> int: - config = self._parse_dict() - unknown_keys = set(config) - {"payload_m"} - if unknown_keys: + def _parse_alter(self) -> AlterCollectionStmt: + self._expect(TokenKind.ALTER) + self._expect(TokenKind.COLLECTION) + collection = self._parse_identifier() + config = self._parse_collection_config_blocks(for_alter=True) + quantization = self._parse_optional_alter_quantization() + if config is None and quantization is None: raise QQLSyntaxError( - "Unknown HNSW parameter " - f"'{sorted(unknown_keys)[0]}'. Expected: payload_m", - 0, + "ALTER COLLECTION requires at least one WITH HNSW/VECTORS/OPTIMIZERS/PARAMS clause or QUANTIZE clause", + self._peek().pos, + ) + return AlterCollectionStmt( + collection=collection, + config=config, + quantization=quantization, + ) + + def _parse_collection_config_blocks(self, *, for_alter: bool) -> CollectionConfig | None: + config: CollectionConfig | None = None + while self._peek().kind == TokenKind.WITH: + self._advance() + block = self._parse_collection_config_clause(for_alter=for_alter) + config = block if config is None else self._merge_collection_config(config, block) + return config + + def _parse_optional_create_quantization(self) -> QuantizationConfig | None: + if self._peek().kind != TokenKind.QUANTIZE: + return None + self._advance() + return self._parse_quantize_clause() + + def _parse_optional_alter_quantization(self) -> QuantizationUpdate | None: + if self._peek().kind != TokenKind.QUANTIZE: + return None + self._advance() + if self._peek().kind == TokenKind.DISABLED: + self._advance() + return QuantizationUpdate(disabled=True) + return QuantizationUpdate(config=self._parse_quantize_clause()) + + def _parse_collection_config_clause(self, *, for_alter: bool) -> CollectionConfig: + tok = self._peek() + if tok.kind == TokenKind.HNSW: + self._advance() + config = self._parse_dict() + unknown_keys = set(config) - { + "m", + "ef_construct", + "full_scan_threshold", + "max_indexing_threads", + "on_disk", + "payload_m", + "inline_storage", + } + if unknown_keys: + raise QQLSyntaxError( + "Unknown HNSW parameter " + f"'{sorted(unknown_keys)[0]}'. Expected: m, ef_construct, full_scan_threshold, max_indexing_threads, on_disk, payload_m, inline_storage", + 0, + ) + return CollectionConfig( + hnsw=HnswRuntimeConfig( + m=self._collection_min_int(config, "m", minimum=4), + ef_construct=self._collection_positive_int(config, "ef_construct"), + full_scan_threshold=self._collection_non_negative_int(config, "full_scan_threshold"), + max_indexing_threads=self._collection_positive_int(config, "max_indexing_threads"), + on_disk=self._collection_bool(config, "on_disk"), + payload_m=self._collection_positive_int(config, "payload_m"), + inline_storage=self._collection_bool(config, "inline_storage"), + ) + ) + if tok.kind == TokenKind.VECTORS: + self._advance() + config = self._parse_dict() + unknown_keys = set(config) - {"on_disk"} + if unknown_keys: + raise QQLSyntaxError( + "Unknown VECTORS parameter " + f"'{sorted(unknown_keys)[0]}'. Expected: on_disk", + 0, + ) + return CollectionConfig( + vectors=VectorsConfig( + on_disk=self._collection_bool(config, "on_disk"), + ) ) - if "payload_m" not in config: - raise QQLSyntaxError("HNSW clause requires payload_m", 0) - payload_m = config["payload_m"] - if not isinstance(payload_m, int) or isinstance(payload_m, bool) or payload_m <= 0: - raise QQLSyntaxError("payload_m must be a positive integer", 0) - return payload_m + if tok.kind == TokenKind.OPTIMIZERS: + self._advance() + config = self._parse_dict() + unknown_keys = set(config) - { + "deleted_threshold", + "vacuum_min_vector_number", + "default_segment_number", + "max_segment_size", + "memmap_threshold", + "indexing_threshold", + "flush_interval_sec", + "max_optimization_threads", + "prevent_unoptimized", + } + if unknown_keys: + raise QQLSyntaxError( + "Unknown OPTIMIZERS parameter " + f"'{sorted(unknown_keys)[0]}'. Expected: deleted_threshold, vacuum_min_vector_number, default_segment_number, max_segment_size, memmap_threshold, indexing_threshold, flush_interval_sec, max_optimization_threads, prevent_unoptimized", + 0, + ) + return CollectionConfig( + optimizers=OptimizersRuntimeConfig( + deleted_threshold=self._collection_float_range( + config, + "deleted_threshold", + minimum=0.0, + maximum=1.0, + ), + vacuum_min_vector_number=self._collection_positive_int(config, "vacuum_min_vector_number"), + default_segment_number=self._collection_positive_int(config, "default_segment_number"), + max_segment_size=self._collection_positive_int(config, "max_segment_size"), + memmap_threshold=self._collection_non_negative_int(config, "memmap_threshold"), + indexing_threshold=self._collection_non_negative_int(config, "indexing_threshold"), + flush_interval_sec=self._collection_positive_int(config, "flush_interval_sec"), + max_optimization_threads=self._collection_max_optimization_threads(config, "max_optimization_threads"), + prevent_unoptimized=self._collection_bool(config, "prevent_unoptimized"), + ) + ) + if tok.kind == TokenKind.PARAMS: + self._advance() + config = self._parse_dict() + unknown_keys = set(config) - { + "replication_factor", + "write_consistency_factor", + "read_fan_out_factor", + "read_fan_out_delay_ms", + "on_disk_payload", + } + if unknown_keys: + raise QQLSyntaxError( + "Unknown PARAMS parameter " + f"'{sorted(unknown_keys)[0]}'. Expected: replication_factor, write_consistency_factor, read_fan_out_factor, read_fan_out_delay_ms, on_disk_payload", + 0, + ) + if not for_alter and ( + "read_fan_out_factor" in config or "read_fan_out_delay_ms" in config + ): + raise QQLSyntaxError( + "WITH PARAMS { read_fan_out_factor, read_fan_out_delay_ms } is supported only for ALTER COLLECTION", + 0, + ) + return CollectionConfig( + params=CollectionParamsConfig( + replication_factor=self._collection_positive_int(config, "replication_factor"), + write_consistency_factor=self._collection_positive_int(config, "write_consistency_factor"), + read_fan_out_factor=self._collection_positive_int(config, "read_fan_out_factor"), + read_fan_out_delay_ms=self._collection_non_negative_int(config, "read_fan_out_delay_ms"), + on_disk_payload=self._collection_bool(config, "on_disk_payload"), + ) + ) + raise QQLSyntaxError( + f"Expected HNSW, VECTORS, OPTIMIZERS, or PARAMS after WITH, got '{tok.value}'", + tok.pos, + ) + + def _merge_collection_config( + self, + current: CollectionConfig, + new: CollectionConfig, + ) -> CollectionConfig: + return CollectionConfig( + vectors=self._merge_config_block("VECTORS", current.vectors, new.vectors), + hnsw=self._merge_config_block("HNSW", current.hnsw, new.hnsw), + optimizers=self._merge_config_block("OPTIMIZERS", current.optimizers, new.optimizers), + params=self._merge_config_block("PARAMS", current.params, new.params), + ) + + def _merge_config_block(self, name: str, current: Any, new: Any) -> Any: + if new is None: + return current + if current is None: + return new + raise QQLSyntaxError( + f"{name} clause may only appear once", + self._peek().pos, + ) + + def _collection_positive_int(self, config: dict[str, Any], key: str) -> int | None: + if key not in config: + return None + value = config[key] + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise QQLSyntaxError(f"{key} must be a positive integer", 0) + return value + + def _collection_min_int( + self, + config: dict[str, Any], + key: str, + minimum: int, + ) -> int | None: + value = self._collection_positive_int(config, key) + if value is not None and value < minimum: + raise QQLSyntaxError(f"{key} must be >= {minimum}", 0) + return value + + def _collection_non_negative_int(self, config: dict[str, Any], key: str) -> int | None: + if key not in config: + return None + value = config[key] + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise QQLSyntaxError(f"{key} must be a non-negative integer", 0) + return value + + def _collection_float_range( + self, + config: dict[str, Any], + key: str, + minimum: float, + maximum: float, + ) -> float | None: + if key not in config: + return None + value = config[key] + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise QQLSyntaxError(f"{key} must be a number", 0) + value = float(value) + if not minimum <= value <= maximum: + raise QQLSyntaxError(f"{key} must be between {minimum} and {maximum}", 0) + return value + + def _collection_max_optimization_threads( + self, + config: dict[str, Any], + key: str, + ) -> int | str | None: + if key not in config: + return None + value = config[key] + if isinstance(value, bool): + raise QQLSyntaxError(f"{key} must be a positive integer or 'auto'", 0) + if isinstance(value, int): + if value <= 0: + raise QQLSyntaxError(f"{key} must be a positive integer or 'auto'", 0) + return value + if isinstance(value, str) and value.lower() == "auto": + return "auto" + raise QQLSyntaxError(f"{key} must be a positive integer or 'auto'", 0) + + def _collection_bool(self, config: dict[str, Any], key: str) -> bool | None: + if key not in config: + return None + value = config[key] + if not isinstance(value, bool): + raise QQLSyntaxError(f"{key} must be true or false", 0) + return value def _parse_quantize_clause(self) -> QuantizationConfig: """Parse: (SCALAR | BINARY | PRODUCT) [QUANTILE ] [ALWAYS RAM] diff --git a/src/qql/script.py b/src/qql/script.py index 1bff1da..ab2de16 100644 --- a/src/qql/script.py +++ b/src/qql/script.py @@ -22,6 +22,7 @@ _STMT_STARTERS = { TokenKind.INSERT, TokenKind.CREATE, + TokenKind.ALTER, TokenKind.DROP, TokenKind.SHOW, TokenKind.SELECT, @@ -38,18 +39,45 @@ def strip_comments(text: str) -> str: - """Remove ``-- ...`` to-end-of-line comments from every line. + """Remove ``-- ...`` comments while preserving string literals.""" + out: list[str] = [] + in_string = False + quote_char = "" + i = 0 + n = len(text) + + while i < n: + ch = text[i] + + if in_string: + out.append(ch) + if ch == "\\" and i + 1 < n: + out.append(text[i + 1]) + i += 2 + continue + if ch == quote_char: + in_string = False + quote_char = "" + i += 1 + continue - The check is byte-level: ``--`` inside a string literal would also be - stripped, but that edge case does not occur in practice for QQL scripts. - """ - lines: list[str] = [] - for line in text.splitlines(): - idx = line.find("--") - if idx != -1: - line = line[:idx] - lines.append(line) - return "\n".join(lines) + if ch in ("'", '"'): + in_string = True + quote_char = ch + out.append(ch) + i += 1 + continue + + if ch == "-" and i + 1 < n and text[i + 1] == "-": + i += 2 + while i < n and text[i] not in "\r\n": + i += 1 + continue + + out.append(ch) + i += 1 + + return "".join(out) def split_statements(tokens: list[Token]) -> list[list[Token]]: diff --git a/tests/test_dumper.py b/tests/test_dumper.py index 9257b63..32ee430 100644 --- a/tests/test_dumper.py +++ b/tests/test_dumper.py @@ -1,6 +1,8 @@ """Tests for the QQL collection dumper (src/qql/dumper.py).""" from __future__ import annotations +from types import SimpleNamespace + import pytest from rich.console import Console @@ -40,11 +42,28 @@ def _make_client(mocker, *, exists=True, hybrid=False, points=None, total=None): # get_collection — return hybrid or dense vector config if hybrid: client.get_collection.return_value.config.params.vectors = {"dense": object()} + client.get_collection.return_value.config.params.sparse_vectors = {"sparse": object()} else: # non-dict → dense-only client.get_collection.return_value.config.params.vectors = mocker.MagicMock( spec=[] # not a dict ) + client.get_collection.return_value.config.params.sparse_vectors = None + client.get_collection.return_value.config.params.replication_factor = None + client.get_collection.return_value.config.params.write_consistency_factor = None + client.get_collection.return_value.config.params.on_disk_payload = None + client.get_collection.return_value.config.hnsw_config = SimpleNamespace( + m=None, + ef_construct=None, + full_scan_threshold=None, + max_indexing_threads=None, + on_disk=None, + payload_m=None, + inline_storage=None, + ) + client.get_collection.return_value.config.quantization_config = None + client.get_collection.return_value.config.optimizer_config = None + client.get_collection.return_value.config.optimizers_config = None # count cnt = mocker.MagicMock() @@ -111,6 +130,7 @@ class TestIsHybrid: def test_dict_vectors_is_hybrid(self, mocker): client = mocker.MagicMock() client.get_collection.return_value.config.params.vectors = {"dense": object()} + client.get_collection.return_value.config.params.sparse_vectors = {"sparse": object()} assert _is_hybrid("col", client) is True def test_scalar_vectors_is_not_hybrid(self, mocker): @@ -118,6 +138,13 @@ def test_scalar_vectors_is_not_hybrid(self, mocker): client.get_collection.return_value.config.params.vectors = mocker.MagicMock( spec=[] ) + client.get_collection.return_value.config.params.sparse_vectors = None + assert _is_hybrid("col", client) is False + + def test_named_dense_without_sparse_is_not_hybrid(self, mocker): + client = mocker.MagicMock() + client.get_collection.return_value.config.params.vectors = {"dense": object()} + client.get_collection.return_value.config.params.sparse_vectors = None assert _is_hybrid("col", client) is False @@ -146,6 +173,41 @@ def test_writes_create_statement_hybrid(self, tmp_path, mocker): content = (tmp_path / "dump.qql").read_text() assert "CREATE COLLECTION col HYBRID" in content + def test_writes_collection_config_and_quantization(self, tmp_path, mocker): + from qdrant_client.models import ( + ScalarQuantization, + ScalarQuantizationConfig, + ScalarType, + VectorParams, + Distance, + ) + + out = str(tmp_path / "dump.qql") + client = _make_client(mocker, points=[{"text": "hello"}]) + info = client.get_collection.return_value + info.config.params.vectors = VectorParams( + size=384, + distance=Distance.COSINE, + on_disk=True, + ) + info.config.params.replication_factor = 2 + info.config.params.write_consistency_factor = 1 + info.config.params.on_disk_payload = True + info.config.hnsw_config.m = 32 + info.config.hnsw_config.payload_m = 24 + info.config.quantization_config = ScalarQuantization( + scalar=ScalarQuantizationConfig( + type=ScalarType.INT8, + always_ram=True, + ) + ) + dump_collection("col", out, client, null_console(), null_console()) + content = (tmp_path / "dump.qql").read_text() + assert "WITH VECTORS { on_disk: true }" in content + assert "WITH HNSW { m: 32, payload_m: 24 }" in content + assert "WITH PARAMS { replication_factor: 2, write_consistency_factor: 1, on_disk_payload: true }" in content + assert "QUANTIZE SCALAR ALWAYS RAM" in content + def test_hybrid_insert_bulk_has_using_hybrid(self, tmp_path, mocker): out = str(tmp_path / "dump.qql") client = _make_client(mocker, hybrid=True, points=[{"text": "hello"}]) diff --git a/tests/test_executor.py b/tests/test_executor.py index 543c45f..41404d7 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -1,13 +1,19 @@ import pytest from qql.ast_nodes import ( + AlterCollectionStmt, + CollectionParamsConfig, + CollectionConfig, CreateCollectionStmt, CreateIndexStmt, DeleteStmt, + HnswRuntimeConfig, DropCollectionStmt, InsertBulkStmt, InsertStmt, + OptimizersRuntimeConfig, QuantizationConfig, + QuantizationUpdate, QuantizationSearchWith, QuantizationType, RecommendStmt, @@ -17,9 +23,11 @@ SearchWith, ShowCollectionStmt, ShowCollectionsStmt, + VectorsConfig, ) from qql.config import QQLConfig from qql.exceptions import QQLRuntimeError +from qql.cli import _format_collection_diagnostics from qql.executor import Executor @@ -247,12 +255,60 @@ def test_create_new_collection(self, executor, mock_client): def test_create_collection_passes_payload_m(self, executor, mock_client): from qdrant_client.models import HnswConfigDiff - node = CreateCollectionStmt(collection="new_col", payload_m=24) + node = CreateCollectionStmt( + collection="new_col", + config=CollectionConfig(hnsw=HnswRuntimeConfig(payload_m=24)), + ) executor.execute(node) kw = mock_client.create_collection.call_args.kwargs assert isinstance(kw["hnsw_config"], HnswConfigDiff) assert kw["hnsw_config"].payload_m == 24 + def test_create_collection_passes_all_new_config_blocks(self, executor, mock_client): + node = CreateCollectionStmt( + collection="new_col", + config=CollectionConfig( + vectors=VectorsConfig(on_disk=True), + hnsw=HnswRuntimeConfig( + m=32, + ef_construct=200, + full_scan_threshold=5000, + max_indexing_threads=2, + on_disk=True, + payload_m=24, + inline_storage=False, + ), + optimizers=OptimizersRuntimeConfig( + indexing_threshold=10000, + memmap_threshold=20000, + deleted_threshold=0.2, + max_optimization_threads="auto", + ), + params=CollectionParamsConfig( + replication_factor=2, + write_consistency_factor=1, + on_disk_payload=True, + ), + ), + ) + executor.execute(node) + kw = mock_client.create_collection.call_args.kwargs + assert kw["vectors_config"].on_disk is True + assert kw["hnsw_config"].m == 32 + assert kw["hnsw_config"].ef_construct == 200 + assert kw["hnsw_config"].full_scan_threshold == 5000 + assert kw["hnsw_config"].max_indexing_threads == 2 + assert kw["hnsw_config"].on_disk is True + assert kw["hnsw_config"].payload_m == 24 + assert kw["hnsw_config"].inline_storage is False + assert kw["optimizers_config"].deleted_threshold == pytest.approx(0.2) + assert kw["optimizers_config"].indexing_threshold == 10000 + assert kw["optimizers_config"].memmap_threshold == 20000 + assert kw["optimizers_config"].max_optimization_threads.value == "auto" + assert kw["replication_factor"] == 2 + assert kw["write_consistency_factor"] == 1 + assert kw["on_disk_payload"] is True + def test_create_existing_collection_is_noop(self, executor, mock_client): mock_client.collection_exists.return_value = True node = CreateCollectionStmt(collection="existing") @@ -261,6 +317,72 @@ def test_create_existing_collection_is_noop(self, executor, mock_client): assert result.success is True assert "already exists" in result.message + def test_alter_collection_passes_all_new_config_blocks(self, executor, mock_client): + mock_client.collection_exists.return_value = True + node = AlterCollectionStmt( + collection="new_col", + config=CollectionConfig( + vectors=VectorsConfig(on_disk=True), + hnsw=HnswRuntimeConfig(full_scan_threshold=5000), + optimizers=OptimizersRuntimeConfig(indexing_threshold=10000), + params=CollectionParamsConfig( + on_disk_payload=False, + read_fan_out_factor=4, + ), + ), + quantization=QuantizationUpdate( + config=QuantizationConfig(type=QuantizationType.BINARY) + ), + ) + executor.execute(node) + kw = mock_client.update_collection.call_args.kwargs + assert kw["vectors_config"][""].on_disk is True + assert kw["hnsw_config"].full_scan_threshold == 5000 + assert kw["optimizers_config"].indexing_threshold == 10000 + assert kw["collection_params"].on_disk_payload is False + assert kw["collection_params"].read_fan_out_factor == 4 + assert kw["quantization_config"].binary is not None + + def test_alter_collection_named_vectors_use_dense_key(self, executor, mock_client, mocker): + from qdrant_client.models import Distance, VectorParams + + mock_client.collection_exists.return_value = True + mock_client.get_collection.return_value.config.params.vectors = { + "dense": VectorParams(size=384, distance=Distance.COSINE) + } + node = AlterCollectionStmt( + collection="named_col", + config=CollectionConfig(vectors=VectorsConfig(on_disk=True)), + ) + executor.execute(node) + kw = mock_client.update_collection.call_args.kwargs + assert kw["vectors_config"]["dense"].on_disk is True + + def test_alter_collection_can_disable_quantization(self, executor, mock_client): + mock_client.collection_exists.return_value = True + node = AlterCollectionStmt( + collection="new_col", + quantization=QuantizationUpdate(disabled=True), + ) + executor.execute(node) + kw = mock_client.update_collection.call_args.kwargs + assert kw["quantization_config"].value == "Disabled" + + def test_collection_is_hybrid_depends_on_sparse_vectors(self, executor, mock_client, mocker): + mock_client.collection_exists.return_value = True + mock_client.get_collection.return_value.config.params.vectors = {"dense": object()} + mock_client.get_collection.return_value.config.params.sparse_vectors = None + assert executor._collection_is_hybrid("named_dense") is False + + def test_insert_named_dense_collection_uses_named_vector_payload(self, executor, mock_client): + mock_client.collection_exists.return_value = True + mock_client.get_collection.return_value.config.params.vectors = {"dense": object()} + mock_client.get_collection.return_value.config.params.sparse_vectors = None + node = InsertStmt(collection="named_dense", values={"text": "hello"}, model=None) + executor.execute(node) + point = mock_client.upsert.call_args.kwargs["points"][0] + assert point.vector == {"dense": FAKE_VECTOR} + class TestCreateIndex: def test_create_index_calls_qdrant(self, executor, mock_client): @@ -454,10 +576,17 @@ def test_show_collection_returns_diagnostics(self, executor, mock_client, mocker mock_info.indexed_vectors_count = 42 mock_info.segments_count = 2 - mock_info.config.params.vectors = VectorParams(size=384, distance=Distance.COSINE) + mock_info.config.params.vectors = VectorParams( + size=384, + distance=Distance.COSINE, + on_disk=True, + ) mock_info.config.params.shard_number = 1 mock_info.config.params.replication_factor = 1 mock_info.config.params.write_consistency_factor = 1 + mock_info.config.params.read_fan_out_factor = None + mock_info.config.params.read_fan_out_delay_ms = None + mock_info.config.params.on_disk_payload = False mock_info.config.params.sparse_vectors = None mock_info.config.hnsw_config.m = 16 mock_info.config.hnsw_config.ef_construct = 100 @@ -480,6 +609,7 @@ def test_show_collection_returns_diagnostics(self, executor, mock_client, mocker assert data["topology"] == "dense" assert data["vectors"][""]["size"] == 384 assert data["vectors"][""]["distance"] == "Cosine" + assert data["vectors"][""]["on_disk"] is True assert data["quantization"] is None assert data["hnsw_config"]["m"] == 16 assert data["hnsw_config"]["ef_construct"] == 100 @@ -512,6 +642,9 @@ def test_show_collection_hybrid(self, executor, mock_client, mocker): mock_info.config.params.shard_number = 1 mock_info.config.params.replication_factor = 1 mock_info.config.params.write_consistency_factor = 1 + mock_info.config.params.read_fan_out_factor = None + mock_info.config.params.read_fan_out_delay_ms = None + mock_info.config.params.on_disk_payload = None mock_info.config.hnsw_config.m = 16 mock_info.config.hnsw_config.ef_construct = 100 mock_info.config.hnsw_config.full_scan_threshold = None @@ -554,6 +687,9 @@ def test_show_collection_named_dense_is_not_reported_as_hybrid(self, executor, m mock_info.config.params.shard_number = 1 mock_info.config.params.replication_factor = 1 mock_info.config.params.write_consistency_factor = 1 + mock_info.config.params.read_fan_out_factor = None + mock_info.config.params.read_fan_out_delay_ms = None + mock_info.config.params.on_disk_payload = None mock_info.config.hnsw_config.m = 16 mock_info.config.hnsw_config.ef_construct = 100 mock_info.config.hnsw_config.full_scan_threshold = None @@ -600,6 +736,9 @@ def test_show_collection_with_payload_schema(self, executor, mock_client, mocker mock_info.config.params.shard_number = 1 mock_info.config.params.replication_factor = 1 mock_info.config.params.write_consistency_factor = 1 + mock_info.config.params.read_fan_out_factor = None + mock_info.config.params.read_fan_out_delay_ms = None + mock_info.config.params.on_disk_payload = None mock_info.config.params.sparse_vectors = None mock_info.config.hnsw_config.m = 16 mock_info.config.hnsw_config.ef_construct = 100 @@ -607,6 +746,7 @@ def test_show_collection_with_payload_schema(self, executor, mock_client, mocker mock_info.config.hnsw_config.max_indexing_threads = None mock_info.config.hnsw_config.on_disk = None mock_info.config.hnsw_config.payload_m = None + mock_info.config.hnsw_config.inline_storage = True mock_info.config.quantization_config = None mock_info.payload_schema = {"category": idx_info} @@ -622,6 +762,37 @@ def test_show_collection_with_payload_schema(self, executor, mock_client, mocker "params": {"is_tenant": True, "on_disk": True}, } } + assert result.data["hnsw_config"]["inline_storage"] is True + + def test_format_collection_diagnostics_does_not_duplicate_replication(self): + text = _format_collection_diagnostics( + { + "name": "docs", + "status": "green", + "points_count": 1, + "indexed_vectors_count": 1, + "segments_count": 1, + "topology": "dense", + "vectors": {"": {"size": 384, "distance": "Cosine", "on_disk": True}}, + "sparse_vectors": None, + "quantization": None, + "hnsw_config": {"m": 16, "ef_construct": 100, "inline_storage": True}, + "payload_schema": None, + "sharding": { + "shard_number": 1, + "replication_factor": 2, + "write_consistency_factor": 1, + "read_fan_out_factor": 4, + "read_fan_out_delay_ms": 10, + "on_disk_payload": False, + }, + } + ) + assert text.count("Replication factor") == 1 + assert text.count("Write consistency") == 1 + assert "Replicas :" not in text + assert "Payload indexes : none" in text + assert "HNSW inline_storage : True" in text def test_show_collection_handles_missing_payload_schema(self, executor, mock_client, mocker): from qdrant_client.models import ( diff --git a/tests/test_lexer.py b/tests/test_lexer.py index a9bfd57..aad9855 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -234,6 +234,26 @@ def test_fusion_keyword(self): ks = kinds("FUSION") assert ks[0] == TokenKind.FUSION + def test_alter_keyword(self): + ks = kinds("ALTER") + assert ks[0] == TokenKind.ALTER + + def test_vectors_keyword(self): + ks = kinds("VECTORS") + assert ks[0] == TokenKind.VECTORS + + def test_optimizers_keyword(self): + ks = kinds("OPTIMIZERS") + assert ks[0] == TokenKind.OPTIMIZERS + + def test_params_keyword(self): + ks = kinds("PARAMS") + assert ks[0] == TokenKind.PARAMS + + def test_disabled_keyword(self): + ks = kinds("DISABLED") + assert ks[0] == TokenKind.DISABLED + def test_hybrid_in_create_statement(self): ks = kinds("CREATE COLLECTION articles HYBRID") assert ks[3] == TokenKind.HYBRID diff --git a/tests/test_parser.py b/tests/test_parser.py index 2c61b66..efa30cd 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,8 +1,11 @@ import pytest from qql.ast_nodes import ( + AlterCollectionStmt, AndExpr, BetweenExpr, + CollectionParamsConfig, + CollectionConfig, CompareExpr, CreateCollectionStmt, CreateIndexStmt, @@ -20,7 +23,9 @@ MatchTextExpr, NotExpr, NotInExpr, + OptimizersRuntimeConfig, OrExpr, + QuantizationUpdate, QuantizationType, QuantizationSearchWith, RecommendStmt, @@ -29,6 +34,8 @@ SearchStmt, ShowCollectionStmt, ShowCollectionsStmt, + VectorsConfig, + HnswRuntimeConfig, ) from qql.exceptions import QQLSyntaxError from qql.lexer import Lexer @@ -666,6 +673,93 @@ def test_create_using_model_case_insensitive(self): assert node.model == "some/model" +class TestCollectionConfig: + def test_create_with_collection_blocks(self): + node = parse( + "CREATE COLLECTION articles " + "WITH VECTORS { on_disk: true } " + "WITH HNSW { m: 32, ef_construct: 200, full_scan_threshold: 5000, " + "max_indexing_threads: 2, on_disk: true, payload_m: 24, inline_storage: false } " + "WITH OPTIMIZERS { indexing_threshold: 10000, memmap_threshold: 20000, deleted_threshold: 0.2, max_optimization_threads: 'auto' } " + "WITH PARAMS { replication_factor: 2, write_consistency_factor: 1, on_disk_payload: true } " + "QUANTIZE SCALAR ALWAYS RAM" + ) + assert isinstance(node, CreateCollectionStmt) + assert node.config == CollectionConfig( + vectors=VectorsConfig(on_disk=True), + hnsw=HnswRuntimeConfig( + m=32, + ef_construct=200, + full_scan_threshold=5000, + max_indexing_threads=2, + on_disk=True, + payload_m=24, + inline_storage=False, + ), + optimizers=OptimizersRuntimeConfig( + indexing_threshold=10000, + memmap_threshold=20000, + deleted_threshold=0.2, + max_optimization_threads="auto", + ), + params=CollectionParamsConfig( + replication_factor=2, + write_consistency_factor=1, + on_disk_payload=True, + ), + ) + assert node.quantization is not None + assert node.quantization.type == QuantizationType.SCALAR + assert node.quantization.always_ram is True + + def test_alter_with_collection_blocks(self): + node = parse( + "ALTER COLLECTION articles " + "WITH VECTORS { on_disk: true } " + "WITH HNSW { full_scan_threshold: 1234 } " + "WITH OPTIMIZERS { indexing_threshold: 4567, prevent_unoptimized: true } " + "WITH PARAMS { on_disk_payload: false, read_fan_out_factor: 4 } " + "QUANTIZE DISABLED" + ) + assert isinstance(node, AlterCollectionStmt) + assert node.config == CollectionConfig( + vectors=VectorsConfig(on_disk=True), + hnsw=HnswRuntimeConfig(full_scan_threshold=1234), + optimizers=OptimizersRuntimeConfig( + indexing_threshold=4567, + prevent_unoptimized=True, + ), + params=CollectionParamsConfig( + on_disk_payload=False, + read_fan_out_factor=4, + ), + ) + assert node.quantization == QuantizationUpdate(disabled=True) + + def test_create_hnsw_without_with_rejected(self): + with pytest.raises(QQLSyntaxError): + parse("CREATE COLLECTION articles HNSW { payload_m: 24 }") + + def test_duplicate_collection_config_rejected(self): + with pytest.raises(QQLSyntaxError): + parse( + "CREATE COLLECTION articles " + "WITH HNSW { payload_m: 24 } " + "WITH HNSW { payload_m: 32 }" + ) + + def test_create_quantize_must_come_after_with_blocks(self): + with pytest.raises(QQLSyntaxError): + parse("CREATE COLLECTION articles QUANTIZE SCALAR WITH HNSW { payload_m: 24 }") + + def test_create_rejects_alter_only_params(self): + with pytest.raises(QQLSyntaxError, match="supported only for ALTER COLLECTION"): + parse( + "CREATE COLLECTION articles " + "WITH PARAMS { read_fan_out_factor: 4 }" + ) + + class TestHybridInsert: def test_insert_using_hybrid_sets_flag(self): node = parse("INSERT INTO COLLECTION col VALUES {'text': 'hi'} USING HYBRID") diff --git a/tests/test_script.py b/tests/test_script.py index 16e5d51..2c4b21b 100644 --- a/tests/test_script.py +++ b/tests/test_script.py @@ -50,6 +50,12 @@ def test_comment_at_start_of_line(self): assert "DROP" in result assert "leading" not in result + def test_preserves_double_dash_inside_string_literal(self): + text = "INSERT INTO COLLECTION x VALUES {'text': 'hello--world'} -- trailing comment" + result = strip_comments(text) + assert "hello--world" in result + assert "trailing comment" not in result + # ── split_statements ────────────────────────────────────────────────────────── @@ -134,6 +140,18 @@ def test_select_starts_new_top_level_statement(self): assert len(chunks) == 3 assert chunks[1][0].kind == TokenKind.SELECT + def test_alter_starts_new_top_level_statement(self): + from qql.lexer import TokenKind + + tokens = tokenize( + "CREATE COLLECTION x\n" + "ALTER COLLECTION x WITH HNSW { payload_m: 24 }\n" + "SHOW COLLECTIONS" + ) + chunks = split_statements(tokens) + assert len(chunks) == 3 + assert chunks[1][0].kind == TokenKind.ALTER + # ── run_script ──────────────────────────────────────────────────────────────── From fdf8cb8e76688e894c9986cbd12761973404fe42 Mon Sep 17 00:00:00 2001 From: Srimon Date: Mon, 18 May 2026 17:23:18 +0530 Subject: [PATCH 2/2] feat: readme update --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 372a8b4..7b15af8 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,8 @@ CREATE COLLECTION articles HYBRID CREATE COLLECTION articles WITH HNSW { payload_m: 16 } CREATE COLLECTION articles WITH VECTORS { on_disk: true } WITH HNSW { full_scan_threshold: 10000 } ALTER COLLECTION articles WITH OPTIMIZERS { indexing_threshold: 10000 } +ALTER COLLECTION articles WITH PARAMS { read_fan_out_factor: 4, on_disk_payload: false } +ALTER COLLECTION articles QUANTIZE DISABLED CREATE COLLECTION articles QUANTIZE SCALAR CREATE COLLECTION articles QUANTIZE TURBO CREATE COLLECTION articles QUANTIZE TURBO BITS 2