Codex/organize database structure by court and year y87cij

README.md

@@ -104,6 +104,24 @@ python main.py --query "artificial intelligence" --retrieval_model all-minilm --
 python main.py --query "research papers" --corpus_dir ./my_documents --retrieval_model colpali
 ```
 
+### ✅ Easy UI (No Command Line)
+
+If you prefer a simple UI, use the Streamlit app:
+
+1. **Install dependencies** (one time):
+   ```bash
+   pip install -r requirements.txt
+   ```
+2. **Double-click** `start_ui.bat` (Windows) to launch the UI.
+3. A browser window will open with the NanoSage UI.
+
+The UI lets you select:
+- Query text
+- FAISS root folder or a single index/meta pair
+- Retrieval model (must match your index embedding model)
+
+**Note for Windows users:** The UI launcher prefers Python 3.10/3.11. If you installed Python 3.13+ (e.g., 3.14) and see `%1 is not a valid Win32 application`, install Python 3.11 and try again. The launcher will skip Python 3.13+ on PATH and fall back automatically if other versions are installed.
+
 **Parameters**:
 - `--query`: Main search query (natural language).
 - `--web_search`: Enables web-based retrieval via Tavily API.
@@ -163,6 +181,29 @@ python main.py --query "AI in finance" \
 
 Now the system searches **both** local docs and web data (if `--web_search` is enabled).
 
+#### ✅ Using a FAISS Index (index.faiss + meta.jsonl)
+
+If you already have FAISS indexes and JSONL metadata (one line per document), you can load them directly:
+
+```bash
+pip install faiss-cpu
+
+python main.py --query "contract law precedents" \
+               --faiss_index_path "C:\path\to\index.faiss" \
+               --faiss_meta_path "C:\path\to\meta.jsonl" \
+               --retrieval_model all-minilm
+```
+
+Make sure the FAISS index was built with embeddings from the same retrieval model you select (e.g., `all-minilm`). The metadata lines should include `file_path` and optional `snippet` fields for best report output.
+
+If you have many year/court folders, point to a root directory that contains multiple `index.faiss` + `meta.jsonl` pairs and NanoSage will load them all:
+
+```bash
+python main.py --query "contract law precedents" \
+               --faiss_root_dir "C:\Users\vasil\Desktop\Ai project\_INDEX" \
+               --retrieval_model all-minilm
+```
+
 #### 🔄 RAG with Gemma 2B
 
 ```bash

app.py

@@ -0,0 +1,125 @@
+# app.py
+
+import asyncio
+import os
+import yaml
+import streamlit as st
+
+from search_session import SearchSession
+
+
+def load_config(config_path: str):
+    if not config_path or not os.path.isfile(config_path):
+        return {}
+    with open(config_path, "r", encoding="utf-8") as f:
+        return yaml.safe_load(f) or {}
+
+
+def run_search_session(
+    query: str,
+    config: dict,
+    corpus_dir: str,
+    device: str,
+    retrieval_model: str,
+    top_k: int,
+    web_search_enabled: bool,
+    personality: str,
+    rag_model: str,
+    max_depth: int,
+    llm_provider: str,
+    llm_model: str,
+    faiss_index_path: str,
+    faiss_meta_path: str,
+    faiss_root_dir: str,
+):
+    session = SearchSession(
+        query=query,
+        config=config,
+        corpus_dir=corpus_dir or None,
+        device=device,
+        retrieval_model=retrieval_model,
+        top_k=top_k,
+        web_search_enabled=web_search_enabled,
+        personality=personality or None,
+        rag_model=rag_model,
+        max_depth=max_depth,
+        llm_provider=llm_provider,
+        llm_model=llm_model or None,
+        faiss_index_path=faiss_index_path or None,
+        faiss_meta_path=faiss_meta_path or None,
+        faiss_root_dir=faiss_root_dir or None,
+    )
+    final_answer = asyncio.run(session.run_session())
+    output_path = session.save_report(final_answer)
+    return final_answer, output_path
+
+
+st.set_page_config(page_title="NanoSage UI", layout="wide")
+st.title("NanoSage UI")
+
+with st.sidebar:
+    st.header("Configuration")
+    config_path = st.text_input("Config path", value="config.yaml")
+    device = st.selectbox("Device", ["cpu", "cuda"], index=0)
+    retrieval_model = st.selectbox(
+        "Retrieval model",
+        ["colpali", "all-minilm", "siglip", "clip"],
+        index=1,
+    )
+    top_k = st.number_input("Top K", min_value=1, max_value=20, value=3, step=1)
+    web_search_enabled = st.checkbox("Enable web search", value=False)
+    personality = st.text_input("Personality (optional)")
+    rag_model = st.text_input("RAG model", value="gemma")
+    max_depth = st.number_input("Max depth", min_value=0, max_value=5, value=1, step=1)
+    llm_provider = st.selectbox("LLM provider", ["ollama", "openai", "anthropic"], index=0)
+    llm_model = st.text_input("LLM model (optional)")
+
+st.subheader("Query")
+query = st.text_area("Enter your question", height=120)
+
+st.subheader("Local Data Sources")
+corpus_dir = st.text_input("Corpus folder (optional)")
+faiss_root_dir = st.text_input("FAISS root folder (optional)")
+faiss_index_path = st.text_input("FAISS index path (optional)")
+faiss_meta_path = st.text_input("FAISS meta.jsonl path (optional)")
+
+run_button = st.button("Run Search")
+
+if run_button:
+    if not query.strip():
+        st.error("Please enter a query.")
+    elif faiss_root_dir and (faiss_index_path or faiss_meta_path):
+        st.error("Provide either a FAISS root folder or a single FAISS index/meta pair, not both.")
+    elif (faiss_index_path and not faiss_meta_path) or (faiss_meta_path and not faiss_index_path):
+        st.error("Provide both FAISS index and metadata paths together.")
+    else:
+        try:
+            config = load_config(config_path)
+            if web_search_enabled:
+                config.update({
+                    "web_concurrency": config.get("web_concurrency", 8),
+                    "include_wikipedia": config.get("include_wikipedia", False),
+                })
+            with st.spinner("Running search..."):
+                answer, output_path = run_search_session(
+                    query=query.strip(),
+                    config=config,
+                    corpus_dir=corpus_dir,
+                    device=device,
+                    retrieval_model=retrieval_model,
+                    top_k=top_k,
+                    web_search_enabled=web_search_enabled,
+                    personality=personality,
+                    rag_model=rag_model,
+                    max_depth=max_depth,
+                    llm_provider=llm_provider,
+                    llm_model=llm_model,
+                    faiss_index_path=faiss_index_path,
+                    faiss_meta_path=faiss_meta_path,
+                    faiss_root_dir=faiss_root_dir,
+                )
+            st.success(f"Report saved to: {output_path}")
+            st.markdown("### Final Answer")
+            st.write(answer)
+        except Exception as exc:
+            st.error(f"Error: {exc}")

knowledge_base.py

@@ -1,9 +1,10 @@
-import os
-import io
-import torch
-import numpy as np
-import fitz  # PyMuPDF
-from PIL import Image
+import os
+import io
+import json
+import torch
+import numpy as np
+import fitz  # PyMuPDF
+from PIL import Image
 
 ############################
 # Load & Configure Retrieval
@@ -16,7 +17,7 @@ def _pick_dtype(device: str):
     return torch.float32
 
 
-def load_retrieval_model(model_choice="colpali", device="cpu"):
+def load_retrieval_model(model_choice="colpali", device="cpu"):
     """
     Backward-compatible loader with extra, faster VLM options.
     Returns: (model, processor, model_type)
@@ -61,7 +62,47 @@ def load_retrieval_model(model_choice="colpali", device="cpu"):
     else:
         raise ValueError(f"Unsupported retrieval model choice: {model_choice}")
 
-    return model, processor, model_type
+    return model, processor, model_type
+
+
+def load_faiss_index(index_path: str, meta_path: str):
+    if not index_path or not meta_path:
+        raise ValueError("Both index_path and meta_path are required to load a FAISS index.")
+    if not os.path.isfile(index_path):
+        raise FileNotFoundError(f"FAISS index file not found: {index_path}")
+    if not os.path.isfile(meta_path):
+        raise FileNotFoundError(f"FAISS metadata file not found: {meta_path}")
+    try:
+        import faiss
+    except ImportError as exc:
+        raise ImportError("faiss is required to load FAISS indexes. Install faiss-cpu.") from exc
+
+    index = faiss.read_index(index_path)
+    metadata = []
+    with open(meta_path, "r", encoding="utf-8") as f:
+        for line_no, line in enumerate(f, start=1):
+            if not line.strip():
+                continue
+            try:
+                metadata.append(json.loads(line))
+            except json.JSONDecodeError as exc:
+                print(f"[WARN] Skipping malformed JSONL line {line_no} in {meta_path}: {exc}")
+    return index, metadata
+
+
+def find_faiss_pairs(root_dir: str, index_name: str = "index.faiss", meta_name: str = "meta.jsonl"):
+    if not root_dir:
+        return []
+    if not os.path.isdir(root_dir):
+        raise FileNotFoundError(f"FAISS root directory not found: {root_dir}")
+
+    pairs = []
+    for dirpath, _, filenames in os.walk(root_dir):
+        if index_name in filenames and meta_name in filenames:
+            pairs.append(
+                (os.path.join(dirpath, index_name), os.path.join(dirpath, meta_name))
+            )
+    return pairs
 
 
 def _l2norm(x: torch.Tensor) -> torch.Tensor:

main.py

@@ -24,12 +24,15 @@ def main():
     parser.add_argument("--web_search", action="store_true", default=False, help="Enable web search")
     parser.add_argument("--personality", type=str, default=None, help="Optional personality for LLM (e.g. cheerful)")
     parser.add_argument("--rag_model", type=str, default="gemma", help="Which model to use for final RAG steps")
-    parser.add_argument("--max_depth", type=int, default=1, help="Depth limit for subquery expansions")
-    parser.add_argument("--llm_provider", type=str, choices=["ollama", "openai", "anthropic"], default="ollama", help="LLM provider to use")
-    parser.add_argument("--llm_model", type=str, default=None, help="Specific LLM model to use (overrides provider default)")
-    parser.add_argument("--web_concurrency", type=int, default=8, help="Concurrent web downloads")
-    parser.add_argument("--include_wikipedia", action="store_true", help="Include Wikipedia in web search")
-    args = parser.parse_args()
+    parser.add_argument("--max_depth", type=int, default=1, help="Depth limit for subquery expansions")
+    parser.add_argument("--llm_provider", type=str, choices=["ollama", "openai", "anthropic"], default="ollama", help="LLM provider to use")
+    parser.add_argument("--llm_model", type=str, default=None, help="Specific LLM model to use (overrides provider default)")
+    parser.add_argument("--web_concurrency", type=int, default=8, help="Concurrent web downloads")
+    parser.add_argument("--include_wikipedia", action="store_true", help="Include Wikipedia in web search")
+    parser.add_argument("--faiss_index_path", type=str, default=None, help="Path to FAISS index file")
+    parser.add_argument("--faiss_meta_path", type=str, default=None, help="Path to FAISS metadata JSONL file")
+    parser.add_argument("--faiss_root_dir", type=str, default=None, help="Root directory containing FAISS index/meta pairs")
+    args = parser.parse_args()
 
     config = load_config(args.config)
 
@@ -49,10 +52,13 @@ def main():
         web_search_enabled=args.web_search,
         personality=args.personality,
         rag_model=args.rag_model,
-        max_depth=args.max_depth,
-        llm_provider=args.llm_provider,
-        llm_model=args.llm_model
-    )
+        max_depth=args.max_depth,
+        llm_provider=args.llm_provider,
+        llm_model=args.llm_model,
+        faiss_index_path=args.faiss_index_path,
+        faiss_meta_path=args.faiss_meta_path,
+        faiss_root_dir=args.faiss_root_dir
+    )
 
     loop = asyncio.get_event_loop()
     final_answer = loop.run_until_complete(session.run_session())

requirements.txt

@@ -30,3 +30,6 @@ ollama
 # Tavily Search API
 langchain-tavily
 python-dotenv
+
+# UI
+streamlit