-incorporated self correcting rag, -all code will defaultly support pdf and txt extentions, -other improvements in docs (#17)

Co-authored-by: pavanmantha <pavan.mantha@thevaslabs.io>

Author: pavanjava

bootstraprag/cli.py

@@ -35,6 +35,7 @@ def create(project_name, framework, template, observability):
             'rag-with-react',
             'rag-with-hyde',
             'rag-with-flare',
+            'rag-with-self-correction',
             'llama-agents-with-simpleq'
         ]
     elif framework == 'None':

bootstraprag/templates/llamaindex/rag_with_flare/base_rag.py

@@ -33,7 +33,7 @@ class BaseRAG:
     ]
 
     def __init__(self, data_path: str, chunk_size: int = 512, chunk_overlap: int = 200,
-                 required_exts: list[str] = ['.pdf'],
+                 required_exts: list[str] = ['.pdf', '.txt'],
                  show_progress: bool = False, similarity_top_k: int = 3, max_iterations: int = 5):
         # load the local data directory and chunk the data for further processing
         self.docs = SimpleDirectoryReader(input_dir=data_path, required_exts=required_exts).load_data(

bootstraprag/templates/llamaindex/rag_with_hyde/base_rag.py

@@ -37,7 +37,7 @@ class BaseRAG:
         Response, StreamingResponse, AsyncStreamingResponse, PydanticResponse
     ]
 
-    def __init__(self, data_path: str, chunk_size: int = 512, chunk_overlap: int = 200, required_exts: list[str] = ['.pdf'],
+    def __init__(self, data_path: str, chunk_size: int = 512, chunk_overlap: int = 200, required_exts: list[str] = ['.pdf', '.txt'],
                  show_progress: bool = False, similarity_top_k: int = 3):
         # load the local data directory and chunk the data for further processing
         self.docs = SimpleDirectoryReader(input_dir=data_path, required_exts=required_exts).load_data(

bootstraprag/templates/llamaindex/rag_with_hyde_with_observability/base_rag.py

@@ -42,7 +42,7 @@ class BaseRAG:
     ]
 
     def __init__(self, data_path: str, chunk_size: int = 512, chunk_overlap: int = 200,
-                 required_exts: list[str] = ['.pdf'],
+                 required_exts: list[str] = ['.pdf', '.txt'],
                  show_progress: bool = False, similarity_top_k: int = 3):
         # load the local data directory and chunk the data for further processing
         self.docs = SimpleDirectoryReader(input_dir=data_path, required_exts=required_exts).load_data(

bootstraprag/templates/llamaindex/rag_with_react/main.py

@@ -14,4 +14,4 @@
         break
 
     response = react_with_engine.query(user_query=user_query)
-    print(response)  
+    print(response)

bootstraprag/templates/llamaindex/rag_with_react/react_agent_with_query_engine.py

@@ -23,12 +23,12 @@
 
 
 class ReActWithQueryEngine:
-
     RESPONSE_TYPE = Union[
         Response, StreamingResponse, AsyncStreamingResponse, PydanticResponse
     ]
 
-    def __init__(self, input_dir: str, similarity_top_k: int = 3, chunk_size: int = 128, chunk_overlap: int = 100, show_progress: bool = False, no_of_iterations: int = 5):
+    def __init__(self, input_dir: str, similarity_top_k: int = 3, chunk_size: int = 128, chunk_overlap: int = 100,
+                 show_progress: bool = False, no_of_iterations: int = 5, required_exts: list[str] = ['.pdf', '.txt']):
         self.index_loaded = False
         self.similarity_top_k = similarity_top_k
         self.input_dir = input_dir
@@ -38,6 +38,7 @@ def __init__(self, input_dir: str, similarity_top_k: int = 3, chunk_size: int =
         self.query_engine_tools = []
         self.show_progress = show_progress
         self.no_of_iterations = no_of_iterations
+        self.required_exts = required_exts
 
         # use your prefered vector embeddings model
         logger.info("initializing the OllamaEmbedding")
@@ -58,7 +59,8 @@ def __init__(self, input_dir: str, similarity_top_k: int = 3, chunk_size: int =
 
         # Create a local Qdrant vector store
         logger.info("initializing the vector store related objects")
-        self.client: qdrant_client.QdrantClient = qdrant_client.QdrantClient(url=os.environ['DB_URL'], api_key=os.environ['DB_API_KEY'])
+        self.client: qdrant_client.QdrantClient = qdrant_client.QdrantClient(url=os.environ['DB_URL'],
+                                                                             api_key=os.environ['DB_API_KEY'])
         self.vector_store = QdrantVectorStore(client=self.client, collection_name=os.environ['COLLECTION_NAME'])
         self._load_data_and_create_engine()
 
@@ -72,13 +74,15 @@ def _load_data_and_create_engine(self):
 
         if not self.index_loaded:
             # load data
-            _docs = SimpleDirectoryReader(input_dir=self.input_dir).load_data(show_progress=self.show_progress)
+            _docs = (SimpleDirectoryReader(input_dir=self.input_dir, required_exts=self.required_exts)
+                     .load_data(show_progress=self.show_progress))
 
             # build and persist index
             storage_context = StorageContext.from_defaults(vector_store=self.vector_store)
             logger.info("indexing the docs in VectorStoreIndex")
-            self._index = VectorStoreIndex.from_documents(documents=_docs, storage_context=storage_context, show_progress=self.show_progress)
-            
+            self._index = VectorStoreIndex.from_documents(documents=_docs, storage_context=storage_context,
+                                                          show_progress=self.show_progress)
+
         self._engine = self._index.as_query_engine(similarity_top_k=self.similarity_top_k)
         self._create_query_engine_tools()
 

bootstraprag/templates/llamaindex/rag_with_react_with_observability/react_agent_with_query_engine.py

@@ -34,7 +34,7 @@ class ReActWithQueryEngine:
     ]
 
     def __init__(self, input_dir: str, similarity_top_k: int = 3, chunk_size: int = 128, chunk_overlap: int = 100,
-                 show_progress: bool = False, no_of_iterations: int = 5):
+                 show_progress: bool = False, no_of_iterations: int = 5, required_exts: list[str] = ['.pdf', '.txt']):
         self.index_loaded = False
         self.similarity_top_k = similarity_top_k
         self.input_dir = input_dir
@@ -44,6 +44,7 @@ def __init__(self, input_dir: str, similarity_top_k: int = 3, chunk_size: int =
         self.query_engine_tools = []
         self.show_progress = show_progress
         self.no_of_iterations = no_of_iterations
+        self.required_exts = required_exts
 
         # use your prefered vector embeddings model
         logger.info("initializing the OllamaEmbedding")
@@ -79,7 +80,8 @@ def _load_data_and_create_engine(self):
 
         if not self.index_loaded:
             # load data
-            _docs = SimpleDirectoryReader(input_dir=self.input_dir).load_data(show_progress=self.show_progress)
+            _docs = (SimpleDirectoryReader(input_dir=self.input_dir, required_exts=self.required_exts)
+                     .load_data(show_progress=self.show_progress))
 
             # build and persist index
             storage_context = StorageContext.from_defaults(vector_store=self.vector_store)

bootstraprag/templates/llamaindex/rag_with_self_correction/.env

@@ -0,0 +1,21 @@
+DB_URL='http://localhost:6333'
+DB_API_KEY='th3s3cr3tk3y'
+COLLECTION_NAME='YOUR_COLLECTION'
+
+OPENAI_API_KEY=''
+OPENAI_EMBED_MODEL=''
+
+# use this incase you are prefering to experiment with local models.
+OLLAMA_BASE_URL='http://localhost:11434'
+OLLAMA_LLM_MODEL='llama3.1'
+OLLAMA_EMBED_MODEL='nomic-embed-text:latest'
+
+# logger can be controlled usiing env
+CRITICAL = 50
+FATAL = 50
+ERROR = 40
+WARNING = 30
+WARN = 30
+INFO = 20
+DEBUG = 10
+NOTSET = 0

bootstraprag/templates/llamaindex/rag_with_self_correction/data/mlops.pdf

@@ -0,0 +1,23 @@
+# driver code
+from self_correction_core import SelfCorrectingRAG
+
+
+self_correcting_rag = SelfCorrectingRAG(input_dir='data', show_progress=True, no_of_retries=3)
+
+# Start a loop to continually get input from the user
+while True:
+    # Get a query from the user
+    user_query = input("Enter your query [type 'bye' to 'exit']: ")
+
+    # Check if the user wants to terminate the loop
+    if user_query.lower() == "bye" or user_query.lower() == "exit":
+        break
+
+    response1 = self_correcting_rag.query_with_retry_query_engine(query=user_query)
+    print(response1)
+
+    response1 = self_correcting_rag.query_with_source_query_engine(query=user_query)
+    print(response1)
+
+    response1 = self_correcting_rag.query_with_guideline_query_engine(query=user_query)
+    print(response1)