support for .md .txt and .docx files

llm/api.py

@@ -5,7 +5,7 @@
 import openai
 from logging import basicConfig, INFO, getLogger
 
-from pypdf import PdfReader
+from django.core.files.uploadedfile import TemporaryUploadedFile
 from pgvector.django import L2Distance
 from django.http import JsonResponse
 from django.forms.models import model_to_dict
@@ -22,7 +22,7 @@
 )
 from llm.utils.general import generate_session_id
 from llm.models import Organization, Embedding, Message, File, KnowledgeCategory
-
+from llm.utils.document import OpenLLMDocumentLoader
 
 basicConfig(level=INFO)
 logger = getLogger()
@@ -61,18 +61,18 @@ def create_chat(request):
             request.data.get("system_prompt", None) or organization.system_prompt
         )
         system_prompt = system_prompt.strip() if system_prompt else None
-        logger.info(f"Using the system prompt : {system_prompt}")
 
         gpt_model = request.data.get("gpt_model", "gpt-3.5-turbo").strip()
         session_id = (request.data.get("session_id") or generate_session_id()).strip()
 
         # 1. Function calling to do language detection of the user's question (1st call to OpenAI)
         response = openai.ChatCompletion.create(
             model=gpt_model,
+            response_format={"type": "json_object"},
             messages=[
                 {
                     "role": "user",
-                    "content": f"Detect the languages in this text: {question}",
+                    "content": f"Detect the languages in this text: {question}. Produce the result in JSON",
                 }
             ],
             functions=[
@@ -184,7 +184,7 @@ def create_chat(request):
 
         # 5. Evaluate if the request asks for it
         evaluation_scores = {}
-        if request.data.get("evaluate"):
+        if request.data.get("evaluate") and request.data.get("evaluate") == "true":
             logger.info("Evaluting the response")
             evaluator_prompts = organization.evaluator_prompts  # { 'coherence': ... }
             for criteria, evaluator_prompt in evaluator_prompts.items():
@@ -250,14 +250,15 @@ def post(self, request, format=None):
 
             openai.api_key = org.openai_key
 
-            request_file = request.data["file"]
+            request_file: TemporaryUploadedFile = request.data["file"]
             file_name = (
                 request.data["filename"].strip()
                 if "filename" in request.data
                 else request_file.name
             )
 
             logger.info("Recognized file with name %s", file_name)
+            logger.info("Upload file is of content type %s", request_file.content_type)
 
             if "category_id" not in request.data:
                 return JsonResponse(
@@ -285,9 +286,11 @@ def post(self, request, format=None):
                 name=file_name,
             )
 
-            pdf_reader = PdfReader(request_file)
-            for page in pdf_reader.pages:
-                page_text = page.extract_text().replace("\n", " ")
+            doc_loader = OpenLLMDocumentLoader(
+                request_file.temporary_file_path(), request_file.content_type
+            )
+            for page in doc_loader.chunks():
+                page_text = page.page_content
 
                 if len(page_text) > 0:
                     response = openai.Embedding.create(

llm/data/glific_sources/links_and_titles.md

@@ -0,0 +1,113 @@
+# Links to Glific documentation
+
+- [Introduction](https://glific.github.io/docs/docs/intro)
+- [Onboarding](https://glific.github.io/docs/docs/category/onboarding)
+- [Glific Onboarding Toolkit](https://glific.github.io/docs/docs/Onboarding/Glific%20Onboarding%20Toolkit)
+- [Facebook Verification Process for WhatsApp Business API](https://glific.github.io/docs/docs/Onboarding/Facebook%20Verification%20Process%20for%20WhatsApp%20Business%20API)
+- [Setup an organisation on GupShup](https://glific.github.io/docs/docs/Onboarding/Setup%20an%20organisation%20on%20GupShup)
+- [Pricing, Billing & Payment](https://glific.github.io/docs/docs/Onboarding/Pricing,%20Billing%20&%20Payment)
+- [GCS Setup](https://glific.github.io/docs/docs/category/gcs-setup)
+- [Google Cloud Storage Setup](https://glific.github.io/docs/docs/Onboarding/GCS%20Setup/Google%20Cloud%20Storage%20Setup)
+- [Create an account in Google Cloud Storage GCS](https://glific.github.io/docs/docs/Onboarding/GCS%20Setup/Create%20an%20account%20in%20Google%20Cloud%20Storage%20GCS)
+- [Upload files on GCS](https://glific.github.io/docs/docs/Onboarding/GCS%20Setup/Upload%20files%20on%20GCS)
+- [Pilot Essentials](https://glific.github.io/docs/docs/Pilot%20Essentials)
+- [Product Features](https://glific.github.io/docs/docs/category/product-features)
+- [Chats](https://glific.github.io/docs/docs/Product%20Features/Chats)
+- [Speed Sends](https://glific.github.io/docs/docs/Product%20Features/Speed%20Sends)
+- [Flows](https://glific.github.io/docs/docs/category/flows)
+- [Flow Overview](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Overview)
+- [Flow Actions](https://glific.github.io/docs/docs/category/flow-actions)
+- [Flow Actions Overview](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Flow%20Actions%20Overview)
+- [Send the contact a message](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Send%20the%20contact%20a%20message)
+- [Send the contact an interactive message](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Send%20the%20contact%20an%20interactive%20message)
+- [Wait for the contact to respond](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Wait%20for%20the%20contact%20to%20respond)
+- [Wait for time](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Wait%20for%20time)
+- [Send a staff member a message](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Send%20a%20staff%20member%20a%20message)
+- [Label the incoming message](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Label%20the%20incoming%20message)
+- [Add or Remove the contact to a collection](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Add%20or%20Remove%20the%20contact%20to%20a%20collection)
+- [Update the contact](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Update%20the%20contact)
+- [Save a result for this flow](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Save%20a%20result%20for%20this%20flow)
+- [Call a webhook](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Call%20a%20webhook)
+- [Link Google Sheets](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Link%20Google%20Sheets)
+- [Enter another flow](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Enter%20another%20flow)
+- [Start somebody else in a flow](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Start%20somebody%20else%20in%20a%20flow)
+- [Wait for result](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Wait%20for%20result)
+- [Open a ticket with a human agent](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Open%20a%20ticket%20with%20a%20human%20agent)
+- [Manage profile](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Manage%20profile)
+- [Split By](https://glific.github.io#)
+- [Custom Expression](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Split%20By/Custom%20Expression)
+- [Contact Field](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Split%20By/Contact%20Field)
+- [Result in the Flow](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Split%20By/Result%20in%20the%20Flow)
+- [Random Chance for AB Tests](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Split%20By/Random%20Chance%20for%20AB%20Tests)
+- [Collection Membership](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Actions/Split%20By/Collection%20Membership)
+- [Flow Variables](https://glific.github.io/docs/docs/category/flow-variables)
+- [Predefined Contact Variables in Glific](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Variables/Predefined%20Contact%20Variables%20in%20Glific)
+- [Flow variables vs Contact variables](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Variables/Flow%20variables%20vs%20Contact%20variables)
+- [List of Custom Contact Variables created in Flows](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Variables/List%20of%20Custom%20Contact%20Variables%20created%20in%20Flows)
+- [Parent and Child variable](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Variables/Parent%20and%20Child%20variable)
+- [Create a Counter Variable in a Flow](https://glific.github.io/docs/docs/Product%20Features/Flows/Flow%20Variables/Create%20a%20Counter%20Variable%20in%20a%20Flow)
+- [Others](https://glific.github.io/docs/docs/category/others)
+- [Flows not working - Troubleshoot checklist](https://glific.github.io/docs/docs/Product%20Features/Flows/Others/Flows%20not%20working%20-%20Troubleshoot%20checklist)
+- [How to identify which keyword triggered the flow](https://glific.github.io/docs/docs/Product%20Features/Flows/Others/How%20to%20identify%20which%20keyword%20triggered%20the%20flow)
+- [Opt-in & Opt-out Flows](https://glific.github.io/docs/docs/Product%20Features/Flows/Others/Configure%20Optin%20&%20Optout%20preferences%20in%20Glific)
+- [Background Flows](https://glific.github.io/docs/docs/Product%20Features/Flows/Others/Background%20Flows)
+- [Create a flow to capture multiple answers for a single question](https://glific.github.io/docs/docs/Product%20Features/Flows/Others/Create%20a%20flow%20to%20capture%20multiple%20answers%20for%20a%20single%20question)
+- [Flows Auto translate](https://glific.github.io/docs/docs/Product%20Features/Flows/Others/Flows%20Auto%20translate)
+- [Triggers](https://glific.github.io/docs/docs/Product%20Features/Triggers)
+- [Searches](https://glific.github.io/docs/docs/Product%20Features/Searches)
+- [Templates](https://glific.github.io/docs/docs/Product%20Features/Templates)
+- [Interactive Messages](https://glific.github.io/docs/docs/Product%20Features/Interactive%20Messages)
+- [Notifications](https://glific.github.io/docs/docs/Product%20Features/Notifications)
+- [Reporting & Dashboard](https://glific.github.io/docs/docs/category/reporting--dashboard)
+- [BigQuery Setup and link with Glific](https://glific.github.io/docs/docs/Product%20Features/Reporting%20&%20Dashboard/BigQuery%20Setup%20and%20link%20with%20Glific)
+- [DataStudio Integration for Visualizations & Reports](https://glific.github.io/docs/docs/Product%20Features/Reporting%20&%20Dashboard/DataStudio%20Integration%20for%20Visualizations%20&%20Reports)
+- [Capturing Data for Reports & Analytics](https://glific.github.io/docs/docs/Product%20Features/Reporting%20&%20Dashboard/Capturing%20Data%20for%20Reports%20&%20Analytics)
+- [Making Custom Reports on DataStudio](https://glific.github.io/docs/docs/Product%20Features/Reporting%20&%20Dashboard/Making%20Custom%20Reports%20on%20DataStudio)
+- [Populate Data in Field Views- User Info Report](https://glific.github.io/docs/docs/Product%20Features/Reporting%20&%20Dashboard/Populate%20Data%20in%20Field%20Views-%20User%20Info%20Report)
+- [Understanding V5 Data Studio Reports](https://glific.github.io/docs/docs/Product%20Features/Reporting%20&%20Dashboard/Understanding%20V5%20Data%20Studio%20Reports)
+- [Help](https://glific.github.io/docs/docs/Product%20Features/Help)
+- [Others](https://glific.github.io/docs/docs/category/others-1)
+- [Getting Started with Glific](https://glific.github.io/docs/docs/Product%20Features/Others/Getting%20Started%20with%20Glific)
+- [Staff Management & Role Management](https://glific.github.io/docs/docs/Product%20Features/Others/Staff%20Management%20&%20Role%20Management)
+- [Languages](https://glific.github.io/docs/docs/Product%20Features/Others/Languages)
+- [Keywords](https://glific.github.io/docs/docs/Product%20Features/Others/Keywords)
+- [New Contact, Default flow Out of office hours notifications](https://glific.github.io/docs/docs/Product%20Features/Others/New%20Contact,%20Default%20flow%20Out%20of%20office%20hours%20notifications)
+- [Collections](https://glific.github.io/docs/docs/Product%20Features/Others/Collections)
+- [Import & Export Flows](https://glific.github.io/docs/docs/Product%20Features/Others/Import%20&%20Export%20Flows)
+- [Contact Profile & History](https://glific.github.io/docs/docs/Product%20Features/Others/Contact%20Profile%20&%20History)
+- [All product features](https://glific.github.io/docs/docs/Product%20Features/Others/All%20product%20features)
+- [Tags](https://glific.github.io/docs/docs/Product%20Features/Others/Tags)
+- [Starter Kit](https://glific.github.io#)
+- [15 Pre-launch Chatbot Checks](https://glific.github.io/docs/docs/Starter%20Kit/15%20Pre-launch%20Chatbot%20Checks)
+- [Integrations](https://glific.github.io/docs/docs/category/integrations)
+- [Bhashini ASR](https://glific.github.io/docs/docs/Integrations/Bhashini%20ASR)
+- [ChatGPT Integration via Jugalbandi](https://glific.github.io/docs/docs/Integrations/ChatGPT%20Integration%20via%20Jugalbandi)
+- [ChatGPT using OpenAI APIs](https://glific.github.io/docs/docs/Integrations/ChatGPT%20using%20OpenAI%20APIs)
+- [Conversing in voice notes over custom knowledge base](https://glific.github.io/docs/docs/Integrations/Conversing%20in%20voice%20notes%20over%20custom%20knowledge%20base)
+- [Dialogflow for Natural Language Processing](https://glific.github.io/docs/docs/Integrations/Dialogflow%20for%20Natural%20Language%20Processing)
+- [Setting up Exotel](https://glific.github.io/docs/docs/Integrations/Setting%20up%20Exotel)
+- [FAQ](https://glific.github.io/docs/docs/category/faq)
+- [Find opt-in link of our NGO](https://glific.github.io/docs/docs/FAQ/Find%20opt-in%20link%20of%20our%20NGO)
+- [Check if contact is not opted in](https://glific.github.io/docs/docs/FAQ/Check%20if%20contact%20is%20not%20opted%20in)
+- [Check if user is opted-in in the flow](https://glific.github.io/docs/docs/FAQ/Check%20if%20user%20is%20opted-in%20in%20the%20flow)
+- [Change admin in our FB business account](https://glific.github.io/docs/docs/FAQ/Change%20admin%20in%20our%20FB%20business%20account)
+- [Update my profile picture of BOT](https://glific.github.io/docs/docs/FAQ/Update%20my%20profile%20picture%20of%20BOT)
+- [Make sure flow is not interrupted if contact sends any keyword in between the flow.](https://glific.github.io/docs/docs/FAQ/Make%20sure%20flow%20is%20not%20interrupted%20if%20contact%20sends%20any%20keyword%20in%20between%20the%20flow.)
+- [Make a flow inactive](https://glific.github.io/docs/docs/FAQ/Make%20a%20flow%20inactive)
+- [Format the text in messages.](https://glific.github.io/docs/docs/FAQ/Format%20the%20text%20in%20messages.)
+- [Send a link of a specific flow to new contacts to join the BOT](https://glific.github.io/docs/docs/FAQ/Send%20a%20link%20of%20a%20specific%20flow%20to%20new%20contacts%20to%20join%20the%20BOT)
+- [Get upload attachment option in Send message](https://glific.github.io/docs/docs/FAQ/Get%20upload%20attachment%20option%20in%20Send%20message)
+- [Give permissions to individuals in Google Cloud Storage GCS](https://glific.github.io/docs/docs/FAQ/Give%20permissions%20to%20individuals%20in%20Google%20Cloud%20Storage%20GCS)
+- [Display Name, Picture & Guidelines for ChatBot](https://glific.github.io/docs/docs/FAQ/Add%20the%20display%20Name%20of%20the%20BOT%20number%20and%20show%20it%20to%20end-users)
+- [Create a flow to capture multiple answers for a single question](https://glific.github.io/docs/docs/FAQ/Create%20a%20flow%20to%20capture%20multiple%20answers%20for%20a%20single%20question)
+- [Check WhatsApp Quality Rating and Messaging Limits](https://glific.github.io/docs/docs/FAQ/Check%20WhatsApp%20Quality%20Rating%20and%20Messaging%20Limits)
+- [Read Receipts](https://glific.github.io/docs/docs/FAQ/Read%20Receipts)
+- [Find the QR Code of BOT for opt-in contacts.](https://glific.github.io/docs/docs/FAQ/Find%20the%20QR%20Code%20of%20BOT%20for%20opt-in%20contacts.)
+- [Gupshup wallet balance and Usage](https://glific.github.io/docs/docs/FAQ/Gupshup%20wallet%20balance%20and%20Usage)
+- [Update collection with bulk contacts or contact fields](https://glific.github.io/docs/docs/FAQ/Update%20collection%20with%20bulk%20contacts%20or%20contact%20fields)
+- [Get a data of a particular flow from BigQuery](https://glific.github.io/docs/docs/FAQ/Get%20a%20data%20of%20a%20particular%20flow%20from%20BigQuery)
+- [Get WABA ID](https://glific.github.io/docs/docs/FAQ/Get%20WABA%20ID)
+- [How to get Green Tick in Whatsapp Business](https://glific.github.io/docs/docs/FAQ/How%20to%20get%20Green%20Tick%20in%20Whatsapp%20Business)
+- [Using Glific APIs for OTP Authentication](https://glific.github.io/docs/docs/FAQ/Using%20Glific%20APIs%20for%20OTP%20Authentication)
+- [Get Glific Support](https://glific.github.io/docs/docs/FAQ/Get%20Glific%20Support)
+- [Join Discord Community](https://glific.github.iohttps://discord.gg/BP8WCy8sZf)

llm/settings.py

@@ -141,3 +141,9 @@
 # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field
 
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+
+
+FILE_UPLOAD_MAX_MEMORY_SIZE = 0
+
+
+FILE_UPLOAD_TEMP_DIR = "temp_files/"

llm/utils/document.py

@@ -0,0 +1,31 @@
+from typing import List
+from langchain_community.document_loaders.base import BaseLoader
+from langchain_community.document_loaders.text import TextLoader
+from langchain_community.document_loaders.pdf import PyPDFLoader
+from langchain_community.document_loaders import Docx2txtLoader
+from langchain_community.document_loaders.markdown import UnstructuredMarkdownLoader
+from langchain.schema import Document
+
+
+class OpenLLMDocumentLoader(BaseLoader):
+    def __init__(self, file_path: str, content_type: str):
+        self.file_path = file_path
+        self.file_type = content_type
+        self.loader: BaseLoader = None
+
+        if content_type == "text/plain":
+            self.loader = TextLoader(file_path)
+        elif content_type == "application/pdf":
+            self.loader = PyPDFLoader(file_path)
+        elif content_type == "text/markdown":
+            self.loader = UnstructuredMarkdownLoader(file_path, mode="elements")
+        elif (
+            content_type
+            == "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+        ):
+            self.loader = Docx2txtLoader(file_path)
+        else:
+            raise ValueError(f"Unsupported file type: {content_type}")
+
+    def chunks(self) -> List[Document]:
+        return self.loader.load_and_split()

requirements.txt

@@ -5,45 +5,75 @@ anyio==3.7.1
 asgiref==3.7.2
 async-timeout==4.0.3
 attrs==23.1.0
+backoff==2.2.1
+beautifulsoup4==4.12.3
 certifi==2023.7.22
+chardet==5.2.0
 charset-normalizer==3.3.0
-dataclasses-json==0.6.1
+click==8.1.7
+dataclasses-json==0.6.5
+deepdiff==7.0.1
 Django==4.2.6
 djangorestframework==3.14.0
+docx2txt==0.8
+emoji==2.11.1
 exceptiongroup==1.1.3
 faiss-cpu==1.7.4
+filetype==1.2.0
 frozenlist==1.4.0
 gunicorn==22.0.0
 idna==3.4
+joblib==1.4.2
 jsonpatch==1.33
+jsonpath-python==1.0.6
 jsonpointer==2.4
-langchain==0.0.315
-langsmith==0.0.44
+langchain==0.1.19
+langchain-community==0.0.38
+langchain-core==0.1.52
+langchain-text-splitters==0.0.1
+langdetect==1.0.9
+langsmith==0.1.56
+lxml==5.2.1
+Markdown==3.6
 marshmallow==3.20.1
 multidict==6.0.4
 mypy-extensions==1.0.0
+nltk==3.8.1
 numexpr==2.8.5
 numpy==1.26.1
 openai==0.28.1
+ordered-set==4.1.0
+orjson==3.10.3
 packaging==23.2
 pgvector==0.2.3
 psycopg==3.1.12
 psycopg2-binary==2.9.9
 pydantic==2.4.2
 pydantic_core==2.10.1
-pypdf==3.16.4
+pypdf==4.2.0
+python-dateutil==2.9.0.post0
+python-docx==1.1.2
 python-dotenv==1.0.0
+python-iso639==2024.4.27
+python-magic==0.4.27
 pytz==2023.3.post1
 PyYAML==6.0.1
+rapidfuzz==3.9.0
 regex==2023.10.3
 requests==2.31.0
+six==1.16.0
 sniffio==1.3.0
+soupsieve==2.5
 SQLAlchemy==2.0.22
 sqlparse==0.4.4
+tabulate==0.9.0
 tenacity==8.2.3
 tiktoken==0.5.1
 tqdm==4.66.1
 typing-inspect==0.9.0
-typing_extensions==4.8.0
+typing_extensions==4.11.0
+unstructured==0.13.7
+unstructured-client==0.22.0
 urllib3==2.0.6
+wrapt==1.16.0
 yarl==1.9.2