refactor: parameterize PINECONE_VECTORSTORE_TEXT_KEY

Author: lpm0073

Makefile

@@ -3,7 +3,7 @@ SHELL := /bin/bash
 ifneq ("$(wildcard .env)","")
     include .env
 else
-    $(shell echo -e "OPENAI_API_ORGANIZATION=PLEASE-ADD-ME\nOPENAI_API_KEY=PLEASE-ADD-ME\nPINECONE_API_KEY=PLEASE-ADD-ME\nPINECONE_ENVIRONMENT=gcp-starter\nPINECONE_INDEX_NAME=hsr\nOPENAI_CHAT_MODEL_NAME=gpt-3.5-turbo\nOPENAI_PROMPT_MODEL_NAME=text-davinci-003\nOPENAI_CHAT_TEMPERATURE=0.0\nOPENAI_CHAT_MAX_RETRIES=3\nDEBUG_MODE=True\n" >> .env)
+    $(shell echo -e "OPENAI_API_ORGANIZATION=PLEASE-ADD-ME\nOPENAI_API_KEY=PLEASE-ADD-ME\nPINECONE_API_KEY=PLEASE-ADD-ME\nPINECONE_ENVIRONMENT=gcp-starter\nPINECONE_INDEX_NAME=hsr\nPINECONE_VECTORSTORE_TEXT_KEY=lc_id\nOPENAI_CHAT_MODEL_NAME=gpt-3.5-turbo\nOPENAI_PROMPT_MODEL_NAME=text-davinci-003\nOPENAI_CHAT_TEMPERATURE=0.0\nOPENAI_CHAT_MAX_RETRIES=3\nDEBUG_MODE=True\n" >> .env)
 endif
 
 .PHONY: analyze init activate test lint clean

models/const.py

@@ -16,6 +16,7 @@
     PINECONE_API_KEY = os.environ["PINECONE_API_KEY"]
     PINECONE_ENVIRONMENT = os.environ["PINECONE_ENVIRONMENT"]
     PINECONE_INDEX_NAME = os.environ.get("PINECONE_INDEX_NAME", "hsr")
+    PINECONE_VECTORSTORE_TEXT_KEY = os.environ.get("PINECONE_VECTORSTORE_TEXT_KEY", "lc_id")
     OPENAI_CHAT_MODEL_NAME = os.environ.get("OPENAI_CHAT_MODEL_NAME", "gpt-3.5-turbo")
     OPENAI_PROMPT_MODEL_NAME = os.environ.get("OPENAI_PROMPT_MODEL_NAME", "text-davinci-003")
     OPENAI_CHAT_TEMPERATURE = float(os.environ.get("OPENAI_CHAT_TEMPERATURE", 0.0))
@@ -35,6 +36,9 @@ class Config:
     OPENAI_CHAT_TEMPERATURE: float = OPENAI_CHAT_TEMPERATURE
     OPENAI_CHAT_MAX_RETRIES: int = OPENAI_CHAT_MAX_RETRIES
     OPENAI_CHAT_CACHE: bool = OPENAI_CHAT_CACHE
+    PINECONE_ENVIRONMENT = PINECONE_ENVIRONMENT
+    PINECONE_INDEX_NAME = PINECONE_INDEX_NAME
+    PINECONE_VECTORSTORE_TEXT_KEY: str = PINECONE_VECTORSTORE_TEXT_KEY
 
 
 class Credentials:
@@ -43,5 +47,3 @@ class Credentials:
     OPENAI_API_KEY = OPENAI_API_KEY
     OPENAI_API_ORGANIZATION = OPENAI_API_ORGANIZATION
     PINECONE_API_KEY = PINECONE_API_KEY
-    PINECONE_ENVIRONMENT = PINECONE_ENVIRONMENT
-    PINECONE_INDEX_NAME = PINECONE_INDEX_NAME

models/hybrid_search_retreiver.py

@@ -54,7 +54,7 @@
 # initializations
 ###############################################################################
 DEFAULT_MODEL_NAME = Config.OPENAI_PROMPT_MODEL_NAME
-pinecone.init(api_key=Credentials.PINECONE_API_KEY, environment=Credentials.PINECONE_ENVIRONMENT)
+pinecone.init(api_key=Credentials.PINECONE_API_KEY, environment=Config.PINECONE_ENVIRONMENT)
 set_llm_cache(InMemoryCache())
 logging.basicConfig(level=logging.DEBUG if Config.DEBUG_MODE else logging.INFO)
 
@@ -92,8 +92,10 @@ class HybridSearchRetriever:
     openai_embeddings = OpenAIEmbeddings(
         api_key=Credentials.OPENAI_API_KEY, organization=Credentials.OPENAI_API_ORGANIZATION
     )
-    pinecone_index = pinecone.Index(index_name=Credentials.PINECONE_INDEX_NAME)
-    vector_store = Pinecone(index=pinecone_index, embedding=openai_embeddings, text_key="lc_id")
+    pinecone_index = pinecone.Index(index_name=Config.PINECONE_INDEX_NAME)
+    vector_store = Pinecone(
+        index=pinecone_index, embedding=openai_embeddings, text_key=Config.PINECONE_VECTORSTORE_TEXT_KEY
+    )
 
     text_splitter = TextSplitter()
     bm25_encoder = BM25Encoder().default()
@@ -135,17 +137,17 @@ def load(self, filepath: str):
         """
         try:
             logging.debug("Deleting index...")
-            pinecone.delete_index(Credentials.PINECONE_INDEX_NAME)
+            pinecone.delete_index(Config.PINECONE_INDEX_NAME)
         except pinecone.exceptions.PineconeException:
             logging.debug("Index does not exist. Continuing...")
 
         metadata_config = {
-            "indexed": ["lc_id", "lc_type"],
+            "indexed": [Config.PINECONE_VECTORSTORE_TEXT_KEY, "lc_type"],
             "context": ["lc_text"],
         }
         logging.debug("Creating index. This may take a few minutes...")
         pinecone.create_index(
-            Credentials.PINECONE_INDEX_NAME, dimension=1536, metric="dotproduct", metadata_config=metadata_config
+            Config.PINECONE_INDEX_NAME, dimension=1536, metric="dotproduct", metadata_config=metadata_config
         )
 
         pdf_files = glob.glob(os.path.join(filepath, "*.pdf"))
@@ -187,11 +189,13 @@ def rag(self, human_message: Union[str, HumanMessage]):
             logging.debug("Converting human_message to HumanMessage")
             human_message = HumanMessage(content=human_message)
 
+        # ---------------------------------------------------------------------
+        # 1.) Retrieve relevant documents from Pinecone vector database
+        # ---------------------------------------------------------------------
         retriever = PineconeHybridSearchRetriever(
             embeddings=self.openai_embeddings, sparse_encoder=self.bm25_encoder, index=self.pinecone_index
         )
         documents = retriever.get_relevant_documents(query=human_message.content)
-        logging.debug("Retrieved %i related documents from Pinecone", len(documents))
 
         # Extract the text from the documents
         document_texts = [doc.page_content for doc in documents]
@@ -202,13 +206,19 @@ def rag(self, human_message: Union[str, HumanMessage]):
             into your responses:\n\n
         """
         )
-        system_message = f"{leader} {'. '.join(document_texts)}"
+        system_message_content = f"{leader} {'. '.join(document_texts)}"
+        system_message = SystemMessage(content=system_message_content)
+        # ---------------------------------------------------------------------
+        # finished with hybrid search setup
+        # ---------------------------------------------------------------------
 
-        logging.debug("System messages contains %i words", len(system_message.split()))
-        logging.debug("Prompt: %s", system_message)
-        system_message = SystemMessage(content=system_message)
+        # 2.) get a response from the chat model
         response = self.cached_chat_request(system_message=system_message, human_message=human_message)
 
+        logging.debug("------------------------------------------------------")
+        logging.debug("Retrieved %i related documents from Pinecone", len(documents))
+        logging.debug("System messages contains %i words", len(system_message.content.split()))
+        logging.debug("Prompt: %s", system_message.content)
         logging.debug("Response:")
         logging.debug("------------------------------------------------------")
         return response.content

models/tests/test_pinecone.py

@@ -9,7 +9,7 @@
 from langchain.embeddings import OpenAIEmbeddings
 from langchain.vectorstores.pinecone import Pinecone
 
-from ..const import Credentials
+from ..const import Config, Credentials
 
 
 class TestPinecone:
@@ -19,22 +19,20 @@ def test_01_test_pinecone_connectivity(self):
         """Ensure that we have connectivity to Pinecone."""
         # pylint: disable=broad-except
         try:
-            pinecone.init(api_key=Credentials.PINECONE_API_KEY, environment=Credentials.PINECONE_ENVIRONMENT)
+            pinecone.init(api_key=Credentials.PINECONE_API_KEY, environment=Config.PINECONE_ENVIRONMENT)
         except Exception as e:
             assert False, f"pinecone.init() failed with exception: {e}"
 
     def test_02_test_pinecone_index(self):
         """Ensure that the Pinecone index exists and that we can connect to it."""
-        pinecone.init(api_key=Credentials.PINECONE_API_KEY, environment=Credentials.PINECONE_ENVIRONMENT)
+        pinecone.init(api_key=Credentials.PINECONE_API_KEY, environment=Config.PINECONE_ENVIRONMENT)
         openai_embedding = OpenAIEmbeddings()
 
         # pylint: disable=broad-except
         try:
             Pinecone.from_existing_index(
-                Credentials.PINECONE_INDEX_NAME,
+                Config.PINECONE_INDEX_NAME,
                 embedding=openai_embedding,
             )
         except Exception as e:
-            assert (
-                False
-            ), f"Pinecone initialization of index {Credentials.PINECONE_INDEX_NAME,} failed with exception: {e}"
+            assert False, f"Pinecone initialization of index {Config.PINECONE_INDEX_NAME,} failed with exception: {e}"