Merge pull request #19 from lpm0073/next

new PineconeIndex class

Author: lpm0073

Makefile

@@ -7,7 +7,7 @@ $(shell echo -e "OPENAI_API_ORGANIZATION=PLEASE-ADD-ME\n\
 OPENAI_API_KEY=PLEASE-ADD-ME\n\
 PINECONE_API_KEY=PLEASE-ADD-ME\n\
 PINECONE_ENVIRONMENT=gcp-starter\n\
-PINECONE_INDEX_NAME=hsr\n\
+PINECONE_INDEX_NAME=rag\n\
 PINECONE_VECTORSTORE_TEXT_KEY=lc_id\n\
 PINECONE_METRIC=dotproduct\n\
 PINECONE_DIMENSIONS=1536\n\

README.md

@@ -42,10 +42,9 @@ python3 -m models.examples.training_services "Microsoft certified Azure AI engin
 # example 4 - prompted assistant
 python3 -m models.examples.training_services_oracle "Oracle database administrator"
 
-# example 5 - Load PDF documents
+# example 5 - Retrieval Augmented Generation
+python3 -m models.examples.pinecone_init
 python3 -m models.examples.load "./data/"
-
-# example 6 - Retrieval Augmented Generation
 python3 -m models.examples.rag "What analytics and accounting courses does Wharton offer?"
 ```
 
@@ -67,7 +66,7 @@ DEBUG_MODE=False
 You'll need to manually create an index with the following characteristics
 
 - Environment: gcp-starter
-- Index name: netec-rag
+- Index name: rag
 - Metric: dotproduct
 - Dimensions: 1536
 - Pod Type: starter

models/__version__.py

@@ -1,2 +1,2 @@
 # -*- coding: utf-8 -*-
-__version__ = "1.1.3"
+__version__ = "1.2.0"

models/const.py

@@ -7,6 +7,14 @@
 from dotenv import find_dotenv, load_dotenv
 
 
+class ConfigurationError(Exception):
+    """Exception raised for errors in the configuration."""
+
+    def __init__(self, message):
+        self.message = message
+        super().__init__(self.message)
+
+
 # pylint: disable=duplicate-code
 dotenv_path = find_dotenv()
 if os.path.exists(dotenv_path):
@@ -25,6 +33,16 @@
     OPENAI_CHAT_MAX_RETRIES = int(os.environ.get("OPENAI_CHAT_MAX_RETRIES", 3))
     OPENAI_CHAT_CACHE = bool(os.environ.get("OPENAI_CHAT_CACHE", True))
     DEBUG_MODE = os.environ.get("DEBUG_MODE", "False") == "True"
+
+    if OPENAI_API_KEY == "PLEASE-ADD-ME":
+        raise ConfigurationError("OPENAI_API_KEY is not set. Please add your OpenAI API key to the .env file.")
+    if OPENAI_API_ORGANIZATION == "PLEASE-ADD-ME":
+        raise ConfigurationError(
+            "OPENAI_API_ORGANIZATION is not set. Please add your OpenAI API organization to the .env file."
+        )
+    if PINECONE_API_KEY == "PLEASE-ADD-ME":
+        raise ConfigurationError("PINECONE_API_KEY is not set. Please add your Pinecone API key to the .env file.")
+
 else:
     raise FileNotFoundError("No .env file found in root directory of repository")
 

models/examples/pinecone_init.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""Sales Support Model (hsr) Retrieval Augmented Generation (RAG)"""
+
+from models.pinecone import PineconeIndex
+
+
+pinecone = PineconeIndex()
+
+if __name__ == "__main__":
+    pinecone.initialize()
+    print("Pinecone index initialized. name: ", pinecone.index_name)
+    print(pinecone.index.describe_index_stats())

models/examples/rag.py

@@ -14,6 +14,6 @@
     parser.add_argument("prompt", type=str, help="A question about the PDF contents")
     args = parser.parse_args()
 
-    human_message = HumanMessage(text=args.prompt)
+    human_message = HumanMessage(content=args.prompt)
     result = hsr.rag(human_message=human_message)
     print(result)

models/hybrid_search_retreiver.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-# pylint: disable=too-few-public-methods
 """
 Hybrid Search Retriever. A class that combines the following:
     - OpenAI prompting and ChatModel
@@ -16,23 +15,16 @@
      https://python.langchain.com/docs/integrations/retrievers/pinecone_hybrid_search
 """
 
-# document loading
-import glob
-
 # general purpose imports
 import logging
-import os
 import textwrap
 from typing import Union
 
 # pinecone integration
-import pinecone
 from langchain.cache import InMemoryCache
 from langchain.chat_models import ChatOpenAI
-from langchain.document_loaders import PyPDFLoader
 
 # embedding
-from langchain.embeddings import OpenAIEmbeddings
 from langchain.globals import set_llm_cache
 
 # prompting and chat
@@ -42,12 +34,11 @@
 # hybrid search capability
 from langchain.retrievers import PineconeHybridSearchRetriever
 from langchain.schema import BaseMessage, HumanMessage, SystemMessage
-from langchain.text_splitter import Document
-from langchain.vectorstores.pinecone import Pinecone
 from pinecone_text.sparse import BM25Encoder
 
 # this project
 from models.const import Config, Credentials
+from models.pinecone import PineconeIndex
 
 
 ###############################################################################
@@ -56,37 +47,25 @@
 logging.basicConfig(level=logging.DEBUG if Config.DEBUG_MODE else logging.INFO)
 
 
-class TextSplitter:
-    """
-    Custom text splitter that adds metadata to the Document object
-    which is required by PineconeHybridSearchRetriever.
-    """
-
-    def create_documents(self, texts):
-        """Create documents"""
-        documents = []
-        for text in texts:
-            # Create a Document object with the text and metadata
-            document = Document(page_content=text, metadata={"context": text})
-            documents.append(document)
-        return documents
-
-
 class HybridSearchRetriever:
     """Hybrid Search Retriever"""
 
     _chat: ChatOpenAI = None
-    _openai_embeddings: OpenAIEmbeddings = None
-    _pinecone_index: pinecone.Index = None
-    _vector_store: Pinecone = None
-    _text_splitter: TextSplitter = None
     _b25_encoder: BM25Encoder = None
+    _pinecone: PineconeIndex = None
+    _retriever: PineconeHybridSearchRetriever = None
 
     def __init__(self):
         """Constructor"""
-        pinecone.init(api_key=Credentials.PINECONE_API_KEY, environment=Config.PINECONE_ENVIRONMENT)
         set_llm_cache(InMemoryCache())
 
+    @property
+    def pinecone(self) -> PineconeIndex:
+        """PineconeIndex lazy read-only property."""
+        if self._pinecone is None:
+            self._pinecone = PineconeIndex()
+        return self._pinecone
+
     # prompting wrapper
     @property
     def chat(self) -> ChatOpenAI:
@@ -102,48 +81,22 @@ def chat(self) -> ChatOpenAI:
             )
         return self._chat
 
-    # embeddings
-    @property
-    def openai_embeddings(self) -> OpenAIEmbeddings:
-        """OpenAIEmbeddings lazy read-only property."""
-        if self._openai_embeddings is None:
-            self._openai_embeddings = OpenAIEmbeddings(
-                api_key=Credentials.OPENAI_API_KEY, organization=Credentials.OPENAI_API_ORGANIZATION
-            )
-        return self._openai_embeddings
-
-    @property
-    def pinecone_index(self) -> pinecone.Index:
-        """pinecone.Index lazy read-only property."""
-        if self._pinecone_index is None:
-            self._pinecone_index = pinecone.Index(index_name=Config.PINECONE_INDEX_NAME)
-        return self._pinecone_index
-
-    @property
-    def vector_store(self) -> Pinecone:
-        """Pinecone lazy read-only property."""
-        if self._vector_store is None:
-            self._vector_store = Pinecone(
-                index=self.pinecone_index,
-                embedding=self.openai_embeddings,
-                text_key=Config.PINECONE_VECTORSTORE_TEXT_KEY,
-            )
-        return self._vector_store
-
-    @property
-    def text_splitter(self) -> TextSplitter:
-        """TextSplitter lazy read-only property."""
-        if self._text_splitter is None:
-            self._text_splitter = TextSplitter()
-        return self._text_splitter
-
     @property
     def bm25_encoder(self) -> BM25Encoder:
         """BM25Encoder lazy read-only property."""
         if self._b25_encoder is None:
             self._b25_encoder = BM25Encoder().default()
         return self._b25_encoder
 
+    @property
+    def retriever(self) -> PineconeHybridSearchRetriever:
+        """PineconeHybridSearchRetriever lazy read-only property."""
+        if self._retriever is None:
+            self._retriever = PineconeHybridSearchRetriever(
+                embeddings=self.pinecone.openai_embeddings, sparse_encoder=self.bm25_encoder, index=self.pinecone.index
+            )
+        return self._retriever
+
     def cached_chat_request(
         self, system_message: Union[str, SystemMessage], human_message: Union[str, HumanMessage]
     ) -> BaseMessage:
@@ -169,54 +122,8 @@ def prompt_with_template(
         return retval
 
     def load(self, filepath: str):
-        """
-        Embed PDF.
-        1. Load PDF document text data
-        2. Split into pages
-        3. Embed each page
-        4. Store in Pinecone
-
-        Note: it's important to make sure that the "context" field that holds the document text
-        in the metadata is not indexed. Currently you need to specify explicitly the fields you
-        do want to index. For more information checkout
-        https://docs.pinecone.io/docs/manage-indexes#selective-metadata-indexing
-        """
-        try:
-            logging.info("Deleting index...")
-            pinecone.delete_index(Config.PINECONE_INDEX_NAME)
-        except pinecone.exceptions.PineconeException:
-            logging.info("Index does not exist. Continuing...")
-
-        metadata_config = {
-            "indexed": [Config.PINECONE_VECTORSTORE_TEXT_KEY, "lc_type"],
-            "context": ["lc_text"],
-        }
-        logging.info("Creating index. This may take a few minutes...")
-        pinecone.create_index(
-            Config.PINECONE_INDEX_NAME,
-            dimension=Config.PINECONE_DIMENSIONS,
-            metric=Config.PINECONE_METRIC,
-            metadata_config=metadata_config,
-        )
-
-        pdf_files = glob.glob(os.path.join(filepath, "*.pdf"))
-        i = 0
-        for pdf_file in pdf_files:
-            i += 1
-            j = len(pdf_files)
-            logging.info("Loading PDF %s of %s: %s", i, j, pdf_file)
-            loader = PyPDFLoader(file_path=pdf_file)
-            docs = loader.load()
-            k = 0
-            for doc in docs:
-                k += 1
-                logging.info(k * "-", end="\r")
-                documents = self.text_splitter.create_documents([doc.page_content])
-                document_texts = [doc.page_content for doc in documents]
-                embeddings = self.openai_embeddings.embed_documents(document_texts)
-                self.vector_store.add_documents(documents=documents, embeddings=embeddings)
-
-        logging.info("Finished loading PDFs")
+        """Pdf loader."""
+        self.pinecone.pdf_loader(filepath=filepath)
 
     def rag(self, human_message: Union[str, HumanMessage]):
         """
@@ -241,10 +148,8 @@ def rag(self, human_message: Union[str, HumanMessage]):
         # ---------------------------------------------------------------------
         # 1.) Retrieve relevant documents from Pinecone vector database
         # ---------------------------------------------------------------------
-        retriever = PineconeHybridSearchRetriever(
-            embeddings=self.openai_embeddings, sparse_encoder=self.bm25_encoder, index=self.pinecone_index
-        )
-        documents = retriever.get_relevant_documents(query=human_message.content)
+        # documents = self.retriever.get_relevant_documents(query=human_message.content)
+        documents = self.pinecone.vector_store.similarity_search(query=human_message.content)
 
         # Extract the text from the documents
         document_texts = [doc.page_content for doc in documents]
@@ -261,14 +166,15 @@ def rag(self, human_message: Union[str, HumanMessage]):
         # finished with hybrid search setup
         # ---------------------------------------------------------------------
 
-        # 2.) get a response from the chat model
-        response = self.cached_chat_request(system_message=system_message, human_message=human_message)
-
         logging.debug("------------------------------------------------------")
         logging.debug("rag() Retrieval Augmented Generation prompt")
         logging.debug("Diagnostic information:")
         logging.debug("  Retrieved %i related documents from Pinecone", len(documents))
         logging.debug("  System messages contains %i words", len(system_message.content.split()))
         logging.debug("  Prompt: %s", system_message.content)
         logging.debug("------------------------------------------------------")
+
+        # 2.) get a response from the chat model
+        response = self.cached_chat_request(system_message=system_message, human_message=human_message)
+
         return response.content