Add external vector index support for Pinecone (neo4j#52)

* Added pinecone-client to poetry

* Copied weaviate populate_dbs.py to pinecone_e2e

* Updated tests/e2e/pinecone_e2e/populate_dbs.py to work with Pinecone

* Copied the Weaviate retriever to use as a base for the Pinecone retriever

* Added first draft of PineconeNeo4jRetriever

* Added input validation to PineconeNeo4jRetriever

* Renamed PineconeModel to PineconeClientModel

* Added Pinecone retriever unit tests

* Added E2E tests for pinecone retriever

* Updated pinecone retriever to work with new return types

* Updated E2E tests to work with new types

* Added __init__.py to E2E tests

* Update src/neo4j_genai/retrievers/external/pinecone/pinecone.py

Co-authored-by: willtai <william.tai@neo4j.com>

* Updated Pinecone retriever with new Exceptions

* Added Pinecone example scripts

* Moved Pinecone Pydantic types to their own file

* Ran import organizer on examples/weaviate/text_search_local_embedder.py

* Added docs for PineconeNeo4jRetriever

* Pinecone E2E test fix

* Pinecone E2E test update

* Pincone E2E test fix

* Moved populate_neo4j to its own file

* Moved populate_neo4j to utils

* Updated PineconeNeo4jRetriever import in test_pinecone_e2e.py

* Removed copyright from Pinecone and Weaviate examples

* Refactored E2E test utils

* Small fix to tests/e2e/utils.py

* Ran poetry lock --no-update

* Small formatting changes

* More formatting changes

* Moved embedding_biology.py to the examples folder

* mypy fixes

* Removed unused imports

* More mypy fixes

* More mypy fixes

* Added .venv to .gitignore

* Fixed mypy pytest conflict

---------

Co-authored-by: willtai <william.tai@neo4j.com>

Author: alexthomas93

.gitignore

@@ -10,3 +10,4 @@ docs/build/
 .vscode/
 .python-version
 .DS_Store
+.venv

docs/source/api.rst

@@ -57,6 +57,13 @@ WeaviateNeo4jRetriever
    :members:
 
 
+PineconeNeo4jRetriever
+======================
+
+.. autoclass:: neo4j_genai.retrievers.external.pinecone.PineconeNeo4jRetriever
+   :members:
+
+
 ******
 Errors
 ******

examples/__init__.py

@@ -0,0 +1,30 @@
+### Usage Instructions
+
+You will need both a Pinecone vector database and a Neo4j database to use this retriever.
+
+### Writing Test Data
+
+Update  `NEO4J_AUTH`, `NEO4J_URL`, and `PC_API_KEY` variables in the `tests/e2e/pinecone_e2e/populate_dbs.py` script then run this from the project root to write test data to both dbs.
+
+```
+poetry run python tests/e2e/pinecone_e2e/populate_dbs.py
+```
+
+### Install Pinecone client
+
+You need to install the `pinecone-client` package to use this retriever.
+
+```bash
+pip install pinecone-client
+```
+
+### Search
+Update the `NEO4J_AUTH`, `NEO4J_URL`, and `PC_API_KEY` variables in each file then run one of the following from the project root to test the retriever.
+
+```
+# Search by vector
+poetry run python -m examples.pinecone.vector_search
+
+# Search by text, with embeddings generated locally
+poetry run python -m examples.pinecone.text_search
+```

examples/weaviate/embedding_biology.py renamed to examples/embedding_biology.py

@@ -0,0 +1,29 @@
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from neo4j import GraphDatabase
+from neo4j_genai.retrievers.external.pinecone import PineconeNeo4jRetriever
+from pinecone import Pinecone
+
+NEO4J_AUTH = ("neo4j", "password")
+NEO4J_URL = "neo4j://localhost:7687"
+PC_API_KEY = "API_KEY"
+
+
+def main() -> None:
+    with GraphDatabase.driver(NEO4J_URL, auth=NEO4J_AUTH) as neo4j_driver:
+        pc_client = Pinecone(PC_API_KEY)
+        embedder = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
+
+        retriever = PineconeNeo4jRetriever(
+            driver=neo4j_driver,
+            client=pc_client,
+            index_name="jeopardy",
+            id_property_neo4j="id",
+            embedder=embedder,  # type: ignore
+        )
+
+        res = retriever.search(query_text="biology", top_k=2)
+        print(res)
+
+
+if __name__ == "__main__":
+    main()

examples/pinecone/README.md

@@ -0,0 +1,27 @@
+from neo4j import GraphDatabase
+from neo4j_genai.retrievers.external.pinecone import PineconeNeo4jRetriever
+from pinecone import Pinecone
+
+from examples.embedding_biology import EMBEDDING_BIOLOGY
+
+NEO4J_AUTH = ("neo4j", "password")
+NEO4J_URL = "neo4j://localhost:7687"
+PC_API_KEY = "API_KEY"
+
+
+def main() -> None:
+    with GraphDatabase.driver(NEO4J_URL, auth=NEO4J_AUTH) as neo4j_driver:
+        pc_client = Pinecone(PC_API_KEY)
+        retriever = PineconeNeo4jRetriever(
+            driver=neo4j_driver,
+            client=pc_client,
+            index_name="jeopardy",
+            id_property_neo4j="id",
+        )
+
+        res = retriever.search(query_vector=EMBEDDING_BIOLOGY, top_k=2)
+        print(res)
+
+
+if __name__ == "__main__":
+    main()

examples/pinecone/__init__.py

@@ -28,11 +28,11 @@ pip install weaviate-client
 
 ```
 # search by vector
-poetry run python examples/weaviate/vector_search.py
+poetry run python -m examples.weaviate.vector_search
 
 # search by text, with embeddings generated locally (via embedder argument)
-poetry run python examples/weaviate/text_search_local_embedder.py
+poetry run python -m examples.weaviate.text_search_local_embedder
 
 # search by text, with embeddings generated on the Weaviate side, via configured vectorizer
-poetry run python examples/weaviate/text_search_remote_embedder.py
+poetry run python -m examples.weaviate.text_search_remote_embedder
 ```