enhance: replace pytidb-experimental with pytidb dep for tidb integration (#3073)

Co-authored-by: Wendong-Fan <133094783+Wendong-Fan@users.noreply.github.com>

Author: Mini256

camel/retrievers/auto_retriever.py

@@ -97,6 +97,7 @@ def _initialize_vector_storage(
                     "URL (database url) and API key required for TiDB storage "
                     "are not provided. Format: "
                     "mysql+pymysql://<username>:<password>@<host>:4000/test"
+                    "You can get the database url from https://tidbcloud.com/console/clusters"
                 )
             return TiDBStorage(
                 vector_dim=self.embedding_model.get_output_dim(),

camel/storages/vectordb_storages/tidb.py

@@ -44,7 +44,7 @@ class TiDBStorage(BaseVectorStorage):
     r"""An implementation of the `BaseVectorStorage` for interacting with TiDB.
 
     The detailed information about TiDB is available at:
-    `TiDB Vector Search <https://ai.pingcap.com/>`_
+    `TiDB Vector Search <https://pingcap.com/ai>`_
 
     Args:
         vector_dim (int): The dimension of storing vectors.
@@ -107,10 +107,10 @@ def _create_client(
         )
 
     def _get_table_model(self, collection_name: str) -> Any:
+        from pytidb.datatype import JSON
         from pytidb.schema import Field, TableModel, VectorField
-        from sqlalchemy import JSON
 
-        class VectorDBRecord(TableModel):
+        class VectorDBRecordBase(TableModel, table=False):
             id: Optional[str] = Field(None, primary_key=True)
             vector: list[float] = VectorField(self.vector_dim)
             payload: Optional[dict[str, Any]] = Field(None, sa_type=JSON)
@@ -119,7 +119,7 @@ class VectorDBRecord(TableModel):
         # class names.
         return type(
             f"VectorDBRecord_{collection_name}",
-            (VectorDBRecord,),
+            (VectorDBRecordBase,),
             {"__tablename__": collection_name},
             table=True,
         )
@@ -128,8 +128,9 @@ def _open_and_create_table(self) -> "Table[Any]":
         r"""Opens an existing table or creates a new table in TiDB."""
         table = self._client.open_table(self.collection_name)
         if table is None:
+            table_model = self._get_table_model(self.collection_name)
             table = self._client.create_table(
-                schema=self._get_table_model(self.collection_name)
+                schema=table_model, if_exists="skip"
             )
         return table
 
@@ -166,6 +167,7 @@ def _get_table_info(self) -> Dict[str, Any]:
                 table.
         """
         vector_count = self._table.rows()
+
         # Get vector dimension from table schema
         columns = self._table.columns()
         dim_value = None
@@ -303,7 +305,7 @@ def query(
         for row in rows:
             query_results.append(
                 VectorDBQueryResult.create(
-                    similarity=float(row['similarity_score']),
+                    similarity=float(row['_score']),
                     id=str(row['id']),
                     payload=row['payload'],
                     vector=row['vector'],

docs/key_modules/storages.md

@@ -45,7 +45,7 @@ The <b>Storage</b> module in CAMEL-AI gives you a **unified interface for saving
     - High scalability, real-time search
 
     **TiDBStorage**
-    - For [TiDB](https://ai.pingcap.com/) (hybrid vector/relational database)
+    - For [TiDB](https://pingcap.com/ai) (hybrid vector/relational database)
     - Handles embeddings, knowledge graphs, ops data
 
     **QdrantStorage**

pyproject.toml

@@ -69,7 +69,7 @@ rag = [
     "numpy>=1.2,<=2.2",
     "qdrant-client>=1.9.0,<2",
     "pymilvus>=2.4.0,<3",
-    "pytidb-experimental==0.0.1.dev4",
+    "pytidb>=0.0.13",
     "faiss-cpu>=1.7.2,<2",
     "weaviate-client>=4.15.0",
     "protobuf>=6.0.0",
@@ -200,7 +200,7 @@ huggingface = [
 storage = [
     "qdrant-client>=1.9.0,<2",
     "pymilvus>=2.4.0,<3",
-    "pytidb-experimental==0.0.1.dev4",
+    "pytidb>=0.0.13",
     "faiss-cpu>=1.7.2,<2",
     "neo4j>=5.18.0,<6",
     "nebula3-python==3.8.2",
@@ -360,7 +360,7 @@ all = [
     "networkx>=3.4.2,<4",
     "qdrant-client>=1.9.0,<2",
     "pymilvus>=2.4.0,<3",
-    "pytidb-experimental==0.0.1.dev4",
+    "pytidb>=0.0.13",
     "faiss-cpu>=1.7.2,<2",
     "cohere>=5.11.0,<6",
     "neo4j>=5.18.0,<6",