tf-idf embeddings for search (#64)

* Adds deletion of tf-idf embeddings

* comment

Author: JWittmeyer

upgrade_logic/business_objects/gateway.py

@@ -12,45 +12,8 @@
 
 
 def gateway_1_15_0() -> bool:
-    # here, we update data for cognition using the gateway pattern
-    # as the corresponding database updates (alembic) are managed using the refinery gateway it is
-    # ensured that these updates are executed at the correct time
-    __gateway_1_15_0_add_cognition_project_file_defaults()
-    __gateway_1_15_0_add_cognition_conversation_file_defaults()
-    __gateway_1_15_0_remove_cognition_step_type_relevance()
-    return True
-
-
-def __gateway_1_15_0_add_cognition_project_file_defaults() -> bool:
-    query = """
-    UPDATE cognition.project
-    SET max_file_size_mb = 3,
-        allow_file_upload = FALSE
-    WHERE max_file_size_mb IS NULL
-    """
-    general.execute(query)
-    general.commit()
-    return True
-
-
-def __gateway_1_15_0_add_cognition_conversation_file_defaults() -> bool:
-    query = """
-    UPDATE cognition.conversation
-    SET has_tmp_files = FALSE,
-        archived = FALSE
-    WHERE has_tmp_files IS NULL
-    """
-    general.execute(query)
-    general.commit()
-    return True
-
-
-def __gateway_1_15_0_remove_cognition_step_type_relevance() -> bool:
-    query = """
-    DELETE FROM cognition.strategy_step WHERE step_type = 'RELEVANCE'
-    """
-    general.execute(query)
-    general.commit()
+    # Note: A previous version had the previous update listed as v1.15.
+    # That was false, the updates already ran through. This is now for the actual 1.15 release
     return True
 
 
@@ -60,6 +23,9 @@ def gateway_1_14_0() -> bool:
     # ensured that these updates are executed at the correct time
     gateway_1_14_0_add_cognition_project_state()
     gateway_1_14_0_add_cognition_strategy_complexity()
+    __gateway_1_14_0_add_cognition_project_file_defaults()
+    __gateway_1_14_0_add_cognition_conversation_file_defaults()
+    __gateway_1_14_0_remove_cognition_step_type_relevance()
     return True
 
 
@@ -94,6 +60,39 @@ def gateway_1_14_0_add_cognition_strategy_complexity() -> bool:
     return True
 
 
+def __gateway_1_14_0_add_cognition_project_file_defaults() -> bool:
+    query = """
+    UPDATE cognition.project
+    SET max_file_size_mb = 3,
+        allow_file_upload = FALSE
+    WHERE max_file_size_mb IS NULL
+    """
+    general.execute(query)
+    general.commit()
+    return True
+
+
+def __gateway_1_14_0_add_cognition_conversation_file_defaults() -> bool:
+    query = """
+    UPDATE cognition.conversation
+    SET has_tmp_files = FALSE,
+        archived = FALSE
+    WHERE has_tmp_files IS NULL
+    """
+    general.execute(query)
+    general.commit()
+    return True
+
+
+def __gateway_1_14_0_remove_cognition_step_type_relevance() -> bool:
+    query = """
+    DELETE FROM cognition.strategy_step WHERE step_type = 'RELEVANCE'
+    """
+    general.execute(query)
+    general.commit()
+    return True
+
+
 def gateway_1_10_1() -> bool:
     __gateway_1_10_1_add_additional_embedding_information()
     return True

upgrade_logic/business_objects/neural_search.py

@@ -1,12 +1,40 @@
 import os
 import requests
 
-from submodules.model.business_objects import embedding
+from submodules.model.business_objects import embedding, general
 
 
 NEURAL_SEARCH = os.getenv("NEURAL_SEARCH")
 
 
+def neural_search_1_15_0() -> bool:
+    neural_search_1_15_0_delete_all_tf_idf_embeddings()
+    return True
+
+
+def neural_search_1_15_0_delete_all_tf_idf_embeddings() -> bool:
+    # previous tf-idf embeddings didn't do anything useful so we can just delete them
+    # used fixed values instead of enum keys to ensure changes dont break
+    query = "SELECT id FROM embedding WHERE platform = 'python' AND model = 'tf-idf'"
+    embedding_ids = general.execute_all(query)
+    if len(embedding_ids) > 0:
+        url_delete = f"{NEURAL_SEARCH}/delete_collection"
+        for embedding_id in embedding_ids:
+            try:
+                params = {"embedding_id": embedding_id[0]}
+                requests.put(url_delete, params=params)
+            except Exception as e:
+                print(
+                    f"Error deleting tf-idf embedding {embedding_id[0]} from qdrant: {e}"
+                )
+
+        # Note that tensors are deleted by cascading
+        query = "DELETE FROM embedding WHERE platform = 'python' AND model = 'tf-idf'"
+        general.execute(query)
+        general.commit()
+    return True
+
+
 def neural_search_1_12_0() -> bool:
     __neural_search_1_12_0_update_qdrant()
     return True