Fix/query embedding (#909)

AlonsoGuevara · KylinMountain · web-flow · commit 4b9f26860421 · 2024-08-12T17:12:51.000-06:00
* fix strategy config in entity_extraction

* should not post token list to the embedding model

* fix embedding in local query

* add sembersioner

* remove strategy

---------

Co-authored-by: KylinMountain &lt;kose2livs@gmail.com&gt;
diff --git a/.semversioner/next-release/patch-20240726113757173607.json b/.semversioner/next-release/patch-20240726113757173607.json
@@ -0,0 +1,4 @@
+{
+  "type": "patch",
+  "description": "fix query embedding"
+}
diff --git a/graphrag/query/llm/text_utils.py b/graphrag/query/llm/text_utils.py
@@ -39,4 +39,4 @@ def chunk_text(
         token_encoder = tiktoken.get_encoding("cl100k_base")
     tokens = token_encoder.encode(text)  # type: ignore
     chunk_iterator = batched(iter(tokens), max_tokens)
-    yield from chunk_iterator
+    yield from (token_encoder.decode(list(chunk)) for chunk in chunk_iterator)

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +{
 +  "type": "patch",
 +  "description": "fix query embedding"
 +}