add 处理maxNodes 防止超出token

Author: xuzeyu91

Directory.Build.props

@@ -1,7 +1,7 @@
 <Project>
   <!-- See https://aka.ms/dotnet/msbuild/customize for more details on customizing your build -->
   <PropertyGroup>
-    <Version>0.1.24</Version>
+    <Version>0.1.25</Version>
     <SKVersion>1.17.1</SKVersion>  
   </PropertyGroup>
 </Project>

README.md

@@ -36,7 +36,8 @@
 "GraphSearch": {
     "SearchMinRelevance": 0.5, //搜索最小相关性
     "SearchLimit": 3, //搜索限制
-    "NodeDepth": 3 //节点深度
+    "NodeDepth": 3 ,//检索节点深度
+    "MaxNodes": 100 //检索最大节点数
 },
 "GraphSys": {
     "RetryCounnt": 2 //重试次数，使用国产模型可能会出现json提取失败，增加重试次数可提高可用性

src/GraphRag.Net.Web/Program.cs

@@ -35,6 +35,7 @@
 
 builder.Configuration.GetSection("GraphOpenAI").Get<GraphOpenAIOption>();
 builder.Configuration.GetSection("TextChunker").Get<TextChunkerOption>();
+builder.Configuration.GetSection("GraphSearch").Get<GraphSearchOption>();
 //builder.Configuration.GetSection("GraphDBConnection").Get<GraphDBConnectionOption>();
 
 builder.Services.AddGraphRagNet();

src/GraphRag.Net.Web/appsettings.json

@@ -42,7 +42,8 @@
   "GraphSearch": {
     "SearchMinRelevance": 0.5,
     "SearchLimit": 3,
-    "NodeDepth": 3
+    "NodeDepth": 3,
+    "MaxNodes":10
   },
   "GraphSys": {
     "RetryCounnt": 2 //重试次数，使用国产模型可能会出现json提取失败，增加重试次数可提高可用性

src/GraphRag.Net/Common/Options/GraphSearchOption.cs

@@ -23,5 +23,10 @@ public class GraphSearchOption
         /// 节点关系检索深度
         /// </summary>
         public static int NodeDepth { get; set; } = 3;
+
+        /// <summary>
+        /// 节点检索最多节点数量
+        /// </summary>
+        public static int MaxNodes { get; set; } = 300;
     }
 }

src/GraphRag.Net/Domain/Service/GraphService.cs

@@ -533,10 +533,10 @@ private GraphModel GetGraphAllRecursion(string index, List<Nodes> initialNodes)
             var allEdges = new List<Edges>();
             var nodesToExplore = new List<Nodes>(initialNodes);
             int i = 0;
+
             while (nodesToExplore.Count > 0)
             {
-                //线的深度，暂时先不处理太远距离的关联
-                if (i > GraphSearchOption.NodeDepth)
+                if (i > GraphSearchOption.NodeDepth || allNodes.Count >= GraphSearchOption.MaxNodes)
                 {
                     break;
                 }
@@ -557,22 +557,30 @@ private GraphModel GetGraphAllRecursion(string index, List<Nodes> initialNodes)
 
                 // 获取新的节点
                 var newNodes = GetNodes(index, newEdges);
-
                 // 找到新获取的节点，并更新 nodesToExplore
                 nodesToExplore = newNodes.Where(n => !allNodes.Any(existingNode => existingNode.Id == n.Id)).ToList();
-
                 // 将新节点加入到 allNodes 中
                 allNodes.AddRange(nodesToExplore);
 
                 i++;
             }
+
+            // 如果节点数超过最大限制，进行截断
+            if (allNodes.Count > GraphSearchOption.MaxNodes)
+            {
+                allNodes = allNodes.Take(GraphSearchOption.MaxNodes).ToList();  
+            }
+            // 需要相应地处理 allEdges，确保边的节点在 allNodes 中 
+            allEdges = allEdges.Where(e => allNodes.Any(p => p.Id == e.Source) && allNodes.Any(p => p.Id == e.Target)).ToList();
+
             return new GraphModel
             {
                 Nodes = allNodes,
                 Edges = allEdges
             };
         }
 
+
         /// <summary>
         /// 通过社区算法检索社区节点
         /// </summary>
@@ -600,6 +608,15 @@ private GraphModel GetGraphAllCommunitiesRecursion(string index, List<Nodes> ini
                     allEdges.Add(edge);
                 }
             }
+
+            // 如果节点数超过最大限制，进行截断
+            if (allNodes.Count > GraphSearchOption.MaxNodes)
+            {
+                allNodes = allNodes.Take(GraphSearchOption.MaxNodes).ToList();
+            }
+            // 需要相应地处理 allEdges，确保边的节点在 allNodes 中 
+            allEdges = allEdges.Where(e => allNodes.Any(p => p.Id == e.Source) && allNodes.Any(p => p.Id == e.Target)).ToList();
+
             return new GraphModel
             {
                 Nodes = allNodes,
@@ -616,7 +633,7 @@ private List<Edges> GetEdges(string index, List<Nodes> nodes)
         {
             var nodeIds = nodes.Select(x => x.Id).ToList();
             var edges = new List<Edges>();
-            edges = _edges_Repositories.GetList(x => x.Index == index && nodeIds.Contains(x.Source) || nodeIds.Contains(x.Target));
+            edges = _edges_Repositories.GetList(x => x.Index == index && nodeIds.Contains(x.Source) && nodeIds.Contains(x.Target));
             return edges;
         }
 
@@ -633,7 +650,7 @@ private List<Nodes> GetNodes(string index, List<Edges> edges)
             nodeIds.AddRange(targets);
             nodeIds.AddRange(sources);
 
-            var nodes = _nodes_Repositories.GetList(p => p.Index == index && nodeIds.Contains(p.Id));
+            var nodes = _nodes_Repositories.GetList(p => p.Index == index || nodeIds.Contains(p.Id));
             return nodes;
         }