query optimizer

Author: david-leifker

metadata-ingestion/examples/perf/lineage_perf_example.py

@@ -406,7 +406,7 @@ def scenario_perf():
         scenario_skip_complex,
     ]
     # Very long load time
-    # scenario_generators.add(scenario_perf)
+    # scenario_generators.append(scenario_perf)
 
     # Chain all generators into a single generator
     chained_generator = itertools.chain.from_iterable(

metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java

@@ -3,6 +3,7 @@
 import static com.linkedin.metadata.aspect.models.graph.Edge.*;
 import static com.linkedin.metadata.graph.elastic.ElasticSearchGraphService.*;
 import static com.linkedin.metadata.search.utils.ESUtils.applyResultLimit;
+import static com.linkedin.metadata.search.utils.ESUtils.queryOptimize;
 
 import com.datahub.util.exception.ESQueryException;
 import com.google.common.annotations.VisibleForTesting;
@@ -178,7 +179,7 @@ private static BoolQueryBuilder getAggregationFilter(
     BoolQueryBuilder subFilter = QueryBuilders.boolQuery();
     TermQueryBuilder relationshipTypeTerm =
         QueryBuilders.termQuery(RELATIONSHIP_TYPE, pair.getValue().getType()).caseInsensitive(true);
-    subFilter.must(relationshipTypeTerm);
+    subFilter.filter(relationshipTypeTerm);
 
     String sourceType;
     String destinationType;
@@ -192,10 +193,10 @@ private static BoolQueryBuilder getAggregationFilter(
 
     TermQueryBuilder sourceTypeTerm =
         QueryBuilders.termQuery(SOURCE_TYPE, sourceType).caseInsensitive(true);
-    subFilter.must(sourceTypeTerm);
+    subFilter.filter(sourceTypeTerm);
     TermQueryBuilder destinationTypeTerm =
         QueryBuilders.termQuery(DESTINATION_TYPE, destinationType).caseInsensitive(true);
-    subFilter.must(destinationTypeTerm);
+    subFilter.filter(destinationTypeTerm);
     return subFilter;
   }
 
@@ -261,6 +262,7 @@ public static BoolQueryBuilder buildQuery(
                   relationshipQuery.should(
                       QueryBuilders.termQuery(RELATIONSHIP_TYPE, relationshipType)));
       relationshipQuery.minimumShouldMatch(1);
+
       finalQuery.filter(relationshipQuery);
     }
 
@@ -1140,6 +1142,10 @@ private List<LineageRelationship> relationshipsGroupQuery(
 
     Set<Urn> entityUrnSet = new HashSet<>(entityUrns);
 
+    if (config.getSearch().getGraph().isQueryOptimization()) {
+      queryOptimize(baseQuery, false);
+    }
+
     // Get search responses as a stream with pagination
     Stream<SearchResponse> responseStream =
         executeGroupByLineageSearchQuery(
@@ -1167,16 +1173,16 @@ private List<LineageRelationship> relationshipsGroupQuery(
   private static BoolQueryBuilder getOutGoingEdgeQuery(
       @Nonnull Set<Urn> urns, @Nonnull Set<EdgeInfo> outgoingEdges) {
     BoolQueryBuilder outgoingEdgeQuery = QueryBuilders.boolQuery();
-    outgoingEdgeQuery.must(buildUrnFilters(urns, SOURCE));
-    outgoingEdgeQuery.must(buildEdgeFilters(outgoingEdges));
+    outgoingEdgeQuery.filter(buildUrnFilters(urns, SOURCE));
+    outgoingEdgeQuery.filter(buildEdgeFilters(outgoingEdges));
     return outgoingEdgeQuery;
   }
 
   private static BoolQueryBuilder getIncomingEdgeQuery(
       @Nonnull Set<Urn> urns, Set<EdgeInfo> incomingEdges) {
     BoolQueryBuilder incomingEdgeQuery = QueryBuilders.boolQuery();
-    incomingEdgeQuery.must(buildUrnFilters(urns, DESTINATION));
-    incomingEdgeQuery.must(buildEdgeFilters(incomingEdges));
+    incomingEdgeQuery.filter(buildUrnFilters(urns, DESTINATION));
+    incomingEdgeQuery.filter(buildEdgeFilters(incomingEdges));
     return incomingEdgeQuery;
   }