Fixed fq config

Author: GrigoriyPA

ydb/tests/tools/fqrun/configuration/fq_config.conf

@@ -4,6 +4,12 @@ EnableTaskCounters: true
 
 CheckpointCoordinator {
   Enabled: true
+  CheckpointingPeriodMillis: 30000
+  MaxInflight: 1
+
+  CheckpointGarbageConfig {
+    Enabled: true
+  }
 
   Storage {
     TablePrefix: "yq/checkpoints"
@@ -16,13 +22,22 @@ CheckpointCoordinator {
 Common {
   YdbMvpCloudEndpoint: "https://ydbc.ydb.cloud.yandex.net:8789/ydbc/cloud-prod"
   MdbGateway: "https://mdb.api.cloud.yandex.net:443"
+  MdbTransformHost: false
   ObjectStorageEndpoint: "https://storage.yandexcloud.net"
   IdsPrefix: "kr"
   QueryArtifactsCompressionMethod: "zstd_6"
   MonitoringEndpoint: "monitoring.api.cloud.yandex.net"
   KeepInternalErrors: true
   UseNativeProtocolForClickHouse: true
   ShowQueryTimeline: true
+  MaxTasksPerOperation: 400
+  MaxTasksPerStage: 50
+  PqReconnectPeriod: "30m"
+
+  YdbDriverConfig {
+    ClientThreadsNum: 6
+    NetworkThreadsNum: 6
+  }
 }
 
 ControlPlaneProxy {
@@ -32,6 +47,15 @@ ControlPlaneProxy {
 ControlPlaneStorage {
   Enabled: true
   StatsMode: STATS_MODE_PROFILE
+  DumpRawStatistics: true
+  TasksBatchSize: 100
+  NumTasksProportion: 4
+  AnalyticsRetryCounterLimit: 20
+  StreamingRetryCounterLimit: 20
+  AnalyticsRetryCounterUpdateTime: "1d"
+  StreamingRetryCounterUpdateTime: "1d"
+  TaskLeaseTtl: "30s"
+  DisableCurrentIam: false
 
   AvailableConnection: "OBJECT_STORAGE"
   AvailableConnection: "DATA_STREAMS"
@@ -78,65 +102,13 @@ Gateways {
 
   Dq {
     DefaultSettings {
-      Name: "EnableComputeActor"
+      Name: "HashShuffleTasksRatio"
       Value: "1"
     }
     DefaultSettings {
-      Name: "ComputeActorType"
-      Value: "async"
-    }
-    DefaultSettings {
-      Name: "AnalyzeQuery"
-      Value: "true"
-    }
-    DefaultSettings {
-      Name: "MaxTasksPerStage"
-      Value: "200"
-    }
-    DefaultSettings {
-      Name: "MaxTasksPerOperation"
-      Value: "200"
-    }
-    DefaultSettings {
-      Name: "EnableInsert"
-      Value: "true"
-    }
-    DefaultSettings {
-      Name: "_EnablePrecompute"
-      Value: "true"
-    }
-    DefaultSettings {
-      Name: "UseAggPhases"
-      Value: "true"
-    }
-    DefaultSettings {
-      Name: "HashJoinMode"
-      Value: "grace"
-    }
-    DefaultSettings {
-      Name: "UseFastPickleTransport"
-      Value: "true"
-    }
-    DefaultSettings {
-      Name: "UseOOBTransport"
+      Name: "UseFinalizeByKey"
       Value: "true"
     }
-    DefaultSettings {
-      Name: "UseWideChannels"
-      Value: "true"
-    }
-    DefaultSettings {
-      Name: "_SkipRevisionCheck"
-      Value: "true"
-    }
-    DefaultSettings {
-      Name: "EnableDqReplicate"
-      Value: "true"
-    }
-    DefaultSettings {
-      Name: "_TableTimeout"
-      Value: "600000"
-    }
   }
 
   Generic {
@@ -173,8 +145,8 @@ Gateways {
       Endpoint: "localhost:2135"
       Database: "local"
       ClusterType: CT_DATA_STREAMS
-      UseSsl: True
-      SharedReading: True
+      UseSsl: true
+      SharedReading: true
       ReadGroup: "fqrun"
     }
   }
@@ -200,6 +172,15 @@ Gateways {
     MinDesiredDirectoriesOfFilesPerQuery: 1000
     RegexpCacheSize: 100
 
+    FormatSizeLimit {
+      Name: "parquet"
+      FileSizeLimit: 52428800
+    }
+    FormatSizeLimit {
+      Name: "raw"
+      FileSizeLimit: 52428800
+    }
+
     DefaultSettings {
       Name: "AtomicUploadCommit"
       Value: "true"
@@ -208,10 +189,6 @@ Gateways {
       Name: "UseBlocksSource"
       Value: "true"
     }
-    DefaultSettings {
-      Name: "UseRuntimeListing"
-      Value: "true"
-    }
   }
 
   YqlCore {
@@ -224,6 +201,10 @@ Gateways {
   }
 }
 
+Health {
+  Enabled: true
+}
+
 NodesManager {
   Enabled: true
 }
@@ -234,6 +215,7 @@ PendingFetcher {
 
 PrivateApi {
   Enabled: true
+  Loopback: true
 }
 
 PrivateProxy {
@@ -242,6 +224,23 @@ PrivateProxy {
 
 QuotasManager {
   Enabled: true
+  QuotaDescriptions {
+    SubjectType: "cloud"
+    MetricName: "yq.cpuPercent.count"
+    HardLimit: 7500
+    DefaultLimit: 3500
+  }
+  QuotaDescriptions {
+    SubjectType: "cloud"
+    MetricName: "yq.streamingQueryDurationMinutes.count"
+    DefaultLimit: 10080
+  }
+  QuotaDescriptions {
+    SubjectType: "cloud"
+    MetricName: "yq.analyticsQueryDurationMinutes.count"
+    HardLimit: 1440
+    DefaultLimit: 30
+  }
 }
 
 RateLimiter {
@@ -261,14 +260,26 @@ RateLimiter {
   }
 }
 
+ReadActorsFactoryConfig {
+  PqReadActorFactoryConfig {
+    CookieCommitMode: false
+  }
+}
+
 ResourceManager {
   Enabled: true
+  MkqlInitialMemoryLimit: 16777216
+  MkqlTotalMemoryLimit: 193273528320
+  MkqlAllocSize: 16777216
+  MkqlTaskHardMemoryLimit: 24696061952
 }
 
 RowDispatcher {
   Enabled: true
   SendStatusPeriodSec: 10
   TimeoutBeforeStartSessionSec: 10
+  MaxSessionUsedMemory: 16000000
+  WithoutConsumer: false
 
   CompileService {
     ParallelCompilationLimit: 20

ydb/tests/tools/fqrun/src/fq_setup.cpp

@@ -111,6 +111,10 @@ class TFqSetup::TImpl {
         nodesMenagerConfig->SetPort(grpcPort);
         nodesMenagerConfig->SetHost("localhost");
 
+        auto* healthConfig = fqConfig.MutableHealth();
+        healthConfig->SetPort(grpcPort);
+        healthConfig->SetDatabase(database);
+
         if (Settings.EmulateS3) {
             fqConfig.MutableCommon()->SetObjectStorageEndpoint("file://");
         }

ydb/tests/tools/fqrun/ya.make

@@ -17,6 +17,7 @@ PEERDIR(
 
 PEERDIR(
     yql/essentials/udfs/common/compress_base
+    yql/essentials/udfs/common/re2
 )
 
 YQL_LAST_ABI_VERSION()