Merge branch 'stable-25-1' into 25-1

Author: iddqdex

.github/actions/test_ya/action.yml

@@ -468,6 +468,7 @@ runs:
               --public_dir "$PUBLIC_DIR" \
               --public_dir_url "$PUBLIC_DIR_URL" \
               --build_preset "$BUILD_PRESET" \
+              --branch "$BRANCH_NAME" \
               --status_report_file statusrep.txt \
               --is_retry $IS_RETRY \
               --is_last_retry $IS_LAST_RETRY \

.github/config/muted_ya.txt

@@ -44,6 +44,7 @@ COPY --from=builder \
 EXPOSE ${GRPC_TLS_PORT:-2135}
 EXPOSE ${GRPC_PORT:-2136}
 EXPOSE ${MON_PORT:-8765}
+EXPOSE ${YDB_KAFKA_PROXY_PORT:-9092}
 
 HEALTHCHECK --start-period=60s --interval=1s CMD sh ./health_check
 

.github/docker/Dockerfile

@@ -7,6 +7,7 @@ export YDB_GRPC_ENABLE_TLS="true"
 export GRPC_TLS_PORT=${GRPC_TLS_PORT:-2135}
 export GRPC_PORT=${GRPC_PORT:-2136}
 export YDB_GRPC_TLS_DATA_PATH="/ydb_certs"
+export YDB_KAFKA_PROXY_PORT=${YDB_KAFKA_PROXY_PORT:-9092}
 
 # Start local_ydb tool. Pass additional arguments for local_ydb
 /local_ydb deploy --ydb-working-dir /ydb_data --ydb-binary-path /ydbd --fixed-ports --dont-use-log-files "$@";

.github/docker/files/initialize_local_ydb

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+import argparse
+import ydb
+import configparser
+import os
+
+# Load configuration
+dir = os.path.dirname(__file__)
+config = configparser.ConfigParser()
+config_file_path = f"{dir}/../../config/ydb_qa_db.ini"
+config.read(config_file_path)
+
+DATABASE_ENDPOINT = config["QA_DB"]["DATABASE_ENDPOINT"]
+DATABASE_PATH = config["QA_DB"]["DATABASE_PATH"]
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Delete a YDB table")
+    parser.add_argument("--table_path", required=True, help="Table path and name to delete")
+    
+    return parser.parse_args()
+
+def check_table_exists(session, table_path):
+    """Check if table exists"""
+    try:
+        session.describe_table(table_path)
+        return True
+    except ydb.SchemeError:
+        return False
+
+def delete_table(session, table_path):
+    """Delete the specified table."""
+    try:
+        session.drop_table(table_path)
+        print(f"Table '{table_path}' successfully deleted.")
+        return True
+    except ydb.Error as e:
+        print(f"Error deleting table: {e}")
+        return False
+
+def main():
+    args = parse_args()
+
+    if "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" not in os.environ:
+        print("Error: Env variable CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS is missing, skipping")
+        return 1
+    else:
+        os.environ["YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"] = os.environ[
+            "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"
+        ]
+
+    table_path = args.table_path
+    full_table_path = f'{DATABASE_PATH}/{table_path}'
+
+    print(f"Connecting to YDB to delete table {full_table_path}")
+
+    with ydb.Driver(
+        endpoint=DATABASE_ENDPOINT,
+        database=DATABASE_PATH,
+        credentials=ydb.credentials_from_env_variables()
+    ) as driver:
+        # Wait until driver is ready
+        driver.wait(timeout=10, fail_fast=True)
+        
+        with ydb.SessionPool(driver) as pool:
+            # Проверяем существование таблицы
+            def check_and_delete(session):
+                exists = check_table_exists(session, full_table_path)
+                if exists:
+                    return delete_table(session, full_table_path)
+                else:
+                    print(f"Table '{full_table_path}' does not exist.")
+                    return False
+            
+            result = pool.retry_operation_sync(check_and_delete)
+            
+            if result:
+                print(f"Table {full_table_path} has been deleted successfully.")
+                return 0
+            else:
+                print(f"No table was deleted.")
+                return 1
+
+if __name__ == "__main__":
+    exit_code = main()
+    exit(exit_code)

.github/scripts/analytics/data_mart_delete_table.py

@@ -0,0 +1,35 @@
+SELECT 
+    build_type , 
+    job_name, 
+    job_id, 
+    commit, 
+    branch, 
+    pull, 
+    run_timestamp, 
+    test_id, 
+    suite_folder, 
+    test_name,
+    cast(suite_folder || '/' || test_name as UTF8)  as full_name, 
+    duration,
+    status,
+    String::ReplaceAll(status_description, ';;', '\n') as status_description,
+    owners,
+    String::ReplaceAll(owners, 'TEAM:@ydb-platform/', '') as owner_team,
+    String::SplitToList(pull,'_A')[0] as pull_raw,
+    cast(COALESCE(String::SplitToList(pull,'_A')[1],"1") as Uint16) as attempt,
+    (cast(pull as String) || '_' || SUBSTRING(cast(commit as String), 1, 8)) as pull_commit,
+    CASE 
+        WHEN String::Contains(test_name, 'chunk chunk') OR String::Contains(test_name, 'chunk+chunk') THEN TRUE
+        ELSE FALSE
+    END as with_cunks
+   
+FROM `test_results/test_runs_column` 
+
+WHERE
+    run_timestamp >= CurrentUtcDate() - 6*Interval("P1D")
+    and build_type = 'relwithdebinfo'
+    and String::Contains(test_name, '.flake8')  = FALSE
+    and (CASE 
+        WHEN String::Contains(test_name, 'chunk chunk') OR String::Contains(test_name, 'chunk+chunk') THEN TRUE
+        ELSE FALSE
+        END) = FALSE

.github/scripts/analytics/data_mart_queries/muted_test_mart.sql

@@ -0,0 +1,25 @@
+SELECT 
+        build_type, 
+        job_name, 
+        job_id, 
+        commit, 
+        branch, 
+        pull, 
+        run_timestamp, 
+        test_id, 
+        suite_folder, 
+        test_name,
+        cast(suite_folder || '/' || test_name as UTF8)  as full_name, 
+        duration,
+        status,
+        cast(String::ReplaceAll(status_description, ';;', '\n')as Utf8) as status_description ,
+        owners
+    FROM `test_results/test_runs_column`  as all_data
+    WHERE
+    run_timestamp >= CurrentUtcDate() - Interval("P1D")
+    and String::Contains(test_name, '.flake8')  = FALSE
+    and (CASE 
+        WHEN String::Contains(test_name, 'chunk chunk') OR String::Contains(test_name, 'chunk+chunk') THEN TRUE
+        ELSE FALSE
+        END) = FALSE
+    and (branch = 'main' or branch like 'stable-%')

.github/scripts/analytics/data_mart_queries/test_history_fast_mart.sql

@@ -0,0 +1,44 @@
+SELECT 
+    state_filtered, 
+    test_name, 
+    suite_folder, 
+    full_name, 
+    date_window, 
+    build_type, 
+    branch, 
+    days_ago_window, 
+    pass_count, 
+    mute_count, 
+    fail_count, 
+    skip_count, 
+    owner, 
+    is_muted, 
+    is_test_chunk, 
+    state, 
+    previous_state, 
+    state_change_date, 
+    days_in_state, 
+    previous_mute_state, 
+    mute_state_change_date, 
+    days_in_mute_state, 
+    previous_state_filtered, 
+    state_change_date_filtered, 
+    days_in_state_filtered,
+    CASE 
+        WHEN (state = 'Skipped' AND days_in_state > 14) THEN 'Skipped'
+        WHEN days_in_mute_state >= 30 THEN 'MUTED: delete candidate'
+        ELSE 'MUTED: in sla'
+    END as resolution,
+    String::ReplaceAll(owner, 'TEAM:@ydb-platform/', '') as owner_team,
+    CASE 
+        WHEN is_muted = 1 OR (state = 'Skipped' AND days_in_state > 14) THEN TRUE
+        ELSE FALSE
+    END as is_muted_or_skipped
+FROM `test_results/analytics/tests_monitor`
+WHERE date_window >= CurrentUtcDate() - 30 * Interval("P1D")
+and ( branch = 'main' or branch like 'stable-%')
+and is_test_chunk = 0
+and (CASE 
+        WHEN is_muted = 1 OR (state = 'Skipped' AND days_in_state > 14) THEN TRUE
+        ELSE FALSE
+    END ) = TRUE

.github/scripts/analytics/data_mart_queries/test_monitor_mart.sql

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+import argparse
+import ydb
+import configparser
+import os
+import time
+
+# Load configuration
+dir = os.path.dirname(__file__)
+config = configparser.ConfigParser()
+config_file_path = f"{dir}/../../config/ydb_qa_db.ini"
+config.read(config_file_path)
+
+DATABASE_ENDPOINT = config["QA_DB"]["DATABASE_ENDPOINT"]
+DATABASE_PATH = config["QA_DB"]["DATABASE_PATH"]
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Delete old records from YDB table")
+    parser.add_argument("--table-path", required=True, help="Table path and name")
+    parser.add_argument("--timestamp-field", required=True, help="Name of the timestamp field")
+    parser.add_argument("--delete-interval", required=True, help="Interval to delete records older than, in ISO 8601 format (https://en.wikipedia.org/wiki/ISO_8601#Durations) without 'P'")
+    
+    return parser.parse_args()
+
+def delete_old_records(session, full_table_path, timestamp_field, delete_interval):
+    """Delete records older than the specified interval."""
+    # First, count the number of records that will be deleted
+    count_query = f"""
+    SELECT COUNT(*) as count
+    FROM `{full_table_path}`
+    WHERE `{timestamp_field}` < CurrentUtcDate() - Interval("P{delete_interval}")
+    """
+    
+    print(f"Counting records to delete...")
+    result_sets = session.transaction().execute(count_query)
+    row_count = result_sets[0].rows[0].count
+    
+    if row_count == 0:
+        print("No records to delete.")
+        return 0
+    
+    print(f"Found {row_count} records older than {delete_interval}.")
+    
+    # Now perform the delete operation
+    delete_query = f"""
+    DELETE FROM `{full_table_path}`
+    WHERE `{timestamp_field}` < CurrentUtcDate() - Interval("P{delete_interval}")
+    """
+    
+    print(f"Executing DELETE query: {delete_query}")
+    start_time = time.time()
+    session.transaction().execute(delete_query, commit_tx=True)
+    end_time = time.time()
+    
+    print(f"Deleted {row_count} records in {end_time - start_time:.2f} seconds.")
+    return row_count
+
+def main():
+    args = parse_args()
+
+    if "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" not in os.environ:
+        print("Error: Env variable CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS is missing, skipping")
+        return 1
+    else:
+        os.environ["YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"] = os.environ[
+            "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"
+        ]
+
+    table_path = args.table_path
+    full_table_path = f'{DATABASE_PATH}/{table_path}'
+    timestamp_field = args.timestamp_field
+    delete_interval = args.delete_interval
+
+    print(f"Connecting to YDB to delete records from {full_table_path}")
+    print(f"Will delete records where {timestamp_field} < CurrentUtcDate() - Interval(\"P{delete_interval}\")")
+
+    with ydb.Driver(
+        endpoint=DATABASE_ENDPOINT,
+        database=DATABASE_PATH,
+        credentials=ydb.credentials_from_env_variables()
+    ) as driver:
+        # Wait until driver is ready
+        driver.wait(timeout=10, fail_fast=True)
+        
+        with ydb.SessionPool(driver) as pool:
+            try:
+                def transaction_delete(session):
+                    return delete_old_records(session, full_table_path, timestamp_field, delete_interval)
+                
+                deleted_count = pool.retry_operation_sync(transaction_delete)
+                
+                print(f"Successfully deleted old records from {full_table_path}")
+                print(f"Total records deleted: {deleted_count}")
+                return 0
+            except ydb.Error as e:
+                print(f"Error deleting records: {e}")
+                return 1
+
+if __name__ == "__main__":
+    exit_code = main()
+    exit(exit_code)

.github/scripts/analytics/data_mart_ttl_analog.py

@@ -78,8 +78,8 @@ def main():
     parser = argparse.ArgumentParser()
 
     parser.add_argument('--days-window', default=1, type=int, help='how many days back we collecting history')
-    parser.add_argument('--build_type',choices=['relwithdebinfo', 'release-asan'], default='relwithdebinfo', type=str, help='build : relwithdebinfo or release-asan')
-    parser.add_argument('--branch', default='main',choices=['main'], type=str, help='branch')
+    parser.add_argument('--build_type', default='relwithdebinfo', type=str, help='build types')
+    parser.add_argument('--branch', default='main', type=str, help='branch')
 
     args, unknown = parser.parse_known_args()
     history_for_n_day = args.days_window
@@ -203,6 +203,9 @@ def main():
                         and job_name in (
                             'Nightly-run',
                             'Regression-run',
+                            'Regression-run_Large',
+                            'Regression-run_Small_and_Medium',
+                            'Regression-run_compatibility',
                             'Regression-whitelist-run',
                             'Postcommit_relwithdebinfo', 
                             'Postcommit_asan'