Skip to content

Commit c859376

Browse files
naspiratonaspirato
authored
Upload testowners analytics in workflow (#8156)
1 parent 9445a62 commit c859376

File tree

7 files changed

+422
-15
lines changed

7 files changed

+422
-15
lines changed

‎.github/TESTOWNERS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
/ydb/core/tx/datashard @ydb-platform/datashard
1414
/ydb/core/mon_alloc @ydb-platform/datashard
1515
/ydb/core/tx/coordinator @ydb-platform/datashard
16+
/ydb/core/statistics @ydb-platform/datashard
1617

1718
#Column Tables Development Team @zverevgeny TEAM:@ydb-platform/cs
1819
/ydb/core/tx/columnshard @ydb-platform/cs

‎.github/actions/test_ya/action.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -384,7 +384,7 @@ runs:
384384

385385
# upload tests results to YDB
386386
ydb_upload_run_name="${TESTMO_RUN_NAME// /"_"}"
387-
result=`.github/scripts/upload_tests_results.py --test-results-file ${CURRENT_JUNIT_XML_PATH} --run-timestamp $(date +%s) --commit $(git rev-parse HEAD) --build-type ${BUILD_PRESET} --pull $ydb_upload_run_name --job-name "${{ github.workflow }}" --job-id "${{ github.run_id }}" --branch ${GITHUB_REF_NAME}`
387+
result=`.github/scripts/analytics/upload_tests_results.py --test-results-file ${CURRENT_JUNIT_XML_PATH} --run-timestamp $(date +%s) --commit $(git rev-parse HEAD) --build-type ${BUILD_PRESET} --pull $ydb_upload_run_name --job-name "${{ github.workflow }}" --job-id "${{ github.run_id }}" --branch ${GITHUB_REF_NAME}`
388388

389389
if [ ${{ inputs.testman_token }} ]; then
390390
# finish testme session

‎.github/scripts/analytics/flaky_tests_history.py

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,17 @@
1515
config_file_path = f"{dir}/../../config/ydb_qa_db.ini"
1616
config.read(config_file_path)
1717

18-
build_preset = os.environ.get("build_preset")
19-
branch = os.environ.get("branch_to_compare")
2018

2119
DATABASE_ENDPOINT = config["QA_DB"]["DATABASE_ENDPOINT"]
2220
DATABASE_PATH = config["QA_DB"]["DATABASE_PATH"]
2321

2422

2523
def create_tables(pool, table_path):
26-
print(f"> create table: {table_path}")
24+
print(f"> create table if not exists:'{table_path}'")
2725

2826
def callee(session):
2927
session.execute_scheme(f"""
30-
CREATE table `{table_path}` (
28+
CREATE table IF NOT EXISTS `{table_path}` (
3129
`test_name` Utf8 NOT NULL,
3230
`suite_folder` Utf8 NOT NULL,
3331
`full_name` Utf8 NOT NULL,
@@ -154,18 +152,18 @@ def main():
154152
from `test_results/test_runs_column`
155153
where
156154
status in ('failure','mute')
157-
and job_name in ('Nightly-run', 'Postcommit_relwithdebinfo')
158-
and build_type = 'relwithdebinfo' and
159-
run_timestamp >= Date('{last_date}') -{history_for_n_day}*Interval("P1D")
155+
and job_name in ('Nightly-run', 'Postcommit_relwithdebinfo','Postcommit_asan')
156+
and branch = 'main'
157+
and run_timestamp >= Date('{last_date}') -{history_for_n_day}*Interval("P1D")
160158
) as tests_with_fails
161159
cross join (
162160
select
163161
DISTINCT DateTime::MakeDate(run_timestamp) as date_base
164162
from `test_results/test_runs_column`
165163
where
166164
status in ('failure','mute')
167-
and job_name in ('Nightly-run', 'Postcommit_relwithdebinfo')
168-
and build_type = 'relwithdebinfo'
165+
and job_name in ('Nightly-run', 'Postcommit_relwithdebinfo','Postcommit_asan')
166+
and branch = 'main'
169167
and run_timestamp>= Date('{last_date}')
170168
) as date_list
171169
) as test_and_date

‎.github/scripts/analytics/flaky_tests_history_n_runs.py

Lines changed: 244 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,244 @@
1+
#!/usr/bin/env python3
2+
3+
import argparse
4+
import configparser
5+
import datetime
6+
import os
7+
import posixpath
8+
import traceback
9+
import time
10+
import ydb
11+
12+
dir = os.path.dirname(__file__)
13+
config = configparser.ConfigParser()
14+
config_file_path = f"{dir}/../../config/ydb_qa_db.ini"
15+
config.read(config_file_path)
16+
17+
build_preset = os.environ.get("build_preset")
18+
branch = os.environ.get("branch_to_compare")
19+
20+
DATABASE_ENDPOINT = config["QA_DB"]["DATABASE_ENDPOINT"]
21+
DATABASE_PATH = config["QA_DB"]["DATABASE_PATH"]
22+
23+
24+
def create_tables(pool, table_path):
25+
print(f"> create table: {table_path}")
26+
27+
def callee(session):
28+
session.execute_scheme(f"""
29+
CREATE table `{table_path}` (
30+
`test_name` Utf8 NOT NULL,
31+
`suite_folder` Utf8 NOT NULL,
32+
`full_name` Utf8 NOT NULL,
33+
`date_window` Date NOT NULL,
34+
`build_type` Utf8 NOT NULL,
35+
`branch` Utf8 NOT NULL,
36+
`runs_window` Uint64 NOT NULL,
37+
`history` String,
38+
`history_class` String,
39+
`pass_count` Uint64,
40+
`mute_count` Uint64,
41+
`fail_count` Uint64,
42+
`skip_count` Uint64,
43+
PRIMARY KEY (`test_name`, `suite_folder`, `full_name`,date_window,runs_window,build_type,branch)
44+
)
45+
PARTITION BY HASH(`full_name`,build_type,branch)
46+
WITH (STORE = COLUMN)
47+
""")
48+
49+
return pool.retry_operation_sync(callee)
50+
51+
52+
def bulk_upsert(table_client, table_path, rows):
53+
print(f"> bulk upsert: {table_path}")
54+
column_types = (
55+
ydb.BulkUpsertColumns()
56+
.add_column("test_name", ydb.OptionalType(ydb.PrimitiveType.Utf8))
57+
.add_column("suite_folder", ydb.OptionalType(ydb.PrimitiveType.Utf8))
58+
.add_column("build_type", ydb.OptionalType(ydb.PrimitiveType.Utf8))
59+
.add_column("branch", ydb.OptionalType(ydb.PrimitiveType.Utf8))
60+
.add_column("full_name", ydb.OptionalType(ydb.PrimitiveType.Utf8))
61+
.add_column("date_window", ydb.OptionalType(ydb.PrimitiveType.Date))
62+
.add_column("runs_window", ydb.OptionalType(ydb.PrimitiveType.Uint64))
63+
.add_column("history", ydb.OptionalType(ydb.PrimitiveType.String))
64+
.add_column("history_class", ydb.OptionalType(ydb.PrimitiveType.String))
65+
.add_column("pass_count", ydb.OptionalType(ydb.PrimitiveType.Uint64))
66+
.add_column("mute_count", ydb.OptionalType(ydb.PrimitiveType.Uint64))
67+
.add_column("fail_count", ydb.OptionalType(ydb.PrimitiveType.Uint64))
68+
.add_column("skip_count", ydb.OptionalType(ydb.PrimitiveType.Uint64))
69+
)
70+
table_client.bulk_upsert(table_path, rows, column_types)
71+
72+
73+
def main():
74+
parser = argparse.ArgumentParser()
75+
parser.add_argument('--runs', default=10,choices=[10, 25, 50], type=int, help='how many runs back we collecting history')
76+
parser.add_argument('--build_type',choices=['relwithdebinfo', 'release-asan'], default='relwithdebinfo', type=str, help='build : relwithdebinfo or release-asan')
77+
parser.add_argument('--branch', default='main',choices=['main'], type=str, help='branch')
78+
79+
args, unknown = parser.parse_known_args()
80+
history_for_n_runs = args.runs
81+
build_type = args.build_type
82+
branch = args.branch
83+
84+
print(f'Getting hostory in window {history_for_n_runs} runs')
85+
86+
87+
if "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" not in os.environ:
88+
print(
89+
"Error: Env variable CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS is missing, skipping"
90+
)
91+
return 1
92+
else:
93+
# Do not set up 'real' variable from gh workflows because it interfere with ydb tests
94+
# So, set up it locally
95+
os.environ["YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"] = os.environ[
96+
"CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"
97+
]
98+
with ydb.Driver(
99+
endpoint=DATABASE_ENDPOINT,
100+
database=DATABASE_PATH,
101+
credentials=ydb.credentials_from_env_variables(),
102+
) as driver:
103+
driver.wait(timeout=10, fail_fast=True)
104+
session = ydb.retry_operation_sync(
105+
lambda: driver.table_client.session().create()
106+
)
107+
108+
# settings, paths, consts
109+
tc_settings = ydb.TableClientSettings().with_native_date_in_result_sets(enabled=True)
110+
table_client = ydb.TableClient(driver, tc_settings)
111+
112+
table_path = f'test_results/analytics/flaky_tests_history_{history_for_n_runs}_runs'
113+
default_start_date = datetime.date(2024, 7, 19)
114+
115+
with ydb.SessionPool(driver) as pool:
116+
create_tables(pool, table_path)
117+
118+
# geting last date from history
119+
last_date_query = f"""select max(date_window) as max_date_window from `{table_path}`
120+
where build_type = '{build_type}' and branch = '{branch}'"""
121+
query = ydb.ScanQuery(last_date_query, {})
122+
it = table_client.scan_query(query)
123+
results = []
124+
while True:
125+
try:
126+
result = next(it)
127+
results = results + result.result_set.rows
128+
except StopIteration:
129+
break
130+
131+
if results[0] and results[0].get( 'max_date_window', default_start_date) is not None:
132+
last_date = results[0].get(
133+
'max_date_window', default_start_date).strftime('%Y-%m-%d')
134+
last_datetime = results[0].get(
135+
'max_date_window', default_start_date)
136+
else:
137+
last_date = default_start_date.strftime('%Y-%m-%d')
138+
last_datetime = default_start_date
139+
140+
print(f'last hisotry date: {last_date}')
141+
today = datetime.date.today()
142+
date_list = [today - datetime.timedelta(days=x) for x in range((today - last_datetime).days+1)]
143+
for date in sorted(date_list):
144+
query_get_history = f"""
145+
select
146+
full_name,
147+
date_base,
148+
build_type,
149+
branch,
150+
history_list,
151+
dist_hist,
152+
suite_folder,
153+
test_name
154+
from (
155+
select
156+
full_name,
157+
date_base,
158+
build_type,
159+
branch,
160+
AGG_LIST(status) as history_list ,
161+
String::JoinFromList( AGG_LIST_DISTINCT(status) ,',') as dist_hist,
162+
suite_folder,
163+
test_name
164+
from (
165+
select * from (
166+
select t1.test_name, t1.suite_folder, t1.full_name,
167+
Date('{date}') as date_base,
168+
'{build_type}' as build_type,
169+
'{branch}' as branch
170+
from `test_results/analytics/testowners` as t1
171+
) as test_and_date
172+
left JOIN (
173+
select * from (
174+
select
175+
suite_folder || '/' || test_name as full_name,
176+
run_timestamp,
177+
status ,
178+
ROW_NUMBER() OVER (PARTITION BY test_name ORDER BY run_timestamp DESC) AS run_number
179+
from `test_results/test_runs_column`
180+
where
181+
run_timestamp <= Date('{date}')
182+
and run_timestamp >= Date('{date}') -14*Interval("P1D")
183+
and job_name in ('Postcommit_relwithdebinfo','Postcommit_asan')
184+
and build_type = '{build_type}'
185+
and status != 'skipped'
186+
and branch = '{branch}'
187+
)
188+
where run_number <= {history_for_n_runs}
189+
) as hist
190+
ON test_and_date.full_name=hist.full_name
191+
)
192+
GROUP BY full_name,suite_folder,test_name,date_base,build_type,branch
193+
194+
)
195+
"""
196+
query = ydb.ScanQuery(query_get_history, {})
197+
# start transaction time
198+
start_time = time.time()
199+
it = driver.table_client.scan_query(query)
200+
# end transaction time
201+
202+
results = []
203+
prepared_for_update_rows = []
204+
while True:
205+
try:
206+
result = next(it)
207+
results = results + result.result_set.rows
208+
except StopIteration:
209+
break
210+
end_time = time.time()
211+
print(f'transaction duration: {end_time - start_time}')
212+
213+
print(f'history data captured, {len(results)} rows')
214+
for row in results:
215+
row['count'] = dict(zip(list(row['history_list']), [list(
216+
row['history_list']).count(i) for i in list(row['history_list'])]))
217+
prepared_for_update_rows.append({
218+
'suite_folder': row['suite_folder'],
219+
'test_name': row['test_name'],
220+
'full_name': row['full_name'],
221+
'date_window': row['date_base'],
222+
'build_type': row['build_type'],
223+
'branch': row['branch'],
224+
'runs_window': history_for_n_runs,
225+
'history': ','.join(row['history_list']).encode('utf8'),
226+
'history_class': row['dist_hist'],
227+
'pass_count': row['count'].get('passed', 0),
228+
'mute_count': row['count'].get('mute', 0),
229+
'fail_count': row['count'].get('failure', 0),
230+
'skip_count': row['count'].get('skipped', 0),
231+
})
232+
print(f'upserting history for date {date}')
233+
with ydb.SessionPool(driver) as pool:
234+
235+
full_path = posixpath.join(DATABASE_PATH, table_path)
236+
bulk_upsert(driver.table_client, full_path,
237+
prepared_for_update_rows)
238+
239+
print('flaky history updated')
240+
print('finished')
241+
242+
243+
if __name__ == "__main__":
244+
main()

0 commit comments

Comments
 (0)