Skip to content

Commit b1c0ea1

Browse files
committed
[yaml-tests] Fix & improve yaml test runtime
1 parent 15f270d commit b1c0ea1

File tree

4 files changed

+107
-49
lines changed

4 files changed

+107
-49
lines changed

Makefile.toml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ STACK_VERSION = { value = "8.0.0-SNAPSHOT", condition = { env_not_set = ["STACK_
88
TEST_SUITE = { value = "free", condition = { env_not_set = ["TEST_SUITE"] }}
99
# Set publish flags to dry-run by default, to force user to explicitly define for publishing
1010
CARGO_MAKE_CARGO_PUBLISH_FLAGS = "--dry-run"
11+
# RUST_BACKTRACE is set to "full" in cargo make's builtin makefiles/stable.toml
12+
RUST_BACKTRACE = { value = "0", condition = { env_not_set = ["RUST_BACKTRACE"]}}
1113

1214
[tasks.set-free-env]
1315
category = "Elasticsearch"
@@ -39,14 +41,14 @@ category = "Elasticsearch"
3941
private = true
4042
condition = { env_set = [ "ELASTICSEARCH_URL" ], env_false = ["CARGO_MAKE_CI"] }
4143
command = "cargo"
42-
args = ["test", "-p", "yaml_test_runner", "--", "--test-threads=1"]
44+
args = ["test", "-p", "yaml_test_runner", "--", "--test-threads", "1"]
4345
dependencies = ["generate-yaml-tests"]
4446

4547
[tasks.test-yaml-test-runner-ci]
4648
category = "Elasticsearch"
4749
private = true
4850
condition = { env_set = [ "ELASTICSEARCH_URL" ], env_true = ["CARGO_MAKE_CI"] }
49-
script = ["cargo test -p yaml_test_runner -- --test-threads=1 -Z unstable-options --format json | tee test_results/results.json"]
51+
script = ["cargo test -p yaml_test_runner -- --test-threads 1 -Z unstable-options --format json | tee test_results/results.json"]
5052
dependencies = ["generate-yaml-tests"]
5153

5254
[tasks.test-elasticsearch]

yaml_test_runner/src/generator.rs

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -413,11 +413,7 @@ pub fn generate_tests_from_yaml(
413413
};
414414

415415
if &test_suite != suite {
416-
info!(
417-
"skipping {}. compiling tests for {:?}",
418-
relative_path.to_slash_lossy(),
419-
suite
420-
);
416+
// Belongs to another test suite
421417
continue;
422418
}
423419

yaml_test_runner/src/step/mod.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,9 @@ impl Expr {
162162
// which should be removed.
163163
if Self::is_string_body(values[0].as_ref()) {
164164
values.remove(0);
165+
} else if values[0] == "" {
166+
// some tests start the json path with a dot, leading to an empty first element
167+
values.remove(0);
165168
}
166169

167170
let mut expr = String::new();

yaml_test_runner/tests/common/client.rs

Lines changed: 99 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
*/
1919
use elasticsearch::{
2020
auth::Credentials,
21-
cat::{CatSnapshotsParts, CatTemplatesParts},
21+
cat::CatTemplatesParts,
2222
cert::CertificateValidation,
2323
cluster::ClusterHealthParts,
2424
http::{
@@ -27,7 +27,10 @@ use elasticsearch::{
2727
Method, StatusCode,
2828
},
2929
ilm::IlmRemovePolicyParts,
30-
indices::{IndicesDeleteParts, IndicesDeleteTemplateParts, IndicesRefreshParts},
30+
indices::{
31+
IndicesDeleteIndexTemplateParts, IndicesDeleteParts, IndicesDeleteTemplateParts,
32+
IndicesRefreshParts,
33+
},
3134
ml::{
3235
MlCloseJobParts, MlDeleteDatafeedParts, MlDeleteJobParts, MlGetDatafeedsParts,
3336
MlGetJobsParts, MlStopDatafeedParts,
@@ -138,49 +141,55 @@ pub async fn read_response(
138141
/// general setup step for an OSS yaml test
139142
pub async fn general_oss_setup() -> Result<(), Error> {
140143
let client = get();
144+
delete_data_streams(client).await?;
141145
delete_indices(client).await?;
142146
delete_templates(client).await?;
147+
delete_snapshots(client).await?;
148+
149+
Ok(())
150+
}
143151

144-
let cat_snapshot_response = client
152+
pub async fn delete_snapshots(client: &Elasticsearch) -> Result<(), Error> {
153+
let cat_repo_response = client
145154
.cat()
146-
.snapshots(CatSnapshotsParts::None)
147-
.h(&["id", "repository"])
155+
.repositories()
156+
.h(&["id"])
148157
.send()
158+
.await?
159+
.error_for_status_code()?
160+
.text()
149161
.await?;
150162

151-
if cat_snapshot_response.status_code().is_success() {
152-
let cat_snapshot_text = cat_snapshot_response.text().await?;
163+
if cat_repo_response.len() > 0 {
164+
let repositories: Vec<&str> = cat_repo_response.split_terminator('\n').collect();
153165

154-
let all_snapshots: Vec<(&str, &str)> = cat_snapshot_text
155-
.split('\n')
156-
.map(|s| s.split(' ').collect::<Vec<&str>>())
157-
.filter(|s| s.len() == 2)
158-
.map(|s| (s[0].trim(), s[1].trim()))
159-
.filter(|(id, repo)| !id.is_empty() && !repo.is_empty())
160-
.collect();
161-
162-
for (id, repo) in all_snapshots {
163-
let _snapshot_response = client
166+
// Delete snapshots in each repository
167+
for repo in repositories {
168+
let delete_snapshots_response = client
164169
.snapshot()
165-
.delete(SnapshotDeleteParts::RepositorySnapshot(&repo, &[&id]))
170+
.delete(SnapshotDeleteParts::RepositorySnapshot(repo, &["*"]))
166171
.send()
167172
.await?;
173+
174+
assert_response_success!(delete_snapshots_response);
168175
}
169-
}
170176

171-
let _delete_repo_response = client
172-
.snapshot()
173-
.delete_repository(SnapshotDeleteRepositoryParts::Repository(&["*"]))
174-
.send()
175-
.await?;
177+
// Delete all snapshot repositories
178+
let delete_repo_response = client
179+
.snapshot()
180+
.delete_repository(SnapshotDeleteRepositoryParts::Repository(&["*"]))
181+
.send()
182+
.await?;
183+
184+
assert_response_success!(delete_repo_response);
185+
}
176186

177187
Ok(())
178188
}
179189

180190
/// general setup step for an xpack yaml test
181191
pub async fn general_xpack_setup() -> Result<(), Error> {
182192
let client = get();
183-
delete_templates(client).await?;
184193

185194
let _delete_watch_response = client
186195
.watcher()
@@ -193,22 +202,26 @@ pub async fn general_xpack_setup() -> Result<(), Error> {
193202
delete_privileges(client).await?;
194203
stop_and_delete_datafeeds(client).await?;
195204

196-
let _ = client
205+
let response = client
197206
.ilm()
198207
.remove_policy(IlmRemovePolicyParts::Index("_all"))
199208
.send()
200209
.await?;
201210

211+
assert_response_success!(response);
212+
202213
close_and_delete_jobs(client).await?;
203214

204215
// TODO: stop and delete rollup jobs once implemented in the client
205216

206217
cancel_tasks(client).await?;
207218
stop_and_delete_transforms(client).await?;
208219
wait_for_yellow_status(client).await?;
220+
delete_data_streams(client).await?;
209221
delete_indices(client).await?;
222+
delete_templates(client).await?;
210223

211-
let _ = client
224+
let response = client
212225
.security()
213226
.put_user(SecurityPutUserParts::Username("x_pack_rest_user"))
214227
.body(json!({
@@ -218,7 +231,9 @@ pub async fn general_xpack_setup() -> Result<(), Error> {
218231
.send()
219232
.await?;
220233

221-
let _ = client
234+
assert_response_success!(response);
235+
236+
let response = client
222237
.indices()
223238
.refresh(IndicesRefreshParts::Index(&["_all"]))
224239
.expand_wildcards(&[
@@ -229,6 +244,8 @@ pub async fn general_xpack_setup() -> Result<(), Error> {
229244
.send()
230245
.await?;
231246

247+
assert_response_success!(response);
248+
232249
wait_for_yellow_status(client).await?;
233250

234251
Ok(())
@@ -246,7 +263,7 @@ async fn wait_for_yellow_status(client: &Elasticsearch) -> Result<(), Error> {
246263
Ok(())
247264
}
248265

249-
async fn delete_indices(client: &Elasticsearch) -> Result<(), Error> {
266+
async fn delete_data_streams(client: &Elasticsearch) -> Result<(), Error> {
250267
// Hand-crafted request as the indices.delete_data_stream spec doesn't yet have the
251268
// "expand_wildcards" parameter that is needed to delete ILM data streams
252269
//
@@ -276,6 +293,10 @@ async fn delete_indices(client: &Elasticsearch) -> Result<(), Error> {
276293

277294
assert_response_success!(delete_response);
278295

296+
Ok(())
297+
}
298+
299+
async fn delete_indices(client: &Elasticsearch) -> Result<(), Error> {
279300
let delete_response = client
280301
.indices()
281302
.delete(IndicesDeleteParts::Index(&["*"]))
@@ -302,17 +323,21 @@ async fn stop_and_delete_transforms(client: &Elasticsearch) -> Result<(), Error>
302323

303324
for transform in transforms_response["transforms"].as_array().unwrap() {
304325
let id = transform["id"].as_str().unwrap();
305-
let _ = client
326+
let response = client
306327
.transform()
307328
.stop_transform(TransformStopTransformParts::TransformId(id))
308329
.send()
309330
.await?;
310331

311-
let _ = client
332+
assert_response_success!(response);
333+
334+
let response = client
312335
.transform()
313336
.delete_transform(TransformDeleteTransformParts::TransformId(id))
314337
.send()
315338
.await?;
339+
340+
assert_response_success!(response);
316341
}
317342

318343
Ok(())
@@ -325,11 +350,13 @@ async fn cancel_tasks(client: &Elasticsearch) -> Result<(), Error> {
325350
for (task_id, task) in nodes["tasks"].as_object().unwrap() {
326351
if let Some(b) = task["cancellable"].as_bool() {
327352
if b {
328-
let _ = client
353+
let response = client
329354
.tasks()
330355
.cancel(TasksCancelParts::TaskId(task_id))
331356
.send()
332357
.await?;
358+
359+
assert_response_success!(response);
333360
}
334361
}
335362
}
@@ -339,6 +366,8 @@ async fn cancel_tasks(client: &Elasticsearch) -> Result<(), Error> {
339366
}
340367

341368
async fn delete_templates(client: &Elasticsearch) -> Result<(), Error> {
369+
// There are "legacy templates and "new templates"
370+
342371
let cat_template_response = client
343372
.cat()
344373
.templates(CatTemplatesParts::Name("*"))
@@ -349,16 +378,32 @@ async fn delete_templates(client: &Elasticsearch) -> Result<(), Error> {
349378
.await?;
350379

351380
let all_templates: Vec<&str> = cat_template_response
352-
.split('\n')
353-
.filter(|s| !s.is_empty() && !s.starts_with('.') && s != &"security-audit-log")
381+
.split_terminator('\n')
382+
.filter(|s| !s.starts_with('.') && s != &"security-audit-log")
354383
.collect();
355384

356385
for template in all_templates {
357-
let _delete_template_response = client
386+
if template == "ilm-history" {
387+
// We may need to extend this to mimic ESRestTestCase.isXPackTemplate() from the ES
388+
// test harness
389+
continue;
390+
}
391+
392+
let mut delete_template_response = client
358393
.indices()
359-
.delete_template(IndicesDeleteTemplateParts::Name(&template))
394+
.delete_index_template(IndicesDeleteIndexTemplateParts::Name(&template))
360395
.send()
361396
.await?;
397+
398+
if delete_template_response.status_code().as_u16() == 404 {
399+
// Certainly an old-style template
400+
delete_template_response = client
401+
.indices()
402+
.delete_template(IndicesDeleteTemplateParts::Name(&template))
403+
.send()
404+
.await?;
405+
}
406+
assert_response_success!(delete_template_response);
362407
}
363408

364409
Ok(())
@@ -376,11 +421,13 @@ async fn delete_users(client: &Elasticsearch) -> Result<(), Error> {
376421
for (k, v) in users_response.as_object().unwrap() {
377422
if let Some(b) = v["metadata"]["_reserved"].as_bool() {
378423
if !b {
379-
let _ = client
424+
let response = client
380425
.security()
381426
.delete_user(SecurityDeleteUserParts::Username(k))
382427
.send()
383428
.await?;
429+
430+
assert_response_success!(response);
384431
}
385432
}
386433
}
@@ -400,11 +447,13 @@ async fn delete_roles(client: &Elasticsearch) -> Result<(), Error> {
400447
for (k, v) in roles_response.as_object().unwrap() {
401448
if let Some(b) = v["metadata"]["_reserved"].as_bool() {
402449
if !b {
403-
let _ = client
450+
let response = client
404451
.security()
405452
.delete_role(SecurityDeleteRoleParts::Name(k))
406453
.send()
407454
.await?;
455+
456+
assert_response_success!(response);
408457
}
409458
}
410459
}
@@ -424,11 +473,13 @@ async fn delete_privileges(client: &Elasticsearch) -> Result<(), Error> {
424473
for (k, v) in privileges_response.as_object().unwrap() {
425474
if let Some(b) = v["metadata"]["_reserved"].as_bool() {
426475
if !b {
427-
let _ = client
476+
let response = client
428477
.security()
429478
.delete_privileges(SecurityDeletePrivilegesParts::ApplicationName(k, "_all"))
430479
.send()
431480
.await?;
481+
482+
assert_response_success!(response);
432483
}
433484
}
434485
}
@@ -437,12 +488,14 @@ async fn delete_privileges(client: &Elasticsearch) -> Result<(), Error> {
437488
}
438489

439490
async fn stop_and_delete_datafeeds(client: &Elasticsearch) -> Result<(), Error> {
440-
let _stop_data_feed_response = client
491+
let stop_data_feed_response = client
441492
.ml()
442493
.stop_datafeed(MlStopDatafeedParts::DatafeedId("_all"))
443494
.send()
444495
.await?;
445496

497+
assert_response_success!(stop_data_feed_response);
498+
446499
let get_data_feeds_response = client
447500
.ml()
448501
.get_datafeeds(MlGetDatafeedsParts::None)
@@ -464,12 +517,14 @@ async fn stop_and_delete_datafeeds(client: &Elasticsearch) -> Result<(), Error>
464517
}
465518

466519
async fn close_and_delete_jobs(client: &Elasticsearch) -> Result<(), Error> {
467-
let _ = client
520+
let response = client
468521
.ml()
469522
.close_job(MlCloseJobParts::JobId("_all"))
470523
.send()
471524
.await?;
472525

526+
assert_response_success!(response);
527+
473528
let get_jobs_response = client
474529
.ml()
475530
.get_jobs(MlGetJobsParts::JobId("_all"))
@@ -480,11 +535,13 @@ async fn close_and_delete_jobs(client: &Elasticsearch) -> Result<(), Error> {
480535

481536
for job in get_jobs_response["jobs"].as_array().unwrap() {
482537
let id = job["job_id"].as_str().unwrap();
483-
let _ = client
538+
let response = client
484539
.ml()
485540
.delete_job(MlDeleteJobParts::JobId(id))
486541
.send()
487542
.await?;
543+
544+
assert_response_success!(response);
488545
}
489546

490547
Ok(())

0 commit comments

Comments
 (0)