Skip to content

Commit 65e376d

Browse files
jdisantijdisanti
authored
Add SRA benchmark to compare the last release against the current release (#2648)
## Motivation and Context This PR adds a benchmark to make sure we didn't regress the old middleware client implementation while making changes to it for the new orchestrator implementation. ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._
1 parent b1ce5e3 commit 65e376d

File tree

2 files changed

+108
-73
lines changed

2 files changed

+108
-73
lines changed

‎aws/sra-test/integration-tests/aws-sdk-s3/Cargo.toml

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,20 @@ publish = false
66
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
77

88
[dependencies]
9-
aws-credential-types = { path = "../../../rust-runtime/aws-credential-types", features = ["test-util"] }
109
aws-http = { path = "../../../rust-runtime/aws-http" }
1110
aws-runtime = { path = "../../../rust-runtime/aws-runtime" }
1211
aws-sdk-s3 = { path = "../../build/sdk/aws-sdk-s3", features = ["test-util"] }
13-
aws-sigv4 = { path = "../../../rust-runtime/aws-sigv4" }
14-
aws-types = { path = "../../../rust-runtime/aws-types" }
15-
aws-smithy-async = { path = "../../../../rust-runtime/aws-smithy-async", features = ["rt-tokio"] }
1612
aws-smithy-client = { path = "../../../../rust-runtime/aws-smithy-client", features = ["test-util", "rustls"] }
17-
aws-smithy-types = { path = "../../../../rust-runtime/aws-smithy-types" }
18-
aws-smithy-http = { path = "../../../../rust-runtime/aws-smithy-http" }
19-
aws-smithy-runtime = { path = "../../../../rust-runtime/aws-smithy-runtime", features = ["test-util"] }
2013
aws-smithy-runtime-api = { path = "../../../../rust-runtime/aws-smithy-runtime-api" }
14+
aws-types = { path = "../../../rust-runtime/aws-types" }
2115
criterion = { version = "0.4", features = ["async_tokio"] }
16+
http = "0.2.3"
17+
http-body = "0.4.5"
18+
last-release-smithy-client = { version = "0.55", package = "aws-smithy-client", features = ["test-util", "rustls"] }
19+
last-release-s3 = { version = "0.26", package = "aws-sdk-s3", features = ["test-util"] }
2220
tokio = { version = "1.23.1", features = ["macros", "test-util", "rt-multi-thread"] }
2321
tracing = "0.1.37"
2422
tracing-subscriber = { version = "0.3.15", features = ["env-filter", "json"] }
25-
http = "0.2.3"
26-
http-body = "0.4.5"
2723

2824
[profile.release]
2925
debug = 1

‎aws/sra-test/integration-tests/aws-sdk-s3/benches/middleware_vs_orchestrator.rs

Lines changed: 103 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -6,21 +6,84 @@
66
#[macro_use]
77
extern crate criterion;
88
use aws_sdk_s3 as s3;
9-
use aws_smithy_client::erase::DynConnector;
10-
use aws_smithy_client::test_connection::infallible_connection_fn;
119
use aws_smithy_runtime_api::client::runtime_plugin::RuntimePlugin;
1210
use aws_smithy_runtime_api::config_bag::ConfigBag;
13-
use criterion::Criterion;
14-
use s3::endpoint::Params;
11+
use criterion::{BenchmarkId, Criterion};
1512

16-
async fn middleware(client: &s3::Client) {
17-
client
18-
.list_objects_v2()
19-
.bucket("test-bucket")
20-
.prefix("prefix~")
21-
.send()
22-
.await
23-
.expect("successful execution");
13+
macro_rules! test_connection {
14+
(head) => {
15+
test_connection!(aws_smithy_client)
16+
};
17+
(last_release) => {
18+
test_connection!(last_release_smithy_client)
19+
};
20+
($package:ident) => {
21+
$package::test_connection::infallible_connection_fn(|req| {
22+
assert_eq!(
23+
"https://test-bucket.s3.us-east-1.amazonaws.com/?list-type=2&prefix=prefix~",
24+
req.uri().to_string()
25+
);
26+
assert!(req.headers().contains_key("authorization"));
27+
http::Response::builder()
28+
.status(200)
29+
.body(
30+
r#"<?xml version="1.0" encoding="UTF-8"?>
31+
<ListBucketResult>
32+
<Name>test-bucket</Name>
33+
<Prefix>prefix~</Prefix>
34+
<KeyCount>1</KeyCount>
35+
<MaxKeys>1000</MaxKeys>
36+
<IsTruncated>false</IsTruncated>
37+
<Contents>
38+
<Key>some-file.file</Key>
39+
<LastModified>2009-10-12T17:50:30.000Z</LastModified>
40+
<Size>434234</Size>
41+
<StorageClass>STANDARD</StorageClass>
42+
</Contents>
43+
</ListBucketResult>
44+
"#,
45+
)
46+
.unwrap()
47+
})
48+
};
49+
}
50+
51+
macro_rules! create_client {
52+
(head) => {
53+
create_client!(head, aws_sdk_s3)
54+
};
55+
(last_release) => {
56+
create_client!(last_release, last_release_s3)
57+
};
58+
($original:ident, $package:ident) => {{
59+
let conn = test_connection!($original);
60+
let config = $package::Config::builder()
61+
.credentials_provider($package::config::Credentials::for_tests())
62+
.region($package::config::Region::new("us-east-1"))
63+
.http_connector(conn.clone())
64+
.build();
65+
$package::Client::from_conf(config)
66+
}};
67+
}
68+
69+
macro_rules! middleware_bench_fn {
70+
($fn_name:ident, head) => {
71+
middleware_bench_fn!($fn_name, aws_sdk_s3)
72+
};
73+
($fn_name:ident, last_release) => {
74+
middleware_bench_fn!($fn_name, last_release_s3)
75+
};
76+
($fn_name:ident, $package:ident) => {
77+
async fn $fn_name(client: &$package::Client) {
78+
client
79+
.list_objects_v2()
80+
.bucket("test-bucket")
81+
.prefix("prefix~")
82+
.send()
83+
.await
84+
.expect("successful execution");
85+
}
86+
};
2487
}
2588

2689
async fn orchestrator(client: &s3::Client) {
@@ -32,7 +95,7 @@ async fn orchestrator(client: &s3::Client) {
3295
&self,
3396
cfg: &mut ConfigBag,
3497
) -> Result<(), aws_smithy_runtime_api::client::runtime_plugin::BoxError> {
35-
let params_builder = Params::builder()
98+
let params_builder = s3::endpoint::Params::builder()
3699
.set_region(Some(self.region.clone()))
37100
.bucket("test-bucket");
38101

@@ -55,61 +118,37 @@ async fn orchestrator(client: &s3::Client) {
55118
.expect("successful execution");
56119
}
57120

58-
fn test_connection() -> DynConnector {
59-
infallible_connection_fn(|req| {
60-
assert_eq!(
61-
"https://test-bucket.s3.us-east-1.amazonaws.com/?list-type=2&prefix=prefix~",
62-
req.uri().to_string()
63-
);
64-
assert!(req.headers().contains_key("authorization"));
65-
http::Response::builder()
66-
.status(200)
67-
.body(
68-
r#"<?xml version="1.0" encoding="UTF-8"?>
69-
<ListBucketResult>
70-
<Name>test-bucket</Name>
71-
<Prefix>prefix~</Prefix>
72-
<KeyCount>1</KeyCount>
73-
<MaxKeys>1000</MaxKeys>
74-
<IsTruncated>false</IsTruncated>
75-
<Contents>
76-
<Key>some-file.file</Key>
77-
<LastModified>2009-10-12T17:50:30.000Z</LastModified>
78-
<Size>434234</Size>
79-
<StorageClass>STANDARD</StorageClass>
80-
</Contents>
81-
</ListBucketResult>
82-
"#,
83-
)
84-
.unwrap()
85-
})
86-
}
87-
88-
fn client() -> s3::Client {
89-
let conn = test_connection();
90-
let config = s3::Config::builder()
91-
.credentials_provider(s3::config::Credentials::for_tests())
92-
.region(s3::config::Region::new("us-east-1"))
93-
.http_connector(conn.clone())
94-
.build();
95-
s3::Client::from_conf(config)
96-
}
121+
fn bench(c: &mut Criterion) {
122+
let head_client = create_client!(head);
123+
middleware_bench_fn!(middleware_head, head);
97124

98-
fn middleware_bench(c: &mut Criterion) {
99-
let client = client();
100-
c.bench_function("middleware", move |b| {
101-
b.to_async(tokio::runtime::Runtime::new().unwrap())
102-
.iter(|| async { middleware(&client).await })
103-
});
104-
}
125+
let last_release_client = create_client!(last_release);
126+
middleware_bench_fn!(middleware_last_release, last_release);
105127

106-
fn orchestrator_bench(c: &mut Criterion) {
107-
let client = client();
108-
c.bench_function("orchestrator", move |b| {
128+
let mut group = c.benchmark_group("compare");
129+
let param = "S3 ListObjectsV2";
130+
group.bench_with_input(
131+
BenchmarkId::new("middleware (HEAD)", param),
132+
param,
133+
|b, _| {
134+
b.to_async(tokio::runtime::Runtime::new().unwrap())
135+
.iter(|| async { middleware_head(&head_client).await })
136+
},
137+
);
138+
group.bench_with_input(
139+
BenchmarkId::new("middleware (last_release)", param),
140+
param,
141+
|b, _| {
142+
b.to_async(tokio::runtime::Runtime::new().unwrap())
143+
.iter(|| async { middleware_last_release(&last_release_client).await })
144+
},
145+
);
146+
group.bench_with_input(BenchmarkId::new("orchestrator", param), param, |b, _| {
109147
b.to_async(tokio::runtime::Runtime::new().unwrap())
110-
.iter(|| async { orchestrator(&client).await })
148+
.iter(|| async { orchestrator(&head_client).await })
111149
});
150+
group.finish();
112151
}
113152

114-
criterion_group!(benches, middleware_bench, orchestrator_bench);
153+
criterion_group!(benches, bench);
115154
criterion_main!(benches);

0 commit comments

Comments
 (0)