From 1c5614bd22a8177ab28f36107fd03a78f9754d96 Mon Sep 17 00:00:00 2001 From: Daniel Carl Jones Date: Thu, 22 Aug 2024 14:33:10 +0100 Subject: [PATCH 01/13] Update mountpoint-s3-crt-sys crate excludes to reduce package size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Carl Jones Signed-off-by: Burak Varlı --- mountpoint-s3-crt-sys/Cargo.toml | 4 +++- mountpoint-s3-crt-sys/crt/aws-c-auth | 2 +- mountpoint-s3-crt-sys/crt/aws-c-cal | 2 +- mountpoint-s3-crt-sys/crt/aws-c-common | 2 +- mountpoint-s3-crt-sys/crt/aws-c-http | 2 +- mountpoint-s3-crt-sys/crt/aws-c-io | 2 +- mountpoint-s3-crt-sys/crt/aws-c-sdkutils | 2 +- mountpoint-s3-crt-sys/crt/aws-lc | 2 +- mountpoint-s3-crt-sys/crt/s2n-tls | 2 +- 9 files changed, 11 insertions(+), 9 deletions(-) diff --git a/mountpoint-s3-crt-sys/Cargo.toml b/mountpoint-s3-crt-sys/Cargo.toml index c85af4a32..1dee338c1 100644 --- a/mountpoint-s3-crt-sys/Cargo.toml +++ b/mountpoint-s3-crt-sys/Cargo.toml @@ -62,7 +62,9 @@ exclude = [ ] [build-dependencies] -bindgen = { version = "0.66.1", default-features = false, features = ["runtime"] } +bindgen = { version = "0.66.1", default-features = false, features = [ + "runtime", +] } cc = "1.0.73" cmake = "0.1.48" rustflags = "0.1.1" diff --git a/mountpoint-s3-crt-sys/crt/aws-c-auth b/mountpoint-s3-crt-sys/crt/aws-c-auth index d78952529..52bf59161 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-auth +++ b/mountpoint-s3-crt-sys/crt/aws-c-auth @@ -1 +1 @@ -Subproject commit d7895252915259d037f10be223369eb99e837471 +Subproject commit 52bf591613d1a001c43ec99af7376f871759c5fe diff --git a/mountpoint-s3-crt-sys/crt/aws-c-cal b/mountpoint-s3-crt-sys/crt/aws-c-cal index bc0d71b66..77ca3aea8 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-cal +++ b/mountpoint-s3-crt-sys/crt/aws-c-cal @@ -1 +1 @@ -Subproject commit bc0d71b66deea9e1a2d3a073c9f1ece9286b9e60 +Subproject commit 77ca3aea879bc768082fe7ec715adcde8e98c332 diff --git a/mountpoint-s3-crt-sys/crt/aws-c-common b/mountpoint-s3-crt-sys/crt/aws-c-common index 67601bbbc..672cc0032 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-common +++ b/mountpoint-s3-crt-sys/crt/aws-c-common @@ -1 +1 @@ -Subproject commit 67601bbbce6ea1c26191f235afc50007a4e796c5 +Subproject commit 672cc0032eb28d69fbdd22c9463253c89d7a6f30 diff --git a/mountpoint-s3-crt-sys/crt/aws-c-http b/mountpoint-s3-crt-sys/crt/aws-c-http index a2fb16c43..4e74ab1e3 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-http +++ b/mountpoint-s3-crt-sys/crt/aws-c-http @@ -1 +1 @@ -Subproject commit a2fb16c431152bd677093f9910f405a37533ad5a +Subproject commit 4e74ab1e3702763e0b87bd1752f5a37c2f0400ac diff --git a/mountpoint-s3-crt-sys/crt/aws-c-io b/mountpoint-s3-crt-sys/crt/aws-c-io index e5fe40e11..c345d7727 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-io +++ b/mountpoint-s3-crt-sys/crt/aws-c-io @@ -1 +1 @@ -Subproject commit e5fe40e11d97221c25b3b40ad2949822b3422993 +Subproject commit c345d77274db83c0c2e30331814093e7c84c45e2 diff --git a/mountpoint-s3-crt-sys/crt/aws-c-sdkutils b/mountpoint-s3-crt-sys/crt/aws-c-sdkutils index 8c7af71f9..4658412a6 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-sdkutils +++ b/mountpoint-s3-crt-sys/crt/aws-c-sdkutils @@ -1 +1 @@ -Subproject commit 8c7af71f91ed5b9d2a043d51f120495f43723f80 +Subproject commit 4658412a61ad5749db92a8d1e0717cb5e76ada1c diff --git a/mountpoint-s3-crt-sys/crt/aws-lc b/mountpoint-s3-crt-sys/crt/aws-lc index 47333e181..057477806 160000 --- a/mountpoint-s3-crt-sys/crt/aws-lc +++ b/mountpoint-s3-crt-sys/crt/aws-lc @@ -1 +1 @@ -Subproject commit 47333e18117875148fc737c38c2d5586b45c7dfc +Subproject commit 05747780676652f41d0b9c570a495e4bb6608560 diff --git a/mountpoint-s3-crt-sys/crt/s2n-tls b/mountpoint-s3-crt-sys/crt/s2n-tls index 138e3ece9..79c0f1b43 160000 --- a/mountpoint-s3-crt-sys/crt/s2n-tls +++ b/mountpoint-s3-crt-sys/crt/s2n-tls @@ -1 +1 @@ -Subproject commit 138e3ece9e457c2f824a85b63095737f30d624a9 +Subproject commit 79c0f1b434742d9f1152c48d3781433649f6f8fe From e55e6c660b8467651d52aece44c67d7be6491966 Mon Sep 17 00:00:00 2001 From: Daniel Carl Jones Date: Mon, 19 Aug 2024 17:50:18 +0100 Subject: [PATCH 02/13] Update CRT submodules to latest releases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Carl Jones Signed-off-by: Burak Varlı --- mountpoint-s3/CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mountpoint-s3/CHANGELOG.md b/mountpoint-s3/CHANGELOG.md index 28b370b51..32f368c69 100644 --- a/mountpoint-s3/CHANGELOG.md +++ b/mountpoint-s3/CHANGELOG.md @@ -1,3 +1,10 @@ +## Unreleased + +### Other changes + +* Fix an issue where `credential_process` field would not be picked up correctly when using `source_profile`. ([awslabs/aws-c-auth#245](https://github.com/awslabs/aws-c-auth/pull/245)) +* Fix an issue where `credential_process` field would not be picked up correctly when using `--profile `. ([awslabs/aws-c-auth#245](https://github.com/awslabs/aws-c-auth/pull/245)) + ## v1.8.0 ### New features From f3f008185bbc7786ed8e59e9c00dced4e93baa47 Mon Sep 17 00:00:00 2001 From: Daniel Carl Jones Date: Mon, 2 Sep 2024 20:45:06 +0100 Subject: [PATCH 03/13] WIP: Add testing for https://github.com/awslabs/mountpoint-s3/issues/927 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Burak Varlı --- mountpoint-s3-client/tests/auth.rs | 103 +++++++++++++++++++++++++++++ mountpoint-s3/CHANGELOG.md | 2 + 2 files changed, 105 insertions(+) diff --git a/mountpoint-s3-client/tests/auth.rs b/mountpoint-s3-client/tests/auth.rs index d92fcf482..ef4bfa0a3 100644 --- a/mountpoint-s3-client/tests/auth.rs +++ b/mountpoint-s3-client/tests/auth.rs @@ -4,7 +4,9 @@ pub mod common; use std::io::Write; use std::option::Option::None; +use std::writeln; +use aws_sdk_s3::config; use aws_sdk_s3::primitives::ByteStream; use bytes::Bytes; #[cfg(not(feature = "s3express_tests"))] @@ -240,6 +242,100 @@ async fn test_profile_provider_assume_role_async() { check_get_result(result, None, &body[..]).await; } +async fn test_credential_process_behind_source_profile_async() { + let sdk_client = get_test_sdk_client().await; + let (bucket, prefix) = get_test_bucket_and_prefix("test_credential_process_behind_source_profile"); + + let key = format!("{prefix}/hello"); + let body = b"hello world!"; + sdk_client + .put_object() + .bucket(&bucket) + .key(&key) + .body(ByteStream::from(Bytes::from_static(body))) + .send() + .await + .unwrap(); + + let mut config_file = NamedTempFile::new().unwrap(); + + // Firstly, let's configure the credentials to be fetched by the credential process. + // Scope down to the `foo` prefix + let static_profile = "static-profile"; + let policy = r#"{"Statement": [ + {"Effect": "Allow", "Action": "s3:GetObject", "Resource": "arn:aws:s3:::__BUCKET__/__PREFIX__/*"}, + {"Effect": "Allow", "Action": "s3:ListBucket", "Resource": "arn:aws:s3:::__BUCKET__", "Condition": {"StringLike": {"s3:prefix": "__PREFIX__/*"}}} + ]}"#; + let policy = policy + .replace("__BUCKET__", &bucket) + .replace("__PREFIX__", &format!("{prefix}foo")); + let credentials = get_scoped_down_credentials(policy).await; + writeln!(config_file, "[profile {}]", static_profile).unwrap(); + writeln!(config_file, "aws_access_key_id={}", credentials.access_key_id()).unwrap(); + writeln!(config_file, "aws_secret_access_key={}", credentials.secret_access_key()).unwrap(); + if let Some(session_token) = credentials.session_token() { + writeln!(config_file, "aws_session_token={session_token}").unwrap(); + } + + let mut some_other_file = NamedTempFile::new().unwrap(); + let json_response = r#"{ + "Version": 1, + "AccessKeyId": "__AWS_ACCESS_KEY_ID__", + "SecretAccessKey": "__AWS_SECRET_ACCESS_KEY__", + "SessionToken": "__AWS_SESSION_TOKEN__", + "Expiration": "2099-08-20T00:05:35+00:00" + }"#; + let json_response = json_response + .replace("__AWS_ACCESS_KEY_ID__", &credentials.access_key_id()) + .replace("__AWS_SECRET_ACCESS_KEY__", &credentials.secret_access_key()) + .replace("__AWS_SESSION_TOKEN__", &credentials.session_token().unwrap()); + some_other_file.write_all(json_response.as_bytes()).unwrap(); + + let source_profile = "source-profile"; + writeln!(config_file, "[profile {}]", source_profile).unwrap(); + writeln!( + config_file, + "credential_process = cat {}", + some_other_file.path().to_string_lossy() + ) + .unwrap(); + + let configured_profile = "test-profile"; + writeln!(config_file, "[profile {}]", configured_profile).unwrap(); + writeln!(config_file, "source_profile={}", source_profile).unwrap(); + writeln!(config_file, "region={}", &get_test_region()).unwrap(); + config_file.flush().unwrap(); + + // Set up the environment variables to use this new config file. This is only OK to do because + // this test is run in a forked process, so won't affect any other concurrently running tests. + std::env::set_var("AWS_CONFIG_FILE", config_file.path().as_os_str()); + std::env::remove_var("AWS_ACCESS_KEY_ID"); + std::env::remove_var("AWS_SECRET_ACCESS_KEY"); + std::env::remove_var("AWS_SESSION_TOKEN"); + + let config = S3ClientConfig::new() + .auth_config(S3ClientAuthConfig::Profile(configured_profile.to_owned())) + .endpoint_config(EndpointConfig::new(&get_test_region())); + let client = S3CrtClient::new(config).unwrap(); + + // Inside the prefix, things should be fine + let _result = client + .list_objects(&bucket, None, "/", 10, &format!("{prefix}foo/")) + .await + .expect("list_objects should succeed"); + + // Outside the prefix, requests should fail with permissions errors + let err = client + .list_objects(&bucket, None, "/", 10, &format!("{prefix}/")) + .await + .expect_err("should fail in different prefix"); + assert!(matches!( + err, + ObjectClientError::ClientError(S3RequestError::Forbidden(_, _)) + )); + drop(config_file); +} + rusty_fork_test! { #[test] fn test_profile_provider_static() { @@ -254,6 +350,13 @@ rusty_fork_test! { let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); runtime.block_on(test_profile_provider_assume_role_async()); } + + #[test] + fn test_credential_process_behind_source_profile() { + // rusty_fork doesn't support async tests, so build an SDK-usable runtime manually + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + runtime.block_on(test_credential_process_behind_source_profile_async()); + } } /// Test using a client with scoped-down credentials diff --git a/mountpoint-s3/CHANGELOG.md b/mountpoint-s3/CHANGELOG.md index 32f368c69..8213e6a67 100644 --- a/mountpoint-s3/CHANGELOG.md +++ b/mountpoint-s3/CHANGELOG.md @@ -3,7 +3,9 @@ ### Other changes * Fix an issue where `credential_process` field would not be picked up correctly when using `source_profile`. ([awslabs/aws-c-auth#245](https://github.com/awslabs/aws-c-auth/pull/245)) + * Fix an issue where `credential_process` field would not be picked up correctly when using `--profile `. ([awslabs/aws-c-auth#245](https://github.com/awslabs/aws-c-auth/pull/245)) + ## v1.8.0 From 02e456b8afa715a1c46269805fc8a073c31cad76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Varl=C4=B1?= Date: Tue, 3 Sep 2024 09:44:47 +0100 Subject: [PATCH 04/13] Gate scoped credential test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Burak Varlı --- mountpoint-s3-client/tests/auth.rs | 136 ++++++++++++++++------------- 1 file changed, 74 insertions(+), 62 deletions(-) diff --git a/mountpoint-s3-client/tests/auth.rs b/mountpoint-s3-client/tests/auth.rs index ef4bfa0a3..fd914e8aa 100644 --- a/mountpoint-s3-client/tests/auth.rs +++ b/mountpoint-s3-client/tests/auth.rs @@ -242,79 +242,89 @@ async fn test_profile_provider_assume_role_async() { check_get_result(result, None, &body[..]).await; } -async fn test_credential_process_behind_source_profile_async() { - let sdk_client = get_test_sdk_client().await; - let (bucket, prefix) = get_test_bucket_and_prefix("test_credential_process_behind_source_profile"); +// S3 Express One Zone doesn't support scoped credentials +#[cfg(not(feature = "s3express_tests"))] +async fn test_credential_process_behind_source_profile_with_scoped_credentials_async() { + let (bucket, prefix) = + get_test_bucket_and_prefix("test_credential_process_behind_source_profile_with_scoped_credentials"); + + // Create a test file in "{prefix}/hello" + { + let sdk_client = get_test_sdk_client().await; + let key = format!("{prefix}/hello"); + let body = b"hello world!"; + sdk_client + .put_object() + .bucket(&bucket) + .key(&key) + .body(ByteStream::from(Bytes::from_static(body))) + .send() + .await + .unwrap(); + } - let key = format!("{prefix}/hello"); - let body = b"hello world!"; - sdk_client - .put_object() - .bucket(&bucket) - .key(&key) - .body(ByteStream::from(Bytes::from_static(body))) - .send() - .await - .unwrap(); + // Generate temporary credentials scoped down to the `foo` prefix. + let credentials = { + let policy = r#"{"Statement": [ + {"Effect": "Allow", "Action": "s3:GetObject", "Resource": "arn:aws:s3:::__BUCKET__/__PREFIX__/*"}, + {"Effect": "Allow", "Action": "s3:ListBucket", "Resource": "arn:aws:s3:::__BUCKET__", "Condition": {"StringLike": {"s3:prefix": "__PREFIX__/*"}}} + ]}"#; + let policy = policy + .replace("__BUCKET__", &bucket) + .replace("__PREFIX__", &format!("{prefix}foo")); + get_scoped_down_credentials(policy).await + }; + + // Write temporary credentials to a file to be used in `credential_process`. + let credential_file = { + let mut credential_file = NamedTempFile::new().unwrap(); + let json_response = r#"{ + "Version": 1, + "AccessKeyId": "__AWS_ACCESS_KEY_ID__", + "SecretAccessKey": "__AWS_SECRET_ACCESS_KEY__", + "SessionToken": "__AWS_SESSION_TOKEN__", + "Expiration": "2099-08-20T00:05:35+00:00" + }"#; + let json_response = json_response + .replace("__AWS_ACCESS_KEY_ID__", &credentials.access_key_id()) + .replace("__AWS_SECRET_ACCESS_KEY__", &credentials.secret_access_key()) + .replace("__AWS_SESSION_TOKEN__", &credentials.session_token().unwrap()); + credential_file.write_all(json_response.as_bytes()).unwrap(); + credential_file + }; let mut config_file = NamedTempFile::new().unwrap(); - // Firstly, let's configure the credentials to be fetched by the credential process. - // Scope down to the `foo` prefix - let static_profile = "static-profile"; - let policy = r#"{"Statement": [ - {"Effect": "Allow", "Action": "s3:GetObject", "Resource": "arn:aws:s3:::__BUCKET__/__PREFIX__/*"}, - {"Effect": "Allow", "Action": "s3:ListBucket", "Resource": "arn:aws:s3:::__BUCKET__", "Condition": {"StringLike": {"s3:prefix": "__PREFIX__/*"}}} - ]}"#; - let policy = policy - .replace("__BUCKET__", &bucket) - .replace("__PREFIX__", &format!("{prefix}foo")); - let credentials = get_scoped_down_credentials(policy).await; - writeln!(config_file, "[profile {}]", static_profile).unwrap(); - writeln!(config_file, "aws_access_key_id={}", credentials.access_key_id()).unwrap(); - writeln!(config_file, "aws_secret_access_key={}", credentials.secret_access_key()).unwrap(); - if let Some(session_token) = credentials.session_token() { - writeln!(config_file, "aws_session_token={session_token}").unwrap(); - } + // Create a source profile that provides generated temporary credentials using `credential_process`. + let source_profile = { + let source_profile = "source-profile"; + writeln!(config_file, "[profile {}]", source_profile).unwrap(); + writeln!( + config_file, + "credential_process = cat {}", + credential_file.path().to_string_lossy() + ) + .unwrap(); + source_profile + }; - let mut some_other_file = NamedTempFile::new().unwrap(); - let json_response = r#"{ - "Version": 1, - "AccessKeyId": "__AWS_ACCESS_KEY_ID__", - "SecretAccessKey": "__AWS_SECRET_ACCESS_KEY__", - "SessionToken": "__AWS_SESSION_TOKEN__", - "Expiration": "2099-08-20T00:05:35+00:00" - }"#; - let json_response = json_response - .replace("__AWS_ACCESS_KEY_ID__", &credentials.access_key_id()) - .replace("__AWS_SECRET_ACCESS_KEY__", &credentials.secret_access_key()) - .replace("__AWS_SESSION_TOKEN__", &credentials.session_token().unwrap()); - some_other_file.write_all(json_response.as_bytes()).unwrap(); - - let source_profile = "source-profile"; - writeln!(config_file, "[profile {}]", source_profile).unwrap(); - writeln!( - config_file, - "credential_process = cat {}", - some_other_file.path().to_string_lossy() - ) - .unwrap(); + // Create a test profile that uses `source_profile`. + let test_profile = { + let test_profile = "test-profile"; + writeln!(config_file, "[profile {}]", test_profile).unwrap(); + writeln!(config_file, "source_profile={}", source_profile).unwrap(); + writeln!(config_file, "region={}", &get_test_region()).unwrap(); + test_profile + }; - let configured_profile = "test-profile"; - writeln!(config_file, "[profile {}]", configured_profile).unwrap(); - writeln!(config_file, "source_profile={}", source_profile).unwrap(); - writeln!(config_file, "region={}", &get_test_region()).unwrap(); config_file.flush().unwrap(); // Set up the environment variables to use this new config file. This is only OK to do because // this test is run in a forked process, so won't affect any other concurrently running tests. std::env::set_var("AWS_CONFIG_FILE", config_file.path().as_os_str()); - std::env::remove_var("AWS_ACCESS_KEY_ID"); - std::env::remove_var("AWS_SECRET_ACCESS_KEY"); - std::env::remove_var("AWS_SESSION_TOKEN"); let config = S3ClientConfig::new() - .auth_config(S3ClientAuthConfig::Profile(configured_profile.to_owned())) + .auth_config(S3ClientAuthConfig::Profile(test_profile.to_owned())) .endpoint_config(EndpointConfig::new(&get_test_region())); let client = S3CrtClient::new(config).unwrap(); @@ -352,10 +362,12 @@ rusty_fork_test! { } #[test] - fn test_credential_process_behind_source_profile() { + // S3 Express One Zone doesn't support scoped credentials + #[cfg(not(feature = "s3express_tests"))] + fn test_credential_process_behind_source_profile_with_scoped_credentials() { // rusty_fork doesn't support async tests, so build an SDK-usable runtime manually let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); - runtime.block_on(test_credential_process_behind_source_profile_async()); + runtime.block_on(test_credential_process_behind_source_profile_with_scoped_credentials_async()); } } From 3a6eead51443928c0d9a219a15e9e79705227388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Varl=C4=B1?= Date: Tue, 3 Sep 2024 13:19:29 +0100 Subject: [PATCH 05/13] Update `test_credential_process_behind_source_profile` to use role MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Burak Varlı --- mountpoint-s3-client/tests/auth.rs | 115 +++++++++++++++++------------ 1 file changed, 69 insertions(+), 46 deletions(-) diff --git a/mountpoint-s3-client/tests/auth.rs b/mountpoint-s3-client/tests/auth.rs index fd914e8aa..a3ed43664 100644 --- a/mountpoint-s3-client/tests/auth.rs +++ b/mountpoint-s3-client/tests/auth.rs @@ -242,11 +242,8 @@ async fn test_profile_provider_assume_role_async() { check_get_result(result, None, &body[..]).await; } -// S3 Express One Zone doesn't support scoped credentials -#[cfg(not(feature = "s3express_tests"))] -async fn test_credential_process_behind_source_profile_with_scoped_credentials_async() { - let (bucket, prefix) = - get_test_bucket_and_prefix("test_credential_process_behind_source_profile_with_scoped_credentials"); +async fn test_credential_process_behind_source_profile_async() { + let (bucket, prefix) = get_test_bucket_and_prefix("test_credential_process_behind_source_profile"); // Create a test file in "{prefix}/hello" { @@ -263,21 +260,14 @@ async fn test_credential_process_behind_source_profile_with_scoped_credentials_a .unwrap(); } - // Generate temporary credentials scoped down to the `foo` prefix. - let credentials = { - let policy = r#"{"Statement": [ - {"Effect": "Allow", "Action": "s3:GetObject", "Resource": "arn:aws:s3:::__BUCKET__/__PREFIX__/*"}, - {"Effect": "Allow", "Action": "s3:ListBucket", "Resource": "arn:aws:s3:::__BUCKET__", "Condition": {"StringLike": {"s3:prefix": "__PREFIX__/*"}}} - ]}"#; - let policy = policy - .replace("__BUCKET__", &bucket) - .replace("__PREFIX__", &format!("{prefix}foo")); - get_scoped_down_credentials(policy).await - }; + // Get some static credentials by just using the SDK's default provider, which we know works. + let credentials = get_sdk_default_chain_creds().await; - // Write temporary credentials to a file to be used in `credential_process`. - let credential_file = { - let mut credential_file = NamedTempFile::new().unwrap(); + // Create two credential files to be used in `credential_process`, + // one with correct credentials and one with incorrect credentials. + let (correct_credential_file, incorrect_credential_file) = { + let mut correct = NamedTempFile::new().unwrap(); + let mut incorrect = NamedTempFile::new().unwrap(); let json_response = r#"{ "Version": 1, "AccessKeyId": "__AWS_ACCESS_KEY_ID__", @@ -285,36 +275,66 @@ async fn test_credential_process_behind_source_profile_with_scoped_credentials_a "SessionToken": "__AWS_SESSION_TOKEN__", "Expiration": "2099-08-20T00:05:35+00:00" }"#; - let json_response = json_response - .replace("__AWS_ACCESS_KEY_ID__", &credentials.access_key_id()) - .replace("__AWS_SECRET_ACCESS_KEY__", &credentials.secret_access_key()) - .replace("__AWS_SESSION_TOKEN__", &credentials.session_token().unwrap()); - credential_file.write_all(json_response.as_bytes()).unwrap(); - credential_file + + correct + .write_all( + json_response + .replace("__AWS_ACCESS_KEY_ID__", &credentials.access_key_id()) + .replace("__AWS_SECRET_ACCESS_KEY__", &credentials.secret_access_key()) + .replace("__AWS_SESSION_TOKEN__", &credentials.session_token().unwrap()) + .as_bytes(), + ) + .unwrap(); + + incorrect + .write_all( + json_response + .replace("__AWS_ACCESS_KEY_ID__", &credentials.access_key_id()[..10]) + .replace("__AWS_SECRET_ACCESS_KEY__", &credentials.secret_access_key()) + .replace("__AWS_SESSION_TOKEN__", &credentials.session_token().unwrap()) + .as_bytes(), + ) + .unwrap(); + + (correct, incorrect) }; let mut config_file = NamedTempFile::new().unwrap(); - // Create a source profile that provides generated temporary credentials using `credential_process`. - let source_profile = { - let source_profile = "source-profile"; - writeln!(config_file, "[profile {}]", source_profile).unwrap(); + // Create two source profiles to provide credentials from previously created files using `credential_process`. + let (correct_source_profile, incorrect_source_profile) = { + let correct = "correct-source-profile"; + writeln!(config_file, "[profile {}]", correct).unwrap(); + writeln!( + config_file, + "credential_process=cat {}", + correct_credential_file.path().to_string_lossy() + ) + .unwrap(); + let incorrect = "incorrect-source-profile"; + writeln!(config_file, "[profile {}]", incorrect).unwrap(); writeln!( config_file, - "credential_process = cat {}", - credential_file.path().to_string_lossy() + "credential_process=cat {}", + incorrect_credential_file.path().to_string_lossy() ) .unwrap(); - source_profile + (correct, incorrect) }; - // Create a test profile that uses `source_profile`. - let test_profile = { - let test_profile = "test-profile"; - writeln!(config_file, "[profile {}]", test_profile).unwrap(); - writeln!(config_file, "source_profile={}", source_profile).unwrap(); + // Create two profiles to assume our test role with previously created source profiles. + let (correct_profile, incorrect_profile) = { + let correct = "correct-profile"; + writeln!(config_file, "[profile {}]", correct).unwrap(); + writeln!(config_file, "role_arn={}", get_subsession_iam_role()).unwrap(); + writeln!(config_file, "source_profile={}", correct_source_profile).unwrap(); + writeln!(config_file, "region={}", &get_test_region()).unwrap(); + let incorrect = "incorrect-profile"; + writeln!(config_file, "[profile {}]", incorrect).unwrap(); + writeln!(config_file, "role_arn={}", get_subsession_iam_role()).unwrap(); + writeln!(config_file, "source_profile={}", incorrect_source_profile).unwrap(); writeln!(config_file, "region={}", &get_test_region()).unwrap(); - test_profile + (correct, incorrect) }; config_file.flush().unwrap(); @@ -323,25 +343,28 @@ async fn test_credential_process_behind_source_profile_with_scoped_credentials_a // this test is run in a forked process, so won't affect any other concurrently running tests. std::env::set_var("AWS_CONFIG_FILE", config_file.path().as_os_str()); + // With correct profile, things should be fine let config = S3ClientConfig::new() - .auth_config(S3ClientAuthConfig::Profile(test_profile.to_owned())) + .auth_config(S3ClientAuthConfig::Profile(correct_profile.to_owned())) .endpoint_config(EndpointConfig::new(&get_test_region())); let client = S3CrtClient::new(config).unwrap(); - - // Inside the prefix, things should be fine let _result = client .list_objects(&bucket, None, "/", 10, &format!("{prefix}foo/")) .await .expect("list_objects should succeed"); - // Outside the prefix, requests should fail with permissions errors + // With incorrect profile, requests should fail with no signing credentials + let config = S3ClientConfig::new() + .auth_config(S3ClientAuthConfig::Profile(incorrect_profile.to_owned())) + .endpoint_config(EndpointConfig::new(&get_test_region())); + let client = S3CrtClient::new(config).unwrap(); let err = client .list_objects(&bucket, None, "/", 10, &format!("{prefix}/")) .await .expect_err("should fail in different prefix"); assert!(matches!( - err, - ObjectClientError::ClientError(S3RequestError::Forbidden(_, _)) + dbg!(err), + ObjectClientError::ClientError(S3RequestError::NoSigningCredentials) )); drop(config_file); } @@ -364,10 +387,10 @@ rusty_fork_test! { #[test] // S3 Express One Zone doesn't support scoped credentials #[cfg(not(feature = "s3express_tests"))] - fn test_credential_process_behind_source_profile_with_scoped_credentials() { + fn test_credential_process_behind_source_profile() { // rusty_fork doesn't support async tests, so build an SDK-usable runtime manually let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); - runtime.block_on(test_credential_process_behind_source_profile_with_scoped_credentials_async()); + runtime.block_on(test_credential_process_behind_source_profile_async()); } } From 5f73d935c334ce1c3ebba288755cdc20f80cbbea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Varl=C4=B1?= Date: Tue, 3 Sep 2024 13:22:28 +0100 Subject: [PATCH 06/13] Remove unused import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Burak Varlı --- mountpoint-s3-client/tests/auth.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/mountpoint-s3-client/tests/auth.rs b/mountpoint-s3-client/tests/auth.rs index a3ed43664..83fda61c8 100644 --- a/mountpoint-s3-client/tests/auth.rs +++ b/mountpoint-s3-client/tests/auth.rs @@ -6,7 +6,6 @@ use std::io::Write; use std::option::Option::None; use std::writeln; -use aws_sdk_s3::config; use aws_sdk_s3::primitives::ByteStream; use bytes::Bytes; #[cfg(not(feature = "s3express_tests"))] From 10ebbd7e900ae22c13a0c66c08701621b4a0c0fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Varl=C4=B1?= Date: Tue, 3 Sep 2024 13:54:22 +0100 Subject: [PATCH 07/13] Remove TODOs from CHANGELOG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Burak Varlı --- mountpoint-s3/CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/mountpoint-s3/CHANGELOG.md b/mountpoint-s3/CHANGELOG.md index 8213e6a67..32f368c69 100644 --- a/mountpoint-s3/CHANGELOG.md +++ b/mountpoint-s3/CHANGELOG.md @@ -3,9 +3,7 @@ ### Other changes * Fix an issue where `credential_process` field would not be picked up correctly when using `source_profile`. ([awslabs/aws-c-auth#245](https://github.com/awslabs/aws-c-auth/pull/245)) - * Fix an issue where `credential_process` field would not be picked up correctly when using `--profile `. ([awslabs/aws-c-auth#245](https://github.com/awslabs/aws-c-auth/pull/245)) - ## v1.8.0 From 48dcbd998a85efd828801aec846264dda38692fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Varl=C4=B1?= Date: Tue, 3 Sep 2024 14:25:06 +0100 Subject: [PATCH 08/13] Update CRT submodules to latest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Burak Varlı --- mountpoint-s3-crt-sys/crt/aws-c-auth | 2 +- mountpoint-s3-crt-sys/crt/aws-c-cal | 2 +- mountpoint-s3-crt-sys/crt/aws-c-compression | 2 +- mountpoint-s3-crt-sys/crt/aws-lc | 2 +- mountpoint-s3-crt-sys/crt/s2n-tls | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mountpoint-s3-crt-sys/crt/aws-c-auth b/mountpoint-s3-crt-sys/crt/aws-c-auth index 52bf59161..877c029fc 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-auth +++ b/mountpoint-s3-crt-sys/crt/aws-c-auth @@ -1 +1 @@ -Subproject commit 52bf591613d1a001c43ec99af7376f871759c5fe +Subproject commit 877c029fc4e93d205f9c6855188c3c51f6b497b4 diff --git a/mountpoint-s3-crt-sys/crt/aws-c-cal b/mountpoint-s3-crt-sys/crt/aws-c-cal index 77ca3aea8..2cb1d2eac 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-cal +++ b/mountpoint-s3-crt-sys/crt/aws-c-cal @@ -1 +1 @@ -Subproject commit 77ca3aea879bc768082fe7ec715adcde8e98c332 +Subproject commit 2cb1d2eac925e2dbc45025eb89af82bd790c23a0 diff --git a/mountpoint-s3-crt-sys/crt/aws-c-compression b/mountpoint-s3-crt-sys/crt/aws-c-compression index ea1d421a4..f36d01672 160000 --- a/mountpoint-s3-crt-sys/crt/aws-c-compression +++ b/mountpoint-s3-crt-sys/crt/aws-c-compression @@ -1 +1 @@ -Subproject commit ea1d421a421ad83a540309a94c38d50b6a5d836b +Subproject commit f36d01672d61e49d96a777870d456f66fa391cd4 diff --git a/mountpoint-s3-crt-sys/crt/aws-lc b/mountpoint-s3-crt-sys/crt/aws-lc index 057477806..2f1879759 160000 --- a/mountpoint-s3-crt-sys/crt/aws-lc +++ b/mountpoint-s3-crt-sys/crt/aws-lc @@ -1 +1 @@ -Subproject commit 05747780676652f41d0b9c570a495e4bb6608560 +Subproject commit 2f1879759b2e0fc70592665bdf10087b64f44b7d diff --git a/mountpoint-s3-crt-sys/crt/s2n-tls b/mountpoint-s3-crt-sys/crt/s2n-tls index 79c0f1b43..87f4a0585 160000 --- a/mountpoint-s3-crt-sys/crt/s2n-tls +++ b/mountpoint-s3-crt-sys/crt/s2n-tls @@ -1 +1 @@ -Subproject commit 79c0f1b434742d9f1152c48d3781433649f6f8fe +Subproject commit 87f4a0585dc3056433f193b9305f587cff239be3 From d45b476609b00b5b47e7b663348f4cddbb71cde3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Varl=C4=B1?= Date: Tue, 3 Sep 2024 14:32:38 +0100 Subject: [PATCH 09/13] Remove feature gate from `test_credential_process_behind_source_profile` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Burak Varlı --- mountpoint-s3-client/tests/auth.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/mountpoint-s3-client/tests/auth.rs b/mountpoint-s3-client/tests/auth.rs index 83fda61c8..c5a3ae189 100644 --- a/mountpoint-s3-client/tests/auth.rs +++ b/mountpoint-s3-client/tests/auth.rs @@ -384,8 +384,6 @@ rusty_fork_test! { } #[test] - // S3 Express One Zone doesn't support scoped credentials - #[cfg(not(feature = "s3express_tests"))] fn test_credential_process_behind_source_profile() { // rusty_fork doesn't support async tests, so build an SDK-usable runtime manually let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); From 50b8265bab0807f72e72d1ece1b0fb8c5af0afa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Varl=C4=B1?= Date: Tue, 3 Sep 2024 14:46:02 +0100 Subject: [PATCH 10/13] Fix Clippy failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Burak Varlı --- mountpoint-s3-client/tests/auth.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/mountpoint-s3-client/tests/auth.rs b/mountpoint-s3-client/tests/auth.rs index c5a3ae189..74234dd6b 100644 --- a/mountpoint-s3-client/tests/auth.rs +++ b/mountpoint-s3-client/tests/auth.rs @@ -14,9 +14,7 @@ use common::creds::{get_sdk_default_chain_creds, get_subsession_iam_role}; use common::*; use futures::StreamExt; use mountpoint_s3_client::config::{EndpointConfig, S3ClientAuthConfig, S3ClientConfig}; -#[cfg(not(feature = "s3express_tests"))] use mountpoint_s3_client::error::ObjectClientError; -#[cfg(not(feature = "s3express_tests"))] use mountpoint_s3_client::S3RequestError; use mountpoint_s3_client::{ObjectClient, S3CrtClient}; use mountpoint_s3_crt::auth::credentials::{CredentialsProvider, CredentialsProviderStaticOptions}; From 898b27b61b7f2f862907319750645560bbd09e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Varl=C4=B1?= Date: Wed, 4 Sep 2024 08:14:03 +0100 Subject: [PATCH 11/13] More Clippy fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Burak Varlı --- mountpoint-s3-client/tests/auth.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/mountpoint-s3-client/tests/auth.rs b/mountpoint-s3-client/tests/auth.rs index 74234dd6b..2d853c938 100644 --- a/mountpoint-s3-client/tests/auth.rs +++ b/mountpoint-s3-client/tests/auth.rs @@ -15,6 +15,7 @@ use common::*; use futures::StreamExt; use mountpoint_s3_client::config::{EndpointConfig, S3ClientAuthConfig, S3ClientConfig}; use mountpoint_s3_client::error::ObjectClientError; +#[cfg(not(feature = "s3express_tests"))] use mountpoint_s3_client::S3RequestError; use mountpoint_s3_client::{ObjectClient, S3CrtClient}; use mountpoint_s3_crt::auth::credentials::{CredentialsProvider, CredentialsProviderStaticOptions}; @@ -276,9 +277,9 @@ async fn test_credential_process_behind_source_profile_async() { correct .write_all( json_response - .replace("__AWS_ACCESS_KEY_ID__", &credentials.access_key_id()) - .replace("__AWS_SECRET_ACCESS_KEY__", &credentials.secret_access_key()) - .replace("__AWS_SESSION_TOKEN__", &credentials.session_token().unwrap()) + .replace("__AWS_ACCESS_KEY_ID__", credentials.access_key_id()) + .replace("__AWS_SECRET_ACCESS_KEY__", credentials.secret_access_key()) + .replace("__AWS_SESSION_TOKEN__", credentials.session_token().unwrap()) .as_bytes(), ) .unwrap(); @@ -287,8 +288,8 @@ async fn test_credential_process_behind_source_profile_async() { .write_all( json_response .replace("__AWS_ACCESS_KEY_ID__", &credentials.access_key_id()[..10]) - .replace("__AWS_SECRET_ACCESS_KEY__", &credentials.secret_access_key()) - .replace("__AWS_SESSION_TOKEN__", &credentials.session_token().unwrap()) + .replace("__AWS_SECRET_ACCESS_KEY__", credentials.secret_access_key()) + .replace("__AWS_SESSION_TOKEN__", credentials.session_token().unwrap()) .as_bytes(), ) .unwrap(); @@ -350,7 +351,7 @@ async fn test_credential_process_behind_source_profile_async() { .await .expect("list_objects should succeed"); - // With incorrect profile, requests should fail with no signing credentials + // With incorrect profile, requests should fail with a client error let config = S3ClientConfig::new() .auth_config(S3ClientAuthConfig::Profile(incorrect_profile.to_owned())) .endpoint_config(EndpointConfig::new(&get_test_region())); @@ -360,8 +361,8 @@ async fn test_credential_process_behind_source_profile_async() { .await .expect_err("should fail in different prefix"); assert!(matches!( - dbg!(err), - ObjectClientError::ClientError(S3RequestError::NoSigningCredentials) + err, + ObjectClientError::ClientError(_) )); drop(config_file); } From 36c6ca7c3796ceb903a642c37966c545b32ae6e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Varl=C4=B1?= Date: Wed, 4 Sep 2024 08:17:44 +0100 Subject: [PATCH 12/13] Fix formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Burak Varlı --- mountpoint-s3-client/tests/auth.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mountpoint-s3-client/tests/auth.rs b/mountpoint-s3-client/tests/auth.rs index 2d853c938..7edda24c9 100644 --- a/mountpoint-s3-client/tests/auth.rs +++ b/mountpoint-s3-client/tests/auth.rs @@ -360,10 +360,7 @@ async fn test_credential_process_behind_source_profile_async() { .list_objects(&bucket, None, "/", 10, &format!("{prefix}/")) .await .expect_err("should fail in different prefix"); - assert!(matches!( - err, - ObjectClientError::ClientError(_) - )); + assert!(matches!(err, ObjectClientError::ClientError(_))); drop(config_file); } From 55779fc844ced3c06d379408e4595311048c8189 Mon Sep 17 00:00:00 2001 From: Burak Date: Wed, 4 Sep 2024 08:44:21 +0100 Subject: [PATCH 13/13] Update test failure message Co-authored-by: Daniel Carl Jones Signed-off-by: Burak --- mountpoint-s3-client/tests/auth.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mountpoint-s3-client/tests/auth.rs b/mountpoint-s3-client/tests/auth.rs index 7edda24c9..857bb0d62 100644 --- a/mountpoint-s3-client/tests/auth.rs +++ b/mountpoint-s3-client/tests/auth.rs @@ -359,7 +359,7 @@ async fn test_credential_process_behind_source_profile_async() { let err = client .list_objects(&bucket, None, "/", 10, &format!("{prefix}/")) .await - .expect_err("should fail in different prefix"); + .expect_err("should fail when using invalid credentials"); assert!(matches!(err, ObjectClientError::ClientError(_))); drop(config_file); }