Skip to content

Commit d149037

Browse files
authored
bug(storage-azdls): Fix inferred WASB endpoint (#1417)
## Which issue does this PR close? - Relates to #1360. I tested again fileIO operations with the `wasbs://` scheme, and the current implementation mistakenly used the `blob` endpoint which is lacking the ADLS APIs. ## What changes are included in this PR? The code now always uses the `dfs` endpoint, regardless of whether a user passes `wasb[s]://` or `abfs[s]://`. Because the `AzureStoragePath::storage_service` field isn't read, I've removed it. ## Are these changes tested? Yes, I added a new unit test for the `AzureStoragePath::endpoint` method. --------- Signed-off-by: DerGut <jannik.steinmann@gmx.de> Signed-off-by: Jannik Steinmann <jannik.steinmann@datadoghq.com>
1 parent c74a27a commit d149037

File tree

1 file changed

+46
-8
lines changed

1 file changed

+46
-8
lines changed

crates/iceberg/src/io/storage_azdls.rs

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -230,9 +230,6 @@ struct AzureStoragePath {
230230

231231
account_name: String,
232232

233-
/// Either `blob` or `dfs` for Blob Storage and ADLSv2 respectively.
234-
storage_service: String,
235-
236233
/// The endpoint suffix, e.g., `core.windows.net` for the public cloud
237234
/// endpoint.
238235
endpoint_suffix: String,
@@ -249,10 +246,9 @@ impl AzureStoragePath {
249246
/// This is possible because the path is fully qualified.
250247
fn as_endpoint(&self) -> String {
251248
format!(
252-
"{}://{}.{}.{}",
249+
"{}://{}.dfs.{}",
253250
self.scheme.as_http_scheme(),
254251
self.account_name,
255-
self.storage_service,
256252
self.endpoint_suffix
257253
)
258254
}
@@ -278,7 +274,6 @@ impl FromStr for AzureStoragePath {
278274
scheme,
279275
filesystem: filesystem.to_string(),
280276
account_name: account_name.to_string(),
281-
storage_service: storage_service.to_string(),
282277
endpoint_suffix: endpoint_suffix.to_string(),
283278
path: url.path().to_string(),
284279
})
@@ -505,7 +500,7 @@ mod tests {
505500
}
506501

507502
#[test]
508-
fn test_parse_azure_storage_path() {
503+
fn test_azure_storage_path_parse() {
509504
let test_cases = vec![
510505
(
511506
"succeeds",
@@ -514,7 +509,6 @@ mod tests {
514509
scheme: AzureStorageScheme::Abfss,
515510
filesystem: "somefs".to_string(),
516511
account_name: "myaccount".to_string(),
517-
storage_service: "dfs".to_string(),
518512
endpoint_suffix: "core.windows.net".to_string(),
519513
path: "/path/to/file.parquet".to_string(),
520514
}),
@@ -549,4 +543,48 @@ mod tests {
549543
}
550544
}
551545
}
546+
547+
#[test]
548+
fn test_azure_storage_path_endpoint() {
549+
let test_cases = vec![
550+
(
551+
"abfss uses https",
552+
AzureStoragePath {
553+
scheme: AzureStorageScheme::Abfss,
554+
filesystem: "myfs".to_string(),
555+
account_name: "myaccount".to_string(),
556+
endpoint_suffix: "core.windows.net".to_string(),
557+
path: "/path/to/file.parquet".to_string(),
558+
},
559+
"https://myaccount.dfs.core.windows.net",
560+
),
561+
(
562+
"abfs uses http",
563+
AzureStoragePath {
564+
scheme: AzureStorageScheme::Abfs,
565+
filesystem: "myfs".to_string(),
566+
account_name: "myaccount".to_string(),
567+
endpoint_suffix: "core.windows.net".to_string(),
568+
path: "/path/to/file.parquet".to_string(),
569+
},
570+
"http://myaccount.dfs.core.windows.net",
571+
),
572+
(
573+
"wasbs uses https and dfs",
574+
AzureStoragePath {
575+
scheme: AzureStorageScheme::Abfss,
576+
filesystem: "myfs".to_string(),
577+
account_name: "myaccount".to_string(),
578+
endpoint_suffix: "core.windows.net".to_string(),
579+
path: "/path/to/file.parquet".to_string(),
580+
},
581+
"https://myaccount.dfs.core.windows.net",
582+
),
583+
];
584+
585+
for (name, path, expected) in test_cases {
586+
let endpoint = path.as_endpoint();
587+
assert_eq!(endpoint, expected, "Test case: {}", name);
588+
}
589+
}
552590
}

0 commit comments

Comments
 (0)