Skip to content

Commit bb3dfa5

Browse files
thovollthovoll
authored
Add rename_file_if_not_exists operation to data_lake (#506)
* Add "rename file if not exists". No e2e test yet. * Refactor e2e test. * Code format. * Uncomment code. * Better names. * Better file names. * Upload 2 chunks. * Extract functions. * Inline variable. * Extract function. * Fix merge error.
1 parent 222f1d4 commit bb3dfa5

File tree

8 files changed

+397
-150
lines changed

8 files changed

+397
-150
lines changed

‎sdk/storage/Cargo.toml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,5 +131,14 @@ required-features=["blob"]
131131
name="stream_blob_01"
132132
required-features=["blob"]
133133
[[example]]
134-
name="data_lake_00"
134+
name="data_lake_00_file_system"
135+
required-features=["data_lake"]
136+
[[example]]
137+
name="data_lake_01_file_create"
138+
required-features=["data_lake"]
139+
[[example]]
140+
name="data_lake_02_file_upload"
141+
required-features=["data_lake"]
142+
[[example]]
143+
name="data_lake_03_file_rename"
135144
required-features=["data_lake"]

‎sdk/storage/examples/data_lake_00.rs renamed to ‎sdk/storage/examples/data_lake_00_file_system.rs

Lines changed: 30 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -10,34 +10,9 @@ use std::num::NonZeroU32;
1010

1111
#[tokio::main]
1212
async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
13-
let account = std::env::var("ADLSGEN2_STORAGE_ACCOUNT")
14-
.expect("Set env variable ADLSGEN2_STORAGE_ACCOUNT first!");
15-
let master_key = std::env::var("ADLSGEN2_STORAGE_MASTER_KEY")
16-
.expect("Set env variable ADLSGEN2_STORAGE_MASTER_KEY first!");
17-
18-
let now = Utc::now();
19-
let file_system_name = format!("azurerustsdk-datalake-example00-{}", now.timestamp());
20-
21-
let http_client = new_http_client();
22-
23-
let storage_account_client =
24-
StorageAccountClient::new_access_key(http_client.clone(), &account, &master_key);
25-
26-
let resource_id = "https://storage.azure.com/";
27-
println!("getting bearer token for '{}'...", resource_id);
28-
let bearer_token = DefaultAzureCredential::default()
29-
.get_token(resource_id)
30-
.await?;
31-
println!("token expires on {}\n", bearer_token.expires_on);
32-
33-
let storage_client = storage_account_client.as_storage_client();
34-
let data_lake_client = DataLakeClient::new(
35-
storage_client,
36-
account,
37-
bearer_token.token.secret().to_owned(),
38-
None,
39-
);
13+
let data_lake_client = create_data_lake_client().await.unwrap();
4014

15+
let file_system_name = format!("azurerustsdk-datalake-example00-{}", Utc::now().timestamp());
4116
let file_system_client = data_lake_client
4217
.clone()
4318
.into_file_system_client(file_system_name.to_string());
@@ -93,7 +68,33 @@ async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
9368
let delete_fs_response = file_system_client.delete().execute().await?;
9469
println!("delete file system response == {:?}\n", delete_fs_response);
9570

96-
println!("data lake example 00 done.");
97-
9871
Ok(())
9972
}
73+
74+
async fn create_data_lake_client() -> Result<DataLakeClient, Box<dyn Error + Send + Sync>> {
75+
let account = std::env::var("ADLSGEN2_STORAGE_ACCOUNT")
76+
.expect("Set env variable ADLSGEN2_STORAGE_ACCOUNT first!");
77+
let master_key = std::env::var("ADLSGEN2_STORAGE_MASTER_KEY")
78+
.expect("Set env variable ADLSGEN2_STORAGE_MASTER_KEY first!");
79+
80+
let http_client = new_http_client();
81+
82+
let storage_account_client =
83+
StorageAccountClient::new_access_key(http_client.clone(), &account, &master_key);
84+
85+
let resource_id = "https://storage.azure.com/";
86+
println!("getting bearer token for '{}'...", resource_id);
87+
let bearer_token = DefaultAzureCredential::default()
88+
.get_token(resource_id)
89+
.await?;
90+
println!("token expires on {}\n", bearer_token.expires_on);
91+
92+
let storage_client = storage_account_client.as_storage_client();
93+
94+
Ok(DataLakeClient::new(
95+
storage_client,
96+
account,
97+
bearer_token.token.secret().to_owned(),
98+
None,
99+
))
100+
}

‎sdk/storage/examples/data_lake_01_file_create.rs

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
use azure_core::prelude::*;
2+
use azure_identity::token_credentials::DefaultAzureCredential;
3+
use azure_identity::token_credentials::TokenCredential;
4+
use azure_storage::core::prelude::*;
5+
use azure_storage::data_lake::prelude::*;
6+
use chrono::Utc;
7+
use std::error::Error;
8+
9+
#[tokio::main]
10+
async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
11+
let data_lake_client = create_data_lake_client().await.unwrap();
12+
13+
let file_system_name = format!("azurerustsdk-datalake-example01-{}", Utc::now().timestamp());
14+
let file_system_client = data_lake_client
15+
.clone()
16+
.into_file_system_client(file_system_name.to_string());
17+
18+
println!("creating file system '{}'...", &file_system_name);
19+
let create_fs_response = file_system_client.create().execute().await?;
20+
println!("create file system response == {:?}\n", create_fs_response);
21+
22+
let file_path = "some/path/example-file.txt";
23+
24+
println!("creating file '{}'...", file_path);
25+
let create_file_response = file_system_client
26+
.create_file(Context::default(), file_path, FileCreateOptions::default())
27+
.await?;
28+
println!("create file response == {:?}\n", create_file_response);
29+
30+
println!("creating file '{}' if not exists...", file_path);
31+
let create_file_if_not_exists_result = file_system_client
32+
.create_file_if_not_exists(Context::default(), file_path)
33+
.await;
34+
println!(
35+
"create file result (should fail) == {:?}\n",
36+
create_file_if_not_exists_result
37+
);
38+
39+
println!("creating file '{}' (overwrite)...", file_path);
40+
let create_file_response = file_system_client
41+
.create_file(Context::default(), file_path, FileCreateOptions::default())
42+
.await?;
43+
println!("create file response == {:?}\n", create_file_response);
44+
45+
println!("deleting file system...");
46+
let delete_fs_response = file_system_client.delete().execute().await?;
47+
println!("delete file system response == {:?}\n", delete_fs_response);
48+
49+
Ok(())
50+
}
51+
52+
async fn create_data_lake_client() -> Result<DataLakeClient, Box<dyn Error + Send + Sync>> {
53+
let account = std::env::var("ADLSGEN2_STORAGE_ACCOUNT")
54+
.expect("Set env variable ADLSGEN2_STORAGE_ACCOUNT first!");
55+
let master_key = std::env::var("ADLSGEN2_STORAGE_MASTER_KEY")
56+
.expect("Set env variable ADLSGEN2_STORAGE_MASTER_KEY first!");
57+
58+
let http_client = new_http_client();
59+
60+
let storage_account_client =
61+
StorageAccountClient::new_access_key(http_client.clone(), &account, &master_key);
62+
63+
let resource_id = "https://storage.azure.com/";
64+
println!("getting bearer token for '{}'...", resource_id);
65+
let bearer_token = DefaultAzureCredential::default()
66+
.get_token(resource_id)
67+
.await?;
68+
println!("token expires on {}\n", bearer_token.expires_on);
69+
70+
let storage_client = storage_account_client.as_storage_client();
71+
72+
Ok(DataLakeClient::new(
73+
storage_client,
74+
account,
75+
bearer_token.token.secret().to_owned(),
76+
None,
77+
))
78+
}

‎sdk/storage/examples/data_lake_01.rs renamed to ‎sdk/storage/examples/data_lake_02_file_upload.rs

Lines changed: 52 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -8,34 +8,9 @@ use std::error::Error;
88

99
#[tokio::main]
1010
async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
11-
let account = std::env::var("ADLSGEN2_STORAGE_ACCOUNT")
12-
.expect("Set env variable ADLSGEN2_STORAGE_ACCOUNT first!");
13-
let master_key = std::env::var("ADLSGEN2_STORAGE_MASTER_KEY")
14-
.expect("Set env variable ADLSGEN2_STORAGE_MASTER_KEY first!");
15-
16-
let now = Utc::now();
17-
let file_system_name = format!("azurerustsdk-datalake-example01-{}", now.timestamp());
18-
19-
let http_client = new_http_client();
20-
21-
let storage_account_client =
22-
StorageAccountClient::new_access_key(http_client.clone(), &account, &master_key);
23-
24-
let resource_id = "https://storage.azure.com/";
25-
println!("getting bearer token for '{}'...", resource_id);
26-
let bearer_token = DefaultAzureCredential::default()
27-
.get_token(resource_id)
28-
.await?;
29-
println!("token expires on {}\n", bearer_token.expires_on);
30-
31-
let storage_client = storage_account_client.as_storage_client();
32-
let data_lake_client = DataLakeClient::new(
33-
storage_client,
34-
account,
35-
bearer_token.token.secret().to_owned(),
36-
None,
37-
);
11+
let data_lake_client = create_data_lake_client().await.unwrap();
3812

13+
let file_system_name = format!("azurerustsdk-datalake-example01-{}", Utc::now().timestamp());
3914
let file_system_client = data_lake_client
4015
.clone()
4116
.into_file_system_client(file_system_name.to_string());
@@ -52,67 +27,81 @@ async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
5227
.await?;
5328
println!("create file response == {:?}\n", create_file_response);
5429

55-
println!("creating file '{}' (overwrite)...", file_path);
56-
let create_file_response = file_system_client
57-
.create_file(Context::default(), file_path, FileCreateOptions::default())
58-
.await?;
59-
println!("create file response == {:?}\n", create_file_response);
30+
let string1 = "some data";
31+
let data1 = bytes::Bytes::from(string1);
32+
let data1_length = data1.len() as i64;
33+
34+
let string2 = "some more data";
35+
let data2 = bytes::Bytes::from(string2);
36+
let data2_length = data2.len() as i64;
6037

61-
println!("creating file '{}' if not exists...", file_path);
62-
let create_file_if_not_exists_result = file_system_client
63-
.create_file_if_not_exists(Context::default(), file_path)
64-
.await;
65-
println!(
66-
"create file result (should fail) == {:?}\n",
67-
create_file_if_not_exists_result
68-
);
69-
70-
println!("appending to file '{}'...", file_path);
71-
let bytes = bytes::Bytes::from("some data");
72-
let file_length = bytes.len() as i64;
38+
println!("appending '{}' to file '{}'...", string1, file_path);
7339
let append_to_file_response = file_system_client
7440
.append_to_file(
7541
Context::default(),
7642
file_path,
77-
bytes,
43+
data1,
7844
0,
7945
FileAppendOptions::default(),
8046
)
8147
.await?;
8248
println!("append to file response == {:?}\n", append_to_file_response);
8349

50+
println!("appending '{}' to file '{}'...", string2, file_path);
51+
let append_to_file_response = file_system_client
52+
.append_to_file(
53+
Context::default(),
54+
file_path,
55+
data2,
56+
data1_length,
57+
FileAppendOptions::default(),
58+
)
59+
.await?;
60+
println!("append to file response == {:?}\n", append_to_file_response);
61+
8462
println!("flushing file '{}'...", file_path);
8563
let flush_file_response = file_system_client
8664
.flush_file(
8765
Context::default(),
8866
file_path,
89-
file_length,
67+
data1_length + data2_length,
9068
true,
9169
FileFlushOptions::default(),
9270
)
9371
.await?;
9472
println!("flush file response == {:?}\n", flush_file_response);
9573

96-
let destination_file_path = "some/path/example-file-renamed.txt";
97-
println!(
98-
"renaming file '{}' to '{}'...",
99-
file_path, destination_file_path
100-
);
101-
let rename_file_response = file_system_client
102-
.rename_file(
103-
Context::default(),
104-
file_path,
105-
destination_file_path,
106-
FileRenameOptions::default(),
107-
)
108-
.await?;
109-
println!("rename file response == {:?}\n", rename_file_response);
110-
11174
println!("deleting file system...");
11275
let delete_fs_response = file_system_client.delete().execute().await?;
11376
println!("delete file system response == {:?}\n", delete_fs_response);
11477

115-
println!("data lake example 01 done.");
116-
11778
Ok(())
11879
}
80+
81+
async fn create_data_lake_client() -> Result<DataLakeClient, Box<dyn Error + Send + Sync>> {
82+
let account = std::env::var("ADLSGEN2_STORAGE_ACCOUNT")
83+
.expect("Set env variable ADLSGEN2_STORAGE_ACCOUNT first!");
84+
let master_key = std::env::var("ADLSGEN2_STORAGE_MASTER_KEY")
85+
.expect("Set env variable ADLSGEN2_STORAGE_MASTER_KEY first!");
86+
87+
let http_client = new_http_client();
88+
89+
let storage_account_client =
90+
StorageAccountClient::new_access_key(http_client.clone(), &account, &master_key);
91+
92+
let resource_id = "https://storage.azure.com/";
93+
println!("getting bearer token for '{}'...", resource_id);
94+
let bearer_token = DefaultAzureCredential::default()
95+
.get_token(resource_id)
96+
.await?;
97+
println!("token expires on {}\n", bearer_token.expires_on);
98+
99+
let storage_client = storage_account_client.as_storage_client();
100+
101+
Ok(DataLakeClient::new(
102+
storage_client,
103+
account,
104+
bearer_token.token.secret().to_owned(),
105+
None,
106+
))
107+
}

0 commit comments

Comments
 (0)