Skip to content

Commit dc659ce

Browse files
smkleinsmklein
committed
Support Bundles: Multi-part upload
1 parent 7f1e07f commit dc659ce

File tree

9 files changed

+656
-269
lines changed

9 files changed

+656
-269
lines changed

‎nexus/src/app/background/tasks/support_bundle_collector.rs

Lines changed: 61 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -494,8 +494,12 @@ impl BundleCollection {
494494

495495
// Create the zipfile as a temporary file
496496
let mut zipfile = tokio::fs::File::from_std(bundle_to_zipfile(&dir)?);
497+
let total_len = zipfile.metadata().await?.len();
497498

498-
// Verify the hash locally before we send it over the network
499+
// Collect the hash locally before we send it over the network
500+
//
501+
// We'll use this later during finalization to confirm the bundle
502+
// has been stored successfully.
499503
zipfile.seek(SeekFrom::Start(0)).await?;
500504
let hash = sha2_hash(&mut zipfile).await?;
501505

@@ -515,23 +519,65 @@ impl BundleCollection {
515519
)
516520
.await?;
517521

518-
// Stream the zipfile to the sled where it should be kept
519-
zipfile.seek(SeekFrom::Start(0)).await?;
520-
let file_access = hyper_staticfile::vfs::TokioFileAccess::new(zipfile);
521-
let file_stream =
522-
hyper_staticfile::util::FileBytesStream::new(file_access);
523-
let body =
524-
reqwest::Body::wrap(hyper_staticfile::Body::Full(file_stream));
522+
let zpool = ZpoolUuid::from(self.bundle.zpool_id);
523+
let dataset = DatasetUuid::from(self.bundle.dataset_id);
524+
let support_bundle = SupportBundleUuid::from(self.bundle.id);
525+
526+
// Tell this sled to create the bundle.
527+
let creation_result = sled_client
528+
.support_bundle_start_creation(&zpool, &dataset, &support_bundle)
529+
.await
530+
.with_context(|| "Support bundle failed to start creation")?;
531+
532+
if matches!(
533+
creation_result.state,
534+
sled_agent_client::types::SupportBundleState::Complete
535+
) {
536+
// Early exit case: the bundle was already created -- we must have either
537+
// crashed or failed between "finalizing" and "writing to the database that we
538+
// finished".
539+
info!(&self.log, "Support bundle was already collected"; "bundle" => %self.bundle.id);
540+
return Ok(report);
541+
}
542+
info!(&self.log, "Support bundle creation started"; "bundle" => %self.bundle.id);
543+
544+
const CHUNK_SIZE: u64 = 1024 * 1024 * 1024;
545+
546+
let mut offset = 0;
547+
while offset < total_len {
548+
// Stream the zipfile to the sled where it should be kept
549+
let mut file = zipfile
550+
.try_clone()
551+
.await
552+
.with_context(|| "Failed to clone zipfile")?;
553+
file.seek(SeekFrom::Start(offset)).await.with_context(|| {
554+
format!("Failed to seek to offset {offset} / {total_len} within zipfile")
555+
})?;
556+
557+
// Only stream at most CHUNK_SIZE bytes at once
558+
let remaining = std::cmp::min(CHUNK_SIZE, total_len - offset);
559+
let limited_file = file.take(remaining);
560+
let stream = tokio_util::io::ReaderStream::new(limited_file);
561+
let body = reqwest::Body::wrap_stream(stream);
562+
563+
sled_client.support_bundle_transfer(
564+
&zpool, &dataset, &support_bundle, offset, body
565+
).await.with_context(|| {
566+
format!("Failed to transfer bundle: {remaining}@{offset} of {total_len} to sled")
567+
})?;
568+
569+
offset += CHUNK_SIZE;
570+
}
525571

526572
sled_client
527-
.support_bundle_create(
528-
&ZpoolUuid::from(self.bundle.zpool_id),
529-
&DatasetUuid::from(self.bundle.dataset_id),
530-
&SupportBundleUuid::from(self.bundle.id),
573+
.support_bundle_finalize(
574+
&zpool,
575+
&dataset,
576+
&support_bundle,
531577
&hash.to_string(),
532-
body,
533578
)
534-
.await?;
579+
.await
580+
.with_context(|| "Failed to finalize bundle")?;
535581

536582
// Returning from this method should drop all temporary storage
537583
// allocated locally for this support bundle.
@@ -795,7 +841,7 @@ impl BackgroundTask for SupportBundleCollector {
795841
Ok(report) => collection_report = Some(report),
796842
Err(err) => {
797843
collection_err =
798-
Some(json!({ "collect_error": err.to_string() }))
844+
Some(json!({ "collect_error": InlineErrorChain::new(err.as_ref()).to_string() }))
799845
}
800846
};
801847

‎openapi/sled-agent.json

Lines changed: 140 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -875,8 +875,9 @@
875875
},
876876
"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}": {
877877
"post": {
878-
"summary": "Create a support bundle within a particular dataset",
879-
"operationId": "support_bundle_create",
878+
"summary": "Starts creation of a support bundle within a particular dataset",
879+
"description": "Callers should transfer chunks of the bundle with \"support_bundle_transfer\", and then call \"support_bundle_finalize\" once the bundle has finished transferring.\n\nIf a support bundle was previously created without being finalized successfully, this endpoint will reset the state.\n\nIf a support bundle was previously created and finalized successfully, this endpoint will return metadata indicating that it already exists.",
880+
"operationId": "support_bundle_start_creation",
880881
"parameters": [
881882
{
882883
"in": "path",
@@ -904,28 +905,8 @@
904905
"schema": {
905906
"$ref": "#/components/schemas/TypedUuidForZpoolKind"
906907
}
907-
},
908-
{
909-
"in": "query",
910-
"name": "hash",
911-
"required": true,
912-
"schema": {
913-
"type": "string",
914-
"format": "hex string (32 bytes)"
915-
}
916908
}
917909
],
918-
"requestBody": {
919-
"content": {
920-
"application/octet-stream": {
921-
"schema": {
922-
"type": "string",
923-
"format": "binary"
924-
}
925-
}
926-
},
927-
"required": true
928-
},
929910
"responses": {
930911
"201": {
931912
"description": "successful creation",
@@ -1216,6 +1197,69 @@
12161197
}
12171198
}
12181199
},
1200+
"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/finalize": {
1201+
"post": {
1202+
"summary": "Finalizes the creation of a support bundle",
1203+
"description": "If the requested hash matched the bundle, the bundle is created. Otherwise, an error is returned.",
1204+
"operationId": "support_bundle_finalize",
1205+
"parameters": [
1206+
{
1207+
"in": "path",
1208+
"name": "dataset_id",
1209+
"description": "The dataset on which this support bundle was provisioned",
1210+
"required": true,
1211+
"schema": {
1212+
"$ref": "#/components/schemas/TypedUuidForDatasetKind"
1213+
}
1214+
},
1215+
{
1216+
"in": "path",
1217+
"name": "support_bundle_id",
1218+
"description": "The ID of the support bundle itself",
1219+
"required": true,
1220+
"schema": {
1221+
"$ref": "#/components/schemas/TypedUuidForSupportBundleKind"
1222+
}
1223+
},
1224+
{
1225+
"in": "path",
1226+
"name": "zpool_id",
1227+
"description": "The zpool on which this support bundle was provisioned",
1228+
"required": true,
1229+
"schema": {
1230+
"$ref": "#/components/schemas/TypedUuidForZpoolKind"
1231+
}
1232+
},
1233+
{
1234+
"in": "query",
1235+
"name": "hash",
1236+
"required": true,
1237+
"schema": {
1238+
"type": "string",
1239+
"format": "hex string (32 bytes)"
1240+
}
1241+
}
1242+
],
1243+
"responses": {
1244+
"201": {
1245+
"description": "successful creation",
1246+
"content": {
1247+
"application/json": {
1248+
"schema": {
1249+
"$ref": "#/components/schemas/SupportBundleMetadata"
1250+
}
1251+
}
1252+
}
1253+
},
1254+
"4XX": {
1255+
"$ref": "#/components/responses/Error"
1256+
},
1257+
"5XX": {
1258+
"$ref": "#/components/responses/Error"
1259+
}
1260+
}
1261+
}
1262+
},
12191263
"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/index": {
12201264
"get": {
12211265
"summary": "Fetch the index (list of files within a support bundle)",
@@ -1320,6 +1364,80 @@
13201364
}
13211365
}
13221366
},
1367+
"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/transfer": {
1368+
"put": {
1369+
"summary": "Transfers a chunk of a support bundle within a particular dataset",
1370+
"operationId": "support_bundle_transfer",
1371+
"parameters": [
1372+
{
1373+
"in": "path",
1374+
"name": "dataset_id",
1375+
"description": "The dataset on which this support bundle was provisioned",
1376+
"required": true,
1377+
"schema": {
1378+
"$ref": "#/components/schemas/TypedUuidForDatasetKind"
1379+
}
1380+
},
1381+
{
1382+
"in": "path",
1383+
"name": "support_bundle_id",
1384+
"description": "The ID of the support bundle itself",
1385+
"required": true,
1386+
"schema": {
1387+
"$ref": "#/components/schemas/TypedUuidForSupportBundleKind"
1388+
}
1389+
},
1390+
{
1391+
"in": "path",
1392+
"name": "zpool_id",
1393+
"description": "The zpool on which this support bundle was provisioned",
1394+
"required": true,
1395+
"schema": {
1396+
"$ref": "#/components/schemas/TypedUuidForZpoolKind"
1397+
}
1398+
},
1399+
{
1400+
"in": "query",
1401+
"name": "offset",
1402+
"required": true,
1403+
"schema": {
1404+
"type": "integer",
1405+
"format": "uint64",
1406+
"minimum": 0
1407+
}
1408+
}
1409+
],
1410+
"requestBody": {
1411+
"content": {
1412+
"application/octet-stream": {
1413+
"schema": {
1414+
"type": "string",
1415+
"format": "binary"
1416+
}
1417+
}
1418+
},
1419+
"required": true
1420+
},
1421+
"responses": {
1422+
"201": {
1423+
"description": "successful creation",
1424+
"content": {
1425+
"application/json": {
1426+
"schema": {
1427+
"$ref": "#/components/schemas/SupportBundleMetadata"
1428+
}
1429+
}
1430+
}
1431+
},
1432+
"4XX": {
1433+
"$ref": "#/components/responses/Error"
1434+
},
1435+
"5XX": {
1436+
"$ref": "#/components/responses/Error"
1437+
}
1438+
}
1439+
}
1440+
},
13231441
"/switch-ports": {
13241442
"post": {
13251443
"operationId": "uplink_ensure",

‎sled-agent/api/src/lib.rs

Lines changed: 45 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -166,19 +166,53 @@ pub trait SledAgentApi {
166166
path_params: Path<SupportBundleListPathParam>,
167167
) -> Result<HttpResponseOk<Vec<SupportBundleMetadata>>, HttpError>;
168168

169-
/// Create a support bundle within a particular dataset
169+
/// Starts creation of a support bundle within a particular dataset
170+
///
171+
/// Callers should transfer chunks of the bundle with
172+
/// "support_bundle_transfer", and then call "support_bundle_finalize"
173+
/// once the bundle has finished transferring.
174+
///
175+
/// If a support bundle was previously created without being finalized
176+
/// successfully, this endpoint will reset the state.
177+
///
178+
/// If a support bundle was previously created and finalized successfully,
179+
/// this endpoint will return metadata indicating that it already exists.
170180
#[endpoint {
171181
method = POST,
172-
path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}",
182+
path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}"
183+
}]
184+
async fn support_bundle_start_creation(
185+
rqctx: RequestContext<Self::Context>,
186+
path_params: Path<SupportBundlePathParam>,
187+
) -> Result<HttpResponseCreated<SupportBundleMetadata>, HttpError>;
188+
189+
/// Transfers a chunk of a support bundle within a particular dataset
190+
#[endpoint {
191+
method = PUT,
192+
path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/transfer",
173193
request_body_max_bytes = SUPPORT_BUNDLE_MAX_BYTES,
174194
}]
175-
async fn support_bundle_create(
195+
async fn support_bundle_transfer(
176196
rqctx: RequestContext<Self::Context>,
177197
path_params: Path<SupportBundlePathParam>,
178-
query_params: Query<SupportBundleCreateQueryParams>,
198+
query_params: Query<SupportBundleTransferQueryParams>,
179199
body: StreamingBody,
180200
) -> Result<HttpResponseCreated<SupportBundleMetadata>, HttpError>;
181201

202+
/// Finalizes the creation of a support bundle
203+
///
204+
/// If the requested hash matched the bundle, the bundle is created.
205+
/// Otherwise, an error is returned.
206+
#[endpoint {
207+
method = POST,
208+
path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/finalize"
209+
}]
210+
async fn support_bundle_finalize(
211+
rqctx: RequestContext<Self::Context>,
212+
path_params: Path<SupportBundlePathParam>,
213+
query_params: Query<SupportBundleFinalizeQueryParams>,
214+
) -> Result<HttpResponseCreated<SupportBundleMetadata>, HttpError>;
215+
182216
/// Fetch a support bundle from a particular dataset
183217
#[endpoint {
184218
method = GET,
@@ -760,9 +794,15 @@ pub struct SupportBundleFilePathParam {
760794
pub file: String,
761795
}
762796

797+
/// Metadata about a support bundle transfer
798+
#[derive(Deserialize, Serialize, JsonSchema)]
799+
pub struct SupportBundleTransferQueryParams {
800+
pub offset: u64,
801+
}
802+
763803
/// Metadata about a support bundle
764804
#[derive(Deserialize, Serialize, JsonSchema)]
765-
pub struct SupportBundleCreateQueryParams {
805+
pub struct SupportBundleFinalizeQueryParams {
766806
pub hash: ArtifactHash,
767807
}
768808

0 commit comments

Comments
 (0)