Skip to content

Commit cbc04e3

Browse files
smkleinsmklein
committed
Support Bundles: Multi-part upload
1 parent 77d097a commit cbc04e3

File tree

9 files changed

+656
-269
lines changed

9 files changed

+656
-269
lines changed

‎nexus/src/app/background/tasks/support_bundle_collector.rs

Lines changed: 61 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -494,8 +494,12 @@ impl BundleCollection {
494494

495495
// Create the zipfile as a temporary file
496496
let mut zipfile = tokio::fs::File::from_std(bundle_to_zipfile(&dir)?);
497+
let total_len = zipfile.metadata().await?.len();
497498

498-
// Verify the hash locally before we send it over the network
499+
// Collect the hash locally before we send it over the network
500+
//
501+
// We'll use this later during finalization to confirm the bundle
502+
// has been stored successfully.
499503
zipfile.seek(SeekFrom::Start(0)).await?;
500504
let hash = sha2_hash(&mut zipfile).await?;
501505

@@ -515,23 +519,65 @@ impl BundleCollection {
515519
)
516520
.await?;
517521

518-
// Stream the zipfile to the sled where it should be kept
519-
zipfile.seek(SeekFrom::Start(0)).await?;
520-
let file_access = hyper_staticfile::vfs::TokioFileAccess::new(zipfile);
521-
let file_stream =
522-
hyper_staticfile::util::FileBytesStream::new(file_access);
523-
let body =
524-
reqwest::Body::wrap(hyper_staticfile::Body::Full(file_stream));
522+
let zpool = ZpoolUuid::from(self.bundle.zpool_id);
523+
let dataset = DatasetUuid::from(self.bundle.dataset_id);
524+
let support_bundle = SupportBundleUuid::from(self.bundle.id);
525+
526+
// Tell this sled to create the bundle.
527+
let creation_result = sled_client
528+
.support_bundle_start_creation(&zpool, &dataset, &support_bundle)
529+
.await
530+
.with_context(|| "Support bundle failed to start creation")?;
531+
532+
if matches!(
533+
creation_result.state,
534+
sled_agent_client::types::SupportBundleState::Complete
535+
) {
536+
// Early exit case: the bundle was already created -- we must have either
537+
// crashed or failed between "finalizing" and "writing to the database that we
538+
// finished".
539+
info!(&self.log, "Support bundle was already collected"; "bundle" => %self.bundle.id);
540+
return Ok(report);
541+
}
542+
info!(&self.log, "Support bundle creation started"; "bundle" => %self.bundle.id);
543+
544+
const CHUNK_SIZE: u64 = 1024 * 1024 * 1024;
545+
546+
let mut offset = 0;
547+
while offset < total_len {
548+
// Stream the zipfile to the sled where it should be kept
549+
let mut file = zipfile
550+
.try_clone()
551+
.await
552+
.with_context(|| "Failed to clone zipfile")?;
553+
file.seek(SeekFrom::Start(offset)).await.with_context(|| {
554+
format!("Failed to seek to offset {offset} / {total_len} within zipfile")
555+
})?;
556+
557+
// Only stream at most CHUNK_SIZE bytes at once
558+
let remaining = std::cmp::min(CHUNK_SIZE, total_len - offset);
559+
let limited_file = file.take(remaining);
560+
let stream = tokio_util::io::ReaderStream::new(limited_file);
561+
let body = reqwest::Body::wrap_stream(stream);
562+
563+
sled_client.support_bundle_transfer(
564+
&zpool, &dataset, &support_bundle, offset, body
565+
).await.with_context(|| {
566+
format!("Failed to transfer bundle: {remaining}@{offset} of {total_len} to sled")
567+
})?;
568+
569+
offset += CHUNK_SIZE;
570+
}
525571

526572
sled_client
527-
.support_bundle_create(
528-
&ZpoolUuid::from(self.bundle.zpool_id),
529-
&DatasetUuid::from(self.bundle.dataset_id),
530-
&SupportBundleUuid::from(self.bundle.id),
573+
.support_bundle_finalize(
574+
&zpool,
575+
&dataset,
576+
&support_bundle,
531577
&hash.to_string(),
532-
body,
533578
)
534-
.await?;
579+
.await
580+
.with_context(|| "Failed to finalize bundle")?;
535581

536582
// Returning from this method should drop all temporary storage
537583
// allocated locally for this support bundle.
@@ -795,7 +841,7 @@ impl BackgroundTask for SupportBundleCollector {
795841
Ok(report) => collection_report = Some(report),
796842
Err(err) => {
797843
collection_err =
798-
Some(json!({ "collect_error": err.to_string() }))
844+
Some(json!({ "collect_error": InlineErrorChain::new(err.as_ref()).to_string() }))
799845
}
800846
};
801847

‎openapi/sled-agent.json

Lines changed: 140 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1000,8 +1000,9 @@
10001000
},
10011001
"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}": {
10021002
"post": {
1003-
"summary": "Create a support bundle within a particular dataset",
1004-
"operationId": "support_bundle_create",
1003+
"summary": "Starts creation of a support bundle within a particular dataset",
1004+
"description": "Callers should transfer chunks of the bundle with \"support_bundle_transfer\", and then call \"support_bundle_finalize\" once the bundle has finished transferring.\n\nIf a support bundle was previously created without being finalized successfully, this endpoint will reset the state.\n\nIf a support bundle was previously created and finalized successfully, this endpoint will return metadata indicating that it already exists.",
1005+
"operationId": "support_bundle_start_creation",
10051006
"parameters": [
10061007
{
10071008
"in": "path",
@@ -1029,28 +1030,8 @@
10291030
"schema": {
10301031
"$ref": "#/components/schemas/TypedUuidForZpoolKind"
10311032
}
1032-
},
1033-
{
1034-
"in": "query",
1035-
"name": "hash",
1036-
"required": true,
1037-
"schema": {
1038-
"type": "string",
1039-
"format": "hex string (32 bytes)"
1040-
}
10411033
}
10421034
],
1043-
"requestBody": {
1044-
"content": {
1045-
"application/octet-stream": {
1046-
"schema": {
1047-
"type": "string",
1048-
"format": "binary"
1049-
}
1050-
}
1051-
},
1052-
"required": true
1053-
},
10541035
"responses": {
10551036
"201": {
10561037
"description": "successful creation",
@@ -1341,6 +1322,69 @@
13411322
}
13421323
}
13431324
},
1325+
"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/finalize": {
1326+
"post": {
1327+
"summary": "Finalizes the creation of a support bundle",
1328+
"description": "If the requested hash matched the bundle, the bundle is created. Otherwise, an error is returned.",
1329+
"operationId": "support_bundle_finalize",
1330+
"parameters": [
1331+
{
1332+
"in": "path",
1333+
"name": "dataset_id",
1334+
"description": "The dataset on which this support bundle was provisioned",
1335+
"required": true,
1336+
"schema": {
1337+
"$ref": "#/components/schemas/TypedUuidForDatasetKind"
1338+
}
1339+
},
1340+
{
1341+
"in": "path",
1342+
"name": "support_bundle_id",
1343+
"description": "The ID of the support bundle itself",
1344+
"required": true,
1345+
"schema": {
1346+
"$ref": "#/components/schemas/TypedUuidForSupportBundleKind"
1347+
}
1348+
},
1349+
{
1350+
"in": "path",
1351+
"name": "zpool_id",
1352+
"description": "The zpool on which this support bundle was provisioned",
1353+
"required": true,
1354+
"schema": {
1355+
"$ref": "#/components/schemas/TypedUuidForZpoolKind"
1356+
}
1357+
},
1358+
{
1359+
"in": "query",
1360+
"name": "hash",
1361+
"required": true,
1362+
"schema": {
1363+
"type": "string",
1364+
"format": "hex string (32 bytes)"
1365+
}
1366+
}
1367+
],
1368+
"responses": {
1369+
"201": {
1370+
"description": "successful creation",
1371+
"content": {
1372+
"application/json": {
1373+
"schema": {
1374+
"$ref": "#/components/schemas/SupportBundleMetadata"
1375+
}
1376+
}
1377+
}
1378+
},
1379+
"4XX": {
1380+
"$ref": "#/components/responses/Error"
1381+
},
1382+
"5XX": {
1383+
"$ref": "#/components/responses/Error"
1384+
}
1385+
}
1386+
}
1387+
},
13441388
"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/index": {
13451389
"get": {
13461390
"summary": "Fetch the index (list of files within a support bundle)",
@@ -1445,6 +1489,80 @@
14451489
}
14461490
}
14471491
},
1492+
"/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/transfer": {
1493+
"put": {
1494+
"summary": "Transfers a chunk of a support bundle within a particular dataset",
1495+
"operationId": "support_bundle_transfer",
1496+
"parameters": [
1497+
{
1498+
"in": "path",
1499+
"name": "dataset_id",
1500+
"description": "The dataset on which this support bundle was provisioned",
1501+
"required": true,
1502+
"schema": {
1503+
"$ref": "#/components/schemas/TypedUuidForDatasetKind"
1504+
}
1505+
},
1506+
{
1507+
"in": "path",
1508+
"name": "support_bundle_id",
1509+
"description": "The ID of the support bundle itself",
1510+
"required": true,
1511+
"schema": {
1512+
"$ref": "#/components/schemas/TypedUuidForSupportBundleKind"
1513+
}
1514+
},
1515+
{
1516+
"in": "path",
1517+
"name": "zpool_id",
1518+
"description": "The zpool on which this support bundle was provisioned",
1519+
"required": true,
1520+
"schema": {
1521+
"$ref": "#/components/schemas/TypedUuidForZpoolKind"
1522+
}
1523+
},
1524+
{
1525+
"in": "query",
1526+
"name": "offset",
1527+
"required": true,
1528+
"schema": {
1529+
"type": "integer",
1530+
"format": "uint64",
1531+
"minimum": 0
1532+
}
1533+
}
1534+
],
1535+
"requestBody": {
1536+
"content": {
1537+
"application/octet-stream": {
1538+
"schema": {
1539+
"type": "string",
1540+
"format": "binary"
1541+
}
1542+
}
1543+
},
1544+
"required": true
1545+
},
1546+
"responses": {
1547+
"201": {
1548+
"description": "successful creation",
1549+
"content": {
1550+
"application/json": {
1551+
"schema": {
1552+
"$ref": "#/components/schemas/SupportBundleMetadata"
1553+
}
1554+
}
1555+
}
1556+
},
1557+
"4XX": {
1558+
"$ref": "#/components/responses/Error"
1559+
},
1560+
"5XX": {
1561+
"$ref": "#/components/responses/Error"
1562+
}
1563+
}
1564+
}
1565+
},
14481566
"/switch-ports": {
14491567
"post": {
14501568
"operationId": "uplink_ensure",

‎sled-agent/api/src/lib.rs

Lines changed: 45 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -170,19 +170,53 @@ pub trait SledAgentApi {
170170
path_params: Path<SupportBundleListPathParam>,
171171
) -> Result<HttpResponseOk<Vec<SupportBundleMetadata>>, HttpError>;
172172

173-
/// Create a support bundle within a particular dataset
173+
/// Starts creation of a support bundle within a particular dataset
174+
///
175+
/// Callers should transfer chunks of the bundle with
176+
/// "support_bundle_transfer", and then call "support_bundle_finalize"
177+
/// once the bundle has finished transferring.
178+
///
179+
/// If a support bundle was previously created without being finalized
180+
/// successfully, this endpoint will reset the state.
181+
///
182+
/// If a support bundle was previously created and finalized successfully,
183+
/// this endpoint will return metadata indicating that it already exists.
174184
#[endpoint {
175185
method = POST,
176-
path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}",
186+
path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}"
187+
}]
188+
async fn support_bundle_start_creation(
189+
rqctx: RequestContext<Self::Context>,
190+
path_params: Path<SupportBundlePathParam>,
191+
) -> Result<HttpResponseCreated<SupportBundleMetadata>, HttpError>;
192+
193+
/// Transfers a chunk of a support bundle within a particular dataset
194+
#[endpoint {
195+
method = PUT,
196+
path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/transfer",
177197
request_body_max_bytes = SUPPORT_BUNDLE_MAX_BYTES,
178198
}]
179-
async fn support_bundle_create(
199+
async fn support_bundle_transfer(
180200
rqctx: RequestContext<Self::Context>,
181201
path_params: Path<SupportBundlePathParam>,
182-
query_params: Query<SupportBundleCreateQueryParams>,
202+
query_params: Query<SupportBundleTransferQueryParams>,
183203
body: StreamingBody,
184204
) -> Result<HttpResponseCreated<SupportBundleMetadata>, HttpError>;
185205

206+
/// Finalizes the creation of a support bundle
207+
///
208+
/// If the requested hash matched the bundle, the bundle is created.
209+
/// Otherwise, an error is returned.
210+
#[endpoint {
211+
method = POST,
212+
path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}/finalize"
213+
}]
214+
async fn support_bundle_finalize(
215+
rqctx: RequestContext<Self::Context>,
216+
path_params: Path<SupportBundlePathParam>,
217+
query_params: Query<SupportBundleFinalizeQueryParams>,
218+
) -> Result<HttpResponseCreated<SupportBundleMetadata>, HttpError>;
219+
186220
/// Fetch a support bundle from a particular dataset
187221
#[endpoint {
188222
method = GET,
@@ -796,9 +830,15 @@ pub struct SupportBundleFilePathParam {
796830
pub file: String,
797831
}
798832

833+
/// Metadata about a support bundle transfer
834+
#[derive(Deserialize, Serialize, JsonSchema)]
835+
pub struct SupportBundleTransferQueryParams {
836+
pub offset: u64,
837+
}
838+
799839
/// Metadata about a support bundle
800840
#[derive(Deserialize, Serialize, JsonSchema)]
801-
pub struct SupportBundleCreateQueryParams {
841+
pub struct SupportBundleFinalizeQueryParams {
802842
pub hash: ArtifactHash,
803843
}
804844

0 commit comments

Comments
 (0)