Migrate to a new SDK example workspace structure (#2811)

## Motivation and Context
When the WebAssembly SDK example was added some months ago, we changed
the build process to make each SDK example its own Cargo workspace. This
allowed the `.cargo/config.toml` that changed the compiler target to
work correctly. However, this has led to other issues: dependency
compilation is no longer shared between examples which greatly increases
the time it takes for that CI step to run, and now it is even running
out of disk space on the GitHub Actions runners.

This PR adds support for a new workspace layout where the
[`aws-doc-sdk-examples`](https://github.com/awsdocs/aws-doc-sdk-examples)
repo gets to decide how the workspaces are logically grouped. If a
`Cargo.toml` file exists at the example root, then the build system
assumes that the _old_ "one example, one workspace" layout should be
used. If there is no root `Cargo.toml`, then it assumes the new layout
should be used.

The `sdk-versioner` tool had to be adapted to support more complex
relative path resolution to make this work, and the `publisher
fix-manifests` subcommand had to be fixed to ignore workspace-only
`Cargo.toml` files.

The build system in this PR needs to work for both the old and new
examples layout so that the `sdk-sync` process will succeed. #2810 has
been filed to track removing the old example layout at a later date.


[aws-doc-sdk-examples#4997](awsdocs/aws-doc-sdk-examples#4997)
changes the workspace structure of the actual examples to the new one.

## Testing
- [x] Generated a full SDK with the old example layout, manually
examined the output, and spot checked that some examples compile
- [x] Generated a full SDK with the new example layout, manually
examined the output, and spot checked that some examples compile
- [x] Examples pass in CI with the old example layout
- [x] Examples pass in CI with the new example layout

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

Author: jdisanti

aws/sdk/build.gradle.kts

@@ -3,6 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import aws.sdk.AwsExamplesLayout
 import aws.sdk.AwsServices
 import aws.sdk.Membership
 import aws.sdk.discoverServices
@@ -201,6 +202,7 @@ tasks.register("relocateExamples") {
                 }
                 into(outputDir)
                 exclude("**/target")
+                exclude("**/rust-toolchain.toml")
                 filter { line -> line.replace("build/aws-sdk/sdk/", "sdk/") }
             }
         }
@@ -242,13 +244,22 @@ tasks.register<ExecRustBuildTool>("fixExampleManifests") {
 
     toolPath = sdkVersionerToolPath
     binaryName = "sdk-versioner"
-    arguments = listOf(
-        "use-path-and-version-dependencies",
-        "--isolate-crates",
-        "--sdk-path", "../../sdk",
-        "--versions-toml", outputDir.resolve("versions.toml").absolutePath,
-        outputDir.resolve("examples").absolutePath,
-    )
+    arguments = when (AwsExamplesLayout.detect(project)) {
+        AwsExamplesLayout.Flat -> listOf(
+            "use-path-and-version-dependencies",
+            "--isolate-crates",
+            "--sdk-path", "../../sdk",
+            "--versions-toml", outputDir.resolve("versions.toml").absolutePath,
+            outputDir.resolve("examples").absolutePath,
+        )
+        AwsExamplesLayout.Workspaces -> listOf(
+            "use-path-and-version-dependencies",
+            "--isolate-crates",
+            "--sdk-path", sdkOutputDir.absolutePath,
+            "--versions-toml", outputDir.resolve("versions.toml").absolutePath,
+            outputDir.resolve("examples").absolutePath,
+        )
+    }
 
     outputs.dir(outputDir)
     dependsOn("relocateExamples", "generateVersionManifest")

buildSrc/src/main/kotlin/aws/sdk/ServiceLoader.kt

@@ -19,6 +19,38 @@ data class RootTest(
     val manifestName: String,
 )
 
+// TODO(https://github.com/awslabs/smithy-rs/issues/2810): We can remove the `Flat` layout after the switch
+// to `Workspaces` has been released. This can be checked by looking at the `examples/` directory in aws-sdk-rust's
+// main branch.
+//
+// The `Flat` layout is retained for backwards compatibility so that the next release process can succeed.
+enum class AwsExamplesLayout {
+    /**
+     * Directory layout for examples used prior to June 26, 2023,
+     * where each example was in the `rust_dev_preview/` root directory and
+     * was considered to be its own workspace.
+     *
+     * This layout had issues with CI in terms of time to compile and disk space required
+     * since the dependencies would get recompiled for every example.
+     */
+    Flat,
+
+    /**
+     * Current directory layout where there are a small number of workspaces
+     * rooted in `rust_dev_preview/`.
+     */
+    Workspaces,
+    ;
+
+    companion object {
+        fun detect(project: Project): AwsExamplesLayout = if (project.projectDir.resolve("examples/Cargo.toml").exists()) {
+            AwsExamplesLayout.Flat
+        } else {
+            AwsExamplesLayout.Workspaces
+        }
+    }
+}
+
 class AwsServices(
     private val project: Project,
     services: List<AwsService>,
@@ -44,10 +76,16 @@ class AwsServices(
     }
 
     val examples: List<String> by lazy {
-        project.projectDir.resolve("examples")
-            .listFiles { file -> !file.name.startsWith(".") }.orEmpty().toList()
-            .filter { file -> manifestCompatibleWithGeneratedServices(file) }
-            .map { "examples/${it.name}" }
+        val examplesRoot = project.projectDir.resolve("examples")
+        if (AwsExamplesLayout.detect(project) == AwsExamplesLayout.Flat) {
+            examplesRoot.listFiles { file -> !file.name.startsWith(".") }.orEmpty().toList()
+                .filter { file -> manifestCompatibleWithGeneratedServices(file) }
+                .map { "examples/${it.name}" }
+        } else {
+            examplesRoot.listFiles { file ->
+                !file.name.startsWith(".") && file.isDirectory() && file.resolve("Cargo.toml").exists()
+            }.orEmpty().toList().map { "examples/${it.name}" }
+        }
     }
 
     /**

tools/ci-build/publisher/src/package.rs

@@ -145,8 +145,7 @@ pub async fn discover_and_validate_package_batches(
     fs: Fs,
     path: impl AsRef<Path>,
 ) -> Result<(Vec<PackageBatch>, PackageStats)> {
-    let manifest_paths = discover_package_manifests(path.as_ref().into()).await?;
-    let packages = read_packages(fs, manifest_paths)
+    let packages = discover_packages(fs, path.as_ref().into())
         .await?
         .into_iter()
         .filter(|package| package.publish == Publish::Allowed)
@@ -176,7 +175,7 @@ pub enum Error {
 
 /// Discovers all Cargo.toml files under the given path recursively
 #[async_recursion::async_recursion]
-pub async fn discover_package_manifests(path: PathBuf) -> Result<Vec<PathBuf>> {
+pub async fn discover_manifests(path: PathBuf) -> Result<Vec<PathBuf>> {
     let mut manifests = Vec::new();
     let mut read_dir = fs::read_dir(&path).await?;
     while let Some(entry) = read_dir.next_entry().await? {
@@ -185,14 +184,19 @@ pub async fn discover_package_manifests(path: PathBuf) -> Result<Vec<PathBuf>> {
             let manifest_path = package_path.join("Cargo.toml");
             if manifest_path.exists() {
                 manifests.push(manifest_path);
-            } else {
-                manifests.extend(discover_package_manifests(package_path).await?.into_iter());
             }
+            manifests.extend(discover_manifests(package_path).await?.into_iter());
         }
     }
     Ok(manifests)
 }
 
+/// Discovers and parses all Cargo.toml files that are packages (as opposed to being exclusively workspaces)
+pub async fn discover_packages(fs: Fs, path: PathBuf) -> Result<Vec<Package>> {
+    let manifest_paths = discover_manifests(path).await?;
+    read_packages(fs, manifest_paths).await
+}
+
 /// Parses a semver version number and adds additional error context when parsing fails.
 pub fn parse_version(manifest_path: &Path, version: &str) -> Result<Version, Error> {
     Version::parse(version)
@@ -219,26 +223,33 @@ fn read_dependencies(path: &Path, dependencies: &DepsSet) -> Result<Vec<PackageH
     Ok(result)
 }
 
-fn read_package(path: &Path, manifest_bytes: &[u8]) -> Result<Package> {
+/// Returns `Ok(None)` when the Cargo.toml is a workspace rather than a package
+fn read_package(path: &Path, manifest_bytes: &[u8]) -> Result<Option<Package>> {
     let manifest = Manifest::from_slice(manifest_bytes)
         .with_context(|| format!("failed to load package manifest for {:?}", path))?;
-    let package = manifest
-        .package
-        .ok_or_else(|| Error::InvalidManifest(path.into()))
-        .context("crate manifest doesn't have a `[package]` section")?;
-    let name = package.name;
-    let version = parse_version(path, &package.version)?;
-    let handle = PackageHandle { name, version };
-    let publish = match package.publish {
-        cargo_toml::Publish::Flag(true) => Publish::Allowed,
-        _ => Publish::NotAllowed,
-    };
-
-    let mut local_dependencies = BTreeSet::new();
-    local_dependencies.extend(read_dependencies(path, &manifest.dependencies)?.into_iter());
-    local_dependencies.extend(read_dependencies(path, &manifest.dev_dependencies)?.into_iter());
-    local_dependencies.extend(read_dependencies(path, &manifest.build_dependencies)?.into_iter());
-    Ok(Package::new(handle, path, local_dependencies, publish))
+    if let Some(package) = manifest.package {
+        let name = package.name;
+        let version = parse_version(path, &package.version)?;
+        let handle = PackageHandle { name, version };
+        let publish = match package.publish {
+            cargo_toml::Publish::Flag(true) => Publish::Allowed,
+            _ => Publish::NotAllowed,
+        };
+
+        let mut local_dependencies = BTreeSet::new();
+        local_dependencies.extend(read_dependencies(path, &manifest.dependencies)?.into_iter());
+        local_dependencies.extend(read_dependencies(path, &manifest.dev_dependencies)?.into_iter());
+        local_dependencies
+            .extend(read_dependencies(path, &manifest.build_dependencies)?.into_iter());
+        Ok(Some(Package::new(
+            handle,
+            path,
+            local_dependencies,
+            publish,
+        )))
+    } else {
+        Ok(None)
+    }
 }
 
 /// Validates that all of the publishable crates use consistent version numbers
@@ -275,7 +286,9 @@ pub async fn read_packages(fs: Fs, manifest_paths: Vec<PathBuf>) -> Result<Vec<P
     let mut result = Vec::new();
     for path in &manifest_paths {
         let contents: Vec<u8> = fs.read_file(path).await?;
-        result.push(read_package(path, &contents)?);
+        if let Some(package) = read_package(path, &contents)? {
+            result.push(package);
+        }
     }
     Ok(result)
 }
@@ -350,7 +363,9 @@ mod tests {
         "#;
         let path: PathBuf = "test/Cargo.toml".into();
 
-        let package = read_package(&path, manifest).expect("parse success");
+        let package = read_package(&path, manifest)
+            .expect("parse success")
+            .expect("is a package");
         assert_eq!("test", package.handle.name);
         assert_eq!(version("1.2.0-preview"), package.handle.version);
 

tools/ci-build/publisher/src/subcommand/claim_crate_names.rs

@@ -3,12 +3,10 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 use crate::fs::Fs;
-use crate::package::{discover_package_manifests, Error, PackageHandle};
+use crate::package::{discover_packages, PackageHandle, Publish};
 use crate::publish::{has_been_published_on_crates_io, publish};
 use crate::subcommand::publish::correct_owner;
 use crate::{cargo, SDK_REPO_NAME};
-use anyhow::Context;
-use cargo_toml::Manifest;
 use clap::Parser;
 use dialoguer::Confirm;
 use semver::Version;
@@ -79,20 +77,11 @@ async fn discover_publishable_crate_names(repository_root: &Path) -> anyhow::Res
         fs: Fs,
         path: PathBuf,
     ) -> anyhow::Result<HashSet<String>> {
-        let manifest_paths = discover_package_manifests(path).await?;
+        let packages = discover_packages(fs, path).await?;
         let mut publishable_package_names = HashSet::new();
-        for manifest_path in manifest_paths {
-            let contents: Vec<u8> = fs.read_file(&manifest_path).await?;
-            let manifest = Manifest::from_slice(&contents).with_context(|| {
-                format!("failed to load package manifest for {:?}", manifest_path)
-            })?;
-            let package = manifest
-                .package
-                .ok_or(Error::InvalidManifest(manifest_path))
-                .context("crate manifest doesn't have a `[package]` section")?;
-            let name = package.name;
-            if let cargo_toml::Publish::Flag(true) = package.publish {
-                publishable_package_names.insert(name);
+        for package in packages {
+            if let Publish::Allowed = package.publish {
+                publishable_package_names.insert(package.handle.name);
             }
         }
         Ok(publishable_package_names)

tools/ci-build/publisher/src/subcommand/fix_manifests.rs

@@ -10,7 +10,7 @@
 //! version numbers in addition to the dependency path.
 
 use crate::fs::Fs;
-use crate::package::{discover_package_manifests, parse_version};
+use crate::package::{discover_manifests, parse_version};
 use crate::SDK_REPO_NAME;
 use anyhow::{bail, Context, Result};
 use clap::Parser;
@@ -20,6 +20,7 @@ use std::collections::BTreeMap;
 use std::ffi::OsStr;
 use std::path::{Path, PathBuf};
 use toml::value::Table;
+use toml::Value;
 use tracing::info;
 
 mod validate;
@@ -55,7 +56,7 @@ pub async fn subcommand_fix_manifests(
         true => Mode::Check,
         false => Mode::Execute,
     };
-    let manifest_paths = discover_package_manifests(location.into()).await?;
+    let manifest_paths = discover_manifests(location.into()).await?;
     let mut manifests = read_manifests(Fs::Real, manifest_paths).await?;
     let versions = package_versions(&manifests)?;
 
@@ -91,6 +92,15 @@ fn package_versions(manifests: &[Manifest]) -> Result<BTreeMap<String, Version>>
             Some(package) => package,
             None => continue,
         };
+        // ignore non-publishable crates
+        if let Some(Value::Boolean(false)) = manifest
+            .metadata
+            .get("package")
+            .expect("checked above")
+            .get("publish")
+        {
+            continue;
+        }
         let name = package
             .get("name")
             .and_then(|name| name.as_str())

tools/ci-build/publisher/src/subcommand/generate_version_manifest.rs

@@ -4,7 +4,7 @@
  */
 
 use crate::fs::Fs;
-use crate::package::{discover_package_manifests, read_packages};
+use crate::package::discover_packages;
 use anyhow::{bail, Context, Result};
 use clap::Parser;
 use semver::Version;
@@ -70,10 +70,7 @@ pub async fn subcommand_generate_version_manifest(
         (None, Some(output_location)) => output_location,
         _ => bail!("Only one of `--location` or `--output-location` should be provided"),
     };
-    let manifests = discover_package_manifests(input_location.into())
-        .await
-        .context("discover package manifests")?;
-    let packages = read_packages(Fs::Real, manifests)
+    let packages = discover_packages(Fs::Real, input_location.into())
         .await
         .context("read packages")?;
 

tools/ci-build/sdk-versioner/Cargo.lock

@@ -15,6 +15,7 @@ opt-level = 0
 [dependencies]
 anyhow = "1.0"
 clap = { version = "~3.1.18", features = ["derive"] }
+pathdiff = "0.2.1"
 smithy-rs-tool-common = { version = "0.1", path = "../smithy-rs-tool-common" }
 toml_edit = { version = "0.19.6" }