Fix runtime crate versions (#3260)

## Motivation and Context
When the stable/unstable crate versions were split, this caused a
regression in determining the crate version during codegen. This
enhances our crate-version forwarding code to publish all the crate
versions. For impact, see codegen-client-test.

## Description
- Change the build artifact to be a JSON blob will all required
versions.

## Testing
- [ ] Audit diff

## Checklist
<!--- If a checkbox below is not applicable, then please DELETE it
rather than leaving it unchecked -->
- [ ] I have updated `CHANGELOG.next.toml` if I made changes to the
smithy-rs codegen or runtime crates
- [ ] I have updated `CHANGELOG.next.toml` if I made changes to the AWS
SDK, generated SDK code, or SDK runtime crates

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

Author: rcoh

aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCargoDependency.kt

@@ -10,7 +10,7 @@ import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig
 import software.amazon.smithy.rust.codegen.core.smithy.crateLocation
 
 fun RuntimeConfig.awsRuntimeCrate(name: String, features: Set<String> = setOf()): CargoDependency =
-    CargoDependency(name, awsRoot().crateLocation(null), features = features)
+    CargoDependency(name, awsRoot().crateLocation(name), features = features)
 
 object AwsCargoDependency {
     fun awsConfig(runtimeConfig: RuntimeConfig) = runtimeConfig.awsRuntimeCrate("aws-config")

buildSrc/src/main/kotlin/CrateSet.kt

@@ -79,4 +79,6 @@ object CrateSet {
     )
 
     val ENTIRE_SMITHY_RUNTIME = (AWS_SDK_SMITHY_RUNTIME + SERVER_SMITHY_RUNTIME).toSortedSet(compareBy { it.name })
+
+    val ALL_CRATES = AWS_SDK_RUNTIME + ENTIRE_SMITHY_RUNTIME
 }

codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/generators/config/ServiceConfigGenerator.kt

@@ -476,6 +476,7 @@ class ServiceConfigGenerator(
             docs("Apply test defaults to the builder")
             rustBlock("pub fn apply_test_defaults(&mut self) -> &mut Self") {
                 customizations.forEach { it.section(ServiceConfig.DefaultForTests("self"))(this) }
+                rustTemplate("self.behavior_version = #{Some}(crate::config::BehaviorVersion::latest());", *preludeScope)
                 rust("self")
             }
 

codegen-core/build.gradle.kts

@@ -53,18 +53,47 @@ fun gitCommitHash(): String {
 }
 
 val generateSmithyRuntimeCrateVersion by tasks.registering {
+    fun kv(key: String, value: String) = "\"$key\": \"$value\""
     // generate the version of the runtime to use as a resource.
     // this keeps us from having to manually change version numbers in multiple places
     val resourcesDir = "$buildDir/resources/main/software/amazon/smithy/rust/codegen/core"
     val versionFile = file("$resourcesDir/runtime-crate-version.txt")
     outputs.file(versionFile)
-    val crateVersion = project.properties["smithy.rs.runtime.crate.version"].toString()
-    inputs.property("crateVersion", crateVersion)
+    val stableCrateVersion = project.properties["smithy.rs.runtime.crate.stable.version"].toString()
+    val unstableCrateVersion = project.properties["smithy.rs.runtime.crate.unstable.version"].toString()
+    inputs.property("crateVersion", stableCrateVersion)
     // version format must be in sync with `software.amazon.smithy.rust.codegen.core.Version`
-    val version = "$crateVersion\n${gitCommitHash()}"
+    val version = StringBuilder().append("{")
+    version.append(kv("githash", gitCommitHash())).append(",")
+    version.append(kv("stableVersion", stableCrateVersion)).append(",")
+    version.append(kv("unstableVersion", unstableCrateVersion)).append(",")
+    // hack for internal build
+    val smithyStableCrates = listOf(
+        // AWS crates
+        "aws-config",
+        "aws-credential-types",
+        "aws-runtime",
+        "aws-runtime-api",
+        "aws-sigv4",
+        "aws-types",
+
+        // smithy crates
+        "aws-smithy-async",
+        "aws-smithy-runtime-api",
+        "aws-smithy-runtime",
+        "aws-smithy-types",
+    )
+
+    val runtimeCrates =
+        smithyStableCrates.joinToString(separator = ",", prefix = "{", postfix = "}") { crate ->
+            kv(crate, stableCrateVersion)
+        }
+
+    version.append(""""runtimeCrates": $runtimeCrates""").append("}")
+
     sourceSets.main.get().output.dir(resourcesDir)
     doLast {
-        versionFile.writeText(version)
+        versionFile.writeText(version.toString())
     }
 }
 

codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/Version.kt

@@ -6,29 +6,49 @@
 package software.amazon.smithy.rust.codegen.core
 
 import software.amazon.smithy.codegen.core.CodegenException
+import software.amazon.smithy.model.node.Node
 
 // generated as part of the build, see codegen-core/build.gradle.kts
 private const val VERSION_FILENAME = "runtime-crate-version.txt"
 
-data class Version(val fullVersion: String, val crateVersion: String) {
+data class Version(
+    val fullVersion: String,
+    val stableCrateVersion: String,
+    val unstableCrateVersion: String,
+    val crates: Map<String, String>,
+) {
     companion object {
         // Version must be in the "{smithy_rs_version}\n{git_commit_hash}" format
         fun parse(content: String): Version {
-            val lines = content.lines()
-            if (lines.size != 2) {
-                throw IllegalArgumentException("Invalid version format, it should contain `2` lines but contains `${lines.size}` line(s)")
-            }
-            return Version(lines.joinToString("-"), lines.first())
+            val node = Node.parse(content).expectObjectNode()
+            val githash = node.expectStringMember("githash").value
+            val stableVersion = node.expectStringMember("stableVersion").value
+            val unstableVersion = node.expectStringMember("unstableVersion").value
+            return Version(
+                "$stableVersion-$githash",
+                stableCrateVersion = stableVersion,
+                unstableCrateVersion = unstableVersion,
+                node.expectObjectMember("runtimeCrates").members.map {
+                    it.key.value to it.value.expectStringNode().value
+                }.toMap(),
+            )
         }
 
         // Returns full version in the "{smithy_rs_version}-{git_commit_hash}" format
         fun fullVersion(): String =
             fromDefaultResource().fullVersion
 
-        fun crateVersion(): String =
-            fromDefaultResource().crateVersion
+        fun stableCrateVersion(): String =
+            fromDefaultResource().stableCrateVersion
 
-        private fun fromDefaultResource(): Version = parse(
+        fun unstableCrateVersion(): String =
+            fromDefaultResource().unstableCrateVersion
+
+        fun crateVersion(crate: String): String {
+            val version = fromDefaultResource()
+            return version.crates[crate] ?: version.unstableCrateVersion
+        }
+        fun fromDefaultResource(): Version = parse(
             Version::class.java.getResource(VERSION_FILENAME)?.readText()
                 ?: throw CodegenException("$VERSION_FILENAME does not exist"),
         )

codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/smithy/RuntimeType.kt

@@ -40,19 +40,21 @@ data class RuntimeCrateLocation(val path: String?, val versions: CrateVersionMap
     }
 }
 
-fun RuntimeCrateLocation.crateLocation(crateName: String?): DependencyLocation {
-    val version = crateName.let { versions.map[crateName] } ?: versions.map[DEFAULT_KEY]
+fun RuntimeCrateLocation.crateLocation(crateName: String): DependencyLocation {
+    val version = crateName.let {
+        versions.map[crateName]
+    } ?: Version.crateVersion(crateName)
     return when (this.path) {
         // CratesIo needs an exact version. However, for local runtime crates we do not
         // provide a detected version unless the user explicitly sets one via the `versions` map.
-        null -> CratesIo(version ?: defaultRuntimeCrateVersion())
-        else -> Local(this.path, version)
+        null -> CratesIo(version)
+        else -> Local(this.path)
     }
 }
 
 fun defaultRuntimeCrateVersion(): String {
     try {
-        return Version.crateVersion()
+        return Version.stableCrateVersion()
     } catch (ex: Exception) {
         throw CodegenException("failed to get crate version which sets the default client-runtime version", ex)
     }
@@ -94,10 +96,6 @@ data class RuntimeConfig(
         }
     }
 
-    val crateSrcPrefix: String = cratePrefix.replace("-", "_")
-
-    fun runtimeCratesPath(): String? = runtimeCrateLocation.path
-
     fun smithyRuntimeCrate(
         runtimeCrateName: String,
         optional: Boolean = false,
@@ -324,7 +322,9 @@ data class RuntimeType(val path: String, val dependency: RustDependency? = null)
         fun smithyQuery(runtimeConfig: RuntimeConfig) = CargoDependency.smithyQuery(runtimeConfig).toType()
         fun smithyRuntime(runtimeConfig: RuntimeConfig) = CargoDependency.smithyRuntime(runtimeConfig).toType()
         fun smithyRuntimeApi(runtimeConfig: RuntimeConfig) = CargoDependency.smithyRuntimeApi(runtimeConfig).toType()
-        fun smithyRuntimeApiClient(runtimeConfig: RuntimeConfig) = CargoDependency.smithyRuntimeApiClient(runtimeConfig).toType()
+        fun smithyRuntimeApiClient(runtimeConfig: RuntimeConfig) =
+            CargoDependency.smithyRuntimeApiClient(runtimeConfig).toType()
+
         fun smithyTypes(runtimeConfig: RuntimeConfig) = CargoDependency.smithyTypes(runtimeConfig).toType()
         fun smithyXml(runtimeConfig: RuntimeConfig) = CargoDependency.smithyXml(runtimeConfig).toType()
         private fun smithyProtocolTest(runtimeConfig: RuntimeConfig) =

codegen-core/src/test/kotlin/software/amazon/smithy/rust/codegen/core/VersionTest.kt

@@ -21,7 +21,7 @@ class VersionTest {
     ) {
         val version = Version.parse(content)
         version.fullVersion shouldBe fullVersion
-        version.crateVersion shouldBe crateVersion
+        version.stableCrateVersion shouldBe crateVersion
     }
 
     @ParameterizedTest()
@@ -36,30 +36,15 @@ class VersionTest {
         @JvmStatic
         fun versionProvider() = listOf(
             Arguments.of(
-                "0.47.0\n0198d26096eb1af510ce24766c921ffc5e4c191e",
-                "0.47.0-0198d26096eb1af510ce24766c921ffc5e4c191e",
-                "0.47.0",
+                """{ "stableVersion": "1.0.1", "unstableVersion": "0.60.1","githash": "0198d26096eb1af510ce24766c921ffc5e4c191e", "runtimeCrates": {} }""",
+                "1.0.1-0198d26096eb1af510ce24766c921ffc5e4c191e",
+                "1.0.1",
             ),
             Arguments.of(
-                "release-2022-08-04\ndb48039065bec890ef387385773b37154b555b14",
+                """{ "unstableVersion": "0.60.1", "stableVersion": "release-2022-08-04", "githash": "db48039065bec890ef387385773b37154b555b14", "runtimeCrates": {} }""",
                 "release-2022-08-04-db48039065bec890ef387385773b37154b555b14",
                 "release-2022-08-04",
             ),
-            Arguments.of(
-                "0.30.0-alpha\na1dbbe2947de3c8bbbef9446eb442e298f83f200",
-                "0.30.0-alpha-a1dbbe2947de3c8bbbef9446eb442e298f83f200",
-                "0.30.0-alpha",
-            ),
-            Arguments.of(
-                "0.6-rc1.cargo\nc281800a185b34600b05f8b501a0322074184123",
-                "0.6-rc1.cargo-c281800a185b34600b05f8b501a0322074184123",
-                "0.6-rc1.cargo",
-            ),
-            Arguments.of(
-                "0.27.0-alpha.1\n643f2ee",
-                "0.27.0-alpha.1-643f2ee",
-                "0.27.0-alpha.1",
-            ),
         )
 
         @JvmStatic

codegen-core/src/test/kotlin/software/amazon/smithy/rust/codegen/core/smithy/RuntimeTypeTest.kt

@@ -11,8 +11,8 @@ import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.Arguments
 import org.junit.jupiter.params.provider.MethodSource
 import software.amazon.smithy.model.node.Node
+import software.amazon.smithy.rust.codegen.core.Version
 import software.amazon.smithy.rust.codegen.core.rustlang.CratesIo
-import software.amazon.smithy.rust.codegen.core.rustlang.DependencyLocation
 import software.amazon.smithy.rust.codegen.core.rustlang.Local
 import java.util.Optional
 
@@ -35,22 +35,20 @@ class RuntimeTypesTest {
         cfg.runtimeCrateLocation shouldBe RuntimeCrateLocation(null, CrateVersionMap(mapOf()))
     }
 
-    @ParameterizedTest
-    @MethodSource("runtimeCrateLocationProvider")
-    fun `runtimeCrateLocation provides dependency location`(
-        path: String?,
-        versions: CrateVersionMap,
-        crateName: String?,
-        expectedDependencyLocation: DependencyLocation,
-    ) {
-        val crateLoc = RuntimeCrateLocation(path, versions)
-        val depLoc = crateLoc.crateLocation(crateName)
-        depLoc shouldBe expectedDependencyLocation
+    @Test
+    fun `runtimeCrateLocation provides dependency location`() {
+        val crateLoc = RuntimeCrateLocation("/foo", CrateVersionMap(mapOf("aws-smithy-runtime-api" to "999.999")))
+        crateLoc.crateLocation("aws-smithy-runtime") shouldBe Local("/foo", null)
+        crateLoc.crateLocation("aws-smithy-runtime-api") shouldBe Local("/foo", null)
+        crateLoc.crateLocation("aws-smithy-http") shouldBe Local("/foo", null)
+
+        val crateLocVersioned = RuntimeCrateLocation(null, CrateVersionMap(mapOf("aws-smithy-runtime-api" to "999.999")))
+        crateLocVersioned.crateLocation("aws-smithy-runtime") shouldBe CratesIo(Version.stableCrateVersion())
+        crateLocVersioned.crateLocation("aws-smithy-runtime-api") shouldBe CratesIo("999.999")
+        crateLocVersioned.crateLocation("aws-smithy-http") shouldBe CratesIo(Version.unstableCrateVersion())
     }
 
     companion object {
-        @JvmStatic
-        private val defaultVersion = defaultRuntimeCrateVersion()
 
         @JvmStatic
         fun runtimeConfigProvider() = listOf(
@@ -90,97 +88,5 @@ class RuntimeTypesTest {
                 RuntimeCrateLocation("/path", CrateVersionMap(mapOf("a" to "1.0", "b" to "2.0"))),
             ),
         )
-
-        @JvmStatic
-        fun runtimeCrateLocationProvider() = listOf(
-            // If user specifies `relativePath` in `runtimeConfig`, then that always takes precedence over versions.
-            Arguments.of(
-                "/path",
-                mapOf<String, String>(),
-                null,
-                Local("/path"),
-            ),
-            Arguments.of(
-                "/path",
-                mapOf("a" to "1.0", "b" to "2.0"),
-                null,
-                Local("/path"),
-            ),
-            Arguments.of(
-                "/path",
-                mapOf("DEFAULT" to "0.1", "a" to "1.0", "b" to "2.0"),
-                null,
-                Local("/path", "0.1"),
-            ),
-
-            // User does not specify the versions object.
-            // The version number of the code-generator should be used as the version for all runtime crates.
-            Arguments.of(
-                null,
-                mapOf<String, String>(),
-                null,
-                CratesIo(defaultVersion),
-            ),
-            Arguments.of(
-                null,
-                mapOf<String, String>(),
-                "a",
-                CratesIo(defaultVersion),
-            ),
-
-            // User specifies versions object, setting explicit version numbers for some runtime crates.
-            // Then the rest of the runtime crates use the code-generator's version as their version.
-            Arguments.of(
-                null,
-                mapOf("a" to "1.0", "b" to "2.0"),
-                null,
-                CratesIo(defaultVersion),
-            ),
-            Arguments.of(
-                null,
-                mapOf("a" to "1.0", "b" to "2.0"),
-                "a",
-                CratesIo("1.0"),
-            ),
-            Arguments.of(
-                null,
-                mapOf("a" to "1.0", "b" to "2.0"),
-                "b",
-                CratesIo("2.0"),
-            ),
-            Arguments.of(
-                null,
-                mapOf("a" to "1.0", "b" to "2.0"),
-                "c",
-                CratesIo(defaultVersion),
-            ),
-
-            // User specifies versions object, setting DEFAULT and setting version numbers for some runtime crates.
-            // Then the specified version in DEFAULT is used for all runtime crates, except for those where the user specified a value for in the map.
-            Arguments.of(
-                null,
-                mapOf("DEFAULT" to "0.1", "a" to "1.0", "b" to "2.0"),
-                null,
-                CratesIo("0.1"),
-            ),
-            Arguments.of(
-                null,
-                mapOf("DEFAULT" to "0.1", "a" to "1.0", "b" to "2.0"),
-                "a",
-                CratesIo("1.0"),
-            ),
-            Arguments.of(
-                null,
-                mapOf("DEFAULT" to "0.1", "a" to "1.0", "b" to "2.0"),
-                "b",
-                CratesIo("2.0"),
-            ),
-            Arguments.of(
-                null,
-                mapOf("DEFAULT" to "0.1", "a" to "1.0", "b" to "2.0"),
-                "c",
-                CratesIo("0.1"),
-            ),
-        )
     }
 }

codegen-server/python/src/main/kotlin/software/amazon/smithy/rust/codegen/server/python/smithy/generators/PythonServerModuleGenerator.kt

@@ -237,7 +237,7 @@ class PythonServerModuleGenerator(
 
     // Render the codegeneration version as module attribute.
     private fun RustWriter.renderCodegenVersion() {
-        rust("""m.add("CODEGEN_VERSION", "${Version.crateVersion()}")?;""")
+        rust("""m.add("CODEGEN_VERSION", "${Version.stableCrateVersion()}")?;""")
     }
 
     // Convert to symbol and check the namespace to figure out where they should be imported from.