From 58512c2be1b61f1bef705bf60a144588bd2ab6b1 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Fri, 21 Apr 2023 10:16:20 +0000
Subject: [PATCH 01/52] - add python packages for compiletime benchmark - add
 scripts

---
 tools/ci-build/Dockerfile              |  2 ++
 tools/ci-scripts/compiletime-benchmark | 30 +++++++++++++++++++++
 tools/compiletime-benchmark/format.py  | 36 ++++++++++++++++++++++++++
 3 files changed, 68 insertions(+)
 create mode 100644 tools/ci-scripts/compiletime-benchmark
 create mode 100644 tools/compiletime-benchmark/format.py

diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile
index 0e64f746a9f..1db1677d142 100644
--- a/tools/ci-build/Dockerfile
+++ b/tools/ci-build/Dockerfile
@@ -198,6 +198,8 @@ ENV PATH=/opt/cargo/bin:$PATH \
 # they will assume the tools are on the PATH, but if outside of the image, they will `cargo run` the tools.
 ENV SMITHY_RS_DOCKER_BUILD_IMAGE=1
 RUN pip3 install --no-cache-dir mypy==0.991
+# We are adding this for compile time benchmark
+RUN pip3 install pandas polars pyarrow
 WORKDIR /home/build
 COPY sanity-test /home/build/sanity-test
 RUN /home/build/sanity-test
diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
new file mode 100644
index 00000000000..50f99ccdff3
--- /dev/null
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+set -eux
+
+export TIMEFORMAT=%R
+function compile() {
+        cd $1 &&
+        export CARGORUSTFLAG="" &&
+        cargo build  &> /dev/null && cargo clean &&
+        time cargo build  &> /dev/null && cargo clean &&
+        time cargo build --release  &> /dev/null && cargo clean &&
+        export CARGORUSTFLAG="--cfg aws_sdk_unstable" &&
+        time cargo build --all-features  &> /dev/null && cargo clean &&
+        time cargo build --all-features --release  &> /dev/null && cargo clean
+}
+
+./gradlew :aws:sdk:assemble
+DIR=$PWD
+for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do
+    echo "$DIR/aws/sdk/build/aws-sdk/sdk/$variable";
+    if [[ $variable != *"aws-"* ]]; then
+        echo "$variable" &>> /tmp/compiletime-benchmark.txt
+        compile "$DIR/aws/sdk/build/aws-sdk/sdk/$variable"  &>> /tmp/compiletime-benchmark.txt
+    fi
+done
+
+python3 tools/compiletime-benchmark/format.py
diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py
new file mode 100644
index 00000000000..a4b1262a0cd
--- /dev/null
+++ b/tools/compiletime-benchmark/format.py
@@ -0,0 +1,36 @@
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+
+import polars as pl
+
+def main():
+    file = open("/tmp/compiletime-benchmark.txt", "r").read()
+    stack = []
+    idx = 0
+    hashmap = {}
+    for i in file.split("\n"):
+        if idx > 4:
+            idx = 0
+
+        if idx == 0:
+            hashmap["sdk"] = i
+        elif idx == 1:
+            hashmap["dev"] = i
+        elif idx == 2:
+            hashmap["release"] = i
+        elif idx == 3:
+            hashmap["dev-w-all-features"] = i
+        elif idx == 4:
+            hashmap["release-w-all-features"] = i
+
+        idx+=1
+
+    df = pl.DataFrame(stack).sort("sdk").select(pl.all().exclude("sdk").cast(pl.Float64))
+    dev = df.filter(pl.col("sdk") == "s3").to_dicts()[0]["dev"]
+    df = df.with_columns(pl.col(pl.Float64)/dev)
+    print(df)
+    df.to_pandas().to_markdown("./benchmark.md")
+
+main()

From 1d9cd835febea8577556d18e68aafe3eeb4b0da6 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Fri, 21 Apr 2023 11:55:28 +0000
Subject: [PATCH 02/52] remove polars

---
 tools/ci-build/Dockerfile             |  2 --
 tools/compiletime-benchmark/format.py | 42 +++++++++++++++++----------
 2 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile
index 1db1677d142..0e64f746a9f 100644
--- a/tools/ci-build/Dockerfile
+++ b/tools/ci-build/Dockerfile
@@ -198,8 +198,6 @@ ENV PATH=/opt/cargo/bin:$PATH \
 # they will assume the tools are on the PATH, but if outside of the image, they will `cargo run` the tools.
 ENV SMITHY_RS_DOCKER_BUILD_IMAGE=1
 RUN pip3 install --no-cache-dir mypy==0.991
-# We are adding this for compile time benchmark
-RUN pip3 install pandas polars pyarrow
 WORKDIR /home/build
 COPY sanity-test /home/build/sanity-test
 RUN /home/build/sanity-test
diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py
index a4b1262a0cd..74401e906fe 100644
--- a/tools/compiletime-benchmark/format.py
+++ b/tools/compiletime-benchmark/format.py
@@ -2,35 +2,45 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
-
-import polars as pl
-
 def main():
     file = open("/tmp/compiletime-benchmark.txt", "r").read()
     stack = []
     idx = 0
     hashmap = {}
     for i in file.split("\n"):
-        if idx > 4:
-            idx = 0
+        idx += 1
+        if idx > 5:
+            idx = 1
 
-        if idx == 0:
+        if idx == 1:
             hashmap["sdk"] = i
-        elif idx == 1:
+            continue
+
+        i = float(i)
+        if idx == 2:
             hashmap["dev"] = i
-        elif idx == 2:
-            hashmap["release"] = i
         elif idx == 3:
-            hashmap["dev-w-all-features"] = i
+            hashmap["release"] = i
         elif idx == 4:
+            hashmap["dev-w-all-features"] = i
+        elif idx == 5:
             hashmap["release-w-all-features"] = i
+            stack.append(hashmap)
+            hashmap = {}
 
-        idx+=1
+    header = "|".join(stack[0].keys())
+    header = f"|{header}|"
+    table = [header]
+    for hashmap in stack:
+        row = []
+        for i in hashmap.keys():
+            row.append(str(hashmap[i]))
+        inner = "|".join(row)
+        inner = f"|{inner}|"
+        table.append(inner)
 
-    df = pl.DataFrame(stack).sort("sdk").select(pl.all().exclude("sdk").cast(pl.Float64))
-    dev = df.filter(pl.col("sdk") == "s3").to_dicts()[0]["dev"]
-    df = df.with_columns(pl.col(pl.Float64)/dev)
-    print(df)
-    df.to_pandas().to_markdown("./benchmark.md")
+    markdown = "\n".join(table)
+    print(markdown)
+    # send markdown to somewhere
 
 main()

From 72cbd5dafebf40f4ae8e7b860a6199d92662c0ab Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Fri, 21 Apr 2023 12:10:31 +0000
Subject: [PATCH 03/52] update action

---
 .github/workflows/benchmark.yml | 18 ++++++++++++++++++
 .github/workflows/ci.yml        |  2 --
 2 files changed, 18 insertions(+), 2 deletions(-)
 create mode 100644 .github/workflows/benchmark.yml

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
new file mode 100644
index 00000000000..a9f268eb659
--- /dev/null
+++ b/.github/workflows/benchmark.yml
@@ -0,0 +1,18 @@
+name: compiletime benchmark
+run-name: compile time benchmark
+on:
+  push:
+    branches:
+      - main
+jobs:
+  compiletime-benchmark:
+    name: Compile time benchmark
+    runs-on: ${{ matrix.test.runner }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: smithy-rs
+          ref: ${{ inputs.git_ref }}
+      - name: compiletime-benchmark
+        run: bash tools/ci-scripts/compiletime-benchmark
\ No newline at end of file
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c36d5ba08ca..941298298d5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -159,8 +159,6 @@ jobs:
       with:
         action: ${{ matrix.test.action }}
 
-
-
   test-rust-windows:
     name: Rust Tests on Windows
     runs-on: windows-latest

From c97dfee4d87074fdaeab0db68b0c302c80ae6803 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <68596478+thomas-k-cameron@users.noreply.github.com>
Date: Sat, 22 Apr 2023 08:18:08 +0900
Subject: [PATCH 04/52] Update ci.yml

---
 .github/workflows/ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 336f02b6552..54ccda52ff9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -159,6 +159,8 @@ jobs:
       with:
         action: ${{ matrix.test.action }}
 
+
+
   test-rust-windows:
     name: Rust Tests on Windows
     runs-on: windows-latest

From be405e4a112ebc9d98a46794679dbf23a0a725a4 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Fri, 5 May 2023 07:12:41 +0000
Subject: [PATCH 05/52] updater

---
 .github/workflows/benchmark.yml | 18 ------------------
 .github/workflows/ci.yml        | 15 ++++++++++++++-
 2 files changed, 14 insertions(+), 19 deletions(-)
 delete mode 100644 .github/workflows/benchmark.yml

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
deleted file mode 100644
index a9f268eb659..00000000000
--- a/.github/workflows/benchmark.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-name: compiletime benchmark
-run-name: compile time benchmark
-on:
-  push:
-    branches:
-      - main
-jobs:
-  compiletime-benchmark:
-    name: Compile time benchmark
-    runs-on: ${{ matrix.test.runner }}
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          path: smithy-rs
-          ref: ${{ inputs.git_ref }}
-      - name: compiletime-benchmark
-        run: bash tools/ci-scripts/compiletime-benchmark
\ No newline at end of file
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 54ccda52ff9..76b468480a7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -296,6 +296,19 @@ jobs:
       shell: bash
       run: cross test --target ${{ matrix.target }} --manifest-path "aws/rust-runtime/Cargo.toml" ${{ matrix.test_aws_exclude }} --workspace
 
+  # compile time benchmark
+  compiletime-benchmark:
+    name: Compile time benchmark
+    runs-on: ${{ matrix.test.runner }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: smithy-rs
+          ref: ${{ inputs.git_ref }}
+      - name: compiletime-benchmark
+        run: bash tools/ci-scripts/compiletime-benchmark
+
   # This job is split out from the rest since it is not required to pass for merge
   check-sdk-examples:
     name: Check SDK Examples
@@ -331,4 +344,4 @@ jobs:
       # Pinned to commit hash of v1.2.2
       uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe
       with:
-        jobs: ${{ toJSON(needs) }}
+        jobs: ${{ toJSON(needs) }}
\ No newline at end of file

From 6d1f1dee148872a9bb4b0e86c913e12bd58b23a7 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <68596478+thomas-k-cameron@users.noreply.github.com>
Date: Fri, 5 May 2023 16:47:21 +0900
Subject: [PATCH 06/52] Update ci.yml

---
 .github/workflows/ci.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 76b468480a7..fa96463d65f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -149,6 +149,8 @@ jobs:
           runner: smithy_ubuntu-latest_8-core
         - action: check-aws-sdk-standalone-integration-tests
           runner: ubuntu-latest
+        - action: compiletime-benchmark
+          runner: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
       with:
@@ -344,4 +346,4 @@ jobs:
       # Pinned to commit hash of v1.2.2
       uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe
       with:
-        jobs: ${{ toJSON(needs) }}
\ No newline at end of file
+        jobs: ${{ toJSON(needs) }}

From 34eaadbd9d321c4571eb004d4e9ecd42cae84aa3 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Mon, 5 Jun 2023 18:37:10 +0000
Subject: [PATCH 07/52] update

---
 tools/ci-scripts/compiletime-benchmark | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 tools/ci-scripts/compiletime-benchmark

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
old mode 100644
new mode 100755

From 09672b09a9525397067557bd32ebd08de4f4f394 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Mon, 5 Jun 2023 18:54:06 +0000
Subject: [PATCH 08/52] update

---
 tools/ci-scripts/compiletime-benchmark | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index 50f99ccdff3..6a41f7398b9 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -17,6 +17,8 @@ function compile() {
         time cargo build --all-features --release  &> /dev/null && cargo clean
 }
 
+cd smithy-rs
+
 ./gradlew :aws:sdk:assemble
 DIR=$PWD
 for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do

From 7dd284ece38345920979737e7d9a8de3abaa2c9c Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Tue, 6 Jun 2023 03:10:08 +0000
Subject: [PATCH 09/52] update

---
 tools/ci-scripts/compiletime-benchmark |  2 +
 tools/compiletime-benchmark/format.py  | 55 +++++++++++---------------
 2 files changed, 25 insertions(+), 32 deletions(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index 6a41f7398b9..ed4c484e234 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -24,8 +24,10 @@ DIR=$PWD
 for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do
     echo "$DIR/aws/sdk/build/aws-sdk/sdk/$variable";
     if [[ $variable != *"aws-"* ]]; then
+        echo "START"
         echo "$variable" &>> /tmp/compiletime-benchmark.txt
         compile "$DIR/aws/sdk/build/aws-sdk/sdk/$variable"  &>> /tmp/compiletime-benchmark.txt
+        echo "END"
     fi
 done
 
diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py
index 74401e906fe..b59e6666738 100644
--- a/tools/compiletime-benchmark/format.py
+++ b/tools/compiletime-benchmark/format.py
@@ -2,45 +2,36 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
+
+import itertools
+
+
 def main():
     file = open("/tmp/compiletime-benchmark.txt", "r").read()
-    stack = []
+    iter = map(lambda x: x.split("END"), file.split("START"))
+    iter = itertools.chain.from_iterable(iter)
+    markdown = """
+    | sdk name | dev | release | dev all features | release all features |
+    | -------- | --- | ------- | ---------------- | -------------------- |
+    """
     idx = 0
-    hashmap = {}
-    for i in file.split("\n"):
+    for i in iter:
         idx += 1
-        if idx > 5:
-            idx = 1
-
-        if idx == 1:
-            hashmap["sdk"] = i
+        print("============")
+        print(idx)
+        print(i)
+        print("============")
+        if i == "\n":
             continue
+        lines = i.splitlines()
+        sdk_name = lines[0]
+        row = "\n|" + sdk_name + \
+            "|".join(map(lambda x: float(x), lines[1:])) + "|"
 
-        i = float(i)
-        if idx == 2:
-            hashmap["dev"] = i
-        elif idx == 3:
-            hashmap["release"] = i
-        elif idx == 4:
-            hashmap["dev-w-all-features"] = i
-        elif idx == 5:
-            hashmap["release-w-all-features"] = i
-            stack.append(hashmap)
-            hashmap = {}
-
-    header = "|".join(stack[0].keys())
-    header = f"|{header}|"
-    table = [header]
-    for hashmap in stack:
-        row = []
-        for i in hashmap.keys():
-            row.append(str(hashmap[i]))
-        inner = "|".join(row)
-        inner = f"|{inner}|"
-        table.append(inner)
-
-    markdown = "\n".join(table)
+        markdown += row
+        
     print(markdown)
     # send markdown to somewhere
 
+
 main()

From 232e331963810bcf3d3232c3359b7cb576e59b43 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Tue, 6 Jun 2023 05:05:12 +0000
Subject: [PATCH 10/52] update

---
 tools/ci-scripts/compiletime-benchmark | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index ed4c484e234..bac228ee38f 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -24,10 +24,10 @@ DIR=$PWD
 for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do
     echo "$DIR/aws/sdk/build/aws-sdk/sdk/$variable";
     if [[ $variable != *"aws-"* ]]; then
-        echo "START"
+        echo "START" &>> /tmp/compiletime-benchmark.txt
         echo "$variable" &>> /tmp/compiletime-benchmark.txt
         compile "$DIR/aws/sdk/build/aws-sdk/sdk/$variable"  &>> /tmp/compiletime-benchmark.txt
-        echo "END"
+        echo "END" &>> /tmp/compiletime-benchmark.txt
     fi
 done
 

From 7484dee14086bc62245f5eb51f0d459c083627d4 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Tue, 6 Jun 2023 13:14:21 +0000
Subject: [PATCH 11/52] FIX

---
 tools/ci-scripts/compiletime-benchmark | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index bac228ee38f..75bb085e4e5 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -31,4 +31,5 @@ for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do
     fi
 done
 
+cd $DIR
 python3 tools/compiletime-benchmark/format.py

From 413a0d2988145d42e9240e5a94c094832461b9ee Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Tue, 6 Jun 2023 17:21:58 +0000
Subject: [PATCH 12/52] update

---
 tools/ci-scripts/compiletime-benchmark | 23 ++++++++++++-----------
 tools/compiletime-benchmark/format.py  |  6 ++++--
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index 75bb085e4e5..1428183adf8 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -6,15 +6,16 @@
 set -eux
 
 export TIMEFORMAT=%R
+
 function compile() {
-        cd $1 &&
+    cd $1 &&
         export CARGORUSTFLAG="" &&
-        cargo build  &> /dev/null && cargo clean &&
-        time cargo build  &> /dev/null && cargo clean &&
-        time cargo build --release  &> /dev/null && cargo clean &&
+        cargo build &>/dev/null && cargo clean && # this is for downloading crates
+        \time --format="%e seconds" bash -c "cargo build  &> /dev/null" && cargo clean &&
+        \time --format="%e seconds" bash -c "cargo build --release  &> /dev/null" && cargo clean &&
         export CARGORUSTFLAG="--cfg aws_sdk_unstable" &&
-        time cargo build --all-features  &> /dev/null && cargo clean &&
-        time cargo build --all-features --release  &> /dev/null && cargo clean
+        \time --format="%e seconds" bash -c "cargo build --all-features &>/dev/null" && cargo clean &&
+        \time --format="%e seconds" bash -c "cargo build --all-features --release &>/dev/null" && cargo clean
 }
 
 cd smithy-rs
@@ -22,12 +23,12 @@ cd smithy-rs
 ./gradlew :aws:sdk:assemble
 DIR=$PWD
 for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do
-    echo "$DIR/aws/sdk/build/aws-sdk/sdk/$variable";
+    echo "$DIR/aws/sdk/build/aws-sdk/sdk/$variable"
     if [[ $variable != *"aws-"* ]]; then
-        echo "START" &>> /tmp/compiletime-benchmark.txt
-        echo "$variable" &>> /tmp/compiletime-benchmark.txt
-        compile "$DIR/aws/sdk/build/aws-sdk/sdk/$variable"  &>> /tmp/compiletime-benchmark.txt
-        echo "END" &>> /tmp/compiletime-benchmark.txt
+        echo "START" &>>/tmp/compiletime-benchmark.txt
+        echo "$variable" &>>/tmp/compiletime-benchmark.txt
+        compile "$DIR/aws/sdk/build/aws-sdk/sdk/$variable" &>>/tmp/compiletime-benchmark.txt
+        echo "END" &>>/tmp/compiletime-benchmark.txt
     fi
 done
 
diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py
index b59e6666738..4066cb048a7 100644
--- a/tools/compiletime-benchmark/format.py
+++ b/tools/compiletime-benchmark/format.py
@@ -21,9 +21,11 @@ def main():
         print(idx)
         print(i)
         print("============")
-        if i == "\n":
-            continue
+        
         lines = i.splitlines()
+        if len(lines) > 1:
+            continue
+
         sdk_name = lines[0]
         row = "\n|" + sdk_name + \
             "|".join(map(lambda x: float(x), lines[1:])) + "|"

From fd4fc8fc61063ec5f6dd44322793425ae3d6eca3 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Tue, 6 Jun 2023 17:54:04 +0000
Subject: [PATCH 13/52] FIX

---
 .github/workflows/ci.yml               | 15 ----------
 .github/workflows/pull-request-bot.yml | 39 +++++++++++++++++++++++++-
 2 files changed, 38 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cf474afb0d4..a3ce1bd51cb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -152,8 +152,6 @@ jobs:
           runner: smithy_ubuntu-latest_8-core
         - action: check-aws-sdk-standalone-integration-tests
           runner: ubuntu-latest
-        - action: compiletime-benchmark
-          runner: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
       with:
@@ -299,19 +297,6 @@ jobs:
       shell: bash
       run: cross test --target ${{ matrix.target }} --manifest-path "aws/rust-runtime/Cargo.toml" ${{ matrix.test_aws_exclude }} --workspace
 
-  # compile time benchmark
-  compiletime-benchmark:
-    name: Compile time benchmark
-    runs-on: ${{ matrix.test.runner }}
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          path: smithy-rs
-          ref: ${{ inputs.git_ref }}
-      - name: compiletime-benchmark
-        run: bash tools/ci-scripts/compiletime-benchmark
-
   # This job is split out from the rest since it is not required to pass for merge
   check-sdk-examples:
     name: Check SDK Examples
diff --git a/.github/workflows/pull-request-bot.yml b/.github/workflows/pull-request-bot.yml
index aafeaf773c4..6fd730eb497 100644
--- a/.github/workflows/pull-request-bot.yml
+++ b/.github/workflows/pull-request-bot.yml
@@ -135,10 +135,45 @@ jobs:
       run: |
         aws s3 cp target/doc "s3://${S3_BUCKET_NAME}/docs/${{ inputs.head_revision }}" --recursive
 
+  compiletime-benchmark:
+    runs-on: ubuntu-latest
+    name: Run Compiletime Benchmark
+    permissions:
+      id-token: write
+      contents: read
+      pull-requests: write
+    outputs:
+      compiletime-benchmark: ${{ steps.compiletime-benchmark.outputs.compiletime-benchmark }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/cache@v3
+      name: Gradle Cache
+      with:
+        path: |
+          ~/.gradle/caches
+          ~/.gradle/wrapper
+        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
+        restore-keys: |
+          ${{ runner.os }}-gradle-
+      # JDK is needed to generate code
+    - name: Set up JDK
+      uses: actions/setup-java@v3
+      with:
+        distribution: corretto
+        java-package: jdk
+        java-version: ${{ env.java_version }}
+    - uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ env.rust_version }}
+    - name: run benchmark
+      id: run-compiletime-benchmark
+      run: bash tools/ci-scripts/compiletime-benchmark && cat /tmp/compiletime-benchmark.txt >> "$GITHUB_OUTPUT"
+
   post-bot-comment:
     needs:
     - generate-diff
     - generate-doc-preview
+    - compiletime-benchmark
     runs-on: ubuntu-latest
     name: Post bot comment
     permissions:
@@ -164,6 +199,8 @@ jobs:
             issue_number: ${{ inputs.issue_number }},
             owner: context.repo.owner,
             repo: context.repo.repo,
-            body: '${{ steps.bot-messages.outputs.codegen-diff }}\n\n' +
+            body: '${{ steps.compiletime-benchmark.outputs.compiletime-benchmark }}\n\n' +
+              '${{ steps.bot-messages.outputs.codegen-diff }}\n\n' +
               '${{ needs.generate-doc-preview.outputs.bot-message }}\n\n'
+              
           })

From d4d37c7e3717432d4692dfdeee9d17b4edd71019 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Wed, 7 Jun 2023 11:34:24 +0000
Subject: [PATCH 14/52] update

---
 tools/ci-scripts/compiletime-benchmark | 3 ---
 tools/compiletime-benchmark/format.py  | 1 -
 2 files changed, 4 deletions(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index 1428183adf8..3b0a18cc7f4 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -18,12 +18,9 @@ function compile() {
         \time --format="%e seconds" bash -c "cargo build --all-features --release &>/dev/null" && cargo clean
 }
 
-cd smithy-rs
-
 ./gradlew :aws:sdk:assemble
 DIR=$PWD
 for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do
-    echo "$DIR/aws/sdk/build/aws-sdk/sdk/$variable"
     if [[ $variable != *"aws-"* ]]; then
         echo "START" &>>/tmp/compiletime-benchmark.txt
         echo "$variable" &>>/tmp/compiletime-benchmark.txt
diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py
index 4066cb048a7..239078a9372 100644
--- a/tools/compiletime-benchmark/format.py
+++ b/tools/compiletime-benchmark/format.py
@@ -33,7 +33,6 @@ def main():
         markdown += row
         
     print(markdown)
-    # send markdown to somewhere
 
 
 main()

From d7ce4d1985ea53ee83136e7fc640a65fec3ab676 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Wed, 7 Jun 2023 11:39:13 +0000
Subject: [PATCH 15/52] update

---
 tools/compiletime-benchmark/format.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py
index 239078a9372..f74e06bd11a 100644
--- a/tools/compiletime-benchmark/format.py
+++ b/tools/compiletime-benchmark/format.py
@@ -7,8 +7,8 @@
 
 
 def main():
-    file = open("/tmp/compiletime-benchmark.txt", "r").read()
-    iter = map(lambda x: x.split("END"), file.split("START"))
+    f = open("/tmp/compiletime-benchmark.txt", "r").read()
+    iter = map(lambda x: x.split("END"), f.split("START"))
     iter = itertools.chain.from_iterable(iter)
     markdown = """
     | sdk name | dev | release | dev all features | release all features |
@@ -21,7 +21,7 @@ def main():
         print(idx)
         print(i)
         print("============")
-        
+
         lines = i.splitlines()
         if len(lines) > 1:
             continue
@@ -31,8 +31,10 @@ def main():
             "|".join(map(lambda x: float(x), lines[1:])) + "|"
 
         markdown += row
-        
-    print(markdown)
 
+    print(markdown)
+    with open("/tmp/compiletime-benchmark.md", "w") as f:
+        f.write(markdown)
+        f.flush()
 
 main()

From b8faed50780a2efa0897b328723f18688fadcf63 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Wed, 7 Jun 2023 20:40:47 +0900
Subject: [PATCH 16/52] Update pull-request-bot.yml

---
 .github/workflows/pull-request-bot.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pull-request-bot.yml b/.github/workflows/pull-request-bot.yml
index 6fd730eb497..e07a79fe92a 100644
--- a/.github/workflows/pull-request-bot.yml
+++ b/.github/workflows/pull-request-bot.yml
@@ -167,7 +167,7 @@ jobs:
         toolchain: ${{ env.rust_version }}
     - name: run benchmark
       id: run-compiletime-benchmark
-      run: bash tools/ci-scripts/compiletime-benchmark && cat /tmp/compiletime-benchmark.txt >> "$GITHUB_OUTPUT"
+      run: bash tools/ci-scripts/compiletime-benchmark && cat /tmp/compiletime-benchmark.md >> "$GITHUB_OUTPUT"
 
   post-bot-comment:
     needs:

From f8862e196ced5eac67a505b3cbfe85df54de4d77 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Wed, 14 Jun 2023 19:03:59 +0000
Subject: [PATCH 17/52] fix

---
 tools/ci-scripts/compiletime-benchmark | 10 ++---
 tools/compiletime-benchmark/format.py  | 57 ++++++++++++++++++--------
 2 files changed, 45 insertions(+), 22 deletions(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index 3b0a18cc7f4..4fefd482830 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -5,17 +5,15 @@
 #
 set -eux
 
-export TIMEFORMAT=%R
-
 function compile() {
     cd $1 &&
         export CARGORUSTFLAG="" &&
         cargo build &>/dev/null && cargo clean && # this is for downloading crates
-        \time --format="%e seconds" bash -c "cargo build  &> /dev/null" && cargo clean &&
-        \time --format="%e seconds" bash -c "cargo build --release  &> /dev/null" && cargo clean &&
+        \time --format="%e" bash -c "cargo build  &> /dev/null" && cargo clean &&
+        \time --format="%e" bash -c "cargo build --release  &> /dev/null" && cargo clean &&
         export CARGORUSTFLAG="--cfg aws_sdk_unstable" &&
-        \time --format="%e seconds" bash -c "cargo build --all-features &>/dev/null" && cargo clean &&
-        \time --format="%e seconds" bash -c "cargo build --all-features --release &>/dev/null" && cargo clean
+        \time --format="%e" bash -c "cargo build --all-features &>/dev/null" && cargo clean &&
+        \time --format="%e" bash -c "cargo build --all-features --release &>/dev/null" && cargo clean
 }
 
 ./gradlew :aws:sdk:assemble
diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py
index f74e06bd11a..8cc81225872 100644
--- a/tools/compiletime-benchmark/format.py
+++ b/tools/compiletime-benchmark/format.py
@@ -7,34 +7,59 @@
 
 
 def main():
+    markdown = parser(read_file())
+    print(markdown)
+    # write file
+    with open("/tmp/compiletime-benchmark.md", "w") as f:
+        f.write(markdown)
+        f.flush()
+        f.close()
+
+
+def read_file() -> itertools.chain[str]:
+    # read file
     f = open("/tmp/compiletime-benchmark.txt", "r").read()
     iter = map(lambda x: x.split("END"), f.split("START"))
     iter = itertools.chain.from_iterable(iter)
+    return iter
+
+
+def parser(iter: itertools.chain[str]) -> str:
+    # I could've used a dataframe like pandas but this works.
     markdown = """
     | sdk name | dev | release | dev all features | release all features |
     | -------- | --- | ------- | ---------------- | -------------------- |
     """
-    idx = 0
+
     for i in iter:
-        idx += 1
-        print("============")
-        print(idx)
+        outputs = []
         print(i)
-        print("============")
+        for l in i.splitlines():
+            if not "+" in l:
+                outputs.append(l.replace(" seconds", ""))
 
-        lines = i.splitlines()
-        if len(lines) > 1:
+        if len(outputs) != 6:
             continue
-
-        sdk_name = lines[0]
-        row = "\n|" + sdk_name + \
-            "|".join(map(lambda x: float(x), lines[1:])) + "|"
+        outputs = outputs[1:]
+        sdk_name = outputs[0]
+        row = f"|{sdk_name}|" + \
+            "|".join(outputs[1:]) + "|"
 
         markdown += row
 
-    print(markdown)
-    with open("/tmp/compiletime-benchmark.md", "w") as f:
-        f.write(markdown)
-        f.flush()
+    return markdown
+
+
+def test():
+    s = """
+    | sdk name | dev | release | dev all features | release all features |
+    | -------- | --- | ------- | ---------------- | -------------------- |
+    |iam|201.92|217.43|215.10|187.71|
+    """.strip()
+    s2 = parser(read_file()).strip()
+    assert s2 == s, f"{s2} \n== \n{s}"
+
 
-main()
+if __name__ == '__main__':
+    test()
+    main()

From efd96f0491421face5abb0b8a99b671d91659625 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Wed, 14 Jun 2023 19:05:22 +0000
Subject: [PATCH 18/52] fix

---
 tools/compiletime-benchmark/format.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py
index 8cc81225872..89db67b3559 100644
--- a/tools/compiletime-benchmark/format.py
+++ b/tools/compiletime-benchmark/format.py
@@ -5,7 +5,6 @@
 
 import itertools
 
-
 def main():
     markdown = parser(read_file())
     print(markdown)
@@ -50,16 +49,5 @@ def parser(iter: itertools.chain[str]) -> str:
     return markdown
 
 
-def test():
-    s = """
-    | sdk name | dev | release | dev all features | release all features |
-    | -------- | --- | ------- | ---------------- | -------------------- |
-    |iam|201.92|217.43|215.10|187.71|
-    """.strip()
-    s2 = parser(read_file()).strip()
-    assert s2 == s, f"{s2} \n== \n{s}"
-
-
 if __name__ == '__main__':
-    test()
     main()

From 377aa449ee1e46c147009a26d19ed1b8ccd4d0b2 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Wed, 14 Jun 2023 19:11:14 +0000
Subject: [PATCH 19/52] update

---
 tools/ci-scripts/compiletime-benchmark-test | 34 +++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 tools/ci-scripts/compiletime-benchmark-test

diff --git a/tools/ci-scripts/compiletime-benchmark-test b/tools/ci-scripts/compiletime-benchmark-test
new file mode 100644
index 00000000000..38d39f15d79
--- /dev/null
+++ b/tools/ci-scripts/compiletime-benchmark-test
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+set -eux
+
+
+touch /tmp/compiletime-benchmark.txt
+
+echo "START
+iam
++ cd /mnt/rust/projects/smithy-rs/aws/sdk/build/aws-sdk/sdk/iam
++ export CARGORUSTFLAG=
++ CARGORUSTFLAG=
++ cargo build
++ cargo clean
++ time '--format=%e seconds' bash -c 'cargo build  &> /dev/null'
+201.92 seconds
++ cargo clean
++ time '--format=%e seconds' bash -c 'cargo build --release  &> /dev/null'
+217.43 seconds
++ cargo clean
++ export 'CARGORUSTFLAG=--cfg aws_sdk_unstable'
++ CARGORUSTFLAG='--cfg aws_sdk_unstable'
++ time '--format=%e seconds' bash -c 'cargo build --all-features &>/dev/null'
+215.10 seconds
++ cargo clean
++ time '--format=%e seconds' bash -c 'cargo build --all-features --release &>/dev/null'
+187.71 seconds
++ cargo clean
+END" >> /tmp/compiletime-benchmark.txt
+
+python3.9 tools/compiletime-benchmark/format.py
\ No newline at end of file

From 31eb747a1c51eb965dd4a4523fb379561a09380f Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Wed, 14 Jun 2023 19:12:15 +0000
Subject: [PATCH 20/52] asdf

---
 tools/ci-scripts/compiletime-benchmark-test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/ci-scripts/compiletime-benchmark-test b/tools/ci-scripts/compiletime-benchmark-test
index 38d39f15d79..d911692c434 100644
--- a/tools/ci-scripts/compiletime-benchmark-test
+++ b/tools/ci-scripts/compiletime-benchmark-test
@@ -31,4 +31,4 @@ iam
 + cargo clean
 END" >> /tmp/compiletime-benchmark.txt
 
-python3.9 tools/compiletime-benchmark/format.py
\ No newline at end of file
+python3 tools/compiletime-benchmark/format.py
\ No newline at end of file

From d4add322e673b7ba8ee095f6c6d53ebe0678c1f3 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Wed, 14 Jun 2023 19:13:48 +0000
Subject: [PATCH 21/52] fix

---
 tools/ci-scripts/compiletime-benchmark-test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 mode change 100644 => 100755 tools/ci-scripts/compiletime-benchmark-test

diff --git a/tools/ci-scripts/compiletime-benchmark-test b/tools/ci-scripts/compiletime-benchmark-test
old mode 100644
new mode 100755
index d911692c434..12e5c69c6a9
--- a/tools/ci-scripts/compiletime-benchmark-test
+++ b/tools/ci-scripts/compiletime-benchmark-test
@@ -31,4 +31,4 @@ iam
 + cargo clean
 END" >> /tmp/compiletime-benchmark.txt
 
-python3 tools/compiletime-benchmark/format.py
\ No newline at end of file
+python3 tools/compiletime-benchmark/format.py

From 1ead68d56f05e7fcce493b408277a4a85b6ed761 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Wed, 14 Jun 2023 19:25:08 +0000
Subject: [PATCH 22/52] update

---
 .github/workflows/ci.yml                                        | 2 ++
 .../{compiletime-benchmark-test => check-compiletime-benchmark} | 0
 2 files changed, 2 insertions(+)
 rename tools/ci-scripts/{compiletime-benchmark-test => check-compiletime-benchmark} (100%)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fa47463eabc..4bfc652e993 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -82,6 +82,8 @@ jobs:
       matrix:
         # These correspond to scripts in tools/ci-scripts that will be run in the Docker build image
         test:
+        - action: check-compiletime-benchmark
+          runner: ubuntu-latest
         - action: check-aws-sdk-adhoc-tests
           runner: ubuntu-latest
         # TODO(enableNewSmithyRuntime): Remove `check-aws-sdk-orchestrator-impl` when cleaning up middleware
diff --git a/tools/ci-scripts/compiletime-benchmark-test b/tools/ci-scripts/check-compiletime-benchmark
similarity index 100%
rename from tools/ci-scripts/compiletime-benchmark-test
rename to tools/ci-scripts/check-compiletime-benchmark

From 3ebefe42f753ea342d466b8240f44a5d44a89279 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Thu, 15 Jun 2023 02:37:27 +0000
Subject: [PATCH 23/52] fix

---
 .github/workflows/ci.yml                     |  3 +-
 tools/ci-scripts/check-compiletime-benchmark | 34 --------------------
 tools/ci-scripts/compiletime-benchmark       |  3 +-
 3 files changed, 4 insertions(+), 36 deletions(-)
 delete mode 100755 tools/ci-scripts/check-compiletime-benchmark

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 84dbd1780a3..9ad09c0995b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -82,7 +82,8 @@ jobs:
       matrix:
         # These correspond to scripts in tools/ci-scripts that will be run in the Docker build image
         test:
-        - action: check-compiletime-benchmark
+        # put this to test whether the compiletime benchmark would work or not
+        - action: compiletime-benchmark
           runner: ubuntu-latest
         - action: check-aws-sdk-adhoc-tests
           runner: ubuntu-latest
diff --git a/tools/ci-scripts/check-compiletime-benchmark b/tools/ci-scripts/check-compiletime-benchmark
deleted file mode 100755
index 12e5c69c6a9..00000000000
--- a/tools/ci-scripts/check-compiletime-benchmark
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-#
-# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-set -eux
-
-
-touch /tmp/compiletime-benchmark.txt
-
-echo "START
-iam
-+ cd /mnt/rust/projects/smithy-rs/aws/sdk/build/aws-sdk/sdk/iam
-+ export CARGORUSTFLAG=
-+ CARGORUSTFLAG=
-+ cargo build
-+ cargo clean
-+ time '--format=%e seconds' bash -c 'cargo build  &> /dev/null'
-201.92 seconds
-+ cargo clean
-+ time '--format=%e seconds' bash -c 'cargo build --release  &> /dev/null'
-217.43 seconds
-+ cargo clean
-+ export 'CARGORUSTFLAG=--cfg aws_sdk_unstable'
-+ CARGORUSTFLAG='--cfg aws_sdk_unstable'
-+ time '--format=%e seconds' bash -c 'cargo build --all-features &>/dev/null'
-215.10 seconds
-+ cargo clean
-+ time '--format=%e seconds' bash -c 'cargo build --all-features --release &>/dev/null'
-187.71 seconds
-+ cargo clean
-END" >> /tmp/compiletime-benchmark.txt
-
-python3 tools/compiletime-benchmark/format.py
diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index 4fefd482830..c10464f4a7d 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -3,7 +3,7 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
-set -eux
+set -e
 
 function compile() {
     cd $1 &&
@@ -24,6 +24,7 @@ for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do
         echo "$variable" &>>/tmp/compiletime-benchmark.txt
         compile "$DIR/aws/sdk/build/aws-sdk/sdk/$variable" &>>/tmp/compiletime-benchmark.txt
         echo "END" &>>/tmp/compiletime-benchmark.txt
+        break
     fi
 done
 

From 44ecdc5ac9a5b0f6188b6111d0f26d562f4e34c8 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Fri, 16 Jun 2023 08:33:31 +0000
Subject: [PATCH 24/52] update

---
 tools/ci-scripts/compiletime-benchmark | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index c10464f4a7d..8c7239e0a64 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -3,7 +3,12 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
-set -e
+C_YELLOW='\033[1;33m'
+C_RESET='\033[0m'
+set -ex
+
+cd smithy-rs
+DIR=$PWD
 
 function compile() {
     cd $1 &&
@@ -17,14 +22,13 @@ function compile() {
 }
 
 ./gradlew :aws:sdk:assemble
-DIR=$PWD
+
 for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do
     if [[ $variable != *"aws-"* ]]; then
         echo "START" &>>/tmp/compiletime-benchmark.txt
         echo "$variable" &>>/tmp/compiletime-benchmark.txt
         compile "$DIR/aws/sdk/build/aws-sdk/sdk/$variable" &>>/tmp/compiletime-benchmark.txt
         echo "END" &>>/tmp/compiletime-benchmark.txt
-        break
     fi
 done
 

From 3d22fd215c82e80b7c238c7e09b803851e96b4bf Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Fri, 16 Jun 2023 08:48:22 +0000
Subject: [PATCH 25/52] fix

---
 tools/ci-scripts/compiletime-benchmark | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index 8c7239e0a64..2719d68848c 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -10,15 +10,25 @@ set -ex
 cd smithy-rs
 DIR=$PWD
 
+function log_time() {
+    \time --format="%e" bash -c "$1 &> /dev/null" >> /tmp/compiletime-benchmark.txt
+}
+
 function compile() {
     cd $1 &&
         export CARGORUSTFLAG="" &&
-        cargo build &>/dev/null && cargo clean && # this is for downloading crates
-        \time --format="%e" bash -c "cargo build  &> /dev/null" && cargo clean &&
-        \time --format="%e" bash -c "cargo build --release  &> /dev/null" && cargo clean &&
+        cargo clean &&
+        cargo build &>/dev/null && 
+        cargo clean && # this is for downloading crates
+        log_time "cargo build" &&
+        cargo clean &&
+        log_time "cargo build --release" &&
+        cargo clean &&
         export CARGORUSTFLAG="--cfg aws_sdk_unstable" &&
-        \time --format="%e" bash -c "cargo build --all-features &>/dev/null" && cargo clean &&
-        \time --format="%e" bash -c "cargo build --all-features --release &>/dev/null" && cargo clean
+        log_time "cargo build --all-features" &&
+        cargo clean &&
+        log_time "cargo build --release --all-features" &&
+        cargo clean
 }
 
 ./gradlew :aws:sdk:assemble
@@ -32,5 +42,7 @@ for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do
     fi
 done
 
+echo /tmp/compiletime-benchmark.txt
+
 cd $DIR
 python3 tools/compiletime-benchmark/format.py

From 12b3e56d533f1ea3ebd5e015a3cb2bc6132ef758 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Fri, 16 Jun 2023 09:12:25 +0000
Subject: [PATCH 26/52] test

---
 tools/ci-scripts/compiletime-benchmark | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index 2719d68848c..cca8a7a6863 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -5,13 +5,13 @@
 #
 C_YELLOW='\033[1;33m'
 C_RESET='\033[0m'
-set -ex
+set -x
 
 cd smithy-rs
 DIR=$PWD
 
 function log_time() {
-    \time --format="%e" bash -c "$1 &> /dev/null" >> /tmp/compiletime-benchmark.txt
+    \time --format="%e" bash -c "$1 &> /dev/null" &>> /tmp/compiletime-benchmark.txt
 }
 
 function compile() {
@@ -38,7 +38,7 @@ for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do
         echo "START" &>>/tmp/compiletime-benchmark.txt
         echo "$variable" &>>/tmp/compiletime-benchmark.txt
         compile "$DIR/aws/sdk/build/aws-sdk/sdk/$variable" &>>/tmp/compiletime-benchmark.txt
-        echo "END" &>>/tmp/compiletime-benchmark.txt
+        echo "END" &>> /tmp/compiletime-benchmark.txt
     fi
 done
 

From 8e294f9ce3a40d0e6a86539b4703e1e3edacd38b Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Fri, 16 Jun 2023 09:15:16 +0000
Subject: [PATCH 27/52] asdf

---
 tools/ci-scripts/compiletime-benchmark | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index cca8a7a6863..bc2d8b29ede 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -37,7 +37,7 @@ for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do
     if [[ $variable != *"aws-"* ]]; then
         echo "START" &>>/tmp/compiletime-benchmark.txt
         echo "$variable" &>>/tmp/compiletime-benchmark.txt
-        compile "$DIR/aws/sdk/build/aws-sdk/sdk/$variable" &>>/tmp/compiletime-benchmark.txt
+        compile "$DIR/aws/sdk/build/aws-sdk/sdk/$variable"
         echo "END" &>> /tmp/compiletime-benchmark.txt
     fi
 done

From 0204c94385b3f4127692284a33bca45cde9b3293 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Fri, 16 Jun 2023 15:41:48 +0000
Subject: [PATCH 28/52] fix

---
 tools/compiletime-benchmark/format.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py
index 89db67b3559..977c2edd6ec 100644
--- a/tools/compiletime-benchmark/format.py
+++ b/tools/compiletime-benchmark/format.py
@@ -15,15 +15,12 @@ def main():
         f.close()
 
 
-def read_file() -> itertools.chain[str]:
+def parser() -> str:
     # read file
     f = open("/tmp/compiletime-benchmark.txt", "r").read()
     iter = map(lambda x: x.split("END"), f.split("START"))
     iter = itertools.chain.from_iterable(iter)
-    return iter
-
-
-def parser(iter: itertools.chain[str]) -> str:
+    
     # I could've used a dataframe like pandas but this works.
     markdown = """
     | sdk name | dev | release | dev all features | release all features |

From a0c0cf2f428008da12d33ccd431171eea7148f67 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Fri, 16 Jun 2023 15:43:25 +0000
Subject: [PATCH 29/52] fix

---
 tools/compiletime-benchmark/format.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py
index 977c2edd6ec..311d0eb3b13 100644
--- a/tools/compiletime-benchmark/format.py
+++ b/tools/compiletime-benchmark/format.py
@@ -6,7 +6,7 @@
 import itertools
 
 def main():
-    markdown = parser(read_file())
+    markdown = parser()
     print(markdown)
     # write file
     with open("/tmp/compiletime-benchmark.md", "w") as f:
@@ -20,7 +20,7 @@ def parser() -> str:
     f = open("/tmp/compiletime-benchmark.txt", "r").read()
     iter = map(lambda x: x.split("END"), f.split("START"))
     iter = itertools.chain.from_iterable(iter)
-    
+
     # I could've used a dataframe like pandas but this works.
     markdown = """
     | sdk name | dev | release | dev all features | release all features |

From cd25822f1c55ca2946bcd8ae632ffaae24e4ff85 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Fri, 16 Jun 2023 17:01:57 +0000
Subject: [PATCH 30/52] update

---
 tools/ci-scripts/compiletime-benchmark | 2 +-
 tools/compiletime-benchmark/format.py  | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index bc2d8b29ede..ddbe8f69a17 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -11,7 +11,7 @@ cd smithy-rs
 DIR=$PWD
 
 function log_time() {
-    \time --format="%e" bash -c "$1 &> /dev/null" &>> /tmp/compiletime-benchmark.txt
+    time "bash -c $1 &> /dev/null" | grep "real" &>> /tmp/compiletime-benchmark.txt
 }
 
 function compile() {
diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py
index 311d0eb3b13..585f0fe932d 100644
--- a/tools/compiletime-benchmark/format.py
+++ b/tools/compiletime-benchmark/format.py
@@ -32,10 +32,11 @@ def parser() -> str:
         print(i)
         for l in i.splitlines():
             if not "+" in l:
-                outputs.append(l.replace(" seconds", ""))
+                outputs.append(l.replace("real", "").replace(" ", ""))
 
         if len(outputs) != 6:
             continue
+        
         outputs = outputs[1:]
         sdk_name = outputs[0]
         row = f"|{sdk_name}|" + \

From 34356eccf299ff7416d39f3c4c577bbdeb5662c9 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Fri, 16 Jun 2023 17:18:21 +0000
Subject: [PATCH 31/52] asdf

---
 tools/ci-scripts/compiletime-benchmark | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index ddbe8f69a17..f16c1ba813d 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -11,7 +11,7 @@ cd smithy-rs
 DIR=$PWD
 
 function log_time() {
-    time "bash -c $1 &> /dev/null" | grep "real" &>> /tmp/compiletime-benchmark.txt
+    { time "bash -c $1 &> /dev/null"; } 2>&1 | grep real &>> /tmp/compiletime-benchmark.txt
 }
 
 function compile() {

From f4a230c2c47f9371cc6c0af5d37f358fe2de59cf Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Sat, 17 Jun 2023 07:02:58 +0000
Subject: [PATCH 32/52] fix

---
 tools/ci-scripts/compiletime-benchmark | 5 ++++-
 tools/compiletime-benchmark/format.py  | 6 ++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index f16c1ba813d..0fcf7bbb35c 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -11,7 +11,10 @@ cd smithy-rs
 DIR=$PWD
 
 function log_time() {
-    { time "bash -c $1 &> /dev/null"; } 2>&1 | grep real &>> /tmp/compiletime-benchmark.txt
+    time "bash -c $1 &> /dev/null" &> /tmp/temp-log.txt &&
+    cat /tmp/temp-log.txt 
+    real_time=$(tail /tmp/temp-log.txt | grep real )
+    echo $real_time >> /tmp/compiletime-benchmark.txt
 }
 
 function compile() {
diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py
index 585f0fe932d..7f08a1aca7d 100644
--- a/tools/compiletime-benchmark/format.py
+++ b/tools/compiletime-benchmark/format.py
@@ -5,6 +5,7 @@
 
 import itertools
 
+
 def main():
     markdown = parser()
     print(markdown)
@@ -26,7 +27,7 @@ def parser() -> str:
     | sdk name | dev | release | dev all features | release all features |
     | -------- | --- | ------- | ---------------- | -------------------- |
     """
-
+    markdown += "\n"
     for i in iter:
         outputs = []
         print(i)
@@ -36,13 +37,14 @@ def parser() -> str:
 
         if len(outputs) != 6:
             continue
-        
+
         outputs = outputs[1:]
         sdk_name = outputs[0]
         row = f"|{sdk_name}|" + \
             "|".join(outputs[1:]) + "|"
 
         markdown += row
+        markdown += "\n"
 
     return markdown
 

From 1ea1fc338882cf0218c8a7a02ca6e48084aa3c2f Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Sat, 17 Jun 2023 08:04:27 +0000
Subject: [PATCH 33/52] asdf

---
 tools/ci-scripts/compiletime-benchmark | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index 0fcf7bbb35c..7646f3bc79c 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -11,7 +11,9 @@ cd smithy-rs
 DIR=$PWD
 
 function log_time() {
-    time "bash -c $1 &> /dev/null" &> /tmp/temp-log.txt &&
+    rm /tmp/temp-log.txt && 
+    touch /tmp/temp-log.txt &&
+    { time "bash -c $1"; } &> /tmp/temp-log.txt &&
     cat /tmp/temp-log.txt 
     real_time=$(tail /tmp/temp-log.txt | grep real )
     echo $real_time >> /tmp/compiletime-benchmark.txt

From 3e6fd7e6994c863463da89b30eb4afad493be280 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Sat, 17 Jun 2023 08:14:41 +0000
Subject: [PATCH 34/52] dsfg

---
 tools/ci-scripts/compiletime-benchmark | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index 7646f3bc79c..ba0eca9fe16 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -11,12 +11,11 @@ cd smithy-rs
 DIR=$PWD
 
 function log_time() {
-    rm /tmp/temp-log.txt && 
-    touch /tmp/temp-log.txt &&
     { time "bash -c $1"; } &> /tmp/temp-log.txt &&
     cat /tmp/temp-log.txt 
     real_time=$(tail /tmp/temp-log.txt | grep real )
     echo $real_time >> /tmp/compiletime-benchmark.txt
+    rm /tmp/temp-log.txt 
 }
 
 function compile() {

From 094cea7193a2fd561ad12cb1da452012e3bcfc07 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Sat, 17 Jun 2023 10:22:10 +0000
Subject: [PATCH 35/52] asdf

---
 tools/ci-scripts/compiletime-benchmark | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index ba0eca9fe16..d5edec858e1 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -9,40 +9,38 @@ set -x
 
 cd smithy-rs
 DIR=$PWD
-
+TMP_LOG="/tmp/temp-log.txt"
 function log_time() {
-    { time "bash -c $1"; } &> /tmp/temp-log.txt &&
-    cat /tmp/temp-log.txt 
-    real_time=$(tail /tmp/temp-log.txt | grep real )
+    cargo clean
+    echo "$1"
+    touch $TMP_LOG
+    { time $1 &>$TMP_LOG; } &> $TMP_LOG
+    echo $(cat /tmp/temp-log.txt)
+    real_time=$(tail $TMP_LOG | grep real)
     echo $real_time >> /tmp/compiletime-benchmark.txt
-    rm /tmp/temp-log.txt 
 }
 
 function compile() {
     cd $1 &&
         export CARGORUSTFLAG="" &&
+        cargo build && # this is for downloading crates
         cargo clean &&
-        cargo build &>/dev/null && 
-        cargo clean && # this is for downloading crates
         log_time "cargo build" &&
-        cargo clean &&
         log_time "cargo build --release" &&
-        cargo clean &&
         export CARGORUSTFLAG="--cfg aws_sdk_unstable" &&
         log_time "cargo build --all-features" &&
-        cargo clean &&
-        log_time "cargo build --release --all-features" &&
-        cargo clean
+        log_time "cargo build --release --all-features"
 }
 
 ./gradlew :aws:sdk:assemble
 
 for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do
+    echo $variable
     if [[ $variable != *"aws-"* ]]; then
         echo "START" &>>/tmp/compiletime-benchmark.txt
         echo "$variable" &>>/tmp/compiletime-benchmark.txt
         compile "$DIR/aws/sdk/build/aws-sdk/sdk/$variable"
-        echo "END" &>> /tmp/compiletime-benchmark.txt
+        echo "END" &>>/tmp/compiletime-benchmark.txt
     fi
 done
 

From 18d74e19401132369898ce11bb02d4e18436ab93 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Sat, 17 Jun 2023 11:06:09 +0000
Subject: [PATCH 36/52] fix

---
 tools/ci-scripts/compiletime-benchmark | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index d5edec858e1..4e33237eaa8 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -5,19 +5,17 @@
 #
 C_YELLOW='\033[1;33m'
 C_RESET='\033[0m'
-set -x
+set -xe
 
 cd smithy-rs
 DIR=$PWD
-TMP_LOG="/tmp/temp-log.txt"
+
 function log_time() {
     cargo clean
     echo "$1"
-    touch $TMP_LOG
-    { time $1 &>$TMP_LOG; } &> $TMP_LOG
-    echo $(cat /tmp/temp-log.txt)
-    real_time=$(tail $TMP_LOG | grep real)
-    echo $real_time >> /tmp/compiletime-benchmark.txt
+    cargo build
+    { time $1; } &> tmp_time_log.txt
+    cat tmp_time_log.txt | grep real >> /tmp/compiletime-benchmark.txt
 }
 
 function compile() {
@@ -40,7 +38,7 @@ for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do
         echo "START" &>>/tmp/compiletime-benchmark.txt
         echo "$variable" &>>/tmp/compiletime-benchmark.txt
         compile "$DIR/aws/sdk/build/aws-sdk/sdk/$variable"
-        echo "END" &>>/tmp/compiletime-benchmark.txt
+        echo "END" &>> /tmp/compiletime-benchmark.txt
     fi
 done
 

From a89a23fbc98308c956ab4afc06e34a236b3bf7a5 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Sat, 17 Jun 2023 16:57:25 +0000
Subject: [PATCH 37/52] asdf

---
 tools/ci-scripts/compiletime-benchmark | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index 4e33237eaa8..a614f532801 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -15,6 +15,7 @@ function log_time() {
     echo "$1"
     cargo build
     { time $1; } &> tmp_time_log.txt
+    cat tmp_time_log.txt
     cat tmp_time_log.txt | grep real >> /tmp/compiletime-benchmark.txt
 }
 

From 8cd5e2045d37fe1f3e80a430829e42b56ab529d0 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Sat, 17 Jun 2023 17:25:42 +0000
Subject: [PATCH 38/52] fix

---
 .github/workflows/ci.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 333f8ffc9ff..0cd2c8877b7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -82,9 +82,6 @@ jobs:
       matrix:
         # These correspond to scripts in tools/ci-scripts that will be run in the Docker build image
         test:
-        # put this to test whether the compiletime benchmark would work or not
-        - action: compiletime-benchmark
-          runner: ubuntu-latest
         - action: check-aws-sdk-adhoc-tests
           runner: ubuntu-latest
         # TODO(enableNewSmithyRuntimeCleanup): Remove `check-aws-sdk-orchestrator-impl` when cleaning up middleware

From e1ff66ac8033bab6b5bc3a166b2c4b8f81731c6a Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Sun, 18 Jun 2023 03:07:05 +0000
Subject: [PATCH 39/52] fix

---
 tools/ci-scripts/compiletime-benchmark |  4 ++--
 tools/compiletime-benchmark/format.py  | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index a614f532801..1835a90a007 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -21,12 +21,12 @@ function log_time() {
 
 function compile() {
     cd $1 &&
-        export CARGORUSTFLAG="" &&
+        export RUSTFLAGS="" &&
         cargo build && # this is for downloading crates
         cargo clean &&
         log_time "cargo build" &&
         log_time "cargo build --release" &&
-        export CARGORUSTFLAG="--cfg aws_sdk_unstable" &&
+        export RUSTFLAGS="--cfg aws_sdk_unstable" &&
         log_time "cargo build --all-features" &&
         log_time "cargo build --release --all-features"
 }
diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py
index 7f08a1aca7d..d50fad59e4e 100644
--- a/tools/compiletime-benchmark/format.py
+++ b/tools/compiletime-benchmark/format.py
@@ -23,17 +23,17 @@ def parser() -> str:
     iter = itertools.chain.from_iterable(iter)
 
     # I could've used a dataframe like pandas but this works.
-    markdown = """
-    | sdk name | dev | release | dev all features | release all features |
-    | -------- | --- | ------- | ---------------- | -------------------- |
-    """
+    markdown = "| sdk name | dev | release | dev all features | release all features |"
+    markdown += "\n"
+    markdown += "| -------- | --- | ------- | ---------------- | -------------------- |"
+
     markdown += "\n"
     for i in iter:
         outputs = []
         print(i)
         for l in i.splitlines():
             if not "+" in l:
-                outputs.append(l.replace("real", "").replace(" ", ""))
+                outputs.append(l.replace("real", "").replace(" ", "", 16))
 
         if len(outputs) != 6:
             continue

From 548b139b9d3124d1ba52ed719585a688d485a97a Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Sun, 18 Jun 2023 03:09:48 +0000
Subject: [PATCH 40/52] asdf

---
 tools/ci-scripts/compiletime-benchmark | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/ci-scripts/compiletime-benchmark
index 1835a90a007..f05f4967347 100755
--- a/tools/ci-scripts/compiletime-benchmark
+++ b/tools/ci-scripts/compiletime-benchmark
@@ -43,7 +43,5 @@ for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do
     fi
 done
 
-echo /tmp/compiletime-benchmark.txt
-
 cd $DIR
 python3 tools/compiletime-benchmark/format.py

From d97e899715ca6ac6fe9ccb365f180b5bc44d3d9f Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Thu, 21 Dec 2023 04:00:38 +0900
Subject: [PATCH 41/52] Update pull-request-bot.yml

---
 .github/workflows/pull-request-bot.yml | 34 --------------------------
 1 file changed, 34 deletions(-)

diff --git a/.github/workflows/pull-request-bot.yml b/.github/workflows/pull-request-bot.yml
index 0b0fe4d3911..0b622629a2b 100644
--- a/.github/workflows/pull-request-bot.yml
+++ b/.github/workflows/pull-request-bot.yml
@@ -134,40 +134,6 @@ jobs:
       run: |
         aws s3 cp target/doc "s3://${S3_BUCKET_NAME}/docs/${{ inputs.head_revision }}" --recursive
 
-  compiletime-benchmark:
-    runs-on: ubuntu-latest
-    name: Run Compiletime Benchmark
-    permissions:
-      id-token: write
-      contents: read
-      pull-requests: write
-    outputs:
-      compiletime-benchmark: ${{ steps.compiletime-benchmark.outputs.compiletime-benchmark }}
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/cache@v3
-      name: Gradle Cache
-      with:
-        path: |
-          ~/.gradle/caches
-          ~/.gradle/wrapper
-        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
-        restore-keys: |
-          ${{ runner.os }}-gradle-
-      # JDK is needed to generate code
-    - name: Set up JDK
-      uses: actions/setup-java@v3
-      with:
-        distribution: corretto
-        java-package: jdk
-        java-version: ${{ env.java_version }}
-    - uses: dtolnay/rust-toolchain@master
-      with:
-        toolchain: ${{ env.rust_version }}
-    - name: run benchmark
-      id: run-compiletime-benchmark
-      run: bash tools/ci-scripts/compiletime-benchmark && cat /tmp/compiletime-benchmark.md >> "$GITHUB_OUTPUT"
-
   post-bot-comment:
     needs:
     - generate-diff

From 131b09712fe7ffe84ad8588adfd26c04c5cba538 Mon Sep 17 00:00:00 2001
From: Thomas K Cameron <thomas.k.cameron.1218@gmail.com>
Date: Thu, 21 Dec 2023 04:07:11 +0900
Subject: [PATCH 42/52] update

---
 .github/workflows/pull-request-bot.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/pull-request-bot.yml b/.github/workflows/pull-request-bot.yml
index 0b622629a2b..fabc2181881 100644
--- a/.github/workflows/pull-request-bot.yml
+++ b/.github/workflows/pull-request-bot.yml
@@ -138,7 +138,6 @@ jobs:
     needs:
     - generate-diff
     - generate-doc-preview
-    - compiletime-benchmark
     runs-on: ubuntu-latest
     name: Post bot comment
     permissions:
@@ -164,8 +163,7 @@ jobs:
             issue_number: ${{ inputs.issue_number }},
             owner: context.repo.owner,
             repo: context.repo.repo,
-            body: '${{ steps.compiletime-benchmark.outputs.compiletime-benchmark }}\n\n' +
-              '${{ steps.bot-messages.outputs.codegen-diff }}\n\n' +
+            body: '${{ steps.bot-messages.outputs.codegen-diff }}\n\n' +
               '${{ needs.generate-doc-preview.outputs.bot-message }}\n\n'
               
           })

From c14f06247d8efb2d30aaf16f3a3888a15f28eb40 Mon Sep 17 00:00:00 2001
From: Thomas K Cameron <thomas.k.cameron.1218@gmail.com>
Date: Thu, 21 Dec 2023 04:10:11 +0900
Subject: [PATCH 43/52] update

---
 .github/workflows/ci.yml               | 34 ++++++++++++++++++++++++++
 .github/workflows/pull-request-bot.yml |  3 +--
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4c3c6c4fe7c..3c4806d414b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -317,6 +317,40 @@ jobs:
       with:
         action: check-aws-sdk-examples
 
+  compiletime-benchmark:
+    runs-on: ubuntu-latest
+    name: Run Compiletime Benchmark
+    permissions:
+      id-token: write
+      contents: read
+      pull-requests: write
+    outputs:
+      compiletime-benchmark: ${{ steps.compiletime-benchmark.outputs.compiletime-benchmark }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/cache@v3
+      name: Gradle Cache
+      with:
+        path: |
+          ~/.gradle/caches
+          ~/.gradle/wrapper
+        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
+        restore-keys: |
+          ${{ runner.os }}-gradle-
+      # JDK is needed to generate code
+    - name: Set up JDK
+      uses: actions/setup-java@v3
+      with:
+        distribution: corretto
+        java-package: jdk
+        java-version: ${{ env.java_version }}
+    - uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ env.rust_version }}
+    - name: run benchmark
+      id: run-compiletime-benchmark
+      run: bash tools/ci-scripts/compiletime-benchmark && cat /tmp/compiletime-benchmark.md >> "$GITHUB_OUTPUT"
+
   # Pseudo-job that depends on matrix jobs so that we don't have to enter
   # the myriad of test matrix combinations into GitHub's protected branch rules
   require-all:
diff --git a/.github/workflows/pull-request-bot.yml b/.github/workflows/pull-request-bot.yml
index fabc2181881..40682864e09 100644
--- a/.github/workflows/pull-request-bot.yml
+++ b/.github/workflows/pull-request-bot.yml
@@ -164,6 +164,5 @@ jobs:
             owner: context.repo.owner,
             repo: context.repo.repo,
             body: '${{ steps.bot-messages.outputs.codegen-diff }}\n\n' +
-              '${{ needs.generate-doc-preview.outputs.bot-message }}\n\n'
-              
+              '${{ needs.generate-doc-preview.outputs.bot-message }}\n\n'              
           })

From e007493a1a4f693ea23f59a2eae726311550b3b5 Mon Sep 17 00:00:00 2001
From: Thomas K Cameron <thomas.k.cameron.1218@gmail.com>
Date: Thu, 21 Dec 2023 04:14:15 +0900
Subject: [PATCH 44/52] update

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3c4806d414b..97a343c8833 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -320,6 +320,7 @@ jobs:
   compiletime-benchmark:
     runs-on: ubuntu-latest
     name: Run Compiletime Benchmark
+    needs: generate
     permissions:
       id-token: write
       contents: read

From 5bb6b3cf59b9eff0813703d4d401898f4e6bee96 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Thu, 21 Dec 2023 04:15:31 +0900
Subject: [PATCH 45/52] Update pull-request-bot.yml

---
 .github/workflows/pull-request-bot.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pull-request-bot.yml b/.github/workflows/pull-request-bot.yml
index 40682864e09..ca679a3d2c7 100644
--- a/.github/workflows/pull-request-bot.yml
+++ b/.github/workflows/pull-request-bot.yml
@@ -164,5 +164,5 @@ jobs:
             owner: context.repo.owner,
             repo: context.repo.repo,
             body: '${{ steps.bot-messages.outputs.codegen-diff }}\n\n' +
-              '${{ needs.generate-doc-preview.outputs.bot-message }}\n\n'              
+              '${{ needs.generate-doc-preview.outputs.bot-message }}\n\n'
           })

From 2a2166e3fabf3d28745342c15beb3e44a2361728 Mon Sep 17 00:00:00 2001
From: Thomas K Cameron <thomas.k.cameron.1218@gmail.com>
Date: Thu, 21 Dec 2023 04:33:32 +0900
Subject: [PATCH 46/52] update

---
 .github/workflows/ci.yml | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 97a343c8833..a604b6415f6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -329,22 +329,10 @@ jobs:
       compiletime-benchmark: ${{ steps.compiletime-benchmark.outputs.compiletime-benchmark }}
     steps:
     - uses: actions/checkout@v3
-    - uses: actions/cache@v3
-      name: Gradle Cache
       with:
-        path: |
-          ~/.gradle/caches
-          ~/.gradle/wrapper
-        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
-        restore-keys: |
-          ${{ runner.os }}-gradle-
-      # JDK is needed to generate code
-    - name: Set up JDK
-      uses: actions/setup-java@v3
-      with:
-        distribution: corretto
-        java-package: jdk
-        java-version: ${{ env.java_version }}
+        path: smithy-rs
+        ref: ${{ inputs.git_ref }}
+
     - uses: dtolnay/rust-toolchain@master
       with:
         toolchain: ${{ env.rust_version }}

From e747e052536854c9921c1487383ade2a6f9b52a8 Mon Sep 17 00:00:00 2001
From: Thomas K Cameron <thomas.k.cameron.1218@gmail.com>
Date: Thu, 21 Dec 2023 04:51:21 +0900
Subject: [PATCH 47/52] update

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a604b6415f6..0653fd5c927 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -338,7 +338,7 @@ jobs:
         toolchain: ${{ env.rust_version }}
     - name: run benchmark
       id: run-compiletime-benchmark
-      run: bash tools/ci-scripts/compiletime-benchmark && cat /tmp/compiletime-benchmark.md >> "$GITHUB_OUTPUT"
+      run: bash tools/compiletime-benchmark && cat /tmp/compiletime-benchmark.md >> "$GITHUB_OUTPUT"
 
   # Pseudo-job that depends on matrix jobs so that we don't have to enter
   # the myriad of test matrix combinations into GitHub's protected branch rules

From 3a8fcdffb315128794f72145de7fdc73856307ed Mon Sep 17 00:00:00 2001
From: Thomas K Cameron <thomas.k.cameron.1218@gmail.com>
Date: Thu, 21 Dec 2023 05:19:05 +0900
Subject: [PATCH 48/52] fix

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0653fd5c927..0793a54c41f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -338,7 +338,7 @@ jobs:
         toolchain: ${{ env.rust_version }}
     - name: run benchmark
       id: run-compiletime-benchmark
-      run: bash tools/compiletime-benchmark && cat /tmp/compiletime-benchmark.md >> "$GITHUB_OUTPUT"
+      run: bash smithy-rs/tools/compiletime-benchmark && cat /tmp/compiletime-benchmark.md >> "$GITHUB_OUTPUT"
 
   # Pseudo-job that depends on matrix jobs so that we don't have to enter
   # the myriad of test matrix combinations into GitHub's protected branch rules

From 51314e2ab16cbee676e652e4333e2b4d89e36688 Mon Sep 17 00:00:00 2001
From: Thomas K Cameron <thomas.k.cameron.1218@gmail.com>
Date: Thu, 21 Dec 2023 05:46:37 +0900
Subject: [PATCH 49/52] update

---
 .github/workflows/ci.yml                                        | 2 +-
 .../compiletime-benchmark => compiletime-benchmark/script}      | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename tools/{ci-scripts/compiletime-benchmark => compiletime-benchmark/script} (100%)
 mode change 100755 => 100644

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0793a54c41f..a2e5d8f2315 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -338,7 +338,7 @@ jobs:
         toolchain: ${{ env.rust_version }}
     - name: run benchmark
       id: run-compiletime-benchmark
-      run: bash smithy-rs/tools/compiletime-benchmark && cat /tmp/compiletime-benchmark.md >> "$GITHUB_OUTPUT"
+      run: bash smithy-rs/tools/compiletime-benchmark/main && cat /tmp/compiletime-benchmark.md >> "$GITHUB_OUTPUT"
 
   # Pseudo-job that depends on matrix jobs so that we don't have to enter
   # the myriad of test matrix combinations into GitHub's protected branch rules
diff --git a/tools/ci-scripts/compiletime-benchmark b/tools/compiletime-benchmark/script
old mode 100755
new mode 100644
similarity index 100%
rename from tools/ci-scripts/compiletime-benchmark
rename to tools/compiletime-benchmark/script

From 362ab82fcffa252a148bb409d14db4d963f21c94 Mon Sep 17 00:00:00 2001
From: Thomas K Cameron <thomas.k.cameron.1218@gmail.com>
Date: Thu, 21 Dec 2023 10:02:24 +0900
Subject: [PATCH 50/52] update

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a2e5d8f2315..06c395ee92e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -338,7 +338,7 @@ jobs:
         toolchain: ${{ env.rust_version }}
     - name: run benchmark
       id: run-compiletime-benchmark
-      run: bash smithy-rs/tools/compiletime-benchmark/main && cat /tmp/compiletime-benchmark.md >> "$GITHUB_OUTPUT"
+      run: bash smithy-rs/tools/compiletime-benchmark/script && cat /tmp/compiletime-benchmark.md >> "$GITHUB_OUTPUT"
 
   # Pseudo-job that depends on matrix jobs so that we don't have to enter
   # the myriad of test matrix combinations into GitHub's protected branch rules

From c510d6e0128f98507914eb0f3b2c4ccd98b2cd95 Mon Sep 17 00:00:00 2001
From: Thomas K Cameron <thomas.k.cameron.1218@gmail.com>
Date: Thu, 21 Dec 2023 13:53:24 +0900
Subject: [PATCH 51/52] update

---
 .github/workflows/ci.yml              |  5 +++--
 tools/compiletime-benchmark/format.py | 20 +++++++++-----------
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 06c395ee92e..50f68feb71f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -322,8 +322,8 @@ jobs:
     name: Run Compiletime Benchmark
     needs: generate
     permissions:
-      id-token: write
       contents: read
+      id-token: write
       pull-requests: write
     outputs:
       compiletime-benchmark: ${{ steps.compiletime-benchmark.outputs.compiletime-benchmark }}
@@ -336,9 +336,10 @@ jobs:
     - uses: dtolnay/rust-toolchain@master
       with:
         toolchain: ${{ env.rust_version }}
+
     - name: run benchmark
       id: run-compiletime-benchmark
-      run: bash smithy-rs/tools/compiletime-benchmark/script && cat /tmp/compiletime-benchmark.md >> "$GITHUB_OUTPUT"
+      run: bash smithy-rs/tools/compiletime-benchmark/script
 
   # Pseudo-job that depends on matrix jobs so that we don't have to enter
   # the myriad of test matrix combinations into GitHub's protected branch rules
diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py
index d50fad59e4e..8565627f749 100644
--- a/tools/compiletime-benchmark/format.py
+++ b/tools/compiletime-benchmark/format.py
@@ -8,7 +8,7 @@
 
 def main():
     markdown = parser()
-    print(markdown)
+
     # write file
     with open("/tmp/compiletime-benchmark.md", "w") as f:
         f.write(markdown)
@@ -23,11 +23,11 @@ def parser() -> str:
     iter = itertools.chain.from_iterable(iter)
 
     # I could've used a dataframe like pandas but this works.
-    markdown = "| sdk name | dev | release | dev all features | release all features |"
-    markdown += "\n"
-    markdown += "| -------- | --- | ------- | ---------------- | -------------------- |"
+    markdown_rows = [
+        "| sdk name | dev | release | dev all features | release all features |",
+        "| -------- | --- | ------- | ---------------- | -------------------- |",
 
-    markdown += "\n"
+    ]
     for i in iter:
         outputs = []
         print(i)
@@ -38,13 +38,11 @@ def parser() -> str:
         if len(outputs) != 6:
             continue
 
-        outputs = outputs[1:]
-        sdk_name = outputs[0]
-        row = f"|{sdk_name}|" + \
-            "|".join(outputs[1:]) + "|"
+        row = f"|{'|'.join(outputs)}|"
+        markdown_rows.append(row)
 
-        markdown += row
-        markdown += "\n"
+    markdown = "\n".join(markdown_rows)
+    print(markdown)
 
     return markdown
 

From d46c39ea8610cbe75c04da97e75078d2cea2e8c2 Mon Sep 17 00:00:00 2001
From: Thomas Cameron <thomas.k.cameron.1218@gmail.com>
Date: Tue, 23 Jan 2024 12:15:45 +0900
Subject: [PATCH 52/52] Update format.py

---
 tools/compiletime-benchmark/format.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/compiletime-benchmark/format.py b/tools/compiletime-benchmark/format.py
index 8565627f749..1fffbac5fc2 100644
--- a/tools/compiletime-benchmark/format.py
+++ b/tools/compiletime-benchmark/format.py
@@ -38,7 +38,7 @@ def parser() -> str:
         if len(outputs) != 6:
             continue
 
-        row = f"|{'|'.join(outputs)}|"
+        row = f"|{'|'.join(outputs)}|".replace("||", "|")
         markdown_rows.append(row)
 
     markdown = "\n".join(markdown_rows)