Rfc30/compiletime benchmark to ci

.github/workflows/ci.yml

@@ -320,6 +320,30 @@ jobs:
       with:
         action: check-aws-sdk-examples
 
+  compiletime-benchmark:
+    runs-on: ubuntu-latest
+    name: Run Compiletime Benchmark
+    needs: generate
+    permissions:
+      contents: read
+      id-token: write
+      pull-requests: write
+    outputs:
+      compiletime-benchmark: ${{ steps.compiletime-benchmark.outputs.compiletime-benchmark }}
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        path: smithy-rs
+        ref: ${{ inputs.git_ref }}
+
+    - uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ env.rust_version }}
+
+    - name: run benchmark
+      id: run-compiletime-benchmark
+      run: bash smithy-rs/tools/compiletime-benchmark/script
+
   # Pseudo-job that depends on matrix jobs so that we don't have to enter
   # the myriad of test matrix combinations into GitHub's protected branch rules
   require-all:

tools/compiletime-benchmark/format.py

@@ -0,0 +1,51 @@
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+
+import itertools
+
+
+def main():
+    markdown = parser()
+
+    # write file
+    with open("/tmp/compiletime-benchmark.md", "w") as f:
+        f.write(markdown)
+        f.flush()
+        f.close()
+
+
+def parser() -> str:
+    # read file
+    f = open("/tmp/compiletime-benchmark.txt", "r").read()
+    iter = map(lambda x: x.split("END"), f.split("START"))
+    iter = itertools.chain.from_iterable(iter)
+
+    # I could've used a dataframe like pandas but this works.
+    markdown_rows = [
+        "| sdk name | dev | release | dev all features | release all features |",
+        "| -------- | --- | ------- | ---------------- | -------------------- |",
+
+    ]
+    for i in iter:
+        outputs = []
+        print(i)
+        for l in i.splitlines():
+            if not "+" in l:
+                outputs.append(l.replace("real", "").replace(" ", "", 16))
+
+        if len(outputs) != 6:
+            continue
+
+        row = f"|{'|'.join(outputs)}|".replace("||", "|")
+        markdown_rows.append(row)
+
+    markdown = "\n".join(markdown_rows)
+    print(markdown)
+
+    return markdown
+
+
+if __name__ == '__main__':
+    main()

tools/compiletime-benchmark/script

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+C_YELLOW='\033[1;33m'
+C_RESET='\033[0m'
+set -xe
+
+cd smithy-rs
+DIR=$PWD
+
+function log_time() {
+    cargo clean
+    echo "$1"
+    cargo build
+    { time $1; } &> tmp_time_log.txt
+    cat tmp_time_log.txt
+    cat tmp_time_log.txt | grep real >> /tmp/compiletime-benchmark.txt
+}
+
+function compile() {
+    cd $1 &&
+        export RUSTFLAGS="" &&
+        cargo build && # this is for downloading crates
+        cargo clean &&
+        log_time "cargo build" &&
+        log_time "cargo build --release" &&
+        export RUSTFLAGS="--cfg aws_sdk_unstable" &&
+        log_time "cargo build --all-features" &&
+        log_time "cargo build --release --all-features"
+}
+
+./gradlew :aws:sdk:assemble
+
+for variable in $(dir "aws/sdk/build/aws-sdk/sdk"); do
+    echo $variable
+    if [[ $variable != *"aws-"* ]]; then
+        echo "START" &>>/tmp/compiletime-benchmark.txt
+        echo "$variable" &>>/tmp/compiletime-benchmark.txt
+        compile "$DIR/aws/sdk/build/aws-sdk/sdk/$variable"
+        echo "END" &>> /tmp/compiletime-benchmark.txt
+    fi
+done
+
+cd $DIR
+python3 tools/compiletime-benchmark/format.py