Merge pull request #61 from Mark-Simulacrum/parallelize-s3

Parallelize S3 upload/download more

Author: pietroalbini

local/Dockerfile

@@ -29,5 +29,9 @@ ENV PATH=/root/.cargo/bin:$PATH
 COPY --from=mc /usr/bin/mc /usr/local/bin/mc
 RUN chmod 0755 /usr/local/bin/mc
 
+# The default is 20 but slightly bumping it here. Mostly to test that this
+# command works. In production we set a much higher value (150).
+RUN aws configure set default.s3.max_concurrent_requests 25
+
 ENTRYPOINT ["/src/local/setup.sh"]
 CMD ["/src/local/idle.sh"]

prod/Dockerfile

@@ -56,6 +56,9 @@ RUN curl https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustu
     /root/.cargo/bin/rustup toolchain remove stable
 ENV PATH=/root/.cargo/bin:$PATH
 
+# Roughly 2x number of CPU cores we have in production.
+RUN aws configure set default.s3.max_concurrent_requests 150
+
 COPY --from=build /tmp/source/target/release/promote-release /usr/local/bin/
 COPY prod/load-gpg-keys.sh /usr/local/bin/load-gpg-keys