Skip to content

Commit 80b8894

Browse files
sausageebors-libra
sausagee
authored and
bors-libra
committed
[CI] parallel CI build and test
Currently the end-to-end CI workflow takes 23 minutes to complete. This lengthens the PR land time as Homu lands them serially. When the CI workflow fails due to flaky tests and has to re-run, dev speed and experience suffers. The E2E (smoke) test job is the current critical path. This commit attempts to reduce the CI workflow time to 14 mintues, by parallelizing the build and test. - prefetch cargo crates - pre-build e2e test binaries - distribute e2e tests between containers Future improvements include parallelzing the unit test (ie the next critical path) and generating/storing/analyzing test artifacts. Closes: diem#3025 Approved by: davidiw
1 parent 73aa95d commit 80b8894

File tree

1 file changed

+210
-29
lines changed

1 file changed

+210
-29
lines changed

‎.circleci/config.yml

Lines changed: 210 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@ executors:
55
docker:
66
- image: circleci/rust:buster
77
resource_class: 2xlarge
8+
test-executor:
9+
docker:
10+
- image: circleci/rust:buster
11+
resource_class: xlarge
812
audit-executor:
913
docker:
1014
- image: circleci/rust:buster
@@ -79,6 +83,26 @@ commands:
7983
git diff-tree --no-commit-id --name-only -r "$commit" -- "*.Dockerfile";
8084
done
8185
)' >> $BASH_ENV
86+
save_cargo_package_cache:
87+
description: Save cargo package cache for subsequent jobs
88+
steps:
89+
- run:
90+
name: Save cargo package cache
91+
command: |
92+
mv /usr/local/cargo/git .
93+
mv /usr/local/cargo/registry .
94+
mv /usr/local/cargo/.package-cache .
95+
restore_cargo_package_cache:
96+
description: Restore Cargo package cache from prev job
97+
steps:
98+
- run:
99+
name: Check cargo package cache
100+
command: |
101+
mv git /usr/local/cargo/
102+
mv registry /usr/local/cargo/
103+
mv .package-cache /usr/local/cargo/
104+
ls -all /usr/local/cargo
105+
du -ssh /usr/local/cargo
82106
build_setup:
83107
steps:
84108
- checkout
@@ -93,53 +117,186 @@ commands:
93117
command: ./scripts/changed-files.sh
94118

95119
jobs:
96-
build:
97-
executor: build-executor
98-
parallelism: 5
120+
prefetch-crates:
121+
executor: test-executor
122+
description: Prefetch cargo crates for subsequent jobs.
123+
steps:
124+
- build_setup
125+
- run:
126+
name: Git Hooks and Checks
127+
command: ./scripts/git-checks.sh
128+
- run:
129+
name: Fetch workspace dependencies over network
130+
command: cargo fetch
131+
- save_cargo_package_cache
132+
- persist_to_workspace:
133+
root: /home/circleci/project
134+
paths:
135+
- git
136+
- registry
137+
- .package-cache
138+
lint:
139+
executor: test-executor
140+
description: Run Rust linting tools.
99141
steps:
100142
- build_setup
101143
- run:
102144
name: Git Hooks and Checks
103145
command: ./scripts/git-checks.sh
146+
- attach_workspace:
147+
at: /home/circleci/project
148+
- restore_cargo_package_cache
149+
- run:
150+
name: cargo lint
151+
command: cargo x lint
104152
- run:
105-
name: Linting
153+
name: cargo clippy
154+
command: cargo xclippy --workspace --all-targets
155+
- run:
156+
name: cargo fmt
157+
command: cargo xfmt --check
158+
- run:
159+
name: cargo guppy
106160
command: |
107-
[[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo x lint
108-
[[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo xclippy --workspace --all-targets
109-
[[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo xfmt --check
110-
[[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo install cargo-guppy --git http://github.com/calibra/cargo-guppy --rev 8b2bc45c0cd6323a7a2b8170ddad6d2a5b79047b
111-
[[ $CIRCLE_NODE_INDEX =~ [1234] ]] || [[ -z $(cargo guppy dups --target x86_64-unknown-linux-gnu --kind directthirdparty) ]]
161+
cargo install cargo-guppy \
162+
--git http://github.com/calibra/cargo-guppy \
163+
--rev 8b2bc45c0cd6323a7a2b8170ddad6d2a5b79047b
164+
[[ -z $(cargo guppy dups --target x86_64-unknown-linux-gnu \
165+
--kind directthirdparty) ]]
166+
build-dev:
167+
executor: build-executor
168+
description: Development Build
169+
steps:
170+
- build_setup
171+
- attach_workspace:
172+
at: /home/circleci/project
173+
- restore_cargo_package_cache
174+
- run:
175+
command: RUST_BACKTRACE=1 cargo build -j 16
176+
- run:
177+
command: RUST_BACKTRACE=1 cargo build -j 16 -p libra-swarm
178+
- run:
179+
command: RUST_BACKTRACE=1 cargo build -j 16 -p cluster-test
180+
- run:
181+
command: RUST_BACKTRACE=1 cargo build -j 16 -p libra-fuzzer
112182
- run:
113-
name: Build Release
183+
command: RUST_BACKTRACE=1 cargo build -j 16 -p language_benchmarks
184+
- run:
185+
command: RUST_BACKTRACE=1 cargo build -j 16 -p cost-synthesis
186+
- run:
187+
command: RUST_BACKTRACE=1 cargo build -j 16 -p test-generation
188+
- build_teardown
189+
build-release:
190+
executor: build-executor
191+
description: Release Build
192+
steps:
193+
- build_setup
194+
- attach_workspace:
195+
at: /home/circleci/project
196+
- restore_cargo_package_cache
197+
- run:
198+
name: Build release
199+
command: RUST_BACKTRACE=1 cargo build -j 16 --release
200+
- build_teardown
201+
build-e2e-test:
202+
executor: build-executor
203+
description: Generate a list of E2E test targets to be distributed.
204+
steps:
205+
- build_setup
206+
- attach_workspace:
207+
at: /home/circleci/project
208+
- restore_cargo_package_cache
209+
- run:
210+
name: Find all e2e tests
114211
command: |
115-
[[ $CIRCLE_NODE_INDEX =~ [0234] ]] || RUST_BACKTRACE=1 cargo build -j 16 --release
212+
cargo x test --package testsuite -- --list | grep "::" | \
213+
sed 's/: .*$//' > e2e_tests
214+
cat e2e_tests
215+
- persist_to_workspace:
216+
root: /home/circleci/project
217+
paths:
218+
- e2e_tests
219+
- target/debug/deps/libratest*
220+
run-e2e-test:
221+
executor: build-executor
222+
parallelism: 8
223+
description: Run E2E tests in parallel. Each container runs a subset of
224+
test targets.
225+
steps:
226+
- build_setup
227+
- attach_workspace:
228+
at: /home/circleci/project
116229
- run:
117-
name: Build Dev
230+
name: Determine test targets for this container.
231+
# NOTE Currently the tests are distributed by name order. Once test
232+
# metadata is enabled, the tests can be distributed by run time to
233+
# speed up this job.
118234
command: |
119-
[[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16
120-
[[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p libra-swarm
121-
[[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p cluster-test
122-
[[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p libra-fuzzer
123-
[[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p language_benchmarks
124-
[[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p cost-synthesis
125-
[[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p test-generation
235+
ls -all target/debug/*
236+
echo -e "Found $(wc -l e2e_tests) tests."
237+
cat e2e_tests | circleci tests split > /tmp/tests_to_run
238+
echo -e "This runner will run these tests\n$(cat /tmp/tests_to_run)"
126239
- run:
127-
name: Run All Non Flaky Unit Tests
240+
name: Run E2E tests
128241
command: |
129-
[[ $CIRCLE_NODE_INDEX =~ [0134] ]] || RUST_BACKTRACE=1 $CI_TIMEOUT cargo test --all-features --workspace --exclude libra-node --exclude libra-crypto --exclude testsuite --exclude consensus
242+
set -x
243+
libratest=$(find target/debug/deps/ -name "libratest*" -executable)
244+
for target in $(cat /tmp/tests_to_run) ; do
245+
if [ ! -z "$libratest" ]; then
246+
RUST_BACKTRACE=1 $CI_TIMEOUT $libratest $target \
247+
--test-threads 1 --exact --nocapture
248+
else
249+
RUST_BACKTRACE=1 $CI_TIMEOUT cargo x test $target \
250+
--package testsuite -- --test-threads 1 --exact --nocapture
251+
fi
252+
done
253+
run-unit-test:
254+
executor: build-executor
255+
description: Run all unit tests, excluding E2E and flaky tests that are
256+
explicitly ignored.
257+
steps:
258+
- build_setup
259+
- attach_workspace:
260+
at: /home/circleci/project
261+
- restore_cargo_package_cache
130262
- run:
131-
name: Run Cryptography Unit Tests with the formally verified backend
263+
name: Run all unit tests
132264
command: |
133-
[[ $CIRCLE_NODE_INDEX =~ [0134] ]] || ( RUST_BACKTRACE=1 cd crypto/crypto && $CI_TIMEOUT cargo test --features='std fiat_u64_backend fuzzing' --no-default-features )
265+
RUST_BACKTRACE=1 $CI_TIMEOUT cargo test \
266+
--all-features \
267+
--workspace \
268+
--exclude libra-node \
269+
--exclude libra-crypto \
270+
--exclude testsuite \
271+
--exclude consensus
272+
run-crypto-unit-test:
273+
executor: test-executor
274+
description: Run crypto unit tests
275+
steps:
276+
- build_setup
277+
- attach_workspace:
278+
at: /home/circleci/project
279+
- restore_cargo_package_cache
134280
- run:
135-
name: Run All End to End Tests
281+
name: Run crypto unit tests
136282
command: |
137-
[[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CI_TIMEOUT cargo x test --package testsuite -- --test-threads 1
283+
cd crypto/crypto && \
284+
RUST_BACKTRACE=1 cargo test \
285+
--features='std fiat_u64_backend fuzzing' \
286+
--no-default-features
287+
run-flaky-unit-test:
288+
executor: test-executor
289+
description: Run a list of known flaky tests.
290+
steps:
291+
- build_setup
292+
- attach_workspace:
293+
at: /home/circleci/project
294+
- restore_cargo_package_cache
138295
- run:
139-
name: Run Quarantined Unit Tests 3 (consensus) times
296+
name: Run flaky tests
140297
command: |
141-
[[ $CIRCLE_NODE_INDEX =~ [0123] ]] || timeout 20m ./scripts/run_quarantined.sh -c consensus -r 3 -f
142-
- build_teardown
298+
RUST_BACKTRACE=1 $CI_TIMEOUT \
299+
./scripts/run_quarantined.sh -c consensus -r 3 -f
143300
validate-cluster-test-dockerfile:
144301
description: Validate that committed docker files for cluster test are up to date
145302
executor: build-executor
@@ -233,10 +390,34 @@ jobs:
233390
workflows:
234391
commit-workflow:
235392
jobs:
236-
- build
237393
- build-docker
238394
- validate-cluster-test-dockerfile
239395
- terraform
396+
- prefetch-crates
397+
- lint:
398+
requires:
399+
- prefetch-crates
400+
- build-dev:
401+
requires:
402+
- prefetch-crates
403+
- build-release:
404+
requires:
405+
- prefetch-crates
406+
- build-e2e-test:
407+
requires:
408+
- prefetch-crates
409+
- run-e2e-test:
410+
requires:
411+
- build-e2e-test
412+
- run-unit-test:
413+
requires:
414+
- prefetch-crates
415+
- run-crypto-unit-test:
416+
requires:
417+
- prefetch-crates
418+
- run-flaky-unit-test:
419+
requires:
420+
- prefetch-crates
240421

241422
scheduled-workflow:
242423
triggers:

0 commit comments

Comments
 (0)