Skip to content

Commit 005b3e6

Browse files
sergeytimoshinsergeytimoshin
authored
refactor: redis-based job queue for prover (#1761)
* refactor: integrate Redis-based job queue * Improve error handling for transient vs permanent failures Distinguish between job_not_found errors (permanent, trigger immediate retry at higher level) and transient errors like 5xx HTTP codes (temporary, retry with exponential backoff).
1 parent 0eee5b8 commit 005b3e6

File tree

41 files changed

+2553
-851
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+2553
-851
lines changed

‎.github/workflows/forester-tests.yml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,21 @@ jobs:
7777
name: test-${{ matrix.test-name.name }}
7878
runs-on: warp-ubuntu-latest-x64-4x
7979
timeout-minutes: ${{ matrix.test-name.timeout }}
80+
81+
services:
82+
redis:
83+
image: redis:8.0.1
84+
ports:
85+
- 6379:6379
86+
options: >-
87+
--health-cmd "redis-cli ping"
88+
--health-interval 10s
89+
--health-timeout 5s
90+
--health-retries 5
91+
92+
env:
93+
REDIS_URL: redis://localhost:6379
94+
8095
steps:
8196
- uses: actions/checkout@v4
8297
- name: Setup and build

‎.github/workflows/js.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,20 @@ jobs:
2323
if: github.event.pull_request.draft == false
2424
runs-on: ubuntu-latest
2525

26+
services:
27+
redis:
28+
image: redis:8.0.1
29+
ports:
30+
- 6379:6379
31+
options: >-
32+
--health-cmd "redis-cli ping"
33+
--health-interval 10s
34+
--health-timeout 5s
35+
--health-retries 5
36+
37+
env:
38+
REDIS_URL: redis://localhost:6379
39+
2640
steps:
2741
- name: Checkout sources
2842
uses: actions/checkout@v4

‎.github/workflows/light-examples-tests.yml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,21 @@ jobs:
2929
if: github.event.pull_request.draft == false
3030
runs-on: ubuntu-latest
3131
timeout-minutes: 60
32+
33+
services:
34+
redis:
35+
image: redis:8.0.1
36+
ports:
37+
- 6379:6379
38+
options: >-
39+
--health-cmd "redis-cli ping"
40+
--health-interval 10s
41+
--health-timeout 5s
42+
--health-retries 5
43+
44+
env:
45+
REDIS_URL: redis://localhost:6379
46+
3247
strategy:
3348
matrix:
3449
include:

‎.github/workflows/light-system-programs-tests.yml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,21 @@ jobs:
3737
if: github.event.pull_request.draft == false
3838
runs-on: warp-ubuntu-latest-x64-4x
3939
timeout-minutes: 90
40+
41+
services:
42+
redis:
43+
image: redis:8.0.1
44+
ports:
45+
- 6379:6379
46+
options: >-
47+
--health-cmd "redis-cli ping"
48+
--health-interval 10s
49+
--health-timeout 5s
50+
--health-retries 5
51+
52+
env:
53+
REDIS_URL: redis://localhost:6379
54+
4055
strategy:
4156
matrix:
4257
include:

‎.github/workflows/rust.yml

Lines changed: 44 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ on:
44
push:
55
branches: [main]
66
paths:
7-
- ".cargo/**"
87
- "**/*.rs"
98
- "**/Cargo.*"
9+
- ".cargo/**"
1010
- "prover/client/**"
1111
- ".github/workflows/rust.yml"
1212
pull_request:
@@ -30,39 +30,56 @@ jobs:
3030
test:
3131
if: github.event.pull_request.draft == false
3232
runs-on: warp-ubuntu-latest-x64-4x
33-
outputs:
34-
timing: ${{ steps.test-timing.outputs.timing }}
33+
3534
strategy:
3635
matrix:
3736
group:
3837
- name: concurrent-merkle-tree
39-
packages: >-
40-
light-concurrent-merkle-tree
41-
light-batched-merkle-tree
38+
packages: light-concurrent-merkle-tree light-batched-merkle-tree
39+
test_cmd: |
40+
cargo test -p light-concurrent-merkle-tree
41+
RUST_LOG=light_prover_client=debug cargo test -p light-batched-merkle-tree --features test-only -- --test test_simulate_transactions
4242
- name: program-libs
43-
packages: >-
44-
aligned-sized
45-
light-bloom-filter
46-
light-hasher
47-
light-compressed-account
48-
light-account-checks
49-
light-verifier
50-
light-merkle-tree-metadata
51-
light-zero-copy
52-
light-hash-set
53-
light-indexed-merkle-tree
54-
light-batched-merkle-tree
43+
packages:
44+
aligned-sized light-bloom-filter light-hasher light-compressed-account light-account-checks \
45+
light-verifier light-merkle-tree-metadata light-zero-copy light-hash-set light-indexed-merkle-tree light-batched-merkle-tree
46+
test_cmd: |
47+
cargo test -p aligned-sized
48+
cargo test -p light-bloom-filter
49+
cargo test -p light-hasher
50+
cargo test -p light-compressed-account
51+
cargo test -p light-account-checks
52+
cargo test -p light-verifier
53+
cargo test -p light-merkle-tree-metadata
54+
cargo test -p light-zero-copy --features std
55+
cargo test -p light-hash-set
56+
cargo test -p light-indexed-merkle-tree
57+
cargo test -p light-batched-merkle-tree --features test-only -- --test test_e2e
5558
- name: sdk-libs
56-
packages: >-
57-
light-macros
58-
light-sdk
59-
light-program-test
60-
light-client
61-
light-batched-merkle-tree
59+
packages: light-macros light-sdk light-program-test light-client light-batched-merkle-tree
60+
test_cmd: |
61+
cargo test -p light-macros
62+
cargo test -p light-sdk
63+
cargo test -p light-program-test
64+
cargo test -p light-client
65+
cargo test -p light-batched-merkle-tree --features test-only -- --skip test_simulate_transactions --skip test_e2e
6266
63-
fail-fast: false
6467
name: Test ${{ matrix.group.name }}
6568

69+
services:
70+
redis:
71+
image: redis:8.0.1
72+
ports:
73+
- 6379:6379
74+
options: >-
75+
--health-cmd "redis-cli ping"
76+
--health-interval 10s
77+
--health-timeout 5s
78+
--health-retries 5
79+
80+
env:
81+
REDIS_URL: redis://localhost:6379
82+
6683
steps:
6784
- uses: actions/checkout@v4
6885

@@ -74,86 +91,7 @@ jobs:
7491
source ./scripts/devenv.sh
7592
npx nx build @lightprotocol/zk-compression-cli
7693
77-
- name: Run tests
78-
id: test-timing
94+
- name: Run tests for ${{ matrix.group.name }}
7995
run: |
8096
source ./scripts/devenv.sh
81-
{
82-
echo "Testing group: ${{ matrix.group.name }}"
83-
echo "Packages: ${{ matrix.group.packages }}"
84-
echo "Rust version: $(rustc --version)"
85-
} >> "$GITHUB_STEP_SUMMARY"
86-
87-
# Function to format time duration
88-
format_duration() {
89-
local duration="$1"
90-
local minutes=$((duration / 60))
91-
local seconds=$((duration % 60))
92-
echo "${minutes}m ${seconds}s"
93-
}
94-
95-
# Record group start time
96-
group_start=$(date +%s)
97-
98-
# Convert space-separated packages into array
99-
readarray -t packages <<< "$(echo "${{ matrix.group.packages }}" | tr ' ' '\n')"
100-
101-
# Test each package and measure time
102-
for pkg in "${packages[@]}"; do
103-
if [[ -n "$pkg" ]]; then # Skip empty lines
104-
echo "::group::Testing ${pkg}"
105-
start=$(date +%s)
106-
echo "${name}"
107-
echo "${{ matrix.group.name }}"
108-
109-
if [ "${pkg}" == "light-zero-copy" ]; then
110-
cargo test -p "${pkg}" --features std || exit 1
111-
elif [ "${pkg}" == "light-batched-merkle-tree" ]; then
112-
if [ "${{ matrix.group.name }}" == "sdk-libs" ]; then
113-
# execute simulate transactions test
114-
cargo test -p "${pkg}" --features test-only -- --skip test_simulate_transactions --skip test_e2e || exit 1
115-
elif [ "${{ matrix.group.name }}" == "program-libs" ]; then
116-
# execute e2e test
117-
cargo test -p "${pkg}" --features test-only -- --test test_e2e || exit 1
118-
else
119-
# execute all tests except test_simulate_transactions and test_e2e
120-
cargo test -p "${pkg}" --features test-only -- --test test_simulate_transactions || exit 1
121-
fi
122-
else
123-
cargo test -p "${pkg}" || exit 1
124-
fi
125-
126-
end=$(date +%s)
127-
duration=$((end - start))
128-
formatted_time=$(format_duration "$duration")
129-
echo "Package ${pkg} completed in ${formatted_time}"
130-
echo "::endgroup::"
131-
fi
132-
done
133-
134-
# Record and print group total time
135-
group_end=$(date +%s)
136-
group_duration=$((group_end - group_start))
137-
formatted_group_time=$(format_duration "$group_duration")
138-
139-
# Create timing report with simplified output
140-
echo "timing=${{ matrix.group.name }}:${formatted_group_time}" >> "$GITHUB_OUTPUT"
141-
echo "Group ${{ matrix.group.name }} total time: ${formatted_group_time}" >> "$GITHUB_STEP_SUMMARY"
142-
143-
collect-times:
144-
needs: test
145-
runs-on: ubuntu-latest
146-
if: always()
147-
steps:
148-
- name: Create timing summary
149-
run: |
150-
{
151-
echo "# Test Execution Times"
152-
echo "| Group | Time |"
153-
echo "|-------|------|"
154-
for timing in ${{ needs.test.outputs.timing }}; do
155-
group="${timing%%:*}"
156-
time="${timing#*:}"
157-
echo "| $group | $time |"
158-
done
159-
} >> "$GITHUB_STEP_SUMMARY"
97+
${{ matrix.group.test_cmd }}

‎Cargo.lock

Lines changed: 3 additions & 37 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎cli/.mocharc.json

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,7 @@
11
{
2-
"require": [
3-
"test/helpers/init.js",
4-
"ts-node/register"
5-
],
6-
"watch-extensions": [
7-
"ts"
8-
],
2+
"require": ["test/helpers/init.js", "ts-node/register"],
3+
"watch-extensions": ["ts"],
94
"recursive": true,
105
"reporter": "spec",
116
"timeout": 600000
12-
}
7+
}

0 commit comments

Comments
 (0)