Skip to content

Commit 7ee46a7

Browse files
jamesobjamesob
committed
contrib: add script to demo/test assumeutxo
Add the script to the shellcheck exception list since the quoted variables rule needs to be violated in order to get bitcoind to pick up on $CHAIN_HACK_FLAGS.
1 parent 42cae39 commit 7ee46a7

File tree

2 files changed

+206
-2
lines changed

2 files changed

+206
-2
lines changed

‎contrib/devtools/test_utxo_snapshots.sh

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
#!/usr/bin/env bash
2+
# Demonstrate the creation and usage of UTXO snapshots.
3+
#
4+
# A server node starts up, IBDs up to a certain height, then generates a UTXO
5+
# snapshot at that point.
6+
#
7+
# The server then downloads more blocks (to create a diff from the snapshot).
8+
#
9+
# We bring a client up, load the UTXO snapshot, and we show the client sync to
10+
# the "network tip" and then start a background validation of the snapshot it
11+
# loaded. We see the background validation chainstate removed after validation
12+
# completes.
13+
#
14+
15+
export LC_ALL=C
16+
set -e
17+
18+
BASE_HEIGHT=${1:-30000}
19+
INCREMENTAL_HEIGHT=20000
20+
FINAL_HEIGHT=$(($BASE_HEIGHT + $INCREMENTAL_HEIGHT))
21+
22+
SERVER_DATADIR="$(pwd)/utxodemo-data-server-$BASE_HEIGHT"
23+
CLIENT_DATADIR="$(pwd)/utxodemo-data-client-$BASE_HEIGHT"
24+
UTXO_DAT_FILE="$(pwd)/utxo.$BASE_HEIGHT.dat"
25+
26+
# Chosen to try to not interfere with any running bitcoind processes.
27+
SERVER_PORT=8633
28+
SERVER_RPC_PORT=8632
29+
30+
CLIENT_PORT=8733
31+
CLIENT_RPC_PORT=8732
32+
33+
SERVER_PORTS="-port=${SERVER_PORT} -rpcport=${SERVER_RPC_PORT}"
34+
CLIENT_PORTS="-port=${CLIENT_PORT} -rpcport=${CLIENT_RPC_PORT}"
35+
36+
# Ensure the client exercises all indexes to test that snapshot use works
37+
# properly with indexes.
38+
ALL_INDEXES="-txindex -coinstatsindex -blockfilterindex=1"
39+
40+
if ! command -v jq >/dev/null ; then
41+
echo "This script requires jq to parse JSON RPC output. Please install it."
42+
echo "(e.g. sudo apt install jq)"
43+
exit 1
44+
fi
45+
46+
DUMP_OUTPUT="dumptxoutset-output-$BASE_HEIGHT.json"
47+
48+
finish() {
49+
echo
50+
echo "Killing server and client PIDs ($SERVER_PID, $CLIENT_PID) and cleaning up datadirs"
51+
echo
52+
rm -f "$UTXO_DAT_FILE" "$DUMP_OUTPUT"
53+
rm -rf "$SERVER_DATADIR" "$CLIENT_DATADIR"
54+
kill -9 "$SERVER_PID" "$CLIENT_PID"
55+
}
56+
57+
trap finish EXIT
58+
59+
# Need to specify these to trick client into accepting server as a peer
60+
# it can IBD from, otherwise the default values prevent IBD from the server node.
61+
EARLY_IBD_FLAGS="-maxtipage=9223372036854775207 -minimumchainwork=0x00"
62+
63+
server_rpc() {
64+
./src/bitcoin-cli -rpcport=$SERVER_RPC_PORT -datadir="$SERVER_DATADIR" "$@"
65+
}
66+
client_rpc() {
67+
./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir="$CLIENT_DATADIR" "$@"
68+
}
69+
server_sleep_til_boot() {
70+
while ! server_rpc ping >/dev/null 2>&1; do sleep 0.1; done
71+
}
72+
client_sleep_til_boot() {
73+
while ! client_rpc ping >/dev/null 2>&1; do sleep 0.1; done
74+
}
75+
76+
mkdir -p "$SERVER_DATADIR" "$CLIENT_DATADIR"
77+
78+
echo "Hi, welcome to the assumeutxo demo/test"
79+
echo
80+
echo "We're going to"
81+
echo
82+
echo " - start up a 'server' node, sync it via mainnet IBD to height ${BASE_HEIGHT}"
83+
echo " - create a UTXO snapshot at that height"
84+
echo " - IBD ${INCREMENTAL_HEIGHT} more blocks on top of that"
85+
echo
86+
echo "then we'll demonstrate assumeutxo by "
87+
echo
88+
echo " - starting another node (the 'client') and loading the snapshot in"
89+
echo " * first you'll have to modify the code slightly (chainparams) and recompile"
90+
echo " * don't worry, we'll make it easy"
91+
echo " - observing the client sync ${INCREMENTAL_HEIGHT} blocks on top of the snapshot from the server"
92+
echo " - observing the client validate the snapshot chain via background IBD"
93+
echo
94+
read -p "Press [enter] to continue" _
95+
96+
echo
97+
echo "-- Starting the demo. You might want to run the two following commands in"
98+
echo " separate terminal windows:"
99+
echo
100+
echo " watch -n0.1 tail -n 30 $SERVER_DATADIR/debug.log"
101+
echo " watch -n0.1 tail -n 30 $CLIENT_DATADIR/debug.log"
102+
echo
103+
read -p "Press [enter] to continue" _
104+
105+
echo
106+
echo "-- IBDing the blocks (height=$BASE_HEIGHT) required to the server node..."
107+
./src/bitcoind -logthreadnames=1 $SERVER_PORTS \
108+
-datadir="$SERVER_DATADIR" $EARLY_IBD_FLAGS -stopatheight="$BASE_HEIGHT" >/dev/null
109+
110+
echo
111+
echo "-- Creating snapshot at ~ height $BASE_HEIGHT ($UTXO_DAT_FILE)..."
112+
sleep 2
113+
./src/bitcoind -logthreadnames=1 $SERVER_PORTS \
114+
-datadir="$SERVER_DATADIR" $EARLY_IBD_FLAGS -connect=0 -listen=0 >/dev/null &
115+
SERVER_PID="$!"
116+
117+
server_sleep_til_boot
118+
server_rpc dumptxoutset "$UTXO_DAT_FILE" > "$DUMP_OUTPUT"
119+
cat "$DUMP_OUTPUT"
120+
kill -9 "$SERVER_PID"
121+
122+
RPC_BASE_HEIGHT=$(jq -r .base_height < "$DUMP_OUTPUT")
123+
RPC_AU=$(jq -r .txoutset_hash < "$DUMP_OUTPUT")
124+
RPC_NCHAINTX=$(jq -r .nchaintx < "$DUMP_OUTPUT")
125+
RPC_BLOCKHASH=$(jq -r .base_hash < "$DUMP_OUTPUT")
126+
127+
# Wait for server to shutdown...
128+
while server_rpc ping >/dev/null 2>&1; do sleep 0.1; done
129+
130+
echo
131+
echo "-- Now: add the following to CMainParams::m_assumeutxo_data"
132+
echo " in src/kernel/chainparams.cpp, and recompile:"
133+
echo
134+
echo " {${RPC_BASE_HEIGHT}, AssumeutxoHash{uint256S(\"0x${RPC_AU}\")}, ${RPC_NCHAINTX}, uint256S(\"0x${RPC_BLOCKHASH}\")},"
135+
echo
136+
echo
137+
echo "-- IBDing more blocks to the server node (height=$FINAL_HEIGHT) so there is a diff between snapshot and tip..."
138+
./src/bitcoind $SERVER_PORTS -logthreadnames=1 -datadir="$SERVER_DATADIR" \
139+
$EARLY_IBD_FLAGS -stopatheight="$FINAL_HEIGHT" >/dev/null
140+
141+
echo
142+
echo "-- Starting the server node to provide blocks to the client node..."
143+
./src/bitcoind $SERVER_PORTS -logthreadnames=1 -debug=net -datadir="$SERVER_DATADIR" \
144+
$EARLY_IBD_FLAGS -connect=0 -listen=1 >/dev/null &
145+
SERVER_PID="$!"
146+
server_sleep_til_boot
147+
148+
echo
149+
echo "-- Okay, what you're about to see is the client starting up and activating the snapshot."
150+
echo " I'm going to display the top 14 log lines from the client on top of an RPC called"
151+
echo " getchainstates, which is like getblockchaininfo but for both the snapshot and "
152+
echo " background validation chainstates."
153+
echo
154+
echo " You're going to first see the snapshot chainstate sync to the server's tip, then"
155+
echo " the background IBD chain kicks in to validate up to the base of the snapshot."
156+
echo
157+
echo " Once validation of the snapshot is done, you should see log lines indicating"
158+
echo " that we've deleted the background validation chainstate."
159+
echo
160+
echo " Once everything completes, exit the watch command with CTRL+C."
161+
echo
162+
read -p "When you're ready for all this, hit [enter]" _
163+
164+
echo
165+
echo "-- Starting the client node to get headers from the server, then load the snapshot..."
166+
./src/bitcoind $CLIENT_PORTS $ALL_INDEXES -logthreadnames=1 -datadir="$CLIENT_DATADIR" \
167+
-connect=0 -addnode=127.0.0.1:$SERVER_PORT -debug=net $EARLY_IBD_FLAGS >/dev/null &
168+
CLIENT_PID="$!"
169+
client_sleep_til_boot
170+
171+
echo
172+
echo "-- Initial state of the client:"
173+
client_rpc getchainstates
174+
175+
echo
176+
echo "-- Loading UTXO snapshot into client..."
177+
client_rpc loadtxoutset "$UTXO_DAT_FILE"
178+
179+
watch -n 0.3 "( tail -n 14 $CLIENT_DATADIR/debug.log ; echo ; ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir=$CLIENT_DATADIR getchainstates) | cat"
180+
181+
echo
182+
echo "-- Okay, now I'm going to restart the client to make sure that the snapshot chain reloads "
183+
echo " as the main chain properly..."
184+
echo
185+
echo " Press CTRL+C after you're satisfied to exit the demo"
186+
echo
187+
read -p "Press [enter] to continue"
188+
189+
while kill -0 "$CLIENT_PID"; do
190+
sleep 1
191+
done
192+
./src/bitcoind $CLIENT_PORTS $ALL_INDEXES -logthreadnames=1 -datadir="$CLIENT_DATADIR" -connect=0 \
193+
-addnode=127.0.0.1:$SERVER_PORT "$EARLY_IBD_FLAGS" >/dev/null &
194+
CLIENT_PID="$!"
195+
client_sleep_til_boot
196+
197+
watch -n 0.3 "( tail -n 14 $CLIENT_DATADIR/debug.log ; echo ; ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir=$CLIENT_DATADIR getchainstates) | cat"
198+
199+
echo
200+
echo "-- Done!"

‎test/lint/lint-shell.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,13 @@ def main():
6767
'*.sh',
6868
]
6969
files = get_files(files_cmd)
70-
# remove everything that doesn't match this regex
7170
reg = re.compile(r'src/[leveldb,secp256k1,minisketch]')
72-
files[:] = [file for file in files if not reg.match(file)]
71+
72+
def should_exclude(fname: str) -> bool:
73+
return bool(reg.match(fname)) or 'test_utxo_snapshots.sh' in fname
74+
75+
# remove everything that doesn't match this regex
76+
files[:] = [file for file in files if not should_exclude(file)]
7377

7478
# build the `shellcheck` command
7579
shellcheck_cmd = [

0 commit comments

Comments
 (0)