Skip to content

Commit b145831

Browse files
authored
Add hotreload.sh (#2110)
* Add hotreload.sh This script is inspired by the hotreload mode in the main repo. When using a k8s cluster with access to the local filesystem (like kind, k3s, minikube, etc.), it configures the collector container in the collector daemonset to execute a locally built binary, which is useful for iterating fast or running with sanitizers enabled. The script also provides a --revert option for dropping the configuration changes once they are no longer needed. * Add THREAD_SANITIZER build configuration This new option will compile the collector binary with thread sanitizer support. libtsan is added to the image-dev in order for collector to be able to use it at runtime. libubsan is also added since it was missing. * Add some docs * Fix inconsistent flag handle and check collector is deployed
1 parent d260a52 commit b145831

File tree

5 files changed

+221
-2
lines changed

5 files changed

+221
-2
lines changed

collector/CMakeLists.txt

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,13 @@ set(CMAKE_CXX_FLAGS_RELEASE "-O3 -fno-strict-aliasing -DNDEBUG")
1616
if(ADDRESS_SANITIZER)
1717
set(DISABLE_PROFILING ON)
1818
set(USE_VALGRIND OFF)
19-
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fsanitize=undefined -DGRPC_ASAN_SUPRESSED")
19+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fsanitize=undefined -DGRPC_ASAN_SUPPRESSED")
20+
endif()
21+
22+
if(THREAD_SANITIZER)
23+
set(DISABLE_PROFILING ON)
24+
set(USE_VALGRIND OFF)
25+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread -DGRPC_TSAN_SUPPRESSED")
2026
endif()
2127

2228
if(USE_VALGRIND)

collector/Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ cmake-configure/collector:
2424
-DDISABLE_PROFILING=$(DISABLE_PROFILING) \
2525
-DUSE_VALGRIND=$(USE_VALGRIND) \
2626
-DADDRESS_SANITIZER=$(ADDRESS_SANITIZER) \
27+
-DTHREAD_SANITIZER=$(THREAD_SANITIZER) \
2728
-DTRACE_SINSP_EVENTS=$(TRACE_SINSP_EVENTS) \
2829
-DBPF_DEBUG_MODE=$(BPF_DEBUG_MODE) \
2930
-DCOLLECTOR_VERSION=$(COLLECTOR_VERSION)

collector/container/devel/install.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22
set -eo pipefail
33

44
dnf upgrade -y
5-
dnf install -y libasan elfutils-libelf
5+
dnf install -y libasan libubsan libtsan elfutils-libelf

docs/how-to-start.md

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,24 @@ Or similarly via a CLI parameter:
164164
$ collector --grpc-server=
165165
```
166166

167+
## Hotreload on local k8s cluster
168+
169+
If using some sort of local k8s cluster for development (KinD, k3s, minikube)
170+
that has access to your local filesystem, you can use the
171+
[hotreload.sh](../utilities/hotreload.sh) script to run the collector binary in
172+
the cluster without rebuilding the entire image.
173+
174+
Start by deploying stackrox to your cluster using the `deploy-local.sh` script
175+
from the main repo, the run `hotreload.sh` to have the collector repository
176+
mounted into the collector container and its command changed to run the binary
177+
in `cmake-build/collector/collector`. Once that is done, you can recompile the
178+
binary by using the `make -C collector container/bin/collector` command or by
179+
exec'ing into the builder container directly and compiling from there with
180+
cmake.
181+
182+
This is also an easy way to run collector under thread or address sanitizer, but
183+
you will need to deploy an image built with the `image-dev` make target.
184+
167185
### Development with an IDE (CLion)
168186

169187
#### Setup

utilities/hotreload.sh

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
NAMESPACE="${NAMESPACE:-stackrox}"
6+
REVERT=0
7+
8+
# The following line allows to symlink the script somewhere in PATH for
9+
# easier use, has no effect if run directly.
10+
REAL_SCRIPT="$(realpath "${BASH_SOURCE[0]}")"
11+
SCRIPT_DIR="$(cd -- "$( dirname -- "${REAL_SCRIPT}")" &> /dev/null && pwd)"
12+
COLLECTOR_PATH="$(realpath "${SCRIPT_DIR:-}/..")"
13+
14+
function usage() {
15+
cat << EOF
16+
$(basename "$0") [OPTIONS]
17+
18+
Configure a running collector pod to use a local binary.
19+
20+
OPTIONS:
21+
-h, --help
22+
Show this help.
23+
-r, --revert
24+
Remove any configuration added by this script.
25+
-p, --path
26+
Override the path to the collector repo.
27+
-n, --namespace
28+
Set the namespace the collector daemonset is deployed to.
29+
Default: stackrox
30+
EOF
31+
}
32+
33+
function die() {
34+
echo >&2 "$1"
35+
exit 1
36+
}
37+
38+
function check_command() {
39+
if ! command -v "$1" &> /dev/null; then
40+
die "$1 not found. Make sure it is in your path"
41+
fi
42+
}
43+
44+
function krox() {
45+
kubectl -n "${NAMESPACE}" "$@"
46+
}
47+
48+
function patch_hotreload() {
49+
cat << EOF | jq -Mc \
50+
--arg hostPath "$COLLECTOR_PATH" \
51+
'.spec.template.spec.volumes.[0].hostPath.path |= $hostPath'
52+
{
53+
"spec": {
54+
"template": {
55+
"spec": {
56+
"containers": [
57+
{
58+
"name": "collector",
59+
"command": ["/host/src/cmake-build/collector/collector"],
60+
"volumeMounts": [
61+
{
62+
"mountPath": "/host/src",
63+
"name": "collector-src",
64+
"readOnly": true
65+
}
66+
]
67+
}
68+
],
69+
"volumes": [
70+
{
71+
"hostPath": {
72+
"path": "\$hostPath",
73+
"type": ""
74+
},
75+
"name": "collector-src"
76+
}
77+
]
78+
}
79+
}
80+
}
81+
}
82+
EOF
83+
}
84+
85+
function revert_command() {
86+
cat << EOF | jq -Mc
87+
{
88+
"spec": {
89+
"template": {
90+
"spec": {
91+
"containers": [
92+
{
93+
"name": "collector",
94+
"command": ["collector"]
95+
}
96+
]
97+
}
98+
}
99+
}
100+
}
101+
EOF
102+
}
103+
104+
function remove_mount() {
105+
jq -Mcn \
106+
--arg container "$1" \
107+
--arg volume "$2" \
108+
'[{"op":"remove","path":"/spec/template/spec/containers/\($container)/volumeMounts/\($volume)"}]'
109+
}
110+
111+
function remove_volume() {
112+
jq -Mcn \
113+
--arg volume "$1" \
114+
'[{"op":"remove","path":"/spec/template/spec/volumes/\($volume)"}]'
115+
}
116+
117+
function find_index() {
118+
local element="$1"
119+
local filter="$2"
120+
ret="$(krox get -o json ds/collector | jq "$element | map($filter) | index(true)")"
121+
if [[ "$ret" == "null" ]]; then
122+
die "Filter failed for '$filter'"
123+
fi
124+
echo "$ret"
125+
}
126+
127+
function add_hotreload() {
128+
krox patch ds collector -p "$(patch_hotreload)"
129+
}
130+
131+
function revert_config() {
132+
local collector_idx
133+
collector_idx="$(find_index ".spec.template.spec.containers" '.name == "collector"')"
134+
local mount_idx
135+
mount_idx="$(find_index ".spec.template.spec.containers.[$collector_idx].volumeMounts" '.name == "collector-src"')"
136+
local volume_idx
137+
volume_idx="$(find_index ".spec.template.spec.volumes" '.name == "collector-src"')"
138+
139+
krox patch ds collector -p "$(revert_command)"
140+
krox patch ds collector --type=json -p "$(remove_mount "$collector_idx" "$mount_idx")"
141+
krox patch ds collector --type=json -p "$(remove_volume "$volume_idx")"
142+
}
143+
144+
check_command jq
145+
check_command kubectl
146+
147+
if ! krox get ds/collector &> /dev/null; then
148+
die "collector daemonset not found (did you forget to deploy stackrox?)"
149+
fi
150+
151+
TEMP=$(getopt -o 'hrp:n:' -l 'help,revert,path:,namespace:' -n "$0" -- "$@")
152+
153+
# shellcheck disable=SC2181
154+
if [ $? -ne 0 ]; then
155+
exit 1
156+
fi
157+
158+
eval set -- "$TEMP"
159+
unset TEMP
160+
161+
while true; do
162+
case "${1:-}" in
163+
'-h' | '--help')
164+
usage
165+
exit 0
166+
;;
167+
168+
'-r' | '--revert')
169+
REVERT=1
170+
shift
171+
;;
172+
173+
'-p' | '--path')
174+
COLLECTOR_PATH="$2"
175+
shift 2
176+
;;
177+
178+
'-n' | '--namespace')
179+
NAMESPACE="$2"
180+
shift 2
181+
;;
182+
183+
'--')
184+
shift
185+
break
186+
;;
187+
esac
188+
done
189+
190+
if ((REVERT)); then
191+
revert_config
192+
else
193+
add_hotreload
194+
fi

0 commit comments

Comments
 (0)