From 65ec701a5261472c6a2accf9131dbb60587e71be Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Tue, 27 May 2025 14:23:17 +0100 Subject: [PATCH 1/5] Initial attempt at ingress/egress/both tests for external IPs --- integration-tests/container/QA_TAG | 2 +- .../container/berserker/Dockerfile | 6 +- integration-tests/container/berserker/init.sh | 33 ++++ .../container/berserker/prepare-tap.sh | 101 ++++++++++ .../berserker/workloads/network/client.toml | 15 ++ .../berserker/workloads/network/server.toml | 13 ++ integration-tests/pkg/types/runtime_config.go | 9 +- .../suites/runtime_config_file.go | 184 ++++++++++++++---- 8 files changed, 324 insertions(+), 39 deletions(-) create mode 100755 integration-tests/container/berserker/init.sh create mode 100755 integration-tests/container/berserker/prepare-tap.sh create mode 100644 integration-tests/container/berserker/workloads/network/client.toml create mode 100644 integration-tests/container/berserker/workloads/network/server.toml diff --git a/integration-tests/container/QA_TAG b/integration-tests/container/QA_TAG index e9307ca575..50ffc5aa7f 100644 --- a/integration-tests/container/QA_TAG +++ b/integration-tests/container/QA_TAG @@ -1 +1 @@ -2.0.2 +2.0.3 diff --git a/integration-tests/container/berserker/Dockerfile b/integration-tests/container/berserker/Dockerfile index ea60cbc931..4563d28b9c 100644 --- a/integration-tests/container/berserker/Dockerfile +++ b/integration-tests/container/berserker/Dockerfile @@ -1,6 +1,10 @@ -FROM quay.io/rhacs-eng/qa:berserker-1.0-59-g87ad0d870e +FROM quay.io/rhacs-eng/qa:berserker-1.0-79-g617ec32386 + +RUN sudo dnf install -y which iproute bpftool procps iptables COPY workloads/ /etc/berserker/ +COPY init.sh /scripts/ +COPY prepare-tap.sh /scripts/ ENV PATH="${PATH}:/usr/local/bin" diff --git a/integration-tests/container/berserker/init.sh b/integration-tests/container/berserker/init.sh new file mode 100755 index 0000000000..e35b8feeaf --- /dev/null +++ b/integration-tests/container/berserker/init.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -eo pipefail + +set -x + +IP_BASE="${IP_BASE:-223.42.0.1/16}" + +/scripts/prepare-tap.sh -a "$IP_BASE" -o + +if [[ "$IS_CLIENT" == "true" ]]; then + berserker /etc/berserker/network/client.toml & +else + berserker /etc/berserker/network/server.toml & +fi + +PID=$! + +cleanup() { + echo "Killing $PID" + + kill -9 "$PID" + + ip link delete berserker0 + + exit +} + +trap cleanup SIGINT SIGABRT + +wait -n "$PID" + +ip link delete berserker0 diff --git a/integration-tests/container/berserker/prepare-tap.sh b/integration-tests/container/berserker/prepare-tap.sh new file mode 100755 index 0000000000..e4151c4923 --- /dev/null +++ b/integration-tests/container/berserker/prepare-tap.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -eou pipefail + +# This script helps to prepare an environment for developing berserker network +# workload. It has the following preparatory steps: +# * Create and start up a new tun device for berserker to use +# * Optionally prepare iptables for the device to be visible +# +# The last step is optional, because iptables configuration could be different +# between development environments. Meaning it's not guaranteed that this part of +# the script is suitable for every case. + +stop() { + echo "$*" 1>&2 + exit 1 +} + +which ip &> /dev/null || stop "Don't have the ip tool" +which whoami &> /dev/null || stop "Don't have the whoami tool" +which sysctl &> /dev/null || stop "Don't have the sysctl tool" + +ADDRESS="10.0.0.1/16" +NAME="berserker0" +USER="$(whoami)" +CONFIGURE_IPTABLE="false" +CONFIGURE_FIREWALLD="false" +CONFIGURE_TUNTAP_IF_EXISTS="false" + +while getopts ":a:t:u:i:fo" opt; do + case $opt in + a) + ADDRESS="${OPTARG}" + ;; + t) + NAME="${OPTARG}" + ;; + u) + USER="${OPTARG}" + ;; + i) + CONFIGURE_IPTABLE="true" + ;; + f) + CONFIGURE_FIREWALLD="true" + ;; + o) + CONFIGURE_TUNTAP_IF_EXISTS="true" + ;; + \?) + echo "Invalid option -$OPTARG" >&2 + exit 1 + ;; + esac +done + +echo "Verifying if device ${NAME} is already created..." +if ip tuntap | grep "${NAME}" &> /dev/null; then + echo "The devince ${NAME} already exists!" + if [[ "${CONFIGURE_TUNTAP_IF_EXISTS}" != "true" ]]; then + exit 1 + fi + + ip link delete "${NAME}" +fi + +echo "Creating tun device ${NAME} for user ${USER}..." +ip tuntap add name "${NAME}" mode tun user "${USER}" +ip link set "${NAME}" up + +echo "Assigning address ${ADDRESS} to device ${NAME}..." +ip addr add "${ADDRESS}" dev "${NAME}" + +if [[ "${CONFIGURE_FIREWALLD}" == "true" ]]; then + which firewall-cmd &> /dev/null || stop "Don't have the firewal-cmd tool" + + echo "Adding to the trusted zone..." + firewall-cmd --zone=trusted --add-interface="${NAME}" +fi + +if [[ "${CONFIGURE_IPTABLE}" == "true" ]]; then + which iptables &> /dev/null || stop "Don't have the iptables tool" + + echo "Enabling ip forward..." + sysctl net.ipv4.ip_forward=1 + + echo "Preparing iptable..." + iptables -t nat -A POSTROUTING -s "${ADDRESS}" -j MASQUERADE + iptables -A FORWARD -i "${NAME}" -s "${ADDRESS}" -j ACCEPT + iptables -A FORWARD -o "${NAME}" -d "${ADDRESS}" -j ACCEPT + + RULE_NR=$(iptables -t filter -L INPUT --line-numbers \ + | grep "REJECT all" \ + | awk '{print $1}') + + # Excempt tun device from potentiall reject all rule + if [[ $RULE_NR == "" ]]; then + iptables -I INPUT -i "${NAME}" -s "${ADDRESS}" -j ACCEPT + else + iptables -I INPUT $((RULE_NR - 1)) -i "${NAME}" -s "${ADDRESS}" -j ACCEPT + fi +fi diff --git a/integration-tests/container/berserker/workloads/network/client.toml b/integration-tests/container/berserker/workloads/network/client.toml new file mode 100644 index 0000000000..a73a032b95 --- /dev/null +++ b/integration-tests/container/berserker/workloads/network/client.toml @@ -0,0 +1,15 @@ +restart_interval = 10 +workers = 1 +per_core = false + +[workload] +type = "network" +server = false +address = "223.42.0.1" +target_port = 1337 +arrival_rate = 10 +departure_rate = 10 +connections_static = 0 +connections_dyn_max = 20 +preempt = true +conns_per_addr = 1 diff --git a/integration-tests/container/berserker/workloads/network/server.toml b/integration-tests/container/berserker/workloads/network/server.toml new file mode 100644 index 0000000000..92bb6a56c4 --- /dev/null +++ b/integration-tests/container/berserker/workloads/network/server.toml @@ -0,0 +1,13 @@ +restart_interval = 10 +workers = 1 +per_core = false + +[workload] +type = "network" +server = true +address = "223.42.0.1" +target_port = 1337 +connections_static = 0 +connections_dyn_max = 20 +preempt = false +conns_per_addr = 1 diff --git a/integration-tests/pkg/types/runtime_config.go b/integration-tests/pkg/types/runtime_config.go index c0b29e54d4..db51c4e9c6 100644 --- a/integration-tests/pkg/types/runtime_config.go +++ b/integration-tests/pkg/types/runtime_config.go @@ -5,7 +5,8 @@ import ( ) type ExternalIpsConfig struct { - Enabled string `yaml:"enabled"` + Enabled string `yaml:"enabled"` + Direction string `yaml:"direction,omitempty"` } type NetworkConfig struct { @@ -29,12 +30,12 @@ func (n *RuntimeConfig) Equal(other RuntimeConfig) bool { return n.Networking.ExternalIps.Enabled == other.Networking.ExternalIps.Enabled } -func (n *RuntimeConfig) GetRuntimeConfigStr() (string, error) { +func (n *RuntimeConfig) String() string { yamlBytes, err := yaml.Marshal(n) if err != nil { - return "", err + panic(err) } - return string(yamlBytes), err + return string(yamlBytes) } diff --git a/integration-tests/suites/runtime_config_file.go b/integration-tests/suites/runtime_config_file.go index 1f7267cb34..181c13db9d 100644 --- a/integration-tests/suites/runtime_config_file.go +++ b/integration-tests/suites/runtime_config_file.go @@ -19,7 +19,7 @@ var ( serverPort = 53 externalUrl = fmt.Sprintf("http://%s:%d", externalIp, serverPort) - activeNormalizedConnection = types.NetworkInfo{ + activeNormalizedConnectionEgress = types.NetworkInfo{ LocalAddress: "", RemoteAddress: fmt.Sprintf("%s:%d", normalizedIp, serverPort), Role: "ROLE_CLIENT", @@ -27,7 +27,7 @@ var ( CloseTimestamp: types.NilTimestamp, } - activeUnnormalizedConnection = types.NetworkInfo{ + activeUnnormalizedConnectionEgress = types.NetworkInfo{ LocalAddress: "", RemoteAddress: fmt.Sprintf("%s:%d", externalIp, serverPort), Role: "ROLE_CLIENT", @@ -35,7 +35,7 @@ var ( CloseTimestamp: types.NilTimestamp, } - inactiveNormalizedConnection = types.NetworkInfo{ + inactiveNormalizedConnectionEgress = types.NetworkInfo{ LocalAddress: "", RemoteAddress: fmt.Sprintf("%s:%d", normalizedIp, serverPort), Role: "ROLE_CLIENT", @@ -43,7 +43,7 @@ var ( CloseTimestamp: "Not nill time", } - inactiveUnnormalizedConnection = types.NetworkInfo{ + inactiveUnnormalizedConnectionEgress = types.NetworkInfo{ LocalAddress: "", RemoteAddress: fmt.Sprintf("%s:%d", externalIp, serverPort), Role: "ROLE_CLIENT", @@ -58,20 +58,62 @@ var ( type RuntimeConfigFileTestSuite struct { IntegrationTestSuiteBase - ClientContainer string + EgressClientContainer string + IngressClientContainer string + IngressServerContainer string } -func (s *RuntimeConfigFileTestSuite) setRuntimeConfig(runtimeConfigFile string, configStr string) { +func (s *RuntimeConfigFileTestSuite) writeRuntimeConfig(runtimeConfigFile string, configStr string) { err := os.WriteFile(runtimeConfigFile, []byte(configStr), 0666) s.Require().NoError(err) } -func (s *RuntimeConfigFileTestSuite) setExternalIpsEnabled(runtimeConfigFile string, enabled string) { - var runtimeConfig types.RuntimeConfig - runtimeConfig.Networking.ExternalIps.Enabled = enabled - runtimeConfigStr, err := runtimeConfig.GetRuntimeConfigStr() +func (s *RuntimeConfigFileTestSuite) setRuntimeConfig(config types.RuntimeConfig) { + s.writeRuntimeConfig(runtimeConfigFile, config.String()) +} + +func (s *RuntimeConfigFileTestSuite) runBerserkerContainers() (client, server string) { + containerID, err := s.Executor().StartContainer( + config.ContainerStartConfig{ + Name: "external-connection-ingress-client", + Image: config.Images().QaImageByKey("performance-berserker"), + Privileged: true, + NetworkMode: "host", + Entrypoint: []string{ + "/scripts/init.sh", + }, + Env: map[string]string{ + "RUST_LOG": "DEBUG", + "IS_CLIENT": "true", + }, + }, + ) + s.Require().NoError(err) + client = common.ContainerShortID(containerID) + + containerID, err = s.Executor().StartContainer( + config.ContainerStartConfig{ + Name: "external-connection-ingress-server", + Image: config.Images().QaImageByKey("performance-berserker"), + Privileged: true, + NetworkMode: "host", + Entrypoint: []string{ + "/scripts/init.sh", + }, + Env: map[string]string{ + "RUST_LOG": "DEBUG", + "IS_CLIENT": "false", + }, + }, + ) s.Require().NoError(err) - s.setRuntimeConfig(runtimeConfigFile, runtimeConfigStr) + server = common.ContainerShortID(containerID) + + return client, server +} + +func (s *RuntimeConfigFileTestSuite) teardownBerserkerContainers() { + s.cleanupContainers("external-connection-ingress-server", "external-connection-ingress-client") } // Launches collector and creates the directory for runtime configuration. @@ -82,12 +124,12 @@ func (s *RuntimeConfigFileTestSuite) SetupTest() { containerID, err := s.Executor().StartContainer( config.ContainerStartConfig{ - Name: "external-connection", + Name: "external-connection-egress", Image: config.Images().QaImageByKey("qa-alpine-curl"), Command: []string{"sh", "-c", "while true; do curl " + externalUrl + "; sleep 1; done"}, }) s.Require().NoError(err) - s.ClientContainer = common.ContainerShortID(containerID) + s.EgressClientContainer = common.ContainerShortID(containerID) collectorOptions := collector.StartupOptions{ Env: map[string]string{ @@ -102,7 +144,11 @@ func (s *RuntimeConfigFileTestSuite) SetupTest() { func (s *RuntimeConfigFileTestSuite) AfterTest(suiteName, testName string) { s.StopCollector() - s.cleanupContainers("external-connection") + s.cleanupContainers( + "external-connection-egress", + "external-connection-ingress-client", + "external-connection-ingress-server", + ) s.WritePerfResults() s.deleteFile(runtimeConfigFile) } @@ -116,32 +162,44 @@ func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigFileEnable() { // Default configuration is external IPs disabled. // We expect normalized connections. assert.AssertNoRuntimeConfig(s.T(), collectorIP) - expectedConnections := []types.NetworkInfoBatch{[]types.NetworkInfo{activeNormalizedConnection}} - connectionSuccess := s.Sensor().ExpectSameElementsConnectionsScrapes(s.T(), s.ClientContainer, 10*time.Second, expectedConnections) + expectedConnections := []types.NetworkInfoBatch{[]types.NetworkInfo{activeNormalizedConnectionEgress}} + connectionSuccess := s.Sensor().ExpectSameElementsConnectionsScrapes(s.T(), s.EgressClientContainer, 10*time.Second, expectedConnections) s.Require().True(connectionSuccess) // External IPs enabled. // Normalized connection must be reported as inactive // Unnormalized connection will now be reported. - s.setExternalIpsEnabled(runtimeConfigFile, "ENABLED") + s.setRuntimeConfig(types.RuntimeConfig{ + Networking: types.NetworkConfig{ + ExternalIps: types.ExternalIpsConfig{ + Enabled: "ENABLED", + }, + }, + }) assert.AssertExternalIps(s.T(), "ENABLED", collectorIP) - expectedConnections = append(expectedConnections, []types.NetworkInfo{activeUnnormalizedConnection, inactiveNormalizedConnection}) - connectionSuccess = s.Sensor().ExpectSameElementsConnectionsScrapes(s.T(), s.ClientContainer, 10*time.Second, expectedConnections) + expectedConnections = append(expectedConnections, []types.NetworkInfo{activeUnnormalizedConnectionEgress, inactiveNormalizedConnectionEgress}) + connectionSuccess = s.Sensor().ExpectSameElementsConnectionsScrapes(s.T(), s.EgressClientContainer, 10*time.Second, expectedConnections) s.Require().True(connectionSuccess) // The runtime config file is deleted. This disables external IPs. The normalized connection should be active // and the unnormalized connection shoul be inactive. s.deleteFile(runtimeConfigFile) assert.AssertNoRuntimeConfig(s.T(), collectorIP) - expectedConnections = append(expectedConnections, []types.NetworkInfo{activeNormalizedConnection, inactiveUnnormalizedConnection}) - connectionSuccess = s.Sensor().ExpectSameElementsConnectionsScrapes(s.T(), s.ClientContainer, 10*time.Second, expectedConnections) + expectedConnections = append(expectedConnections, []types.NetworkInfo{activeNormalizedConnectionEgress, inactiveUnnormalizedConnectionEgress}) + connectionSuccess = s.Sensor().ExpectSameElementsConnectionsScrapes(s.T(), s.EgressClientContainer, 10*time.Second, expectedConnections) s.Require().True(connectionSuccess) // Back to having external IPs enabled. - s.setExternalIpsEnabled(runtimeConfigFile, "ENABLED") + s.setRuntimeConfig(types.RuntimeConfig{ + Networking: types.NetworkConfig{ + ExternalIps: types.ExternalIpsConfig{ + Enabled: "ENABLED", + }, + }, + }) assert.AssertExternalIps(s.T(), "ENABLED", collectorIP) - expectedConnections = append(expectedConnections, []types.NetworkInfo{activeUnnormalizedConnection, inactiveNormalizedConnection}) - connectionSuccess = s.Sensor().ExpectSameElementsConnectionsScrapes(s.T(), s.ClientContainer, 10*time.Second, expectedConnections) + expectedConnections = append(expectedConnections, []types.NetworkInfo{activeUnnormalizedConnectionEgress, inactiveNormalizedConnectionEgress}) + connectionSuccess = s.Sensor().ExpectSameElementsConnectionsScrapes(s.T(), s.EgressClientContainer, 10*time.Second, expectedConnections) s.Require().True(connectionSuccess) } @@ -150,23 +208,30 @@ func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigFileDisable() { // Default configuration is external IPs disabled. // We expect normalized connections. assert.AssertNoRuntimeConfig(s.T(), collectorIP) - expectedConnections := []types.NetworkInfo{activeNormalizedConnection} - connectionSuccess := s.Sensor().ExpectSameElementsConnections(s.T(), s.ClientContainer, 10*time.Second, expectedConnections...) + expectedConnections := []types.NetworkInfo{activeNormalizedConnectionEgress} + connectionSuccess := s.Sensor().ExpectSameElementsConnections(s.T(), s.EgressClientContainer, 10*time.Second, expectedConnections...) s.Require().True(connectionSuccess) // The runtime config file is created, but external IPs is disables. // There is no change in the state, so there are no changes to the connections - s.setExternalIpsEnabled(runtimeConfigFile, "DISABLED") + s.setRuntimeConfig(types.RuntimeConfig{ + Networking: types.NetworkConfig{ + ExternalIps: types.ExternalIpsConfig{ + Enabled: "DISABLED", + }, + }, + }) + assert.AssertExternalIps(s.T(), "DISABLED", collectorIP) common.Sleep(3 * time.Second) // Sleep so that collector has a chance to report connections - connectionSuccess = s.Sensor().ExpectSameElementsConnections(s.T(), s.ClientContainer, 10*time.Second, expectedConnections...) + connectionSuccess = s.Sensor().ExpectSameElementsConnections(s.T(), s.EgressClientContainer, 10*time.Second, expectedConnections...) s.Require().True(connectionSuccess) // Back to using default behavior of external IPs disabled with no file. s.deleteFile(runtimeConfigFile) assert.AssertNoRuntimeConfig(s.T(), collectorIP) common.Sleep(3 * time.Second) // Sleep so that collector has a chance to report connections - connectionSuccess = s.Sensor().ExpectSameElementsConnections(s.T(), s.ClientContainer, 10*time.Second, expectedConnections...) + connectionSuccess = s.Sensor().ExpectSameElementsConnections(s.T(), s.EgressClientContainer, 10*time.Second, expectedConnections...) s.Require().True(connectionSuccess) } @@ -175,15 +240,68 @@ func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigFileInvalid() { // Default configuration is external IPs disabled. // We expect normalized connections. assert.AssertNoRuntimeConfig(s.T(), collectorIP) - expectedConnections := []types.NetworkInfo{activeNormalizedConnection} - connectionSuccess := s.Sensor().ExpectSameElementsConnections(s.T(), s.ClientContainer, 10*time.Second, expectedConnections...) + expectedConnections := []types.NetworkInfo{activeNormalizedConnectionEgress} + connectionSuccess := s.Sensor().ExpectSameElementsConnections(s.T(), s.EgressClientContainer, 10*time.Second, expectedConnections...) s.Require().True(connectionSuccess) // Testing an invalid configuration. There should not be a change in the configuration or reported connections invalidConfig := "asdf" - s.setRuntimeConfig(runtimeConfigFile, invalidConfig) + s.writeRuntimeConfig(runtimeConfigFile, invalidConfig) assert.AssertNoRuntimeConfig(s.T(), collectorIP) common.Sleep(3 * time.Second) // Sleep so that collector has a chance to report connections - connectionSuccess = s.Sensor().ExpectSameElementsConnections(s.T(), s.ClientContainer, 10*time.Second, expectedConnections...) + connectionSuccess = s.Sensor().ExpectSameElementsConnections(s.T(), s.EgressClientContainer, 10*time.Second, expectedConnections...) s.Require().True(connectionSuccess) } + +func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigNetworkIngress() { + client, server := s.runBerserkerContainers() + defer s.teardownBerserkerContainers() + + // assert.AssertNoRuntimeConfig(s.T(), collectorIP) + // expectedConnections := []types.NetworkInfo{activeNormalizedConnectionEgress} + // s.Require().True(s.Sensor().ExpectSameElementsConnections(s.T(), server, 10*time.Second, expectedConnections...)) + + s.setRuntimeConfig(types.RuntimeConfig{ + Networking: types.NetworkConfig{ + ExternalIps: types.ExternalIpsConfig{ + Enabled: "ENABLED", + Direction: "INGRESS", + }, + }, + }) + + common.Sleep(45 * time.Second) + + fmt.Println(s.Sensor().Connections(client)) + fmt.Println("===========") + fmt.Println(s.Sensor().Connections(server)) + + // assert.AssertExternalIps(s.T(), "ENABLED", collectorIP) + // expectedConnections = append(expectedConnections, activeUnnormalizedConnectionEgress, inactiveNormalizedConnectionEgress) + // common.Sleep(3 * time.Second) // Sleep so that collector has a chance to report connections + // s.Require().True(s.Sensor().ExpectSameElementsConnections(s.T(), client, 10*time.Second, expectedConnections...)) +} + +func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigNetworkEgress() { + assert.AssertNoRuntimeConfig(s.T(), collectorIP) + expectedConnections := []types.NetworkInfo{activeNormalizedConnectionEgress} + s.Require().True(s.Sensor().ExpectSameElementsConnections(s.T(), s.EgressClientContainer, 10*time.Second, expectedConnections...)) + + s.setRuntimeConfig(types.RuntimeConfig{ + Networking: types.NetworkConfig{ + ExternalIps: types.ExternalIpsConfig{ + Enabled: "ENABLED", + Direction: "EGRESS", + }, + }, + }) + + assert.AssertExternalIps(s.T(), "ENABLED", collectorIP) + expectedConnections = append(expectedConnections, activeUnnormalizedConnectionEgress, inactiveNormalizedConnectionEgress) + common.Sleep(3 * time.Second) // Sleep so that collector has a chance to report connections + s.Require().True(s.Sensor().ExpectSameElementsConnections(s.T(), s.EgressClientContainer, 10*time.Second, expectedConnections...)) +} + +func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigNetworkBoth() { + +} From 1b4bc9096fad849cccbca056d7733edf82c74e0a Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Wed, 28 May 2025 14:46:45 +0100 Subject: [PATCH 2/5] Uses netcat instead of berserker for simplicity, adds ingress/egress/both tests --- .../container/berserker/Dockerfile | 3 +- integration-tests/container/berserker/init.sh | 33 --- .../berserker/workloads/network/client.toml | 15 -- .../berserker/workloads/network/server.toml | 13 -- integration-tests/pkg/types/network.go | 6 + .../suites/runtime_config_file.go | 188 ++++++++++++++---- 6 files changed, 156 insertions(+), 102 deletions(-) delete mode 100755 integration-tests/container/berserker/init.sh delete mode 100644 integration-tests/container/berserker/workloads/network/client.toml delete mode 100644 integration-tests/container/berserker/workloads/network/server.toml diff --git a/integration-tests/container/berserker/Dockerfile b/integration-tests/container/berserker/Dockerfile index 4563d28b9c..138759b436 100644 --- a/integration-tests/container/berserker/Dockerfile +++ b/integration-tests/container/berserker/Dockerfile @@ -1,9 +1,8 @@ FROM quay.io/rhacs-eng/qa:berserker-1.0-79-g617ec32386 -RUN sudo dnf install -y which iproute bpftool procps iptables +RUN sudo dnf install -y which netcat iproute bpftool procps iptables COPY workloads/ /etc/berserker/ -COPY init.sh /scripts/ COPY prepare-tap.sh /scripts/ ENV PATH="${PATH}:/usr/local/bin" diff --git a/integration-tests/container/berserker/init.sh b/integration-tests/container/berserker/init.sh deleted file mode 100755 index e35b8feeaf..0000000000 --- a/integration-tests/container/berserker/init.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -set -x - -IP_BASE="${IP_BASE:-223.42.0.1/16}" - -/scripts/prepare-tap.sh -a "$IP_BASE" -o - -if [[ "$IS_CLIENT" == "true" ]]; then - berserker /etc/berserker/network/client.toml & -else - berserker /etc/berserker/network/server.toml & -fi - -PID=$! - -cleanup() { - echo "Killing $PID" - - kill -9 "$PID" - - ip link delete berserker0 - - exit -} - -trap cleanup SIGINT SIGABRT - -wait -n "$PID" - -ip link delete berserker0 diff --git a/integration-tests/container/berserker/workloads/network/client.toml b/integration-tests/container/berserker/workloads/network/client.toml deleted file mode 100644 index a73a032b95..0000000000 --- a/integration-tests/container/berserker/workloads/network/client.toml +++ /dev/null @@ -1,15 +0,0 @@ -restart_interval = 10 -workers = 1 -per_core = false - -[workload] -type = "network" -server = false -address = "223.42.0.1" -target_port = 1337 -arrival_rate = 10 -departure_rate = 10 -connections_static = 0 -connections_dyn_max = 20 -preempt = true -conns_per_addr = 1 diff --git a/integration-tests/container/berserker/workloads/network/server.toml b/integration-tests/container/berserker/workloads/network/server.toml deleted file mode 100644 index 92bb6a56c4..0000000000 --- a/integration-tests/container/berserker/workloads/network/server.toml +++ /dev/null @@ -1,13 +0,0 @@ -restart_interval = 10 -workers = 1 -per_core = false - -[workload] -type = "network" -server = true -address = "223.42.0.1" -target_port = 1337 -connections_static = 0 -connections_dyn_max = 20 -preempt = false -conns_per_addr = 1 diff --git a/integration-tests/pkg/types/network.go b/integration-tests/pkg/types/network.go index 458daf865e..64056d7603 100644 --- a/integration-tests/pkg/types/network.go +++ b/integration-tests/pkg/types/network.go @@ -10,6 +10,12 @@ import ( const ( NilTimestamp = "" + + // This sentinel value can be used for expected network infos + // where the timestamp is not known ahead of time, but is expected + // to be non-nil for the purposes of assertions (e.g. compared to + // NilTimestamp as seen in IsActive() below) + NotNilTimestamp = "Not Nil" ) type NetworkInfo struct { diff --git a/integration-tests/suites/runtime_config_file.go b/integration-tests/suites/runtime_config_file.go index 181c13db9d..8bd6b35a5d 100644 --- a/integration-tests/suites/runtime_config_file.go +++ b/integration-tests/suites/runtime_config_file.go @@ -40,7 +40,7 @@ var ( RemoteAddress: fmt.Sprintf("%s:%d", normalizedIp, serverPort), Role: "ROLE_CLIENT", SocketFamily: "SOCKET_FAMILY_UNKNOWN", - CloseTimestamp: "Not nill time", + CloseTimestamp: types.NotNilTimestamp, } inactiveUnnormalizedConnectionEgress = types.NetworkInfo{ @@ -48,12 +48,15 @@ var ( RemoteAddress: fmt.Sprintf("%s:%d", externalIp, serverPort), Role: "ROLE_CLIENT", SocketFamily: "SOCKET_FAMILY_UNKNOWN", - CloseTimestamp: "Not nill time", + CloseTimestamp: types.NotNilTimestamp, } runtimeConfigDir = "/tmp/collector-test" runtimeConfigFile = filepath.Join(runtimeConfigDir, "/runtime_config.yaml") collectorIP = "localhost" + + ingressIP = "223.42.0.1" + ingressPort = 1337 ) type RuntimeConfigFileTestSuite struct { @@ -72,47 +75,42 @@ func (s *RuntimeConfigFileTestSuite) setRuntimeConfig(config types.RuntimeConfig s.writeRuntimeConfig(runtimeConfigFile, config.String()) } -func (s *RuntimeConfigFileTestSuite) runBerserkerContainers() (client, server string) { +func (s *RuntimeConfigFileTestSuite) runNetworkDirectionContainers() (client, server string) { + + serverCmd := fmt.Sprintf("/scripts/prepare-tap.sh -a %s -o && nc -lk %s %d", ingressIP, ingressIP, ingressPort) containerID, err := s.Executor().StartContainer( config.ContainerStartConfig{ - Name: "external-connection-ingress-client", + Name: "external-connection-ingress-server", Image: config.Images().QaImageByKey("performance-berserker"), Privileged: true, NetworkMode: "host", Entrypoint: []string{ - "/scripts/init.sh", - }, - Env: map[string]string{ - "RUST_LOG": "DEBUG", - "IS_CLIENT": "true", + "sh", "-c", serverCmd, }, }, ) s.Require().NoError(err) - client = common.ContainerShortID(containerID) + server = common.ContainerShortID(containerID) + clientCmd := fmt.Sprintf("sleep 10; while true; do nc -zv %s %d; sleep 60; done", ingressIP, ingressPort) containerID, err = s.Executor().StartContainer( config.ContainerStartConfig{ - Name: "external-connection-ingress-server", + Name: "external-connection-ingress-client", Image: config.Images().QaImageByKey("performance-berserker"), Privileged: true, NetworkMode: "host", Entrypoint: []string{ - "/scripts/init.sh", - }, - Env: map[string]string{ - "RUST_LOG": "DEBUG", - "IS_CLIENT": "false", + "sh", "-c", clientCmd, }, }, ) s.Require().NoError(err) - server = common.ContainerShortID(containerID) + client = common.ContainerShortID(containerID) return client, server } -func (s *RuntimeConfigFileTestSuite) teardownBerserkerContainers() { +func (s *RuntimeConfigFileTestSuite) teardownNetworkDirectionContainers() { s.cleanupContainers("external-connection-ingress-server", "external-connection-ingress-client") } @@ -254,12 +252,8 @@ func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigFileInvalid() { } func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigNetworkIngress() { - client, server := s.runBerserkerContainers() - defer s.teardownBerserkerContainers() - - // assert.AssertNoRuntimeConfig(s.T(), collectorIP) - // expectedConnections := []types.NetworkInfo{activeNormalizedConnectionEgress} - // s.Require().True(s.Sensor().ExpectSameElementsConnections(s.T(), server, 10*time.Second, expectedConnections...)) + client, server := s.runNetworkDirectionContainers() + defer s.teardownNetworkDirectionContainers() s.setRuntimeConfig(types.RuntimeConfig{ Networking: types.NetworkConfig{ @@ -270,22 +264,51 @@ func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigNetworkIngress() { }, }) - common.Sleep(45 * time.Second) + // Expect both open and close events for the non-aggregated + // ingress connection. If Collector is aggregating to 255.255.255.255 + // this will fail. + // We are not concerned with event ordering in this test. + expectedIngressConnections := []types.NetworkInfo{ + { + LocalAddress: fmt.Sprintf(":%d", ingressPort), + RemoteAddress: ingressIP, + Role: "ROLE_SERVER", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: types.NotNilTimestamp, + }, + { + LocalAddress: fmt.Sprintf(":%d", ingressPort), + RemoteAddress: ingressIP, + Role: "ROLE_SERVER", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: types.NilTimestamp, + }, + } - fmt.Println(s.Sensor().Connections(client)) - fmt.Println("===========") - fmt.Println(s.Sensor().Connections(server)) + expectedEgressConnections := []types.NetworkInfo{ + { + LocalAddress: "", + RemoteAddress: fmt.Sprintf("%s:%d", normalizedIp, ingressPort), + Role: "ROLE_CLIENT", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: types.NotNilTimestamp, + }, + { + LocalAddress: "", + RemoteAddress: fmt.Sprintf("%s:%d", normalizedIp, ingressPort), + Role: "ROLE_CLIENT", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: types.NilTimestamp, + }, + } - // assert.AssertExternalIps(s.T(), "ENABLED", collectorIP) - // expectedConnections = append(expectedConnections, activeUnnormalizedConnectionEgress, inactiveNormalizedConnectionEgress) - // common.Sleep(3 * time.Second) // Sleep so that collector has a chance to report connections - // s.Require().True(s.Sensor().ExpectSameElementsConnections(s.T(), client, 10*time.Second, expectedConnections...)) + s.Sensor().ExpectConnections(s.T(), client, 30*time.Second, expectedEgressConnections...) + s.Sensor().ExpectConnections(s.T(), server, 30*time.Second, expectedIngressConnections...) } func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigNetworkEgress() { - assert.AssertNoRuntimeConfig(s.T(), collectorIP) - expectedConnections := []types.NetworkInfo{activeNormalizedConnectionEgress} - s.Require().True(s.Sensor().ExpectSameElementsConnections(s.T(), s.EgressClientContainer, 10*time.Second, expectedConnections...)) + client, server := s.runNetworkDirectionContainers() + defer s.teardownNetworkDirectionContainers() s.setRuntimeConfig(types.RuntimeConfig{ Networking: types.NetworkConfig{ @@ -296,12 +319,99 @@ func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigNetworkEgress() { }, }) - assert.AssertExternalIps(s.T(), "ENABLED", collectorIP) - expectedConnections = append(expectedConnections, activeUnnormalizedConnectionEgress, inactiveNormalizedConnectionEgress) - common.Sleep(3 * time.Second) // Sleep so that collector has a chance to report connections - s.Require().True(s.Sensor().ExpectSameElementsConnections(s.T(), s.EgressClientContainer, 10*time.Second, expectedConnections...)) + // Expect both open and close events for the non-aggregated + // egress connection. If Collector is aggregating to 255.255.255.255 + // this will fail. + // We are not concerned with event ordering in this test. + expectedEgressConnections := []types.NetworkInfo{ + { + LocalAddress: "", + RemoteAddress: fmt.Sprintf("%s:%d", ingressIP, ingressPort), + Role: "ROLE_CLIENT", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: types.NotNilTimestamp, + }, + { + LocalAddress: "", + RemoteAddress: fmt.Sprintf("%s:%d", ingressIP, ingressPort), + Role: "ROLE_CLIENT", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: types.NilTimestamp, + }, + } + + expectedIngressConnections := []types.NetworkInfo{ + { + LocalAddress: fmt.Sprintf(":%d", ingressPort), + RemoteAddress: normalizedIp, + Role: "ROLE_SERVER", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: types.NotNilTimestamp, + }, + { + LocalAddress: fmt.Sprintf(":%d", ingressPort), + RemoteAddress: normalizedIp, + Role: "ROLE_SERVER", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: types.NilTimestamp, + }, + } + + s.Require().True(s.Sensor().ExpectConnections(s.T(), client, 30*time.Second, expectedEgressConnections...)) + s.Require().True(s.Sensor().ExpectConnections(s.T(), server, 30*time.Second, expectedIngressConnections...)) } func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigNetworkBoth() { + client, server := s.runNetworkDirectionContainers() + defer s.teardownNetworkDirectionContainers() + + s.setRuntimeConfig(types.RuntimeConfig{ + Networking: types.NetworkConfig{ + ExternalIps: types.ExternalIpsConfig{ + Enabled: "ENABLED", + Direction: "BOTH", + }, + }, + }) + + // Expect both open and close events for the non-aggregated + // egress and ingress connections. If Collector is aggregating to 255.255.255.255 + // this will fail. + // We are not concerned with event ordering in this test. + expectedEgressConnections := []types.NetworkInfo{ + { + LocalAddress: "", + RemoteAddress: fmt.Sprintf("%s:%d", ingressIP, ingressPort), + Role: "ROLE_CLIENT", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: types.NotNilTimestamp, + }, + { + LocalAddress: "", + RemoteAddress: fmt.Sprintf("%s:%d", ingressIP, ingressPort), + Role: "ROLE_CLIENT", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: types.NilTimestamp, + }, + } + + expectedIngressConnections := []types.NetworkInfo{ + { + LocalAddress: fmt.Sprintf(":%d", ingressPort), + RemoteAddress: ingressIP, + Role: "ROLE_SERVER", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: types.NotNilTimestamp, + }, + { + LocalAddress: fmt.Sprintf(":%d", ingressPort), + RemoteAddress: ingressIP, + Role: "ROLE_SERVER", + SocketFamily: "SOCKET_FAMILY_UNKNOWN", + CloseTimestamp: types.NilTimestamp, + }, + } + s.Require().True(s.Sensor().ExpectConnections(s.T(), client, 30*time.Second, expectedEgressConnections...)) + s.Require().True(s.Sensor().ExpectConnections(s.T(), server, 30*time.Second, expectedIngressConnections...)) } From d7554599f9942f1c5b84f828c0d443d8b94a1463 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Wed, 28 May 2025 14:55:22 +0100 Subject: [PATCH 3/5] Missed GetRuntimeConfigStr usage --- integration-tests/suites/k8s/config_reload.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/integration-tests/suites/k8s/config_reload.go b/integration-tests/suites/k8s/config_reload.go index 7a9b98ec93..0a1d05db47 100644 --- a/integration-tests/suites/k8s/config_reload.go +++ b/integration-tests/suites/k8s/config_reload.go @@ -18,20 +18,13 @@ var ( ) func init() { - var err error var runtimeConfig types.RuntimeConfig runtimeConfig.Networking.ExternalIps.Enabled = "ENABLED" - EXT_IP_ENABLE, err = runtimeConfig.GetRuntimeConfigStr() - if err != nil { - panic(err) - } - + EXT_IP_ENABLE = runtimeConfig.String() runtimeConfig.Networking.ExternalIps.Enabled = "DISABLED" - EXT_IP_DISABLE, err = runtimeConfig.GetRuntimeConfigStr() - if err != nil { - panic(err) - } + + EXT_IP_DISABLE = runtimeConfig.String() } type K8sConfigReloadTestSuite struct { From 32aa325326dd59c9469cacae209f2e79bb12e968 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Wed, 28 May 2025 15:00:21 +0100 Subject: [PATCH 4/5] Remove sudo --- integration-tests/container/berserker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/container/berserker/Dockerfile b/integration-tests/container/berserker/Dockerfile index 138759b436..f8e26e1afc 100644 --- a/integration-tests/container/berserker/Dockerfile +++ b/integration-tests/container/berserker/Dockerfile @@ -1,6 +1,6 @@ FROM quay.io/rhacs-eng/qa:berserker-1.0-79-g617ec32386 -RUN sudo dnf install -y which netcat iproute bpftool procps iptables +RUN dnf install -y which netcat iproute bpftool procps iptables COPY workloads/ /etc/berserker/ COPY prepare-tap.sh /scripts/ From f9bad77de4559e5d313c356d4ecb57c0c8730d08 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Thu, 29 May 2025 14:30:51 +0100 Subject: [PATCH 5/5] verify config state before proceeding --- integration-tests/pkg/assert/assert.go | 33 ++++++++++++------- .../suites/runtime_config_file.go | 28 +++++++++------- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/integration-tests/pkg/assert/assert.go b/integration-tests/pkg/assert/assert.go index ff8c869860..684b6f2c05 100644 --- a/integration-tests/pkg/assert/assert.go +++ b/integration-tests/pkg/assert/assert.go @@ -8,6 +8,7 @@ import ( "time" "github.com/davecgh/go-spew/spew" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" "github.com/stackrox/collector/integration-tests/pkg/collector" @@ -17,25 +18,35 @@ import ( var ( runtimeConfigErrorMsg = "Runtime configuration was not updated" + + tickTime = 1 * time.Second + timeout = 3 * time.Minute ) -func AssertExternalIps(t *testing.T, enabled string, collectorIP string) { - tickTime := 1 * time.Second - timeout := 3 * time.Minute +func getCollectorRuntimeConfig(t *testing.T, collectorIP string) types.RuntimeConfig { + body, err := collector.IntrospectionQuery(collectorIP, "/state/runtime-config") + assert.NoError(t, err) + var response types.RuntimeConfig + err = json.Unmarshal(body, &response) + assert.NoError(t, err) + return response +} + +func AssertRuntimeConfig(t *testing.T, collectorIP string, config types.RuntimeConfig) { AssertRepeated(t, tickTime, timeout, runtimeConfigErrorMsg, func() bool { - body, err := collector.IntrospectionQuery(collectorIP, "/state/runtime-config") - assert.NoError(t, err) - var response types.RuntimeConfig - err = json.Unmarshal(body, &response) - assert.NoError(t, err) + collectorConfig := getCollectorRuntimeConfig(t, collectorIP) + return cmp.Equal(config, collectorConfig) + }) +} - return response.Networking.ExternalIps.Enabled == enabled +func AssertExternalIps(t *testing.T, enabled string, collectorIP string) { + AssertRepeated(t, tickTime, timeout, runtimeConfigErrorMsg, func() bool { + collectorConfig := getCollectorRuntimeConfig(t, collectorIP) + return collectorConfig.Networking.ExternalIps.Enabled == enabled }) } func AssertNoRuntimeConfig(t *testing.T, collectorIP string) { - tickTime := 1 * time.Second - timeout := 3 * time.Minute AssertRepeated(t, tickTime, timeout, runtimeConfigErrorMsg, func() bool { body, err := collector.IntrospectionQuery(collectorIP, "/state/runtime-config") assert.NoError(t, err) diff --git a/integration-tests/suites/runtime_config_file.go b/integration-tests/suites/runtime_config_file.go index 8bd6b35a5d..49fca5e76e 100644 --- a/integration-tests/suites/runtime_config_file.go +++ b/integration-tests/suites/runtime_config_file.go @@ -61,9 +61,7 @@ var ( type RuntimeConfigFileTestSuite struct { IntegrationTestSuiteBase - EgressClientContainer string - IngressClientContainer string - IngressServerContainer string + EgressClientContainer string } func (s *RuntimeConfigFileTestSuite) writeRuntimeConfig(runtimeConfigFile string, configStr string) { @@ -76,7 +74,6 @@ func (s *RuntimeConfigFileTestSuite) setRuntimeConfig(config types.RuntimeConfig } func (s *RuntimeConfigFileTestSuite) runNetworkDirectionContainers() (client, server string) { - serverCmd := fmt.Sprintf("/scripts/prepare-tap.sh -a %s -o && nc -lk %s %d", ingressIP, ingressIP, ingressPort) containerID, err := s.Executor().StartContainer( config.ContainerStartConfig{ @@ -92,7 +89,7 @@ func (s *RuntimeConfigFileTestSuite) runNetworkDirectionContainers() (client, se s.Require().NoError(err) server = common.ContainerShortID(containerID) - clientCmd := fmt.Sprintf("sleep 10; while true; do nc -zv %s %d; sleep 60; done", ingressIP, ingressPort) + clientCmd := fmt.Sprintf("sleep 20; while true; do nc -zv %s %d; sleep 60; done", ingressIP, ingressPort) containerID, err = s.Executor().StartContainer( config.ContainerStartConfig{ Name: "external-connection-ingress-client", @@ -255,14 +252,17 @@ func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigNetworkIngress() { client, server := s.runNetworkDirectionContainers() defer s.teardownNetworkDirectionContainers() - s.setRuntimeConfig(types.RuntimeConfig{ + config := types.RuntimeConfig{ Networking: types.NetworkConfig{ ExternalIps: types.ExternalIpsConfig{ Enabled: "ENABLED", Direction: "INGRESS", }, }, - }) + } + + s.setRuntimeConfig(config) + assert.AssertRuntimeConfig(s.T(), collectorIP, config) // Expect both open and close events for the non-aggregated // ingress connection. If Collector is aggregating to 255.255.255.255 @@ -310,14 +310,17 @@ func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigNetworkEgress() { client, server := s.runNetworkDirectionContainers() defer s.teardownNetworkDirectionContainers() - s.setRuntimeConfig(types.RuntimeConfig{ + config := types.RuntimeConfig{ Networking: types.NetworkConfig{ ExternalIps: types.ExternalIpsConfig{ Enabled: "ENABLED", Direction: "EGRESS", }, }, - }) + } + + s.setRuntimeConfig(config) + assert.AssertRuntimeConfig(s.T(), collectorIP, config) // Expect both open and close events for the non-aggregated // egress connection. If Collector is aggregating to 255.255.255.255 @@ -365,14 +368,17 @@ func (s *RuntimeConfigFileTestSuite) TestRuntimeConfigNetworkBoth() { client, server := s.runNetworkDirectionContainers() defer s.teardownNetworkDirectionContainers() - s.setRuntimeConfig(types.RuntimeConfig{ + config := types.RuntimeConfig{ Networking: types.NetworkConfig{ ExternalIps: types.ExternalIpsConfig{ Enabled: "ENABLED", Direction: "BOTH", }, }, - }) + } + + s.setRuntimeConfig(config) + assert.AssertRuntimeConfig(s.T(), collectorIP, config) // Expect both open and close events for the non-aggregated // egress and ingress connections. If Collector is aggregating to 255.255.255.255