feat(CAF-565): K8s support for testing with Java media driver

Dockerfile

@@ -19,6 +19,7 @@ RUN apt-get update &&\
   jq \
   lsb-release \
   python3-pip \
+  numactl \
   hwloc &&\
   pip3 install --upgrade --user hdr-plot
 
@@ -30,5 +31,4 @@ RUN mkdir -p ${BENCHMARKS_PATH} &&\
   tar -C ${BENCHMARKS_PATH} -xf /root/benchmarks.tar &&\
   rm -f /root/benchmarks.tar
 
-ENTRYPOINT [ "/opt/aeron-benchmarks/scripts/k8s/k8s-benchmark-entrypoint.sh" ]
 WORKDIR ${BENCHMARKS_PATH}/scripts

README.md

@@ -236,41 +236,50 @@ will produce plots in which the histograms are grouped by test scenario by defau
 
 You will need the following Docker containers built & injected into a repository that you can use.
 
-### Containers
+The tests currently support Aeron Echo testing with either Java or C-DPDK media drivers.
+
+### Components & Containers
 
 **Benchmarks:**
 
-This wraps the code in *this* repository.
+This is the code in *this* repository. It must be built as a Docker container.
 
-eg.
-```
-docker build -t <your_repo>:aeron-benchmarks .
-docker push <your_repo>:aeron-benchmarks
-```
 
-**Aeron DPDK Media driver:**
+**Optional: Aeron DPDK Media driver**
 
 Premium feature.
 
 If required/activated in your test configuration - see https://github.com/real-logic/premium-extensions/ or ask your support contact at [Adaptive](https://weareadaptive.com/)
 
 This is expected to reside in a container called in an accessible repository.
 
-**Aeron C Media driver:**
+**Optional: Aeron C Media driver:**
 
 Support coming soon
 
-### Settings
+### Running the tests
+
 
-1. Update `scripts/k8s/base/settings.yml` with configuration from your test environment.
+1. Build the benchmarks container and push it to a repo that your K8s nodes can pull from:
+    ```
+    docker build -t <your_repo>:aeron-benchmarks .
+    docker push <your_repo>:aeron-benchmarks
+    ```
+2. Update the following files with configuration from your test environment - you can skip scenario config you don't plan to test
+   * `scripts/k8s/base/settings.yml`
+   * `scripts/k8s/base/aeron-echo-dpdk/settings.yml`
+   * `scripts/k8s/base/aeron-echo-java/settings.yml`
 
-2. Make sure your test environment is the active `kubecontext`
+3. Make sure your test environment is the active `kubecontext`
 
-3. If you are attempting to run DPDK tests, make sure you have a DPDK enabled Pod/Host. Setting this up is outside the scope of this documentation, please see https://github.com/AdaptiveConsulting/k8s-dpdk-mgr for an example of how to do this.
+4. If you are attempting to run DPDK tests, make sure you have a DPDK enabled Pod/Host. Setting this up is outside the scope of this documentation, please see https://github.com/AdaptiveConsulting/k8s-dpdk-mgr for an example of how to do this.
 
-4. Ensure you are permissioned to write to a K8s namespace, by default the tooling will use the `default` namespace.
+5. Ensure you are permissioned to write to a K8s namespace, by default the tooling will use the `default` namespace.
 
-5. Run `./scripts/k8s-remote-testing.sh ( -n my_namespace )`
+6. Run:
+   ```
+   ./scripts/k8s-remote-testing.sh (-t aeron-echo-java | aeron-echo-dpdk ) ( -n my_namespace )
+   ```
 
 ## Other benchmarks (single machine)
 Set of latency benchmarks testing round trip time (RTT) between threads or processes (IPC) via FIFO data structures and messaging systems.

scripts/k8s-remote-testing.sh

@@ -9,7 +9,7 @@ function f_log() {
 
 function f_show_help() {
   f_log "Supported arguments are:"
-  echo "${0} (-n|--namespace) '<namespace>' (-t|--test) 'aeron-echo-dpdk'"
+  echo "${0} (-n|--namespace) '<namespace>' (-t|--test) 'aeron-echo-dpdk|aeron-echo-java'"
 }
 
 while [[ $# -gt 0 ]]
@@ -23,17 +23,18 @@ do
       ;;
     -t|--test)
       TEST_TO_RUN="${2}"
-      if [[ "${TEST_TO_RUN}" != "aeron-echo-dpdk" ]]
-      then
-        f_log "Error: only supported test is 'aeron-echo-dpdk' at the moment"
+      if [[ "${TEST_TO_RUN}" == "aeron-echo-dpdk" || "${TEST_TO_RUN}" == "aeron-echo-java"  ]]
+      then true
+      else
+        f_log "Error: only supported tests are 'aeron-echo-dpdk' or 'aeron-echo-java' at the moment"
         exit 1
       fi
       shift
       shift
       ;;
     -h|--help)
       f_show_help
-      EXIT
+      exit 1
       ;;
     *)
       echo "Error, unknown argument: ${option}"
@@ -45,7 +46,7 @@ done
 
 # Standard vars
 K8S_NAMESPACE="${K8S_NAMESPACE:-default}"
-TEST_TO_RUN="${TEST_TO_RUN:-aeron-echo-dpdk}"
+TEST_TO_RUN="${TEST_TO_RUN:-aeron-echo-java}"
 
 TIMESTAMP="$(date +"%Y-%m-%d-%H-%M-%S")"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
@@ -67,8 +68,8 @@ f_cleanup_k8s
 f_log "Generating new benchmark setup for: ${TEST_TO_RUN}"
 
 kubectl -n "${K8S_NAMESPACE}" apply --wait=true -k "k8s/${TEST_TO_RUN}/"
-kubectl -n "${K8S_NAMESPACE}" wait --for=condition=Ready pod/aeron-benchmark-0
-kubectl -n "${K8S_NAMESPACE}" wait --for=condition=Ready pod/aeron-benchmark-1
+kubectl -n "${K8S_NAMESPACE}" wait --timeout=90s --for=condition=Ready pod/aeron-benchmark-0
+kubectl -n "${K8S_NAMESPACE}" wait --timeout=90s --for=condition=Ready pod/aeron-benchmark-1
 
 # DPDK Media Driver
 if [[ "${TEST_TO_RUN}" =~ .*-dpdk$ ]]

scripts/k8s/aeron-echo-dpdk/aeronmd-dpdk.yml

@@ -0,0 +1,36 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: all
+  labels:
+    aeronmd: dpdk
+spec:
+  containers:
+    - name: aeronmd-dpdk
+      imagePullPolicy: Always
+      args:
+        - "--no-telemetry"
+        - "/opt/aeron-config/low-latency-driver.properties"
+      securityContext:
+        capabilities:
+          # Required for DPDK
+          add:
+            - IPC_LOCK
+            - SYS_RAWIO
+            - SYS_ADMIN
+      envFrom:
+        - configMapRef:
+            name: aeron-benchmark-envs-dpdk
+      volumeMounts:
+        - name: hugepage-2mi
+          mountPath: /hugepages-2Mi
+        - name: shm
+          mountPath: /dev/shm
+        - name: config-volume
+          mountPath: /opt/aeron-config
+      resources:
+        limits:
+          cpu: "8"
+          memory: 8G
+          hugepages-2Mi: 1Gi

scripts/k8s/aeron-echo-dpdk/echo-client.yml

@@ -6,14 +6,14 @@ metadata:
 spec:
   containers:
     - name: benchmark
+      command: ["k8s/k8s-echo-client.sh"]
+      # These are options passed to the benchmark-runner script
       args:
-        - "./benchmark-runner"
         - "--output-file"
         - "aeron-echo_c-dpdk-k8s"
         - "--message-rate"
-        - "100K"
+        - "101K"
         - "--message-length"
         - "288"
         - "--iterations"
         - "60"
-        - "aeron/echo-client"

scripts/k8s/aeron-echo-dpdk/kustomization.yml

@@ -3,11 +3,16 @@
 resources:
   - ../base
 patches:
-  - path: echo-server.yml
+  # Media driver specific config
+  - path: aeronmd-dpdk.yml
     target:
       kind: Pod
-      name: aeron-benchmark-0
+  # Settings for the benchmark
   - path: echo-client.yml
     target:
       kind: Pod
       name: aeron-benchmark-1
+  #  Local environment specific overrides
+  - target:
+      kind: Pod
+    path: settings.yml

scripts/k8s/aeron-echo-dpdk/settings.yml

@@ -0,0 +1,17 @@
+---
+# Set test specific config
+# eg:
+# * image refs
+# * DPDK interface specifics
+apiVersion: v1
+kind: Pod
+metadata:
+  name: all
+spec:
+  containers:
+    - name: aeronmd-dpdk
+      image: "< Ref to Aeron DPDK media driver container >"
+      resources:
+        limits:
+          # Give us a DPDK NIC through https://github.com/AdaptiveConsulting/k8s-dpdk-mgr
+          intel.com/aws_dpdk: "1"

scripts/k8s/aeron-echo-java/aeronmd-java.yml

@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: all
+  labels:
+    aeronmd: java
+spec:
+  containers:
+    - name: aeronmd-java
+      imagePullPolicy: Always
+      command: ["/opt/aeron-benchmarks/scripts/k8s/k8s-java-media-driver.sh"]
+      volumeMounts:
+        - name: shm
+          mountPath: /dev/shm
+      resources:
+        limits:
+          cpu: "8"
+          memory: 8G

scripts/k8s/aeron-echo-java/echo-client.yml

@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: all
+spec:
+  containers:
+    - name: benchmark
+      command: ["k8s/k8s-echo-client.sh"]
+      # These are options passed to the benchmark-runner script
+      args:
+        - "--output-file"
+        - "aeron-echo_java-k8s"
+        - "--message-rate"
+        - "101K"
+        - "--message-length"
+        - "288"
+        - "--iterations"
+        - "60"

scripts/k8s/aeron-echo-java/kustomization.yml

@@ -0,0 +1,18 @@
+# This file contains the overrides for aeron-echo-dpdk test
+
+resources:
+  - ../base
+patches:
+  # Media driver specific config
+  - path: aeronmd-java.yml
+    target:
+      kind: Pod
+  # Settings for the benchmark
+  - path: echo-client.yml
+    target:
+      kind: Pod
+      name: aeron-benchmark-1
+  # Local environment specific settings
+  - target:
+      kind: Pod
+    path: settings.yml

scripts/k8s/aeron-echo-java/settings.yml

@@ -0,0 +1,15 @@
+---
+# Set test specific config
+# The image ref must be set
+apiVersion: v1
+kind: Pod
+metadata:
+  name: all
+spec:
+  # Uncomment me to do host network testing
+  # dnsPolicy: ClusterFirstWithHostNet
+  # hostNetwork: true
+  containers:
+    - name: aeronmd-java
+      # This is the benchmarks container with an alternate entrypoint
+      image: "< Ref to benchmark container >"

scripts/k8s/base/base-containers.yml

@@ -0,0 +1,62 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: all
+  labels:
+    app.kubernetes.io/name: aeron-benchmark
+spec:
+  # The benchmark code runs exactly once and isn't restarted
+  restartPolicy: Never
+  terminationGracePeriodSeconds: 5
+  volumes:
+    - name: hugepage-2mi
+      emptyDir:
+        medium: HugePages-2Mi
+    - name: shm
+      emptyDir:
+        medium: Memory
+        sizeLimit: 2Gi
+    # Common properties files
+    - name: config-volume
+      configMap:
+        name: aeron-benchmark
+        items:
+          - key: low-latency-driver.properties
+            path: low-latency-driver.properties
+  containers:
+    # Media driver containers come from the Kustomization files in the scenarios
+    - name: benchmark
+      imagePullPolicy: Always
+      envFrom:
+        - configMapRef:
+            name: aeron-benchmark-envs
+      volumeMounts:
+        - mountPath: /hugepages-2Mi
+          name: hugepage-2mi
+        - mountPath: /dev/shm
+          name: shm
+        - name: config-volume
+          mountPath: /opt/aeron-config
+      resources:
+        limits:
+          cpu: "8"
+          memory: 10G
+          hugepages-2Mi: 1Gi
+
+    - name: results
+      image: busybox
+      command:
+        - "sleep"
+        - "infinity"
+      envFrom:
+        - configMapRef:
+            name: aeron-benchmark-envs
+      volumeMounts:
+        - name: shm
+          mountPath: /dev/shm
+      resources:
+        # Must set limits, as whole Pod must have requests = limits to get into Guarenteed QoS class
+        limits:
+          cpu: "1"
+          memory: 512Mi

scripts/k8s/aeron-echo-dpdk/echo-server.yml → scripts/k8s/base/echo-server.yml

@@ -6,4 +6,4 @@ metadata:
 spec:
   containers:
     - name: benchmark
-      args: ["aeron/echo-server"]
+      command: ["k8s/k8s-echo-server.sh"]