Update documentation about running intrusive tests (#2868)

* Update documentation about running instrusive tests because the newest version of the tooling is managed by the flag and not by env

* Update documentation about running intrusive tests because the newest version of the tooling is managed by the flag and not by env

* Update documentation about running intrusive tests because the newest version of the tooling is managed by the flag and not by env

* Change the flag non-intrusive to intrusive and update the documentation accordingly

Author: rocrisp

.github/workflows/tnf-image.yaml

@@ -27,7 +27,6 @@ env:
   CERTSUITE_IMAGE_NAME_LEGACY: testnetworkfunction/cnf-certification-test
   IMAGE_TAG: latest
   CERTSUITE_CONTAINER_CLIENT: docker
-  CERTSUITE_NON_INTRUSIVE_ONLY: false
   CERTSUITE_ALLOW_PREFLIGHT_INSECURE: false
   CERTSUITE_CONFIG_DIR: /tmp/tnf/config
   CERTSUITE_SRC_URL: 'https://github.com/${{ github.repository }}'

cmd/certsuite/run/run.go

@@ -34,7 +34,7 @@ func NewCommand() *cobra.Command {
 	runCmd.PersistentFlags().String("log-level", "debug", "Sets the log level")
 	runCmd.PersistentFlags().String("offline-db", "", "Set the location of an offline DB to check the certification status of for container images, operators and helm charts")
 	runCmd.PersistentFlags().String("preflight-dockerconfig", "", "Set the dockerconfig file to be used by the Preflight test suite")
-	runCmd.PersistentFlags().Bool("non-intrusive", false, "Run only the test that do not disrupt the test environment")
+	runCmd.PersistentFlags().Bool("intrusive", true, "Run intrusive tests that may disrupt the test environment")
 	runCmd.PersistentFlags().Bool("allow-preflight-insecure", false, "Allow insecure connections in the Preflight test suite")
 	runCmd.PersistentFlags().Bool("include-web-files", false, "Save web files in the configured output folder")
 	runCmd.PersistentFlags().Bool("enable-data-collection", false, "Allow sending test results to an external data collector")
@@ -67,7 +67,7 @@ func initTestParamsFromFlags(cmd *cobra.Command) error {
 	testParams.LogLevel, _ = cmd.Flags().GetString("log-level")
 	testParams.OfflineDB, _ = cmd.Flags().GetString("offline-db")
 	testParams.PfltDockerconfig, _ = cmd.Flags().GetString("preflight-dockerconfig")
-	testParams.NonIntrusiveOnly, _ = cmd.Flags().GetBool("non-intrusive")
+	testParams.Intrusive, _ = cmd.Flags().GetBool("intrusive")
 	testParams.AllowPreflightInsecure, _ = cmd.Flags().GetBool("allow-preflight-insecure")
 	testParams.IncludeWebFilesInOutputFolder, _ = cmd.Flags().GetBool("include-web-files")
 	testParams.EnableDataCollection, _ = cmd.Flags().GetBool("enable-data-collection")

docs/cluster-deploy/README.md

@@ -44,7 +44,7 @@ But there are the three example of modifications included in [./kustomization.ya
 
 1. The namespace and the prefix/suffix of each resource's name. By default, the [./certsuite.yaml](certsuite.yaml) uses the namespace "certsuite" to deploy all the reources (except the cluster role and the cluster role binding), but this can be changed uncommenting the line that starts with `namespace:`. It's highly recommended to uncomment at least one of suffixName/prefixName so unique cluster role & cluster role-bindings can be created for each CertSuite Pod. This way, you could run more than one CertSuite Pod in the same cluster!.
 2. The (ginkgo) labels expression, in case you want to run different test cases. Uncomment the object that starts with "patches:". The commented example changes the command to use the "preflight" label only.
-3. The value of the CERTSUITE_NON_INTRUSIVE_ONLY env var. Uncomment the last object that starts with "patches:". The commented example changes the CERTSUITE_NON_INTRUSIVE_ONLY to false, so all the intrusive TCs will run in case the lifecycle TCs are selected to run by the appropriate labels.
+3. Configuring the --intrusive Flag. Uncomment the last object that starts with "patches:". The commented example changes the --intrusive to true, so all the intrusive TCs will run in case the lifecycle TCs are selected to run by the appropriate labels.
 
 In case both (1) and (2) wants to be used, just create a list of patches like this:
 
@@ -65,6 +65,7 @@ patches:
       name: certsuite
     patch: |
       - op: replace
-        path: /spec/containers/0/env/0/value
-        value: false
+        path: /spec/containers/0/args/1
+        value: |
+        ./certsuite run -l 'preflight' --intrusive=true ; sleep inf
 ```

docs/cluster-deploy/certsuite.yaml

@@ -127,15 +127,13 @@ spec:
       args:
         - "-c"
         - |
-          ./certsuite run -l '!affiliated-certification-container-is-certified-digest && !access-control-security-context' ; sleep inf
+          ./certsuite run -l '!affiliated-certification-container-is-certified-digest && !access-control-security-context' --intrusive=false ; sleep inf
       volumeMounts:
         - name: config-volume
           mountPath: /usr/certsuite/config
         - name: preflight-dockerconfig
           mountPath: /usr/certsuite/config/preflight
       env:
-        - name: CERTSUITE_NON_INTRUSIVE_ONLY
-          value: "true"
         - name: CERTSUITE_ALLOW_PREFLIGHT_INSECURE
           value: "true"
         - name: CERTSUITE_LOG_LEVEL

docs/cluster-deploy/kustomization.yaml

@@ -4,7 +4,6 @@ kind: Kustomization
 
 resources:
   - certsuite.yaml
-
 # Uncomment the next line (namespace transformer) to deploy all the certsuite related
 # resources in a different namespace.
 # namespace: my-custom-ns-name
@@ -34,5 +33,6 @@ resources:
 #       name: certsuite
 #     patch: |
 #       - op: replace
-#         path: /spec/containers/0/env/0/value
-#         value: false
+#         path: /spec/containers/0/args/1
+#         value: |
+#           ./certsuite.sh -l '!affiliated-certification-container-is-certified-digest && !access-control-security-context' --intrusive=true ; sleep inf

docs/developers.md

@@ -15,9 +15,9 @@ To test the newly added test / existing tests locally, follow the steps
 
 - Also, skip intrusive tests
 
-```shell
-export CERTSUITE_NON_INTRUSIVE_ONLY=true
-```
+    ```shell
+    ./certsuite run --intrusive=false
+    ```
 
 - Set K8s config of the cluster where test pods are running
 

docs/runtime-env.md

@@ -20,30 +20,6 @@ oc label namespace/default pod-security.kubernetes.io/enforce=privileged
 oc label namespace/default pod-security.kubernetes.io/enforce-version=latest
 ```
 
-## Disable intrusive tests
-
-To skip intrusive tests which may disrupt cluster operations, issue the
-following:
-
-```shell
-export CERTSUITE_NON_INTRUSIVE_ONLY=true
-```
-
-The intrusive test cases are:
-
-- [lifecycle-deployment-scaling](https://github.com/redhat-best-practices-for-k8s/certsuite/blob/main/CATALOG.md#lifecycle-deployment-scaling)
-- [lifecycle-statefulset-scaling](https://github.com/redhat-best-practices-for-k8s/certsuite/blob/main/CATALOG.md#lifecycle-statefulset-scaling)
-- [lifecycle-crd-scaling](https://github.com/redhat-best-practices-for-k8s/certsuite/blob/main/CATALOG.md#lifecycle-crd-scaling)
-- [lifecycle-pod-recreation](https://github.com/redhat-best-practices-for-k8s/certsuite/blob/main/CATALOG.md#lifecycle-pod-recreation)
-
-Likewise, to enable intrusive tests, set the following:
-
-```shell
-export CERTSUITE_NON_INTRUSIVE_ONLY=false
-```
-
-Intrusive tests are enabled by default.
-
 ## Preflight Integration
 
 When running the `preflight` suite of tests, there are a few environment variables that

docs/test-run.md

@@ -48,6 +48,30 @@ To view which test cases will run for a specific label or label filter use the f
 
 See the [CATALOG.md](CATALOG.md) to find all test labels.
 
+## Disable intrusive tests
+
+To skip intrusive tests which may disrupt cluster operations, issue the
+following:
+
+```shell
+./certsuite run --intrusive=false
+```
+
+The intrusive test cases are:
+
+* [lifecycle-deployment-scaling](https://github.com/redhat-best-practices-for-k8s/certsuite/blob/main/CATALOG.md#lifecycle-deployment-scaling)
+* [lifecycle-statefulset-scaling](https://github.com/redhat-best-practices-for-k8s/certsuite/blob/main/CATALOG.md#lifecycle-statefulset-scaling)
+* [lifecycle-crd-scaling](https://github.com/redhat-best-practices-for-k8s/certsuite/blob/main/CATALOG.md#lifecycle-crd-scaling)
+* [lifecycle-pod-recreation](https://github.com/redhat-best-practices-for-k8s/certsuite/blob/main/CATALOG.md#lifecycle-pod-recreation)
+
+Likewise, to enable intrusive tests, set the following:
+
+```shell
+./certsuite run --intrusive=true
+```
+
+Intrusive tests are enabled by default.
+
 ## Selected flags description
 
 The following is a non-exhaustive list of the most common flags that the `certsuite run` command accepts. To see the complete list use the `-h, --help` flag.

pkg/configuration/configuration.go

@@ -125,7 +125,7 @@ type TestParameters struct {
 	SanitizeClaim                 bool
 	CertSuiteImageRepo            string
 	CertSuiteProbeImage           string
-	NonIntrusiveOnly              bool
+	Intrusive                     bool
 	AllowPreflightInsecure        bool
 	IncludeWebFilesInOutputFolder bool
 	OmitArtifactsZipFile          bool

pkg/provider/provider.go

@@ -579,7 +579,7 @@ func (env *TestEnvironment) SetNeedsRefresh() {
 }
 
 func (env *TestEnvironment) IsIntrusive() bool {
-	return !env.params.NonIntrusiveOnly
+	return env.params.Intrusive
 }
 
 func (env *TestEnvironment) IsPreflightInsecureAllowed() bool {