CR-8961 network test before runtime installation (#250)

* network test first draft

* wip

* added skip-network-test flag

* handle env vars

* small fix

* generated docs

* small refactor of testNetwork

* move functions to util

* delete pod when test finished

* bump

* handle defer errors

* extracted logic to checkPodLastState

* some improvements

* small improve

* gen docs

* refactor reporter

* indentation

Co-authored-by: Noam Gal <noam.gal@codefresh.io>

Author: rotem-codefresh

Makefile

@@ -1,4 +1,4 @@
-VERSION=v0.0.227
+VERSION=v0.0.228
 
 OUT_DIR=dist
 YEAR?=$(shell date +"%Y")

cmd/commands/runtime.go

@@ -77,6 +77,7 @@ type (
 		IngressController              string
 		Insecure                       bool
 		InstallDemoResources           bool
+		SkipClusterChecks                bool
 		Version                        *semver.Version
 		GsCloneOpts                    *git.CloneOptions
 		InsCloneOpts                   *git.CloneOptions
@@ -218,6 +219,7 @@ func NewRuntimeInstallCommand() *cobra.Command {
 	cmd.Flags().StringVar(&installationOpts.GitIntegrationRegistrationOpts.Token, "personal-git-token", "", "The Personal git token for your user")
 	cmd.Flags().StringVar(&installationOpts.versionStr, "version", "", "The runtime version to install (default: latest)")
 	cmd.Flags().BoolVar(&installationOpts.InstallDemoResources, "demo-resources", true, "Installs demo resources (default: true)")
+	cmd.Flags().BoolVar(&installationOpts.SkipClusterChecks, "skip-cluster-checks", false, "Skips the cluster's checks")
 	cmd.Flags().DurationVar(&store.Get().WaitTimeout, "wait-timeout", store.Get().WaitTimeout, "How long to wait for the runtime components to be ready")
 	cmd.Flags().StringVar(&gitIntegrationCreationOpts.APIURL, "provider-api-url", "", "Git provider API url")
 	cmd.Flags().BoolVar(&store.Get().BypassIngressClassCheck, "bypass-ingress-class-check", false, "Disables the ingress class check during pre-installation")
@@ -227,7 +229,7 @@ func NewRuntimeInstallCommand() *cobra.Command {
 	})
 
 	installationOpts.GsCloneOpts = &git.CloneOptions{
-		FS: fs.Create(memfs.New()),
+		FS:               fs.Create(memfs.New()),
 		CreateIfNotExist: true,
 	}
 
@@ -783,6 +785,17 @@ func preInstallationChecks(ctx context.Context, opts *RuntimeInstallOptions) err
 		return fmt.Errorf("existing runtime check failed: %w", err)
 	}
 
+	if !opts.SkipClusterChecks {
+		err = util.RunNetworkTest(ctx, opts.KubeFactory, cfConfig.GetCurrentContext().URL)
+		if err != nil {
+			log.G(ctx).Info("Network test finished successfully")
+		}
+	}
+	handleCliStep(reporter.InstallStepRunPreCheckClusterChecks, "Running cluster checks", err, false)
+	if err != nil {
+		return fmt.Errorf(fmt.Sprintf("cluster network tests failed: %v ", err))
+	}
+
 	err = kubeutil.EnsureClusterRequirements(ctx, opts.KubeFactory, opts.RuntimeName)
 	handleCliStep(reporter.InstallStepRunPreCheckValidateClusterRequirements, "Ensuring cluster requirements", err, false)
 	if err != nil {

docs/commands/cli-v2_runtime_install.md

@@ -41,6 +41,7 @@ cli-v2 runtime install [runtime_name] [flags]
       --provider string             The git provider, one of: gitea|github|gitlab
       --provider-api-url string     Git provider API url
       --repo string                 Repository URL [GIT_REPO]
+      --skip-cluster-checks         Skips the cluster's checks
       --version string              The runtime version to install (default: latest)
       --wait-timeout duration       How long to wait for the runtime components to be ready (default 8m0s)
 ```

docs/releases/release_notes.md

@@ -23,7 +23,7 @@ cf version
 
 ```bash
 # download and extract the binary
-curl -L --output - https://github.com/codefresh-io/cli-v2/releases/download/v0.0.227/cf-linux-amd64.tar.gz | tar zx
+curl -L --output - https://github.com/codefresh-io/cli-v2/releases/download/v0.0.228/cf-linux-amd64.tar.gz | tar zx
 
 # move the binary to your $PATH
 mv ./cf-linux-amd64 /usr/local/bin/cf
@@ -36,7 +36,7 @@ cf version
 
 ```bash
 # download and extract the binary
-curl -L --output - https://github.com/codefresh-io/cli-v2/releases/download/v0.0.227/cf-darwin-amd64.tar.gz | tar zx
+curl -L --output - https://github.com/codefresh-io/cli-v2/releases/download/v0.0.228/cf-darwin-amd64.tar.gz | tar zx
 
 # move the binary to your $PATH
 mv ./cf-darwin-amd64 /usr/local/bin/cf

manifests/runtime.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: "{{ namespace }}"
 spec:
   defVersion: 1.0.0
-  version: 0.0.227
+  version: 0.0.228
   bootstrapSpecifier: github.com/codefresh-io/cli-v2/manifests/argo-cd
   components:
     - name: events

pkg/reporter/reporter.go

@@ -75,6 +75,7 @@ const (
 	InstallStepRunPreCheckEnsureCliVersion            CliStep = "install.run.pre-check.step.ensure-cli-version"
 	InstallStepRunPreCheckRuntimeCollision            CliStep = "install.run.pre-check.step.runtime-collision"
 	InstallStepRunPreCheckExisitingRuntimes           CliStep = "install.run.pre-check.step.existing-runtimes"
+	InstallStepRunPreCheckClusterChecks               CliStep = "install.run.pre-check.step.cluster-checks"
 	InstallStepRunPreCheckValidateClusterRequirements CliStep = "install.run.pre-check.step.validate-cluster-requirements"
 	InstallPhaseRunPreCheckFinish                     CliStep = "install.run.pre-check.phase.finish"
 	InstallPhaseStart                                 CliStep = "install.run.phase.start"

pkg/store/store.go

@@ -130,6 +130,10 @@ type Store struct {
 	RolloutResourceName                  string
 	RolloutReporterServiceAccount        string
 	SegmentWriteKey                      string
+	DefaultNamespace                     string
+	NetworkTesterName                    string
+	NetworkTesterGenerateName            string
+	NetworkTesterImage                   string
 }
 
 // Get returns the global store
@@ -217,6 +221,11 @@ func init() {
 	s.SegmentWriteKey = segmentWriteKey
 	s.RequirementsLink = "https://codefresh.io/csdp-docs/docs/runtime/requirements/"
 	s.DownloadCliLink = "https://codefresh.io/csdp-docs/docs/clients/csdp-cli/"
+	s.DefaultNamespace = "default"
+	s.NetworkTesterName = "cf-network-tester"
+	s.NetworkTesterGenerateName = "cf-network-tester-"
+	s.NetworkTesterImage = "codefresh/cf-venona-network-tester:latest"
+
 	initVersion()
 }
 

pkg/util/kube/kube.go

@@ -22,6 +22,7 @@ import (
 	"github.com/argoproj-labs/argocd-autopilot/pkg/kube"
 	"github.com/codefresh-io/cli-v2/pkg/store"
 	authv1 "k8s.io/api/authorization/v1"
+	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -41,6 +42,16 @@ type (
 		memorySize string
 		rbac       []rbacValidation
 	}
+
+	LaunchJobOptions struct {
+		Client        kubernetes.Interface
+		Namespace     string
+		JobName       *string
+		Image         *string
+		Env           []v1.EnvVar
+		RestartPolicy v1.RestartPolicy
+		BackOffLimit  int32
+	}
 )
 
 func EnsureClusterRequirements(ctx context.Context, kubeFactory kube.Factory, namespace string) error {
@@ -202,3 +213,36 @@ func testNode(n v1.Node, req validationRequest) []string {
 
 	return result
 }
+
+func LaunchJob(ctx context.Context, opts LaunchJobOptions) error {
+	jobs := opts.Client.BatchV1().Jobs(opts.Namespace)
+
+	jobSpec := &batchv1.Job{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      *opts.JobName,
+			Namespace: opts.Namespace,
+		},
+		Spec: batchv1.JobSpec{
+			Template: v1.PodTemplateSpec{
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name:  *opts.JobName,
+							Image: *opts.Image,
+							Env:   opts.Env,
+						},
+					},
+					RestartPolicy: opts.RestartPolicy,
+				},
+			},
+			BackoffLimit: &opts.BackOffLimit,
+		},
+	}
+
+	_, err := jobs.Create(ctx, jobSpec, metav1.CreateOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to create K8s job '%s' : %w", *opts.JobName, err)
+	}
+
+	return nil
+}

pkg/util/util.go

@@ -15,20 +15,28 @@
 package util
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"os/signal"
 	"regexp"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/argoproj-labs/argocd-autopilot/pkg/kube"
 	"github.com/briandowns/spinner"
 	"github.com/codefresh-io/cli-v2/pkg/log"
 	"github.com/codefresh-io/cli-v2/pkg/reporter"
 	"github.com/codefresh-io/cli-v2/pkg/store"
+	kubeutil "github.com/codefresh-io/cli-v2/pkg/util/kube"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	v1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 )
 
@@ -188,3 +196,148 @@ func reportCancel(status reporter.CliStepStatus) {
 	})
 }
 
+func RunNetworkTest(ctx context.Context, kubeFactory kube.Factory, urls ...string) error {
+	const networkTestsTimeout = 120 * time.Second
+	var testerPodName string
+
+	envVars := map[string]string{
+		"URLS": strings.Join(urls, ","),
+		"IN_CLUSTER": "1",
+	}
+	env := prepareEnvVars(envVars)
+
+	client, err := kubeFactory.KubernetesClientSet()
+	if err != nil {
+		return fmt.Errorf("failed to create kubernetes client: %w", err)
+	}
+
+	err = kubeutil.LaunchJob(ctx, kubeutil.LaunchJobOptions{
+		Client:        client,
+		Namespace:     store.Get().DefaultNamespace,
+		JobName:       &store.Get().NetworkTesterName,
+		Image:         &store.Get().NetworkTesterImage,
+		Env:           env,
+		RestartPolicy: v1.RestartPolicyNever,
+		BackOffLimit:  0,
+	})
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		deferErr := client.BatchV1().Jobs(store.Get().DefaultNamespace).Delete(ctx, store.Get().NetworkTesterName, metav1.DeleteOptions{})
+		if deferErr != nil {
+			log.G(ctx).Error("fail to delete job resource '%s': %s", store.Get().NetworkTesterName, deferErr.Error())
+		}
+	}()
+
+	log.G(ctx).Info("Running network test...")
+
+	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
+	var podLastState *v1.Pod
+	timeoutChan := time.After(networkTestsTimeout)
+
+Loop:
+	for {
+		select {
+		case <-ticker.C:
+			log.G(ctx).Debug("Waiting for network tester to finish")
+
+			if testerPodName == "" {
+				testerPodName, err = getTesterPodName(ctx, client)
+				if err != nil {
+					return err
+				}
+			}
+
+			pod, err := client.CoreV1().Pods(store.Get().DefaultNamespace).Get(ctx, testerPodName, metav1.GetOptions{})
+			if err != nil {
+				if statusError, errIsStatusError := err.(*kerrors.StatusError); errIsStatusError {
+					if statusError.ErrStatus.Reason == metav1.StatusReasonNotFound {
+						log.G(ctx).Debug("Network tester pod not found")
+					}
+				}
+			}
+			if len(pod.Status.ContainerStatuses) == 0 {
+				log.G(ctx).Debug("Network tester pod: creating container")
+				continue
+			}
+			if pod.Status.ContainerStatuses[0].State.Running != nil {
+				log.G(ctx).Debug("Network tester pod: running")
+			}
+			if pod.Status.ContainerStatuses[0].State.Waiting != nil {
+				log.G(ctx).Debug("Network tester pod: waiting")
+			}
+			if pod.Status.ContainerStatuses[0].State.Terminated != nil {
+				log.G(ctx).Debug("Network tester pod: terminated")
+				podLastState = pod
+				break Loop
+			}
+		case <-timeoutChan:
+			return fmt.Errorf("network test timeout reached!")
+		}
+	}
+
+	defer func() {
+		deferErr := client.CoreV1().Pods(store.Get().DefaultNamespace).Delete(ctx, testerPodName, metav1.DeleteOptions{})
+		if deferErr != nil {
+			log.G(ctx).Error("fail to delete tester pod '%s': %s", testerPodName, deferErr.Error())
+		}
+	}()
+	
+	return checkPodLastState(ctx, client, testerPodName,podLastState)
+}
+
+func prepareEnvVars(vars map[string]string) []v1.EnvVar {
+	var env []v1.EnvVar
+
+	for key, value := range vars {
+		env = append(env, v1.EnvVar{
+			Name:  key,
+			Value: value,
+		})
+	}
+
+	return env
+}
+
+func getTesterPodName(ctx context.Context, client kubernetes.Interface) (string, error) {
+	pods, err := client.CoreV1().Pods(store.Get().DefaultNamespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return "", fmt.Errorf("failed to get pods from cluster: %w", err)
+	}
+
+	for _, pod := range pods.Items {
+		if pod.ObjectMeta.GenerateName == store.Get().NetworkTesterGenerateName {
+			return pod.ObjectMeta.Name, nil
+		}
+	}
+
+	return "", nil
+}
+
+func checkPodLastState(ctx context.Context, client kubernetes.Interface, name string, podLastState *v1.Pod) error {
+	req := client.CoreV1().Pods(store.Get().DefaultNamespace).GetLogs(name, &v1.PodLogOptions{})
+	podLogs, err := req.Stream(ctx)
+	if err != nil {
+		return fmt.Errorf("Failed to get network-tester pod logs: %w", err)
+	}
+	defer podLogs.Close()
+
+	logsBuf := new(bytes.Buffer)
+	_, err = io.Copy(logsBuf, podLogs)
+	if err != nil {
+		return fmt.Errorf("Failed to read network-tester pod logs: %w", err)
+	}
+	logs := strings.Trim(logsBuf.String(), "\n")
+	log.G(ctx).Debug(logs)
+
+	if podLastState.Status.ContainerStatuses[0].State.Terminated.ExitCode != 0 {
+		terminationMessage := strings.Trim(podLastState.Status.ContainerStatuses[0].State.Terminated.Message, "\n")
+		return fmt.Errorf("Network test failed with: %s", terminationMessage)
+	}
+
+	return nil
+}
+