Refactor charts

Author: andylibrian

agent/config/config.go

@@ -11,7 +11,8 @@ type AgentConfigurations struct {
 	SANamespace              string `envconfig:"SA_NAMESPACE" default:"default"`
 	SAName                   string `envconfig:"SA_NAME" default:"default"`
 	OutdatedInterval         string `envconfig:"OUTDATED_INTERVAL" default:"0"`
-	KubeAllResources         string `envconfig:"KUBE_ALL_RESOURCES" default:"*/30 * * * *"`
+	KubeAllResourcesInterval string `envconfig:"KUBE_ALL_RESOURCES_INTERVAL" default:"*/30 * * * *"`
+	KubeAllResourcesEnabled  bool   `envconfig:"KUBE_ALL_RESOURCES_ENABLED" default:"true"`
 	GetAllInterval           string `envconfig:"GETALL_INTERVAL" default:"*/30 * * * *"`
 	KubeScoreInterval        string `envconfig:"KUBESCORE_INTERVAL" default:"*/40 * * * *"`
 	RakkessInterval          string `envconfig:"RAKKESS_INTERVAL" default:"*/50 * * * *"`

agent/kubviz/k8smetrics_agent.go

@@ -160,8 +160,12 @@ func main() {
 		events.LogErr(err)
 		err = kubepreupgrade.KubePreUpgradeDetector(config, js)
 		events.LogErr(err)
-		err = kubeallresources.PublishAllResources(config)
-		events.LogErr(err)
+
+		if cfg.KubeAllResourcesEnabled {
+			err = kubeallresources.PublishAllResources(config)
+			events.LogErr(err)
+		}
+
 		err = ketall.GetAllResources(config, js)
 		events.LogErr(err)
 		err = rakkess.RakeesOutput(config, js)

agent/kubviz/scheduler/scheduler.go

@@ -104,8 +104,9 @@ func InitScheduler(config *rest.Config, js nats.JetStreamContext, cfg config.Age
 			log.Fatal("failed to do job", err)
 		}
 	}
-	if cfg.GetAllInterval != "" && cfg.GetAllInterval != "0" {
-		sj, err := NewKubeAllResourcesJob(config, cfg.GetAllInterval)
+
+	if cfg.KubeAllResourcesEnabled && cfg.KubeAllResourcesInterval != "" && cfg.KubeAllResourcesInterval != "0" {
+		sj, err := NewKubeAllResourcesJob(config, cfg.KubeAllResourcesInterval)
 		if err != nil {
 			log.Fatal("no time interval", err)
 		}
@@ -114,6 +115,7 @@ func InitScheduler(config *rest.Config, js nats.JetStreamContext, cfg config.Age
 			log.Fatal("failed to do job", err)
 		}
 	}
+
 	if cfg.GetAllInterval != "" && cfg.GetAllInterval != "0" {
 		sj, err := NewKetallJob(config, js, cfg.GetAllInterval)
 		if err != nil {

charts/agent/templates/deployment.yaml

@@ -101,8 +101,12 @@ spec:
             value: {{ .Values.opentelemetry.url }}
           - name : APPLICATION_NAME
             value : {{ .Values.opentelemetry.appName }}
+          {{- if .Values.transportV2.enabled }}
           - name: OTEL_EXPORTER_OTLP_ENDPOINT
-            value: {{ .Values.otelCollector.exporterOtlpEndpoint }}
+            value: {{ .Values.transportV2.exporterOtlpEndpoint }}
+          {{- end }}   
+          - name: KUBE_ALL_RESOURCES_ENABLED
+            value: "{{ .Values.kubeAllResources.enabled }}"
           volumeMounts:
           {{- if .Values.mtls.enabled }}    
           - name: mtls

charts/agent/values.yaml

@@ -267,10 +267,13 @@ opentelemetry:
   url: "otelcollector.local"
   appName: "kubviz"
 
-# TODO: Merge with opentelemetry key above
-otelCollector:
+transportV2:
+  enabled: true
   exporterOtlpEndpoint: grpc://kubviz-client-otel-collector:4317
 
+kubeAllResources:
+  enabled: true
+
 clusterName: "kubviz"
 
 externalSecrets:

charts/client/templates/configmap-otel-collector-config.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.otelCollector.enabled }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -30,3 +31,4 @@ data:
           receivers: [otlp]
           processors: [batch]
           exporters: [nats]
+{{- end }}

charts/client/templates/deployment-otel-collector.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.otelCollector.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -33,4 +34,4 @@ spec:
       - name: otel-collector-config
         configMap:
           name: {{ include "client.fullname" . }}-otel-collector-config
- 
+{{- end }}

charts/client/templates/deployment.yaml

@@ -149,8 +149,10 @@ spec:
                 key: {{ .Values.existingClickhouse.secret.passwordkey }}
             {{- end }}
       {{- end }}
+      {{- if .Values.dgraph.enabled }}
           - name: DGRAPH_GRPC_ADDRESS
-            value: {{ .Values.dgraph.grpc_address }}
+            value: {{ .Values.dgraph.grpcAddress }}
+      {{- end }}
           - name: DB_PORT
             value: "9000"
           - name: TTL_INTERVAL

charts/client/values.yaml

@@ -84,6 +84,7 @@ externalSecrets:
   create: false
 
 otelCollector:
+  enabled: true
   image:
     repository: localhost:5000/kubviz-otel-collector
     pullPolicy: Always
@@ -132,7 +133,8 @@ existingClickhouse:
     # passwordkey: ""
 
 dgraph:
-  grpc_address: dgraph-public:9080
+  enabled: true
+  grpcAddress: dgraph-public:9080
 
 grafana:
   enabled: false

client/pkg/application/application.go

@@ -24,9 +24,10 @@ import (
 )
 
 type Application struct {
-	Config   *config.Config
-	conn     *clients.NATSContext
-	dbClient clickhouse.DBInterface
+	Config     *config.Config
+	conn       *clients.NATSContext
+	dbClient   clickhouse.DBInterface
+	dgraphGrpc *grpc.ClientConn
 }
 
 const (
@@ -72,19 +73,22 @@ func Start() *Application {
 	}
 
 	// K8s Dgraph
+	var dgraphClient *dgo.Dgraph
+	var grpcConn *grpc.ClientConn
 
-	// Initialize Dgraph client
-	grpcConn, err := grpc.Dial(cfg.DgraphGrpcAddress, grpc.WithInsecure())
-	if err != nil {
-		log.Fatalf("Error connecting to Dgraph: %v", err)
-	}
-	defer conn.Close()
-	dgraphClient := dgo.NewDgraphClient(api.NewDgraphClient(grpcConn))
+	if cfg.DgraphGrpcAddress != "" {
+		// Initialize Dgraph client
+		grpcConn, err = grpc.Dial(cfg.DgraphGrpcAddress, grpc.WithInsecure())
+		if err != nil {
+			log.Fatalf("Error connecting to Dgraph: %v", err)
+		}
+		dgraphClient = dgo.NewDgraphClient(api.NewDgraphClient(grpcConn))
 
-	// Setup schema
-	err = dgraph.SetupDgraphSchema(ctx, dgraphClient)
-	if err != nil {
-		log.Fatalf("Failed to set up Dgraph schema: %v", err)
+		// Setup schema
+		err = dgraph.SetupDgraphSchema(ctx, dgraphClient)
+		if err != nil {
+			log.Fatalf("Failed to set up Dgraph schema: %v", err)
+		}
 	}
 
 	// Connect to NATS
@@ -140,17 +144,29 @@ func Start() *Application {
 	}
 
 	return &Application{
-		Config:   cfg,
-		conn:     natsContext,
-		dbClient: dbClient,
+		Config:     cfg,
+		conn:       natsContext,
+		dbClient:   dbClient,
+		dgraphGrpc: grpcConn,
 	}
 }
 
 func (app *Application) Close() {
 	log.Printf("Closing the service gracefully")
-	app.conn.Close()
-	app.dbClient.Close()
+
+	if app.conn != nil {
+		app.conn.Close()
+	}
+
+	if app.dbClient != nil {
+		app.dbClient.Close()
+	}
+
+	if app.dgraphGrpc != nil {
+		app.dgraphGrpc.Close()
+	}
 }
+
 func exportDataForTables(db *sql.DB) error {
 	//pvcMountPath := "/mnt/client/kbz"
 	tables := []string{

client/pkg/clients/kubviz_client.go

@@ -5,23 +5,21 @@ import (
 	"encoding/json"
 	"log"
 
+	"github.com/dgraph-io/dgo/v240"
+	"github.com/intelops/kubviz/client/pkg/clickhouse"
+	"github.com/intelops/kubviz/client/pkg/config"
+	"github.com/intelops/kubviz/client/pkg/dgraph"
+	"github.com/intelops/kubviz/client/pkg/kubernetes"
 	"github.com/intelops/kubviz/constants"
 	"github.com/intelops/kubviz/model"
 	"github.com/intelops/kubviz/pkg/opentelemetry"
 	"github.com/kelseyhightower/envconfig"
 	"github.com/kuberhealthy/kuberhealthy/v2/pkg/health"
 	"github.com/nats-io/nats.go"
+	"go.opentelemetry.io/collector/pdata/plog"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-
-	"github.com/intelops/kubviz/client/pkg/clickhouse"
-	"github.com/intelops/kubviz/client/pkg/config"
-	"github.com/intelops/kubviz/client/pkg/dgraph"
-	"github.com/intelops/kubviz/client/pkg/kubernetes"
-	"go.opentelemetry.io/collector/pdata/plog"
-
-	"github.com/dgraph-io/dgo/v240"
 )
 
 type SubscriptionInfo struct {
@@ -42,12 +40,6 @@ func (n *NATSContext) SubscribeAllKubvizNats(conn clickhouse.DBInterface, dgraph
 		log.Fatalf("Could not parse env Config: %v", err)
 	}
 
-	plogUnmarshaller := &plog.JSONUnmarshaler{}
-
-	// Create a new repository instance
-	repo := dgraph.NewDgraphKubernetesResourceRepository(dgraphClient)
-	relationshipRepo := dgraph.NewDgraphRelationshipRepository(dgraphClient)
-
 	subscriptions := []SubscriptionInfo{
 		{
 			Subject:  constants.KetallSubject,
@@ -216,7 +208,16 @@ func (n *NATSContext) SubscribeAllKubvizNats(conn clickhouse.DBInterface, dgraph
 				log.Println()
 			},
 		},
-		{
+	}
+
+	if dgraphClient != nil {
+		plogUnmarshaller := &plog.JSONUnmarshaler{}
+
+		// Create a new repository instance
+		repo := dgraph.NewDgraphKubernetesResourceRepository(dgraphClient)
+		relationshipRepo := dgraph.NewDgraphRelationshipRepository(dgraphClient)
+
+		subscriptions = append(subscriptions, SubscriptionInfo{
 			Subject:  constants.KubeAllResourcesSubject,
 			Consumer: cfg.KubeAllResourcesConsumer,
 			Handler: func(msg *nats.Msg) {
@@ -237,6 +238,8 @@ func (n *NATSContext) SubscribeAllKubvizNats(conn clickhouse.DBInterface, dgraph
 						sl := sls.At(j)
 						logRecords := sl.LogRecords()
 						for k := 0; k < logRecords.Len(); k++ {
+							ctx := context.Background()
+
 							logRecord := logRecords.At(k)
 
 							// Print log body
@@ -291,7 +294,7 @@ func (n *NATSContext) SubscribeAllKubvizNats(conn clickhouse.DBInterface, dgraph
 					}
 				}
 			},
-		},
+		})
 	}
 
 	for _, sub := range subscriptions {

dockerfiles/agent/kubviz/Dockerfile

@@ -2,10 +2,16 @@
 FROM golang:1.22 as builder
 
 WORKDIR /workspace
+
 # Copy the Go Modules manifests
-COPY ./ ./
+COPY ./go.mod ./
+COPY ./go.sum ./
 RUN go mod download
+
+COPY ./ ./
+
 RUN gofmt -w -r '"github.com/googleapis/gnostic/OpenAPIv2" -> "github.com/googleapis/gnostic/openapiv2"' /go/pkg/mod/sigs.k8s.io/kustomize/
+
 # Build
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o kubviz_agent agent/kubviz/*.go
 

dockerfiles/client/Dockerfile

@@ -2,10 +2,14 @@
 FROM cgr.dev/chainguard/go:latest AS builder
 
 WORKDIR /workspace
+
 # Copy the Go Modules manifests
-COPY ./ ./
+COPY ./go.mod ./
+COPY ./go.sum ./
 RUN go mod download
 
+COPY ./ ./
+
 # Build
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o k8smetrics_client client/main.go