How to use the otel collector as an agent for each deployment and display the k8s metrics of each pod based on the namespace

[Kaua3045]

I'm currently deploying via argoCD, so each environment for each app has N pods, everything is separated by namespaces, for example: invoices-dev and together I deploy a hotel collector as a daemonset, which in the end exports metrics to datadog, I would like to put the k8s metrics together in these traces, so I can see how many healthy replicas I have, cpu usage of each app (java, go, etc...) and other things, all filtered by namespace, my config is currently like this. And I would like to know how to tell my app how to connect to this daemonset, using a service? or in another way?

otel daemonset

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: {{ .Values.otel.name }}-otel
  namespace: {{ .Values.namespace }}
  labels:
    app: {{ .Values.otel.name }}-otel
spec:
  #replicas: {{ .Values.otel.replicas }}
  selector:
    matchLabels:
      app: {{ .Values.otel.name }}-otel
  template:
    metadata:
      labels:
        app: {{ .Values.otel.name }}-otel
    spec:
      serviceAccountName: {{ .Values.otel.rbac.serviceAccountName }}
      containers:
        - name: {{ .Values.otel.name }}-otel
          image: otel/opentelemetry-collector-contrib:latest
          args: ["--config", "/etc/otel/otel-collector-config.yaml"]
          env:
            - name: NAMESPACE
              value: {{ .Values.namespace }}
            - name: KUBELET_HOST
              valueFrom:
                fieldRef:
                  fieldPath: status.hostIP
            - name: ES_ENDPOINT
              value: {{ .Values.otel.elasticsearch.endpoint }}
            - name: ES_API_KEY
              valueFrom:
                secretKeyRef:
                  name: {{ .Values.otel.elasticsearch.secretName }}
                  key: apiKey
            - name: NEW_RELIC_OTLP_ENDPOINT
              value: {{ .Values.otel.newRelic.endpoint }}
            - name: NEW_RELIC_API_KEY
              valueFrom:
                secretKeyRef:
                  name: {{ .Values.otel.newRelic.secretName }}
                  key: apiKey
            - name: DATADOG_API_KEY
              valueFrom:
                secretKeyRef:
                  name: {{ .Values.otel.dataDog.secretName }}
                  key: apiKey
          ports:
            - containerPort: 4317
            - containerPort: 4318
            - containerPort: 13133
          volumeMounts:
            - name: otel-collector-config
              mountPath: /etc/otel
              readOnly: true
      volumes:
        - name: otel-collector-config
          configMap:
            name: {{ .Values.otel.name }}-otel-config

otel config

apiVersion: v1
kind: ConfigMap
metadata:
  name: {{ .Values.otel.name }}-otel-config
  namespace: {{ .Values.namespace }}
data:
  otel-collector-config.yaml: |
    receivers:
      kubeletstats:
        collection_interval: 10s
        auth_type: serviceAccount
        endpoint: https://${KUBELET_HOST}:10250
        insecure_skip_verify: true
        metric_groups:
          - container
          - pod
          # - node
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
          http:
            endpoint: 0.0.0.0:4318

    exporters:
      datadog:
        api:
          site: "us5.datadoghq.com"
          key: ${env:DATADOG_API_KEY}
        metrics:
          resource_attributes_as_tags: true
      otlphttp/newrelic:
        endpoint: ${env:NEW_RELIC_OTLP_ENDPOINT}
        headers:
          api-key: ${env:NEW_RELIC_API_KEY}
        compression: gzip

      elasticsearch:
        endpoint: ${env:ES_ENDPOINT}
        tls:
          insecure_skip_verify: true
        api_key: ${env:ES_API_KEY}

    processors:
      k8sattributes:
        filter:
          namespace: ${env:NAMESPACE}
        auth_type: "serviceAccount"
        passthrough: false
        extract:
          metadata:
            - k8s.pod.name
            - k8s.pod.uid
            - k8s.deployment.name
            - k8s.namespace.name
            # - k8s.node.name
            - k8s.pod.start_time
            # - k8s.cluster.uid
        # Pod association using resource attributes and connection
        pod_association:
          - sources:
              - from: resource_attribute
                name: k8s.pod.uid
              - from: resource_attribute
                name: k8s.pod.ip
              - from: connection
      batch:
        timeout: 5s
        send_batch_size: 512
        send_batch_max_size: 1024
      memory_limiter:
        check_interval: 1s
        limit_mib: 512
        spike_limit_mib: 128

      transform/logs:
         log_statements:
         - context: log
           statements:
             - delete_key(attributes, "code.function")
             - delete_key(attributes, "code.lineno")
         - context: resource
           statements:
             - set(attributes["elasticsearch.index"], Concat(["logs", attributes["service.name"], attributes["deployment.environment.name"]], "-"))
      resource:
        attributes:
          - key: "process.pid"
            action: delete
          - key: "process.executable.path"
            action: delete
          - key: "process.command_line"
            action: delete
          - key: "process.runtime.name"
            action: delete

    extensions:
      health_check: {}

    service:
      extensions: [health_check]
      pipelines:
        traces:
          receivers: [otlp]
          processors: [k8sattributes, resource]
          exporters: [otlphttp/newrelic]
        metrics:
          receivers: [otlp,kubeletstats]
          processors: [memory_limiter, k8sattributes, batch]
          exporters: [datadog]
        logs:
          receivers: [otlp]
          processors: [transform/logs, k8sattributes, resource]
          exporters: [elasticsearch]

otel rbac

apiVersion: v1
kind: ServiceAccount
metadata:
  name: {{ .Values.otel.rbac.serviceAccountName }}
  namespace: {{ .Values.namespace }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: {{ .Values.otel.rbac.roleName }}
  namespace: {{ .Values.namespace }}
rules:
  - apiGroups: [""]
    resources:
      - pods
    verbs:
      - get
      - watch
      - list
  - apiGroups: ["apps"]
    resources:
      - replicasets
      - deployments
    verbs:
      - get
      - list
      - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: {{ .Values.otel.rbac.roleBindingName }}
  namespace: {{ .Values.namespace }}
subjects:
  - kind: ServiceAccount
    name: {{ .Values.otel.rbac.serviceAccountName }}
    namespace: {{ .Values.namespace }}
roleRef:
  kind: Role
  name: {{ .Values.otel.rbac.roleName }}
  apiGroup: rbac.authorization.k8s.io