Is checkpoints.json need to be mounted to hostPath volume ?

[pingping95]

Hi.

I have a question.

I saw Checkpoint is located in {DATA-DIR}/??/checkpoints.json .

Is checkpoint need to be mounted to hostPath volume ?

The reason I'm asking is that I'm wondering if Vector Pod reads Checkpoint as it is if it restarts.

[pingping95]

When i look up, It seems that checkpoints.json file is already mounted with hostPath volume. (default)

**But Some logs are duplicated. Which part should I fix or watch?
**

• 11:36 : Normal Log (Same with Vector timespamp and Application timestamp)

• 12:37 : Not-Normal Log (Not Same with Vector timestamp and Application timestamp)

My vector config.

customConfig:
  data_dir: /vector-data-dir
  api:
    enabled: false
  sources:
    metrics:
      type: internal_metrics
      namespace: vector
      scrape_interval_secs: 15

    k8s:
      type: kubernetes_logs
      exclude_paths_glob_patterns:
        - "**/kube-system*/**"
        - "**/*monitoring*/**"
        - "**/*vector*/**"
        - "**/*istio*/**"
        - "**/*init-download*/**"
        - "**/*slack*/**"
        - "**/*message-proxy-mi*/**"
        - "**/*splus*/**"
        - "**/*kafka-ui*/**"
        - "**/*prometheus*/**"
        - "**/*eck-es*/**"
        - "**/*eck-kb*/**"
        - "**/*agent*/**"
        - "**/*probe*/**"
        - "**/*kibana*/**"
        - "**/*cronjobs*/**"

      glob_minimum_cooldown_ms: 5000
      read_from: beginning
      pod_annotation_fields:
        pod_name: "kubernetes.pod_name"
        pod_owner: ""
        pod_ips: ""
        pod_ip: ""
        pod_annotations: ""
        pod_namespace: "kubernetes.pod_namespace"
        pod_node_name: "kubernetes.pod_node_name"
        pod_uid: ""
        container_name: "kubernetes.container_name"
        container_image_id: ""
        container_image: ""
        container_id: ""
      node_annotation_fields:
        node_labels: ""
      namespace_annotation_fields:
        namespace_labels: ""

  transforms:
    get-attr:
      type: remap
      inputs:
        - k8s
      source: |-
      
        # Get app, env name
        .envPosition = find!(.kubernetes.container_name, r'dev|qa|stage|prod')
        .appName = slice!(.kubernetes.container_name, start: 0, end: .envPosition - 1)
        .env = slice!(.kubernetes.container_name, start: .envPosition)
        .fullName, err = .appName + "-" + .env

        # Remove unneeded fields
        del(.file); del(.envPosition); del(.source_type); del(.stream);
        
        # route
        if match(string!(.appName), r'(xxxx-xxxx|xxxx-xxxx-ui|xxxx-xxxx)') {
          .type = "normal-logs"
        } else {
          .type = "app-logs"
        }

    type-router:
      type: route
      inputs:
        - get-attr
      route:
        normal-logs: .type == "normal-logs"
        app-logs: .type == "app-logs"

    # normal-logs
    # Multiline
    multiline:
      type: reduce
      inputs:
        - type-router.app-logs
      merge_strategies:
        message: concat_newline
      starts_when: match(string!(.message), r'^\d+\-\d+\-\d+')
      expire_after_ms: 3000
      flush_period_ms: 500

    file-grok:
      type: remap
      inputs:
        - multiline
      source: |-
        # grok parser parsing
        if match(string!(.message), r'\d+\-\d+\-\d+') {
          # GROK Parsing
          . |= parse_grok!(.message, "%{TIMESTAMP_ISO8601:timestamp} \\| %{NOTSPACE:logger} \\| %{LOGLEVEL:level} \\| (?<mdc>[^[|]@]+)\\| %{JAVALOGMESSAGE}")

          # MDC Parsing
          . |= parse_key_value!(.mdc, field_delimiter: ", ", key_value_delimiter: "=")
        }

    # Remove unneeded logs
    filtered-logs:
      type: filter
      inputs:
        - file-grok
      condition:
        type: "vrl"
        source: |-
          true != includes(["/", "/actuator/prometheus"], ."http-request-uri")

  sinks:
    metrics-to-mimir:
      type: prometheus_remote_write
      inputs:
        - metrics
      endpoint: https://mimir.xxxx.xxxx/api/v1/push

    normal-logs-to-loki:
      type: loki
      inputs:
        - type-router.normal-logs
      compression: snappy
      endpoint: 'https://loki.xxxx.xxxx'
      out_of_order_action: drop
      path: /loki/api/v1/push
      remove_label_fields: true
      remove_timestamp: true
      encoding:
        codec: raw_message
      labels:
        source: vector
        cluster: MY_CLUSTER_NAME
        pod_name: |-
          {{ print "{{ .kubernetes.pod_name }}" }}
        namespace: |-
          {{ print "{{ .kubernetes.pod_namespace }}" }}
        node_name: |-
          {{ print "{{ .kubernetes.pod_node_name }}" }}
        container_name: |-
          {{ print "{{ .kubernetes.container_name }}" }}

    app-logs-to-loki:
      type: loki
      inputs:
        - filtered-logs
      compression: snappy
      endpoint: 'https://loki.xxxx.xxxx'
      out_of_order_action: drop
      path: /loki/api/v1/push
      remove_label_fields: true
      remove_timestamp: true
      encoding:
        codec: raw_message
      labels:
        source: vector
        cluster: MY_CLUSTER_NAME
        version: |-
          {{ print "{{ .kubernetes.pod_labels.version }}" }}
        logger: |-
          {{ print "{{ .logger }}" }}
        level: |-
          {{ print "{{ .level }}" }}
        app: |-
          {{ print "{{ .fullName }}" }}
        env: |-
          {{ print "{{ .env }}" }}