Fluentd is missing 50% logs in gcs

[dakshrajgoyal]

I am trying to push loki logs in GCS using fluentd using gcs plugin. But after applying all parameters and changes in configuration, still it is missing 50% logs in GCS. Can someone please help on this. Below I am attaching the configmap file for fluentd.

data:
  fluent.conf: |-
    @include pods-fluent.conf
    @include gcs-loki-fluent.conf
  gcs-loki-fluent.conf: |-
    <match kubernetes.**>
        @type gcs
        project "#{ENV['GCS_BUCKET_PROJECT']}"
        keyfile "#{ENV['GCP_CRED']}"
        bucket "#{ENV['GCS_BUCKET_NAME']}"
        object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
        path k8s_prod_logs/${tag}/%Y/%m/%d/%H/$M

          <buffer tag,time>
            @type file
            path /var/log/fluent/gcs
            timekey 1m
            timekey_wait 1m
            timekey_use_utc true
            flush_thread_count 8
            flush_at_shutdown true
            flush_mode interval
            flush_interval 10
            chunk_limit_size 15MB
            retry_max_interval 30
            retry_wait 60
         </buffer>
         <format>
            @type json
         </format>
      </match>
      <match loki.**>
        @type loki
        #######------Your Remote loki-----#######
        url "http://X.X.X.X:3100"
        extra_labels {"cluster":"Prod"}
        <label>
          filename_prod
        </label>
        extract_kubernetes_labels true
        <buffer time,tag>
          flush_at_shutdown true
          flush_mode interval
          flush_interval 10
          timekey 20
          timekey_wait 60
          chunk_limit_size 10MB
        </buffer>
    </match>
  pods-fluent.conf: |-
    <label @FLUENT_LOG>
        <match fluent.*>
            @type null
        </match>
    </label>
    <source>
      @type tail
      read_from_head true
      tag kubernetes.*
      path /var/log/containers/*.log
      pos_file /var/log/fluentd-containers.log.pos
      exclude_path /var/log/containers/fluentd*.log
      path_key filename_prod_gcs
      <parse>
        @type json
        time_type string
      </parse>
    </source>
    <source>
      @type tail
      read_from_head true
      tag loki.*
      path /var/log/containers/*.log
      pos_file /var/log/containers/loki.log.pos
      exclude_path /var/log/containers/fluentd*.log
      path_key filename_prod
      <parse>
         @type json
         time_type string
      </parse>
      </source>