Unable to see application logs of Aks on SPlunk #861
Description
Hi,
I deployed my application on Azure Aks (it's a simple python hello world that logs to stdout). And I used splunk for connect with the following yaml file:
`global:
logLevel: info
splunk:
hec:
host:
port:
token: mytoken
protocol: https
endpoint: /endpoint/
fullUrl: https://fullurl
indexName: kube-logs
insecureSSL:
clientCert:
clientKey:
caFile:
indexRouting:
consume_chunk_on_4xx_errors:
kubernetes:
clusterName: "cluster_name"
prometheus_enabled:
monitoring_agent_enabled:
monitoring_agent_index_name:
monitoring_agent_bind_address:
metrics:
service:
enabled: true
headless: true
serviceMonitor:
enabled: false
metricsPort: 24231
interval: ""
scrapeTimeout: "10s"
additionalLabels: { }
splunk-kubernetes-logging:
enabled: true
logLevel:
namespace:
fluentd:
path: /var/log/containers/*.log
exclude_path:
containers:
path: /var/log
pathDest: /var/lib/docker/containers
logFormatType: cri
logFormat:
refreshInterval:
removeBlankEvents: true
localTime: false
enableStatWatcher: true
k8sMetadata:
podLabels:
- helloworld5-5dc446c649-zhvq5
- app
- k8s-app
- release
watch: true
cache_ttl: 3600
propagate_namespace_labels: false
sourcetypePrefix: "kube"
rbac:
create: true
openshiftPrivilegedSccBinding: false
serviceAccount:
create: true
name:
podSecurityPolicy:
create: false
apparmor_security: true
apiGroup: policy
splunk:
hec:
host:
port:
token:
protocol:
endpoint:
fullUrl:
indexName:
insecureSSL:
clientCert:
clientKey:
caFile:
consume_chunk_on_4xx_errors:
gzip_compression:
ingest_api:
serviceClientIdentifier:
serviceClientSecretKey:
tokenEndpoint:
ingestAuthHost:
ingestAPIHost:
tenant:
eventsEndpoint:
debugIngestAPI:
secret:
create: true
name:
journalLogPath: /run/log/journal
charEncodingUtf8: false
customLogs:
default:
from:
pod: helloworld5-5dc446c649-zhvq5
multiline:
firstline: /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+\s+\w+\s+[/
flushInterval: 30
logs:
docker:
from:
journald:
unit: docker.service
sourcetype: kube:docker
kubelet: &glog
from:
journald:
unit: kubelet.service
multiline:
firstline: /^\w[0-1]\d[0-3]\d/
sourcetype: kube:kubelet
default:
from:
pod: helloworld5-5dc446c649-zhvq5
etcd:
from:
pod: etcd-server
container: etcd-container
etcd-minikube:
from:
pod: etcd-minikube
container: etcd
etcd-events:
from:
pod: etcd-server-events
container: etcd-container
kube-apiserver:
<<: *glog
from:
pod: kube-apiserver
sourcetype: kube:kube-apiserver
kube-scheduler:
<<: *glog
from:
pod: kube-scheduler
sourcetype: kube:kube-scheduler
kube-controller-manager:
<<: *glog
from:
pod: kube-controller-manager
sourcetype: kube:kube-controller-manager
kube-proxy:
<<: *glog
from:
pod: kube-proxy
sourcetype: kube:kube-proxy
kubedns:
<<: *glog
from:
pod: kube-dns
sourcetype: kube:kubedns
dnsmasq:
<<: *glog
from:
pod: kube-dns
sourcetype: kube:dnsmasq
dns-sidecar:
<<: *glog
from:
pod: kube-dns
container: sidecar
sourcetype: kube:kubedns-sidecar
dns-controller:
<<: *glog
from:
pod: dns-controller
sourcetype: kube:dns-controller
kube-dns-autoscaler:
<<: *glog
from:
pod: kube-dns-autoscaler
container: autoscaler
sourcetype: kube:kube-dns-autoscaler
kube-audit:
from:
file:
path: /var/log/kube-apiserver-audit.log
timestampExtraction:
format: "%Y-%m-%dT%H:%M:%SZ"
sourcetype: kube:apiserver-audit
image:
registry: docker.io
name: splunk/fluentd-hec
tag: 1.3.2
pullPolicy: IfNotPresent
usePullSecret: false
pullSecretName:
environmentVar:
podAnnotations:
extraLabels:
resources:
requests:
cpu: 100m
memory: 200Mi
bufferChunkKeys:
- index
buffer:
"@type": memory
total_limit_size: 600m
chunk_limit_size: 20m
chunk_limit_records: 100000
flush_interval: 5s
flush_thread_count: 1
overflow_action: block
retry_max_times: 10
retry_type: periodic
retry_wait: 30
sendAllMetadata: false
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
nodeSelector:
kubernetes.io/os: linux
affinity: {}
extraVolumes: []
extraVolumeMounts: []
priorityClassName:
kubernetes:
clusterName:
securityContext: false
customMetadata:
customMetadataAnnotations:
customFilters: {}
indexFields: []
rollingUpdate:
splunk-kubernetes-objects:
enabled: true
logLevel:
namespace:
rbac:
create: true
serviceAccount:
create: true
name:
usePullSecrets: false
podSecurityPolicy:
create: false
apparmor_security: true
apiGroup: policy
priorityClassName:
kubernetes:
url:
insecureSSL: false
clientCert:
clientKey:
caFile:
bearerTokenFile:
secretDir:
clusterName:
objects:
core:
v1:
- name: pods
- name: namespaces
- name: nodes
- name: events
mode: watch
checkpointFile:
name: kubernetes-objects.pos
volume:
splunk:
hec:
host:
token:
protocol:
port:
fullUrl:
indexName:
insecureSSL:
clientCert:
clientKey:
caFile:
caPath:
indexRouting:
consume_chunk_on_4xx_errors:
secret:
create: true
name:
image:
registry: docker.io
name: splunk/kube-objects
tag: 1.2.2
pullPolicy: IfNotPresent
usePullSecret: false
pullSecretName:
environmentVar:
podAnnotations:
extraLabels:
resources:
requests:
cpu: 100m
memory: 200Mi
buffer:
"@type": memory
total_limit_size: 600m
chunk_limit_size: 20m
chunk_limit_records: 10000
flush_interval: 5s
flush_thread_count: 1
overflow_action: block
retry_max_times: 10
retry_type: periodic
retry_wait: 30
nodeSelector:
kubernetes.io/os: linux
tolerations: []
affinity: {}
customFilters: {}
indexFields: []
splunk-kubernetes-metrics:
enabled: true
logLevel:
namespace:
rbac:
create: true
serviceAccount:
create: true
name:
usePullSecrets: false
podSecurityPolicy:
create: false
apparmor_security: true
apiGroup: policy
splunk:
hec:
host:
port:
token:
protocol:
fullUrl:
indexName:
insecureSSL:
clientCert:
clientKey:
caFile:
consume_chunk_on_4xx_errors:
secret:
create: true
name:
image:
registry: docker.io
name: splunk/k8s-metrics
tag: 1.2.2
pullPolicy: IfNotPresent
usePullSecret: false
pullsecretName:
imageAgg:
registry: docker.io
name: splunk/k8s-metrics-aggr
tag: 1.2.2
pullPolicy: IfNotPresent
usePullSecret: false
pullsecretName:
environmentVar:
environmentVarAgg:
podAnnotations:
podAnnotationsAgg:
extraLabels:
extraLabelsAgg:
resources:
fluent:
limits:
cpu: 200m
memory: 300Mi
requests:
cpu: 200m
memory: 300Mi
buffer:
"@type": memory
total_limit_size: 400m
chunk_limit_size: 10m
chunk_limit_records: 10000
flush_interval: 5s
flush_thread_count: 1
overflow_action: block
retry_max_times: 10
retry_type: periodic
retry_wait: 30
aggregatorBuffer:
"@type": memory
total_limit_size: 400m
chunk_limit_size: 10m
chunk_limit_records: 10000
flush_interval: 5s
flush_thread_count: 1
overflow_action: block
retry_max_times: 10
retry_type: periodic
retry_wait: 30
metricsInterval: 15s
nodeSelector:
kubernetes.io/os: linux
aggregatorNodeSelector:
kubernetes.io/os: linux
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
aggregatorTolerations: {}
priorityClassName:
priorityClassNameAgg:
affinity: {}
kubernetes:
kubeletAddress:
kubeletPort:
kubeletPortAggregator:
useRestClientSSL: true
insecureSSL: false
caFile:
bearerTokenFile:
secretDir:
clusterName:
customFilters: {}
customFiltersAggr: {}
rollingUpdate:
`
Unfortunately I am not able to see my python script logs in Splunk. Any hint?
Thank you
Best Regards
Marta