diff --git a/.gitignore b/.gitignore index 2fdb4be34..1220c9aae 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ build .DS_Store .idea .attach_pid* -bin \ No newline at end of file +bin +/amazon-cloudwatch-agent-operator \ No newline at end of file diff --git a/docs/api.md b/docs/api.md index a5faebc97..885b2413f 100644 --- a/docs/api.md +++ b/docs/api.md @@ -327,6 +327,13 @@ If not specified, the pod priority will be default or zero if there is no default.
false + + prometheus + object + + Prometheus is the raw YAML to be used as the collector's prometheus configuration.
+ + false replicas integer @@ -368,6 +375,13 @@ injected sidecar container.
the operator will not automatically create a ServiceAccount for the collector.
false + + targetAllocator + object + + TargetAllocator indicates a value which determines whether to spawn a target allocation resource or not.
+ + false terminationGracePeriodSeconds integer @@ -9960,6 +9974,68 @@ More info: https://kubernetes.io/docs/concepts/services-networking/service/#defi +### AmazonCloudWatchAgent.spec.prometheus +[↩ Parent](#amazoncloudwatchagentspec) + + + +Prometheus is the raw YAML to be used as the collector's prometheus configuration. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
configobject + AnyConfig represent parts of the config.
+
false
report_extra_scrape_metricsboolean +
+
false
start_time_metric_regexstring +
+
false
target_allocatorobject + AnyConfig represent parts of the config.
+
false
trim_metric_suffixesboolean +
+
false
use_start_time_metricboolean +
+
false
+ + ### AmazonCloudWatchAgent.spec.resources [↩ Parent](#amazoncloudwatchagentspec) @@ -10386,6 +10462,2831 @@ PodSecurityContext, the value specified in SecurityContext takes precedence.
+### AmazonCloudWatchAgent.spec.targetAllocator +[↩ Parent](#amazoncloudwatchagentspec) + + + +TargetAllocator indicates a value which determines whether to spawn a target allocation resource or not. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
affinityobject + If specified, indicates the pod's scheduling constraints
+
false
allocationStrategyenum + AllocationStrategy determines which strategy the target allocator should use for allocation. +The current option is consistent-hashing.
+
+ Enum: consistent-hashing
+
false
enabledboolean + Enabled indicates whether to use a target allocation mechanism for Prometheus targets or not.
+
false
env[]object + ENV vars to set on the OpenTelemetry TargetAllocator's Pods. These can then in certain cases be +consumed in the config file for the TargetAllocator.
+
false
filterStrategystring + FilterStrategy determines how to filter targets before allocating them among the collectors. +The only current option is relabel-config (drops targets based on prom relabel_config). +Filtering is disabled by default.
+
false
imagestring + Image indicates the container image to use for the OpenTelemetry TargetAllocator.
+
false
nodeSelectormap[string]string + NodeSelector to schedule OpenTelemetry TargetAllocator pods.
+
false
prometheusCRobject + PrometheusCR defines the configuration for the retrieval of PrometheusOperator CRDs ( servicemonitor.monitoring.coreos.com/v1 and podmonitor.monitoring.coreos.com/v1 ) retrieval. +All CR instances which the ServiceAccount has access to will be retrieved. This includes other namespaces.
+
false
replicasinteger + Replicas is the number of pod instances for the underlying TargetAllocator. This should only be set to a value +other than 1 if a strategy that allows for high availability is chosen. Currently, the only allocation strategy +that can be run in a high availability mode is consistent-hashing.
+
+ Format: int32
+
false
resourcesobject + Resources to set on the OpenTelemetryTargetAllocator containers.
+
false
securityContextobject + SecurityContext configures the container security context for +the target-allocator.
+
false
serviceAccountstring + ServiceAccount indicates the name of an existing service account to use with this instance. When set, +the operator will not automatically create a ServiceAccount for the TargetAllocator.
+
false
tolerations[]object + Toleration embedded kubernetes pod configuration option, +controls how pods can be scheduled with matching taints
+
false
topologySpreadConstraints[]object + TopologySpreadConstraints embedded kubernetes pod configuration option, +controls how pods are spread across your cluster among failure-domains +such as regions, zones, nodes, and other user-defined topology domains +https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity +[↩ Parent](#amazoncloudwatchagentspectargetallocator) + + + +If specified, indicates the pod's scheduling constraints + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
nodeAffinityobject + Describes node affinity scheduling rules for the pod.
+
false
podAffinityobject + Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).
+
false
podAntiAffinityobject + Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinity) + + + +Describes node affinity scheduling rules for the pod. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
preferredDuringSchedulingIgnoredDuringExecution[]object + The scheduler will prefer to schedule pods to nodes that satisfy +the affinity expressions specified by this field, but it may choose +a node that violates one or more of the expressions. The node that is +most preferred is the one with the greatest sum of weights, i.e. +for each node that meets all of the scheduling requirements (resource +request, requiredDuringScheduling affinity expressions, etc.), +compute a sum by iterating through the elements of this field and adding +"weight" to the sum if the node matches the corresponding matchExpressions; the +node(s) with the highest sum are the most preferred.
+
false
requiredDuringSchedulingIgnoredDuringExecutionobject + If the affinity requirements specified by this field are not met at +scheduling time, the pod will not be scheduled onto the node. +If the affinity requirements specified by this field cease to be met +at some point during pod execution (e.g. due to an update), the system +may or may not try to eventually evict the pod from its node.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinity) + + + +An empty preferred scheduling term matches all objects with implicit weight 0 +(i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
preferenceobject + A node selector term, associated with the corresponding weight.
+
true
weightinteger + Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.
+
+ Format: int32
+
true
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].preference +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinitypreferredduringschedulingignoredduringexecutionindex) + + + +A node selector term, associated with the corresponding weight. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + A list of node selector requirements by node's labels.
+
false
matchFields[]object + A list of node selector requirements by node's fields.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].preference.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinitypreferredduringschedulingignoredduringexecutionindexpreference) + + + +A node selector requirement is a selector that contains values, a key, and an operator +that relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The label key that the selector applies to.
+
true
operatorstring + Represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+
true
values[]string + An array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. If the operator is Gt or Lt, the values +array must have a single element, which will be interpreted as an integer. +This array is replaced during a strategic merge patch.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].preference.matchFields[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinitypreferredduringschedulingignoredduringexecutionindexpreference) + + + +A node selector requirement is a selector that contains values, a key, and an operator +that relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The label key that the selector applies to.
+
true
operatorstring + Represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+
true
values[]string + An array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. If the operator is Gt or Lt, the values +array must have a single element, which will be interpreted as an integer. +This array is replaced during a strategic merge patch.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinity) + + + +If the affinity requirements specified by this field are not met at +scheduling time, the pod will not be scheduled onto the node. +If the affinity requirements specified by this field cease to be met +at some point during pod execution (e.g. due to an update), the system +may or may not try to eventually evict the pod from its node. + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
nodeSelectorTerms[]object + Required. A list of node selector terms. The terms are ORed.
+
true
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinityrequiredduringschedulingignoredduringexecution) + + + +A null or empty node selector term matches no objects. The requirements of +them are ANDed. +The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + A list of node selector requirements by node's labels.
+
false
matchFields[]object + A list of node selector requirements by node's fields.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[index].matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinityrequiredduringschedulingignoredduringexecutionnodeselectortermsindex) + + + +A node selector requirement is a selector that contains values, a key, and an operator +that relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The label key that the selector applies to.
+
true
operatorstring + Represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+
true
values[]string + An array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. If the operator is Gt or Lt, the values +array must have a single element, which will be interpreted as an integer. +This array is replaced during a strategic merge patch.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[index].matchFields[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitynodeaffinityrequiredduringschedulingignoredduringexecutionnodeselectortermsindex) + + + +A node selector requirement is a selector that contains values, a key, and an operator +that relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The label key that the selector applies to.
+
true
operatorstring + Represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+
true
values[]string + An array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. If the operator is Gt or Lt, the values +array must have a single element, which will be interpreted as an integer. +This array is replaced during a strategic merge patch.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinity) + + + +Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
preferredDuringSchedulingIgnoredDuringExecution[]object + The scheduler will prefer to schedule pods to nodes that satisfy +the affinity expressions specified by this field, but it may choose +a node that violates one or more of the expressions. The node that is +most preferred is the one with the greatest sum of weights, i.e. +for each node that meets all of the scheduling requirements (resource +request, requiredDuringScheduling affinity expressions, etc.), +compute a sum by iterating through the elements of this field and adding +"weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the +node(s) with the highest sum are the most preferred.
+
false
requiredDuringSchedulingIgnoredDuringExecution[]object + If the affinity requirements specified by this field are not met at +scheduling time, the pod will not be scheduled onto the node. +If the affinity requirements specified by this field cease to be met +at some point during pod execution (e.g. due to a pod label update), the +system may or may not try to eventually evict the pod from its node. +When there are multiple elements, the lists of nodes corresponding to each +podAffinityTerm are intersected, i.e. all terms must be satisfied.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinity) + + + +The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
podAffinityTermobject + Required. A pod affinity term, associated with the corresponding weight.
+
true
weightinteger + weight associated with matching the corresponding podAffinityTerm, +in the range 1-100.
+
+ Format: int32
+
true
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinitypreferredduringschedulingignoredduringexecutionindex) + + + +Required. A pod affinity term, associated with the corresponding weight. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
topologyKeystring + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching +the labelSelector in the specified namespaces, where co-located is defined as running on a node +whose value of the label with key topologyKey matches that of any node on which any of the +selected pods is running. +Empty topologyKey is not allowed.
+
true
labelSelectorobject + A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods.
+
false
matchLabelKeys[]string + MatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. +Also, MatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+
false
mismatchLabelKeys[]string + MismatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. +Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+
false
namespaceSelectorobject + A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces.
+
false
namespaces[]string + namespaces specifies a static list of namespace names that the term applies to. +The term is applied to the union of the namespaces listed in this field +and the ones selected by namespaceSelector. +null or empty namespaces list and null namespaceSelector means "this pod's namespace".
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.labelSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinityterm) + + + +A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+
false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.labelSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinitytermlabelselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+
true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+
true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.namespaceSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinityterm) + + + +A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+
false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.namespaceSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinitytermnamespaceselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+
true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+
true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinity) + + + +Defines a set of pods (namely those matching the labelSelector +relative to the given namespace(s)) that this pod should be +co-located (affinity) or not co-located (anti-affinity) with, +where co-located is defined as running on a node whose value of +the label with key matches that of any node on which +a pod of the set of pods is running + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
topologyKeystring + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching +the labelSelector in the specified namespaces, where co-located is defined as running on a node +whose value of the label with key topologyKey matches that of any node on which any of the +selected pods is running. +Empty topologyKey is not allowed.
+
true
labelSelectorobject + A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods.
+
false
matchLabelKeys[]string + MatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. +Also, MatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+
false
mismatchLabelKeys[]string + MismatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. +Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+
false
namespaceSelectorobject + A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces.
+
false
namespaces[]string + namespaces specifies a static list of namespace names that the term applies to. +The term is applied to the union of the namespaces listed in this field +and the ones selected by namespaceSelector. +null or empty namespaces list and null namespaceSelector means "this pod's namespace".
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].labelSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinityrequiredduringschedulingignoredduringexecutionindex) + + + +A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+
false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].labelSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinityrequiredduringschedulingignoredduringexecutionindexlabelselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+
true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+
true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].namespaceSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinityrequiredduringschedulingignoredduringexecutionindex) + + + +A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+
false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].namespaceSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodaffinityrequiredduringschedulingignoredduringexecutionindexnamespaceselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+
true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+
true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinity) + + + +Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
preferredDuringSchedulingIgnoredDuringExecution[]object + The scheduler will prefer to schedule pods to nodes that satisfy +the anti-affinity expressions specified by this field, but it may choose +a node that violates one or more of the expressions. The node that is +most preferred is the one with the greatest sum of weights, i.e. +for each node that meets all of the scheduling requirements (resource +request, requiredDuringScheduling anti-affinity expressions, etc.), +compute a sum by iterating through the elements of this field and adding +"weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the +node(s) with the highest sum are the most preferred.
+
false
requiredDuringSchedulingIgnoredDuringExecution[]object + If the anti-affinity requirements specified by this field are not met at +scheduling time, the pod will not be scheduled onto the node. +If the anti-affinity requirements specified by this field cease to be met +at some point during pod execution (e.g. due to a pod label update), the +system may or may not try to eventually evict the pod from its node. +When there are multiple elements, the lists of nodes corresponding to each +podAffinityTerm are intersected, i.e. all terms must be satisfied.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinity) + + + +The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
podAffinityTermobject + Required. A pod affinity term, associated with the corresponding weight.
+
true
weightinteger + weight associated with matching the corresponding podAffinityTerm, +in the range 1-100.
+
+ Format: int32
+
true
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinitypreferredduringschedulingignoredduringexecutionindex) + + + +Required. A pod affinity term, associated with the corresponding weight. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
topologyKeystring + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching +the labelSelector in the specified namespaces, where co-located is defined as running on a node +whose value of the label with key topologyKey matches that of any node on which any of the +selected pods is running. +Empty topologyKey is not allowed.
+
true
labelSelectorobject + A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods.
+
false
matchLabelKeys[]string + MatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. +Also, MatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+
false
mismatchLabelKeys[]string + MismatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. +Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+
false
namespaceSelectorobject + A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces.
+
false
namespaces[]string + namespaces specifies a static list of namespace names that the term applies to. +The term is applied to the union of the namespaces listed in this field +and the ones selected by namespaceSelector. +null or empty namespaces list and null namespaceSelector means "this pod's namespace".
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.labelSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinityterm) + + + +A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+
false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.labelSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinitytermlabelselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+
true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+
true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.namespaceSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinityterm) + + + +A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+
false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[index].podAffinityTerm.namespaceSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinitypreferredduringschedulingignoredduringexecutionindexpodaffinitytermnamespaceselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+
true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+
true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinity) + + + +Defines a set of pods (namely those matching the labelSelector +relative to the given namespace(s)) that this pod should be +co-located (affinity) or not co-located (anti-affinity) with, +where co-located is defined as running on a node whose value of +the label with key matches that of any node on which +a pod of the set of pods is running + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
topologyKeystring + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching +the labelSelector in the specified namespaces, where co-located is defined as running on a node +whose value of the label with key topologyKey matches that of any node on which any of the +selected pods is running. +Empty topologyKey is not allowed.
+
true
labelSelectorobject + A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods.
+
false
matchLabelKeys[]string + MatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key in (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. +Also, MatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+
false
mismatchLabelKeys[]string + MismatchLabelKeys is a set of pod label keys to select which pods will +be taken into consideration. The keys are used to lookup values from the +incoming pod labels, those key-value labels are merged with `LabelSelector` as `key notin (value)` +to select the group of existing pods which pods will be taken into consideration +for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming +pod labels will be ignored. The default value is empty. +The same key is forbidden to exist in both MismatchLabelKeys and LabelSelector. +Also, MismatchLabelKeys cannot be set when LabelSelector isn't set. +This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate.
+
false
namespaceSelectorobject + A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces.
+
false
namespaces[]string + namespaces specifies a static list of namespace names that the term applies to. +The term is applied to the union of the namespaces listed in this field +and the ones selected by namespaceSelector. +null or empty namespaces list and null namespaceSelector means "this pod's namespace".
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].labelSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinityrequiredduringschedulingignoredduringexecutionindex) + + + +A label query over a set of resources, in this case pods. +If it's null, this PodAffinityTerm matches with no Pods. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+
false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].labelSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinityrequiredduringschedulingignoredduringexecutionindexlabelselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+
true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+
true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].namespaceSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinityrequiredduringschedulingignoredduringexecutionindex) + + + +A label query over the set of namespaces that the term applies to. +The term is applied to the union of the namespaces selected by this field +and the ones listed in the namespaces field. +null selector and null or empty namespaces list means "this pod's namespace". +An empty selector ({}) matches all namespaces. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+
false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[index].namespaceSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatoraffinitypodantiaffinityrequiredduringschedulingignoredduringexecutionindexnamespaceselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+
true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+
true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.env[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocator) + + + +EnvVar represents an environment variable present in a Container. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of the environment variable. Must be a C_IDENTIFIER.
+
true
valuestring + Variable references $(VAR_NAME) are expanded +using the previously defined environment variables in the container and +any service environment variables. If a variable cannot be resolved, +the reference in the input string will be unchanged. Double $$ are reduced +to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. +"$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". +Escaped references will never be expanded, regardless of whether the variable +exists or not. +Defaults to "".
+
false
valueFromobject + Source for the environment variable's value. Cannot be used if value is not empty.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.env[index].valueFrom +[↩ Parent](#amazoncloudwatchagentspectargetallocatorenvindex) + + + +Source for the environment variable's value. Cannot be used if value is not empty. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
configMapKeyRefobject + Selects a key of a ConfigMap.
+
false
fieldRefobject + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, +spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.
+
false
resourceFieldRefobject + Selects a resource of the container: only resources limits and requests +(limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.
+
false
secretKeyRefobject + Selects a key of a secret in the pod's namespace
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.env[index].valueFrom.configMapKeyRef +[↩ Parent](#amazoncloudwatchagentspectargetallocatorenvindexvaluefrom) + + + +Selects a key of a ConfigMap. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The key to select.
+
true
namestring + Name of the referent. +More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names +TODO: Add other useful fields. apiVersion, kind, uid?
+
false
optionalboolean + Specify whether the ConfigMap or its key must be defined
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.env[index].valueFrom.fieldRef +[↩ Parent](#amazoncloudwatchagentspectargetallocatorenvindexvaluefrom) + + + +Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, +spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
fieldPathstring + Path of the field to select in the specified API version.
+
true
apiVersionstring + Version of the schema the FieldPath is written in terms of, defaults to "v1".
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.env[index].valueFrom.resourceFieldRef +[↩ Parent](#amazoncloudwatchagentspectargetallocatorenvindexvaluefrom) + + + +Selects a resource of the container: only resources limits and requests +(limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
resourcestring + Required: resource to select
+
true
containerNamestring + Container name: required for volumes, optional for env vars
+
false
divisorint or string + Specifies the output format of the exposed resources, defaults to "1"
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.env[index].valueFrom.secretKeyRef +[↩ Parent](#amazoncloudwatchagentspectargetallocatorenvindexvaluefrom) + + + +Selects a key of a secret in the pod's namespace + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The key of the secret to select from. Must be a valid secret key.
+
true
namestring + Name of the referent. +More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names +TODO: Add other useful fields. apiVersion, kind, uid?
+
false
optionalboolean + Specify whether the Secret or its key must be defined
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.prometheusCR +[↩ Parent](#amazoncloudwatchagentspectargetallocator) + + + +PrometheusCR defines the configuration for the retrieval of PrometheusOperator CRDs ( servicemonitor.monitoring.coreos.com/v1 and podmonitor.monitoring.coreos.com/v1 ) retrieval. +All CR instances which the ServiceAccount has access to will be retrieved. This includes other namespaces. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
enabledboolean + Enabled indicates whether to use a PrometheusOperator custom resources as targets or not.
+
false
podMonitorSelectormap[string]string + PodMonitors to be selected for target discovery. +This is a map of {key,value} pairs. Each {key,value} in the map is going to exactly match a label in a +PodMonitor's meta labels. The requirements are ANDed.
+
false
scrapeIntervalstring + Interval between consecutive scrapes. Equivalent to the same setting on the Prometheus CRD. + + +Default: "30s"
+
+ Format: duration
+ Default: 30s
+
false
serviceMonitorSelectormap[string]string + ServiceMonitors to be selected for target discovery. +This is a map of {key,value} pairs. Each {key,value} in the map is going to exactly match a label in a +ServiceMonitor's meta labels. The requirements are ANDed.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.resources +[↩ Parent](#amazoncloudwatchagentspectargetallocator) + + + +Resources to set on the OpenTelemetryTargetAllocator containers. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
claims[]object + Claims lists the names of resources, defined in spec.resourceClaims, +that are used by this container. + + +This is an alpha field and requires enabling the +DynamicResourceAllocation feature gate. + + +This field is immutable. It can only be set for containers.
+
false
limitsmap[string]int or string + Limits describes the maximum amount of compute resources allowed. +More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+
false
requestsmap[string]int or string + Requests describes the minimum amount of compute resources required. +If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, +otherwise to an implementation-defined value. Requests cannot exceed Limits. +More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.resources.claims[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatorresources) + + + +ResourceClaim references one entry in PodSpec.ResourceClaims. + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name must match the name of one entry in pod.spec.resourceClaims of +the Pod where this field is used. It makes that resource available +inside a container.
+
true
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.securityContext +[↩ Parent](#amazoncloudwatchagentspectargetallocator) + + + +SecurityContext configures the container security context for +the target-allocator. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
fsGroupinteger + A special supplemental group that applies to all containers in a pod. +Some volume types allow the Kubelet to change the ownership of that volume +to be owned by the pod: + + +1. The owning GID will be the FSGroup +2. The setgid bit is set (new files created in the volume will be owned by FSGroup) +3. The permission bits are OR'd with rw-rw---- + + +If unset, the Kubelet will not modify the ownership and permissions of any volume. +Note that this field cannot be set when spec.os.name is windows.
+
+ Format: int64
+
false
fsGroupChangePolicystring + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume +before being exposed inside Pod. This field will only apply to +volume types which support fsGroup based ownership(and permissions). +It will have no effect on ephemeral volume types such as: secret, configmaps +and emptydir. +Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. +Note that this field cannot be set when spec.os.name is windows.
+
false
runAsGroupinteger + The GID to run the entrypoint of the container process. +Uses runtime default if unset. +May also be set in SecurityContext. If set in both SecurityContext and +PodSecurityContext, the value specified in SecurityContext takes precedence +for that container. +Note that this field cannot be set when spec.os.name is windows.
+
+ Format: int64
+
false
runAsNonRootboolean + Indicates that the container must run as a non-root user. +If true, the Kubelet will validate the image at runtime to ensure that it +does not run as UID 0 (root) and fail to start the container if it does. +If unset or false, no such validation will be performed. +May also be set in SecurityContext. If set in both SecurityContext and +PodSecurityContext, the value specified in SecurityContext takes precedence.
+
false
runAsUserinteger + The UID to run the entrypoint of the container process. +Defaults to user specified in image metadata if unspecified. +May also be set in SecurityContext. If set in both SecurityContext and +PodSecurityContext, the value specified in SecurityContext takes precedence +for that container. +Note that this field cannot be set when spec.os.name is windows.
+
+ Format: int64
+
false
seLinuxOptionsobject + The SELinux context to be applied to all containers. +If unspecified, the container runtime will allocate a random SELinux context for each +container. May also be set in SecurityContext. If set in +both SecurityContext and PodSecurityContext, the value specified in SecurityContext +takes precedence for that container. +Note that this field cannot be set when spec.os.name is windows.
+
false
seccompProfileobject + The seccomp options to use by the containers in this pod. +Note that this field cannot be set when spec.os.name is windows.
+
false
supplementalGroups[]integer + A list of groups applied to the first process run in each container, in addition +to the container's primary GID, the fsGroup (if specified), and group memberships +defined in the container image for the uid of the container process. If unspecified, +no additional groups are added to any container. Note that group memberships +defined in the container image for the uid of the container process are still effective, +even if they are not included in this list. +Note that this field cannot be set when spec.os.name is windows.
+
false
sysctls[]object + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported +sysctls (by the container runtime) might fail to launch. +Note that this field cannot be set when spec.os.name is windows.
+
false
windowsOptionsobject + The Windows specific settings applied to all containers. +If unspecified, the options within a container's SecurityContext will be used. +If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. +Note that this field cannot be set when spec.os.name is linux.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.securityContext.seLinuxOptions +[↩ Parent](#amazoncloudwatchagentspectargetallocatorsecuritycontext) + + + +The SELinux context to be applied to all containers. +If unspecified, the container runtime will allocate a random SELinux context for each +container. May also be set in SecurityContext. If set in +both SecurityContext and PodSecurityContext, the value specified in SecurityContext +takes precedence for that container. +Note that this field cannot be set when spec.os.name is windows. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
levelstring + Level is SELinux level label that applies to the container.
+
false
rolestring + Role is a SELinux role label that applies to the container.
+
false
typestring + Type is a SELinux type label that applies to the container.
+
false
userstring + User is a SELinux user label that applies to the container.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.securityContext.seccompProfile +[↩ Parent](#amazoncloudwatchagentspectargetallocatorsecuritycontext) + + + +The seccomp options to use by the containers in this pod. +Note that this field cannot be set when spec.os.name is windows. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
typestring + type indicates which kind of seccomp profile will be applied. +Valid options are: + + +Localhost - a profile defined in a file on the node should be used. +RuntimeDefault - the container runtime default profile should be used. +Unconfined - no profile should be applied.
+
true
localhostProfilestring + localhostProfile indicates a profile defined in a file on the node should be used. +The profile must be preconfigured on the node to work. +Must be a descending path, relative to the kubelet's configured seccomp profile location. +Must be set if type is "Localhost". Must NOT be set for any other type.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.securityContext.sysctls[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatorsecuritycontext) + + + +Sysctl defines a kernel parameter to be set + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of a property to set
+
true
valuestring + Value of a property to set
+
true
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.securityContext.windowsOptions +[↩ Parent](#amazoncloudwatchagentspectargetallocatorsecuritycontext) + + + +The Windows specific settings applied to all containers. +If unspecified, the options within a container's SecurityContext will be used. +If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. +Note that this field cannot be set when spec.os.name is linux. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
gmsaCredentialSpecstring + GMSACredentialSpec is where the GMSA admission webhook +(https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the +GMSA credential spec named by the GMSACredentialSpecName field.
+
false
gmsaCredentialSpecNamestring + GMSACredentialSpecName is the name of the GMSA credential spec to use.
+
false
hostProcessboolean + HostProcess determines if a container should be run as a 'Host Process' container. +All of a Pod's containers must have the same effective HostProcess value +(it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). +In addition, if HostProcess is true then HostNetwork must also be set to true.
+
false
runAsUserNamestring + The UserName in Windows to run the entrypoint of the container process. +Defaults to the user specified in image metadata if unspecified. +May also be set in PodSecurityContext. If set in both SecurityContext and +PodSecurityContext, the value specified in SecurityContext takes precedence.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.tolerations[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocator) + + + +The pod this Toleration is attached to tolerates any taint that matches +the triple using the matching operator . + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
effectstring + Effect indicates the taint effect to match. Empty means match all taint effects. +When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+
false
keystring + Key is the taint key that the toleration applies to. Empty means match all taint keys. +If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+
false
operatorstring + Operator represents a key's relationship to the value. +Valid operators are Exists and Equal. Defaults to Equal. +Exists is equivalent to wildcard for value, so that a pod can +tolerate all taints of a particular category.
+
false
tolerationSecondsinteger + TolerationSeconds represents the period of time the toleration (which must be +of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, +it is not set, which means tolerate the taint forever (do not evict). Zero and +negative values will be treated as 0 (evict immediately) by the system.
+
+ Format: int64
+
false
valuestring + Value is the taint value the toleration matches to. +If the operator is Exists, the value should be empty, otherwise just a regular string.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.topologySpreadConstraints[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocator) + + + +TopologySpreadConstraint specifies how to spread matching pods among the given topology. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
maxSkewinteger + MaxSkew describes the degree to which pods may be unevenly distributed. +When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference +between the number of matching pods in the target topology and the global minimum. +The global minimum is the minimum number of matching pods in an eligible domain +or zero if the number of eligible domains is less than MinDomains. +For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same +labelSelector spread as 2/2/1: +In this case, the global minimum is 1. +| zone1 | zone2 | zone3 | +| P P | P P | P | +- if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; +scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) +violate MaxSkew(1). +- if MaxSkew is 2, incoming pod can be scheduled onto any zone. +When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence +to topologies that satisfy it. +It's a required field. Default value is 1 and 0 is not allowed.
+
+ Format: int32
+
true
topologyKeystring + TopologyKey is the key of node labels. Nodes that have a label with this key +and identical values are considered to be in the same topology. +We consider each as a "bucket", and try to put balanced number +of pods into each bucket. +We define a domain as a particular instance of a topology. +Also, we define an eligible domain as a domain whose nodes meet the requirements of +nodeAffinityPolicy and nodeTaintsPolicy. +e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. +And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. +It's a required field.
+
true
whenUnsatisfiablestring + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy +the spread constraint. +- DoNotSchedule (default) tells the scheduler not to schedule it. +- ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. +A constraint is considered "Unsatisfiable" for an incoming pod +if and only if every possible node assignment for that pod would violate +"MaxSkew" on some topology. +For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same +labelSelector spread as 3/1/1: +| zone1 | zone2 | zone3 | +| P P P | P | P | +If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled +to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies +MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler +won't make it *more* imbalanced. +It's a required field.
+
true
labelSelectorobject + LabelSelector is used to find matching pods. +Pods that match this label selector are counted to determine the number of pods +in their corresponding topology domain.
+
false
matchLabelKeys[]string + MatchLabelKeys is a set of pod label keys to select the pods over which +spreading will be calculated. The keys are used to lookup values from the +incoming pod labels, those key-value labels are ANDed with labelSelector +to select the group of existing pods over which spreading will be calculated +for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. +MatchLabelKeys cannot be set when LabelSelector isn't set. +Keys that don't exist in the incoming pod labels will +be ignored. A null or empty list means only match against labelSelector. + + +This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default).
+
false
minDomainsinteger + MinDomains indicates a minimum number of eligible domains. +When the number of eligible domains with matching topology keys is less than minDomains, +Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. +And when the number of eligible domains with matching topology keys equals or greater than minDomains, +this value has no effect on scheduling. +As a result, when the number of eligible domains is less than minDomains, +scheduler won't schedule more than maxSkew Pods to those domains. +If value is nil, the constraint behaves as if MinDomains is equal to 1. +Valid values are integers greater than 0. +When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + +For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same +labelSelector spread as 2/2/2: +| zone1 | zone2 | zone3 | +| P P | P P | P P | +The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. +In this situation, new pod with the same labelSelector cannot be scheduled, +because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, +it will violate MaxSkew. + + +This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default).
+
+ Format: int32
+
false
nodeAffinityPolicystring + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector +when calculating pod topology spread skew. Options are: +- Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. +- Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + +If this value is nil, the behavior is equivalent to the Honor policy. +This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
+
false
nodeTaintsPolicystring + NodeTaintsPolicy indicates how we will treat node taints when calculating +pod topology spread skew. Options are: +- Honor: nodes without taints, along with tainted nodes for which the incoming pod +has a toleration, are included. +- Ignore: node taints are ignored. All nodes are included. + + +If this value is nil, the behavior is equivalent to the Ignore policy. +This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.topologySpreadConstraints[index].labelSelector +[↩ Parent](#amazoncloudwatchagentspectargetallocatortopologyspreadconstraintsindex) + + + +LabelSelector is used to find matching pods. +Pods that match this label selector are counted to determine the number of pods +in their corresponding topology domain. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
matchExpressions[]object + matchExpressions is a list of label selector requirements. The requirements are ANDed.
+
false
matchLabelsmap[string]string + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels +map is equivalent to an element of matchExpressions, whose key field is "key", the +operator is "In", and the values array contains only "value". The requirements are ANDed.
+
false
+ + +### AmazonCloudWatchAgent.spec.targetAllocator.topologySpreadConstraints[index].labelSelector.matchExpressions[index] +[↩ Parent](#amazoncloudwatchagentspectargetallocatortopologyspreadconstraintsindexlabelselector) + + + +A label selector requirement is a selector that contains values, a key, and an operator that +relates the key and values. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + key is the label key that the selector applies to.
+
true
operatorstring + operator represents a key's relationship to a set of values. +Valid operators are In, NotIn, Exists and DoesNotExist.
+
true
values[]string + values is an array of string values. If the operator is In or NotIn, +the values array must be non-empty. If the operator is Exists or DoesNotExist, +the values array must be empty. This array is replaced during a strategic +merge patch.
+
false
+ + ### AmazonCloudWatchAgent.spec.tolerations[index] [↩ Parent](#amazoncloudwatchagentspec) diff --git a/internal/webhook/namespacemutation/webhookhandler.go b/internal/webhook/namespacemutation/webhookhandler.go index c290ac3b8..ddfc980d3 100644 --- a/internal/webhook/namespacemutation/webhookhandler.go +++ b/internal/webhook/namespacemutation/webhookhandler.go @@ -19,14 +19,14 @@ import ( var _ admission.Handler = (*handler)(nil) type handler struct { - decoder *admission.Decoder - annotationMutators *auto.AnnotationMutators + decoder *admission.Decoder + instrumentationAnnotator auto.InstrumentationAnnotator } -func NewWebhookHandler(decoder *admission.Decoder, annotationMutators *auto.AnnotationMutators) admission.Handler { +func NewWebhookHandler(decoder *admission.Decoder, instrumentationAnnotator auto.InstrumentationAnnotator) admission.Handler { return &handler{ - decoder: decoder, - annotationMutators: annotationMutators, + decoder: decoder, + instrumentationAnnotator: instrumentationAnnotator, } } @@ -36,7 +36,10 @@ func (h *handler) Handle(_ context.Context, req admission.Request) admission.Res if err != nil { return admission.Errored(http.StatusBadRequest, err) } - h.annotationMutators.MutateObject(namespace) + + // do not need to pass in oldObj because it's only used to check for workload pod template diff + h.instrumentationAnnotator.MutateObject(nil, namespace) + marshaledNamespace, err := json.Marshal(namespace) if err != nil { res := admission.Errored(http.StatusInternalServerError, err) diff --git a/internal/webhook/workloadmutation/webhookhandler.go b/internal/webhook/workloadmutation/webhookhandler.go index a2394596f..f6ce1d720 100644 --- a/internal/webhook/workloadmutation/webhookhandler.go +++ b/internal/webhook/workloadmutation/webhookhandler.go @@ -10,6 +10,7 @@ import ( "errors" "net/http" + v1 "k8s.io/api/admission/v1" appsv1 "k8s.io/api/apps/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -29,26 +30,29 @@ type WebhookHandler interface { // the implementation. type workloadMutationWebhook struct { - decoder *admission.Decoder - annotationMutators *auto.AnnotationMutators + decoder *admission.Decoder + instrumentationAnnotator auto.InstrumentationAnnotator } // NewWebhookHandler creates a new WorkloadWebhookHandler. -func NewWebhookHandler(decoder *admission.Decoder, annotationMutators *auto.AnnotationMutators) WebhookHandler { +func NewWebhookHandler(decoder *admission.Decoder, instrumentationAnnotator auto.InstrumentationAnnotator) WebhookHandler { return &workloadMutationWebhook{ - decoder: decoder, - annotationMutators: annotationMutators, + decoder: decoder, + instrumentationAnnotator: instrumentationAnnotator, } } func (p *workloadMutationWebhook) Handle(_ context.Context, req admission.Request) admission.Response { - var obj client.Object + var oldObj, obj client.Object switch objectKind := req.Kind.Kind; objectKind { case "DaemonSet": + oldObj = &appsv1.DaemonSet{} obj = &appsv1.DaemonSet{} case "Deployment": + oldObj = &appsv1.Deployment{} obj = &appsv1.Deployment{} case "StatefulSet": + oldObj = &appsv1.StatefulSet{} obj = &appsv1.StatefulSet{} default: return admission.Errored(http.StatusBadRequest, errors.New("failed to unmarshal request object")) @@ -58,7 +62,16 @@ func (p *workloadMutationWebhook) Handle(_ context.Context, req admission.Reques return admission.Errored(http.StatusBadRequest, err) } - p.annotationMutators.MutateObject(obj) + // populate old object + if req.Operation == v1.Update { + if err := p.decoder.DecodeRaw(req.OldObject, oldObj); err != nil { + p.instrumentationAnnotator.GetLogger().WithName("workload_webhook").Error(err, "failed to unmarshal old object") + return admission.Errored(http.StatusBadRequest, err) + } + } + + p.instrumentationAnnotator.MutateObject(oldObj, obj) + marshaledObject, err := json.Marshal(obj) if err != nil { res := admission.Errored(http.StatusInternalServerError, err) diff --git a/main.go b/main.go index 63b7fccf0..c3777066c 100644 --- a/main.go +++ b/main.go @@ -128,6 +128,7 @@ func main() { autoInstrumentationDotNet string autoInstrumentationNodeJS string autoAnnotationConfigStr string + autoMonitorConfigStr string autoInstrumentationConfigStr string webhookPort int tlsOpt tlsConfig @@ -145,6 +146,7 @@ func main() { stringFlagOrEnv(&autoInstrumentationDotNet, "auto-instrumentation-dotnet-image", "RELATED_IMAGE_AUTO_INSTRUMENTATION_DOTNET", fmt.Sprintf("%s:%s", autoInstrumentationDotNetImageRepository, v.AutoInstrumentationDotNet), "The default OpenTelemetry Dotnet instrumentation image. This image is used when no image is specified in the CustomResource.") stringFlagOrEnv(&autoInstrumentationNodeJS, "auto-instrumentation-nodejs-image", "RELATED_IMAGE_AUTO_INSTRUMENTATION_NODEJS", fmt.Sprintf("%s:%s", autoInstrumentationNodeJSImageRepository, v.AutoInstrumentationNodeJS), "The default OpenTelemetry NodeJS instrumentation image. This image is used when no image is specified in the CustomResource.") stringFlagOrEnv(&autoAnnotationConfigStr, "auto-annotation-config", "AUTO_ANNOTATION_CONFIG", "", "The configuration for auto-annotation.") + pflag.StringVar(&autoMonitorConfigStr, "auto-monitor-config", "", "The configuration for auto-monitor.") pflag.StringVar(&autoInstrumentationConfigStr, "auto-instrumentation-config", "", "The configuration for auto-instrumentation.") stringFlagOrEnv(&dcgmExporterImage, "dcgm-exporter-image", "RELATED_IMAGE_DCGM_EXPORTER", fmt.Sprintf("%s:%s", dcgmExporterImageRepository, v.DcgmExporter), "The default DCGM Exporter image. This image is used when no image is specified in the CustomResource.") stringFlagOrEnv(&neuronMonitorImage, "neuron-monitor-image", "RELATED_IMAGE_NEURON_MONITOR", fmt.Sprintf("%s:%s", neuronMonitorImageRepository, v.NeuronMonitor), "The default Neuron monitor image. This image is used when no image is specified in the CustomResource.") @@ -285,40 +287,27 @@ func main() { decoder := admission.NewDecoder(mgr.GetScheme()) - if os.Getenv("DISABLE_AUTO_ANNOTATION") == "true" || autoAnnotationConfigStr == "" { - setupLog.Info("Auto-annotation is disabled") + instrumentationAnnotator, shouldMonitorAllServices := auto.CreateInstrumentationAnnotator(autoMonitorConfigStr, autoAnnotationConfigStr, ctx, mgr.GetClient(), mgr.GetAPIReader(), setupLog) + + if instrumentationAnnotator != nil { + mgr.GetWebhookServer().Register("/mutate-v1-workload", &webhook.Admission{ + Handler: workloadmutation.NewWebhookHandler(decoder, instrumentationAnnotator), + }) + mgr.GetWebhookServer().Register("/mutate-v1-namespace", &webhook.Admission{ + Handler: namespacemutation.NewWebhookHandler(decoder, instrumentationAnnotator), + }) + + setupLog.Info("Auto-annotation is enabled") + go waitForWebhookServerStart( + ctx, + mgr.GetWebhookServer().StartedChecker(), + func(ctx context.Context) { + setupLog.Info("Applying auto-annotation") + instrumentationAnnotator.MutateAndPatchAll(ctx) + }, + ) } else { - var autoAnnotationConfig auto.AnnotationConfig - if err = json.Unmarshal([]byte(autoAnnotationConfigStr), &autoAnnotationConfig); err != nil { - setupLog.Error(err, "Unable to unmarshal auto-annotation config") - } else { - autoAnnotationMutators := auto.NewAnnotationMutators( - mgr.GetClient(), - mgr.GetAPIReader(), - logger, - autoAnnotationConfig, - instrumentation.NewTypeSet( - instrumentation.TypeJava, - instrumentation.TypePython, - instrumentation.TypeDotNet, - instrumentation.TypeNodeJS, - ), - ) - mgr.GetWebhookServer().Register("/mutate-v1-workload", &webhook.Admission{ - Handler: workloadmutation.NewWebhookHandler(decoder, autoAnnotationMutators)}) - mgr.GetWebhookServer().Register("/mutate-v1-namespace", &webhook.Admission{ - Handler: namespacemutation.NewWebhookHandler(decoder, autoAnnotationMutators), - }) - setupLog.Info("Auto-annotation is enabled") - go waitForWebhookServerStart( - ctx, - mgr.GetWebhookServer().StartedChecker(), - func(ctx context.Context) { - setupLog.Info("Applying auto-annotation") - autoAnnotationMutators.MutateAndPatchAll(ctx) - }, - ) - } + setupLog.Info("Auto-annotation / Auto Monitor is disabled") } if os.Getenv("ENABLE_WEBHOOKS") != "false" { @@ -334,7 +323,7 @@ func main() { Handler: podmutation.NewWebhookHandler(cfg, ctrl.Log.WithName("pod-webhook"), decoder, mgr.GetClient(), []podmutation.PodMutator{ sidecar.NewMutator(logger, cfg, mgr.GetClient()), - instrumentation.NewMutator(logger, mgr.GetClient(), mgr.GetEventRecorderFor("amazon-cloudwatch-agent-operator")), + instrumentation.NewMutator(logger, mgr.GetClient(), mgr.GetEventRecorderFor("amazon-cloudwatch-agent-operator"), shouldMonitorAllServices), }), }) } else { diff --git a/pkg/instrumentation/annotationtype.go b/pkg/instrumentation/annotationtype.go index 75ceb01f1..a6116c178 100644 --- a/pkg/instrumentation/annotationtype.go +++ b/pkg/instrumentation/annotationtype.go @@ -3,12 +3,33 @@ package instrumentation +import ( + "encoding/json" +) + // Type is an enum for instrumentation types. type Type string // TypeSet is a map with Type keys. type TypeSet map[Type]any +func (s *TypeSet) UnmarshalJSON(data []byte) error { + var types []Type + if err := json.Unmarshal(data, &types); err != nil { + return err + } + *s = NewTypeSet(types...) + return nil +} + +func (s TypeSet) MarshalJSON() ([]byte, error) { + var types []Type + for t := range s { + types = append(types, t) + } + return json.Marshal(types) +} + // NewTypeSet creates a new set of Type. func NewTypeSet(types ...Type) TypeSet { s := make(TypeSet, len(types)) @@ -26,6 +47,10 @@ const ( TypeGo Type = "go" ) +var ( + SupportedTypes = NewTypeSet(TypeJava, TypeNodeJS, TypePython, TypeDotNet) +) + // InjectAnnotationKey maps the instrumentation type to the inject annotation. func InjectAnnotationKey(instType Type) string { switch instType { diff --git a/pkg/instrumentation/auto/annotation.go b/pkg/instrumentation/auto/annotation.go index 63c0d82ab..90ed4d310 100644 --- a/pkg/instrumentation/auto/annotation.go +++ b/pkg/instrumentation/auto/annotation.go @@ -40,29 +40,27 @@ type AnnotationMutators struct { injectAnnotations map[string]struct{} } -// RestartNamespace sets the restartedAtAnnotation for each of the namespace's supported resources and patches them. -func (m *AnnotationMutators) RestartNamespace(ctx context.Context, namespace *corev1.Namespace, mutatedAnnotations map[string]string) { - m.rangeObjectList(ctx, &appsv1.DeploymentList{}, client.InNamespace(namespace.Name), - chainCallbacks(m.shouldRestartFunc(mutatedAnnotations), m.patchFunc(ctx, setRestartAnnotation))) - m.rangeObjectList(ctx, &appsv1.DaemonSetList{}, client.InNamespace(namespace.Name), - chainCallbacks(m.shouldRestartFunc(mutatedAnnotations), m.patchFunc(ctx, setRestartAnnotation))) - m.rangeObjectList(ctx, &appsv1.StatefulSetList{}, client.InNamespace(namespace.Name), - chainCallbacks(m.shouldRestartFunc(mutatedAnnotations), m.patchFunc(ctx, setRestartAnnotation))) +func (m *AnnotationMutators) MutateAndPatchAll(ctx context.Context) { + MutateAndPatchWorkloads(m, ctx) + MutateAndPatchNamespaces(m, ctx, true) } -// MutateAndPatchAll runs the mutators for each of the supported resources and patches them. -func (m *AnnotationMutators) MutateAndPatchAll(ctx context.Context) { - m.rangeObjectList(ctx, &appsv1.DeploymentList{}, &client.ListOptions{}, m.patchFunc(ctx, m.mutateObject)) - m.rangeObjectList(ctx, &appsv1.DaemonSetList{}, &client.ListOptions{}, m.patchFunc(ctx, m.mutateObject)) - m.rangeObjectList(ctx, &appsv1.StatefulSetList{}, &client.ListOptions{}, m.patchFunc(ctx, m.mutateObject)) - m.rangeObjectList(ctx, &corev1.NamespaceList{}, &client.ListOptions{}, - chainCallbacks(m.patchFunc(ctx, m.mutateObject), m.restartNamespaceFunc(ctx)), - ) +func (m *AnnotationMutators) GetLogger() logr.Logger { + return m.logger +} + +func (m *AnnotationMutators) GetReader() client.Reader { + return m.clientReader +} + +func (m *AnnotationMutators) GetWriter() client.Writer { + return m.clientWriter } // MutateObject modifies annotations for a single object using the configured mutators. -func (m *AnnotationMutators) MutateObject(obj client.Object) (any, bool) { - return m.mutateObject(obj, nil) +func (m *AnnotationMutators) MutateObject(oldObj client.Object, obj client.Object) any { + mutatedAnnotations, _ := m.mutateObject(obj, nil) + return mutatedAnnotations.(map[string]string) } // mutateObject modifies annotations for a single object using the configured mutators. @@ -81,9 +79,9 @@ func (m *AnnotationMutators) mutateObject(obj client.Object, _ any) (any, bool) } } -func (m *AnnotationMutators) rangeObjectList(ctx context.Context, list client.ObjectList, option client.ListOption, fn objectCallbackFunc) { - if err := m.clientReader.List(ctx, list, option); err != nil { - m.logger.Error(err, "Unable to list objects", +func rangeObjectList(m InstrumentationAnnotator, ctx context.Context, list client.ObjectList, option client.ListOption, fn objectCallbackFunc) { + if err := m.GetReader().List(ctx, list, option); err != nil { + m.GetLogger().Error(err, "Unable to list objects", "kind", fmt.Sprintf("%T", list), ) return @@ -118,6 +116,9 @@ func (m *AnnotationMutators) mutate(name string, mutators map[string]instrumenta } func namespacedName(obj metav1.Object) string { + if _, ok := obj.(*corev1.Namespace); ok { + return obj.GetName() + } return fmt.Sprintf("%s/%s", obj.GetNamespace(), obj.GetName()) } @@ -131,6 +132,7 @@ func NewAnnotationMutators( cfg AnnotationConfig, typeSet instrumentation.TypeSet, ) *AnnotationMutators { + warnNonNamespacedNames(cfg, logger) builder := newMutatorBuilder(typeSet) return &AnnotationMutators{ clientWriter: clientWriter, @@ -145,6 +147,27 @@ func NewAnnotationMutators( } } +func warnNonNamespacedNames(cfg AnnotationConfig, logger logr.Logger) { + for t := range instrumentation.SupportedTypes { + resources := cfg.getResources(t) + for _, deployment := range resources.Deployments { + if !strings.Contains(deployment, "/") { + logger.Info("invalid deployment name, needs to be namespaced", "deployment", deployment) + } + } + for _, daemonSet := range resources.DaemonSets { + if !strings.Contains(daemonSet, "/") { + logger.Info("invalid daemonSet name, needs to be namespaced", "daemonSet", daemonSet) + } + } + for _, statefulSet := range resources.StatefulSets { + if !strings.Contains(statefulSet, "/") { + logger.Info("invalid statefulSet name, needs to be namespaced", "statefulSet", statefulSet) + } + } + } +} + func getResources( cfg AnnotationConfig, typeSet instrumentation.TypeSet, diff --git a/pkg/instrumentation/auto/callback.go b/pkg/instrumentation/auto/callback.go index 34e0ee0d3..b6bb4e0df 100644 --- a/pkg/instrumentation/auto/callback.go +++ b/pkg/instrumentation/auto/callback.go @@ -8,9 +8,12 @@ import ( "encoding/json" "fmt" + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/strategicpatch" "sigs.k8s.io/controller-runtime/pkg/client" @@ -66,11 +69,11 @@ func createPatch(obj client.Object) (client.Patch, error) { return &basicPatch{originalJSON: originalJSON}, nil } -func (m *AnnotationMutators) patchFunc(ctx context.Context, callback objectCallbackFunc) objectCallbackFunc { +func patchFunc(m InstrumentationAnnotator, ctx context.Context, callback objectCallbackFunc) objectCallbackFunc { return func(obj client.Object, _ any) (any, bool) { patch, err := createPatch(obj) if err != nil { - m.logger.Error(err, "Unable to create patch", + m.GetLogger().Error(err, "Unable to create patch", "kind", fmt.Sprintf("%T", obj), "name", obj.GetName(), "namespace", obj.GetNamespace(), @@ -81,8 +84,8 @@ func (m *AnnotationMutators) patchFunc(ctx context.Context, callback objectCallb if !ok { return ret, false } - if err = m.clientWriter.Patch(ctx, obj, patch); err != nil { - m.logger.Error(err, "Unable to send patch", + if err = m.GetWriter().Patch(ctx, obj, patch); err != nil { + m.GetLogger().Error(err, "Unable to send patch", "kind", fmt.Sprintf("%T", obj), "name", obj.GetName(), "namespace", obj.GetNamespace(), @@ -93,8 +96,11 @@ func (m *AnnotationMutators) patchFunc(ctx context.Context, callback objectCallb } } -func (m *AnnotationMutators) restartNamespaceFunc(ctx context.Context) objectCallbackFunc { +func restartNamespaceFunc(m InstrumentationAnnotator, ctx context.Context, shouldRestartNamespace bool) objectCallbackFunc { return func(obj client.Object, previousResult any) (any, bool) { + if !shouldRestartNamespace { + return nil, false + } mutatedAnnotations, ok := previousResult.(map[string]string) if !ok { return nil, false @@ -103,21 +109,21 @@ func (m *AnnotationMutators) restartNamespaceFunc(ctx context.Context) objectCal if !ok { return nil, false } - m.RestartNamespace(ctx, namespace, mutatedAnnotations) + RestartNamespace(m, ctx, namespace, mutatedAnnotations) return nil, true } } // shouldRestartFunc returns a func that determines if a resource should be restarted -func (m *AnnotationMutators) shouldRestartFunc(namespaceMutatedAnnotations map[string]string) objectCallbackFunc { +func shouldRestartFunc(m InstrumentationAnnotator, namespaceMutatedAnnotations map[string]string) objectCallbackFunc { return func(obj client.Object, _ any) (any, bool) { switch o := obj.(type) { case *appsv1.Deployment: - return nil, m.shouldRestartResource(namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) + return nil, shouldRestartResource(namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) case *appsv1.DaemonSet: - return nil, m.shouldRestartResource(namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) + return nil, shouldRestartResource(namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) case *appsv1.StatefulSet: - return nil, m.shouldRestartResource(namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) + return nil, shouldRestartResource(namespaceMutatedAnnotations, o.Spec.Template.GetObjectMeta()) default: return nil, false } @@ -125,13 +131,13 @@ func (m *AnnotationMutators) shouldRestartFunc(namespaceMutatedAnnotations map[s } // shouldRestartResource returns true if a resource requires a restart corresponding to the mutated annotations on its namespace -func (m *AnnotationMutators) shouldRestartResource(namespaceMutatedAnnotations map[string]string, obj metav1.Object) bool { +func shouldRestartResource(namespaceMutatedAnnotations map[string]string, obj metav1.Object) bool { var shouldRestart bool - + injectAnnotations := buildInjectAnnotations(instrumentation.SupportedTypes) if resourceAnnotations := obj.GetAnnotations(); resourceAnnotations != nil { // For each of the namespace mutated annotations, for namespaceMutatedAnnotation, namespaceMutatedAnnotationValue := range namespaceMutatedAnnotations { - if _, ok := m.injectAnnotations[namespaceMutatedAnnotation]; !ok { + if _, ok := injectAnnotations[namespaceMutatedAnnotation]; !ok { // If it is not an inject-* annotation, we can ignore it continue } @@ -152,3 +158,32 @@ func (m *AnnotationMutators) shouldRestartResource(namespaceMutatedAnnotations m return shouldRestart } + +// RestartNamespace sets the restartedAtAnnotation for each of the namespace's supported resources and patches them. +func RestartNamespace(m InstrumentationAnnotator, ctx context.Context, namespace *corev1.Namespace, mutatedAnnotations map[string]string) { + callbackFunc := patchFunc(m, ctx, setRestartAnnotation) + rangeObjectList(m, ctx, &appsv1.DeploymentList{}, client.InNamespace(namespace.Name), chainCallbacks(shouldRestartFunc(m, mutatedAnnotations), callbackFunc)) + rangeObjectList(m, ctx, &appsv1.DaemonSetList{}, client.InNamespace(namespace.Name), chainCallbacks(shouldRestartFunc(m, mutatedAnnotations), callbackFunc)) + rangeObjectList(m, ctx, &appsv1.StatefulSetList{}, client.InNamespace(namespace.Name), chainCallbacks(shouldRestartFunc(m, mutatedAnnotations), callbackFunc)) +} + +// MutateAndPatchWorkloads runs the mutators for all workloads and patches them with the updated injection annotations +func MutateAndPatchWorkloads(m InstrumentationAnnotator, ctx context.Context) { + f := getMutateObjectFunc(m) + callbackFunc := patchFunc(m, ctx, f) + rangeObjectList(m, ctx, &appsv1.DeploymentList{}, &client.ListOptions{}, callbackFunc) + rangeObjectList(m, ctx, &appsv1.DaemonSetList{}, &client.ListOptions{}, callbackFunc) + rangeObjectList(m, ctx, &appsv1.StatefulSetList{}, &client.ListOptions{}, callbackFunc) +} + +// MutateAndPatchNamespaces runs the mutators for all namespaces. +// If restartNamespace is true, RestartNamespace will be called for each affected namespace. +func MutateAndPatchNamespaces(m InstrumentationAnnotator, ctx context.Context, restartNamespace bool) { + rangeObjectList(m, ctx, &corev1.NamespaceList{}, &client.ListOptions{}, chainCallbacks(patchFunc(m, ctx, getMutateObjectFunc(m)), restartNamespaceFunc(m, ctx, restartNamespace))) +} + +func getMutateObjectFunc(m InstrumentationAnnotator) objectCallbackFunc { + return func(obj client.Object, _ any) (any, bool) { + return m.MutateObject(nil, obj), true + } +} diff --git a/pkg/instrumentation/auto/config.go b/pkg/instrumentation/auto/config.go index 12b5e9e6d..e65613df5 100644 --- a/pkg/instrumentation/auto/config.go +++ b/pkg/instrumentation/auto/config.go @@ -3,7 +3,15 @@ package auto -import "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" +import ( + "slices" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" +) // AnnotationConfig details the resources that have enabled // auto-annotation for each instrumentation type. @@ -29,6 +37,70 @@ func (c AnnotationConfig) getResources(instType instrumentation.Type) Annotation } } +// LanguagesOf get languages to annotate for an object +func (c AnnotationConfig) LanguagesOf(obj client.Object, checkNamespace bool) instrumentation.TypeSet { + objName := namespacedName(obj) + typesSelected := instrumentation.TypeSet{} + + types := instrumentation.SupportedTypes + + if checkNamespace { + for t := range types { + if slices.Contains(c.getResources(t).Namespaces, obj.GetNamespace()) { + typesSelected[t] = nil + } + } + } + + switch obj.(type) { + case *appsv1.Deployment: + for t := range types { + if slices.Contains(c.getResources(t).Deployments, objName) { + typesSelected[t] = nil + } + } + case *appsv1.StatefulSet: + for t := range types { + if slices.Contains(c.getResources(t).StatefulSets, objName) { + typesSelected[t] = nil + } + } + case *appsv1.DaemonSet: + for t := range types { + if slices.Contains(c.getResources(t).DaemonSets, objName) { + typesSelected[t] = nil + } + } + case *corev1.Namespace: + for t := range types { + if slices.Contains(c.getResources(t).Namespaces, objName) { + typesSelected[t] = nil + } + } + } + + return typesSelected +} + +func (c AnnotationConfig) Empty() bool { + for t := range instrumentation.SupportedTypes { + resources := c.getResources(t) + if len(resources.DaemonSets) > 0 { + return false + } + if len(resources.StatefulSets) > 0 { + return false + } + if len(resources.Deployments) > 0 { + return false + } + if len(resources.Namespaces) > 0 { + return false + } + } + return true +} + // AnnotationResources contains slices of resource names for each // of the supported workloads. type AnnotationResources struct { diff --git a/pkg/instrumentation/auto/monitor.go b/pkg/instrumentation/auto/monitor.go new file mode 100644 index 000000000..9feb76eec --- /dev/null +++ b/pkg/instrumentation/auto/monitor.go @@ -0,0 +1,369 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package auto + +import ( + "context" + "fmt" + "reflect" + "slices" + "time" + + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" +) + +var excludedNamespaces = []string{"kube-system", "amazon-cloudwatch"} + +// InstrumentationAnnotator is the highest level abstraction used to annotate kubernetes resources for instrumentation +type InstrumentationAnnotator interface { + MutateObject(oldObj client.Object, obj client.Object) any + GetLogger() logr.Logger + GetReader() client.Reader + GetWriter() client.Writer + MutateAndPatchAll(ctx context.Context) +} + +type Monitor struct { + serviceInformer cache.SharedIndexInformer + ctx context.Context + config MonitorConfig + k8sInterface kubernetes.Interface + clientReader client.Reader + clientWriter client.Writer + logger logr.Logger +} + +func (m *Monitor) MutateAndPatchAll(ctx context.Context) { + if m.config.RestartPods { + MutateAndPatchWorkloads(m, ctx) + } + MutateAndPatchNamespaces(m, ctx, m.config.RestartPods) +} + +func (m *Monitor) GetLogger() logr.Logger { + return m.logger +} + +func (m *Monitor) GetReader() client.Reader { + return m.clientReader +} + +func (m *Monitor) GetWriter() client.Writer { + return m.clientWriter +} + +// NewMonitor is used to create an InstrumentationMutator that supports AutoMonitor. +func NewMonitor(ctx context.Context, config MonitorConfig, k8sClient kubernetes.Interface, w client.Writer, r client.Reader, logger logr.Logger) *Monitor { + // Config default values + if len(config.Languages) == 0 { + logger.Info("Setting languages to default") + config.Languages = instrumentation.SupportedTypes + } + + logger.Info("AutoMonitor starting...") + factory := informers.NewSharedInformerFactoryWithOptions(k8sClient, 10*time.Minute, informers.WithTransform(func(obj interface{}) (interface{}, error) { + svc, ok := obj.(*corev1.Service) + if !ok { + return obj, fmt.Errorf("error transforming service: %s not a service", obj) + } + // Return only the fields we need + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: svc.Name, + Namespace: svc.Namespace, + }, + Spec: corev1.ServiceSpec{ + Selector: svc.Spec.Selector, + }, + }, nil + })) + serviceInformer := factory.Core().V1().Services().Informer() + + warnNonNamespacedNames(config.Exclude, logger) + + m := &Monitor{serviceInformer: serviceInformer, ctx: ctx, config: config, k8sInterface: k8sClient, clientReader: r, clientWriter: w} + _, err := serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + m.onServiceEvent(nil, obj.(*corev1.Service)) + }, + UpdateFunc: func(oldObj, obj interface{}) { + m.onServiceEvent(oldObj.(*corev1.Service), obj.(*corev1.Service)) + }, + DeleteFunc: func(obj interface{}) { + m.onServiceEvent(obj.(*corev1.Service), nil) + }, + }) + if err != nil { + logger.Error(err, "failed to start auto monitor") + return nil + } + factory.Start(ctx.Done()) + synced := factory.WaitForCacheSync(ctx.Done()) + for v, ok := range synced { + if !ok { + logger.Error(fmt.Errorf("caches failed to sync: %v", v), "bad cache sync") + } + } + + logger.Info("Initialization complete!") + return m +} + +func (m *Monitor) onServiceEvent(oldService *corev1.Service, service *corev1.Service) { + if !m.config.RestartPods { + return + } + for _, resource := range m.listServiceDeployments(m.ctx, oldService, service) { + mutatedAnnotations := m.MutateObject(&resource, &resource).(map[string]string) + if len(mutatedAnnotations) == 0 { + continue + } + _, err := m.k8sInterface.AppsV1().Deployments(resource.GetNamespace()).Update(m.ctx, &resource, metav1.UpdateOptions{}) + if err != nil { + m.logger.Error(err, "failed to update deployment", "deployment", resource.Name) + } + } + for _, resource := range m.listServiceStatefulSets(m.ctx, oldService, service) { + mutatedAnnotations := m.MutateObject(&resource, &resource).(map[string]string) + if len(mutatedAnnotations) == 0 { + continue + } + _, err := m.k8sInterface.AppsV1().StatefulSets(resource.GetNamespace()).Update(m.ctx, &resource, metav1.UpdateOptions{}) + if err != nil { + m.logger.Error(err, "failed to update statefulset", "statefulset", resource.Name) + } + } + for _, resource := range m.listServiceDaemonSets(m.ctx, oldService, service) { + mutatedAnnotations := m.MutateObject(&resource, &resource).(map[string]string) + if len(mutatedAnnotations) == 0 { + continue + } + _, err := m.k8sInterface.AppsV1().DaemonSets(resource.GetNamespace()).Update(m.ctx, &resource, metav1.UpdateOptions{}) + if err != nil { + m.logger.Error(err, "failed to update daemonset", "daemonset", resource.Name) + } + } +} + +func (m *Monitor) listServiceDeployments(ctx context.Context, services ...*corev1.Service) []appsv1.Deployment { + var deployments []appsv1.Deployment + for _, service := range services { + if service == nil { + continue + } + list, err := m.k8sInterface.AppsV1().Deployments(service.GetNamespace()).List(ctx, metav1.ListOptions{}) + if err != nil { + m.logger.Error(err, "failed to list deployments") + } + serviceSelector := labels.SelectorFromSet(service.Spec.Selector) + trimmed := slices.DeleteFunc(list.Items, func(deployment appsv1.Deployment) bool { + return !serviceSelector.Matches(getTemplateSpecLabels(&deployment)) + }) + deployments = append(deployments, trimmed...) + } + return deployments +} + +func (m *Monitor) listServiceStatefulSets(ctx context.Context, services ...*corev1.Service) []appsv1.StatefulSet { + var statefulSets []appsv1.StatefulSet + for _, service := range services { + if service == nil { + continue + } + list, err := m.k8sInterface.AppsV1().StatefulSets(service.GetNamespace()).List(ctx, metav1.ListOptions{}) + if err != nil { + m.logger.Error(err, "failed to list statefulsets") + } + serviceSelector := labels.SelectorFromSet(service.Spec.Selector) + trimmed := slices.DeleteFunc(list.Items, func(statefulSet appsv1.StatefulSet) bool { + return !serviceSelector.Matches(getTemplateSpecLabels(&statefulSet)) + }) + statefulSets = append(statefulSets, trimmed...) + } + return statefulSets +} + +func (m *Monitor) listServiceDaemonSets(ctx context.Context, services ...*corev1.Service) []appsv1.DaemonSet { + var daemonSets []appsv1.DaemonSet + for _, service := range services { + if service == nil { + continue + } + list, err := m.k8sInterface.AppsV1().DaemonSets(service.GetNamespace()).List(ctx, metav1.ListOptions{}) + if err != nil { + m.logger.Error(err, "failed to list DaemonSets") + } + serviceSelector := labels.SelectorFromSet(service.Spec.Selector) + trimmed := slices.DeleteFunc(list.Items, func(daemonSet appsv1.DaemonSet) bool { + return !serviceSelector.Matches(getTemplateSpecLabels(&daemonSet)) + }) + daemonSets = append(daemonSets, trimmed...) + } + return daemonSets +} + +func getTemplateSpecLabels(obj metav1.Object) labels.Set { + // Check if the object implements the type assertion for PodTemplateSpec + switch t := obj.(type) { + case *appsv1.Deployment: + return t.Spec.Template.Labels + case *appsv1.StatefulSet: + return t.Spec.Template.Labels + case *appsv1.DaemonSet: + return t.Spec.Template.Labels + default: + // Return empty labels.Set if the object type is not supported + return labels.Set{} + } +} + +// MutateObject adds all enabled languages in config. Should only be run if selected by auto monitor or custom selector +func (m *Monitor) MutateObject(oldObj client.Object, obj client.Object) any { + if !safeToMutate(oldObj, obj, m.config.RestartPods) { + return map[string]string{} + } + + languagesToAnnotate := m.config.CustomSelector.LanguagesOf(obj, false) + if m.isWorkloadAutoMonitored(obj) { + for l := range m.config.Languages { + languagesToAnnotate[l] = nil + } + } + + for l := range m.config.Exclude.LanguagesOf(obj, true) { + delete(languagesToAnnotate, l) + } + + return mutate(obj, languagesToAnnotate) +} + +// returns if workload is auto monitored (does not include custom selector) +func (m *Monitor) isWorkloadAutoMonitored(obj client.Object) bool { + if isNamespace(obj) { + return false + } + + if !m.config.MonitorAllServices { + return false + } + + if slices.Contains(excludedNamespaces, obj.GetNamespace()) { + return false + } + // determine if the object is currently selected by a service + objectLabels := getTemplateSpecLabels(obj) + for _, informerObj := range m.serviceInformer.GetStore().List() { + service := informerObj.(*corev1.Service) + if len(service.Spec.Selector) == 0 || service.GetNamespace() != obj.GetNamespace() { + continue + } + serviceSelector := labels.SelectorFromSet(service.Spec.Selector) + + if serviceSelector.Matches(objectLabels) { + m.logger.Info(fmt.Sprintf("setting %s instrumentation annotations to %s because it is owned by service %s", obj.GetName(), m.config.Languages, service.Name)) + return true + } + } + return false +} + +// mutate if object is a workload, mutate the pod template. otherwise, mutate the object's annotations itself. It will add annotations if needsInstrumentation is true. Otherwise, it will remove instrumentation annotations. +func mutate(object client.Object, languagesToMonitor instrumentation.TypeSet) map[string]string { + var obj metav1.Object + podTemplate := getPodTemplate(object) + if podTemplate != nil { + obj = podTemplate + } else { + obj = object + } + + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + + allMutatedAnnotations := map[string]string{} + for language := range instrumentation.SupportedTypes { + insertMutation, removeMutation := buildMutations(language) + var mutatedAnnotations map[string]string + if _, ok := languagesToMonitor[language]; ok { + mutatedAnnotations = insertMutation.Mutate(annotations) + } else { + mutatedAnnotations = removeMutation.Mutate(annotations) + } + for k, v := range mutatedAnnotations { + allMutatedAnnotations[k] = v + } + } + obj.SetAnnotations(annotations) + return allMutatedAnnotations +} + +// safeToMutate returns whether the customer consents to the operator updating their workload's pods. The user consents if any of the following conditions are true: +// +// 1. Auto restart enabled. +// 2. The user was already modifying the pod template spec (aka a restart would already be triggered) +func safeToMutate(oldWorkload client.Object, workload client.Object, restartPods bool) bool { + // always ok to mutate namespace + if isNamespace(workload) { + return true + } + // should only mutate workloads or namespaces + if !isMutableType(workload) { + return false + } + + if restartPods { + return true + } + oldTemplate, newTemplate := getPodTemplate(oldWorkload), getPodTemplate(workload) + return !reflect.DeepEqual(oldTemplate, newTemplate) +} + +func isMutableType(obj client.Object) bool { + if isNamespace(obj) { + return true + } + switch obj.(type) { + case *appsv1.Deployment: + return true + case *appsv1.StatefulSet: + return true + case *appsv1.DaemonSet: + return true + default: + return false + } +} + +func isNamespace(obj client.Object) bool { + switch obj.(type) { + case *corev1.Namespace: + return true + } + return false +} + +func getPodTemplate(obj client.Object) *corev1.PodTemplateSpec { + switch o := obj.(type) { + case *appsv1.Deployment: + return &o.Spec.Template + case *appsv1.StatefulSet: + return &o.Spec.Template + case *appsv1.DaemonSet: + return &o.Spec.Template + default: + return nil + } +} diff --git a/pkg/instrumentation/auto/monitor_config.go b/pkg/instrumentation/auto/monitor_config.go new file mode 100644 index 000000000..e151632c7 --- /dev/null +++ b/pkg/instrumentation/auto/monitor_config.go @@ -0,0 +1,16 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package auto + +import "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" + +// AnnotationConfig details the resources that have enabled +// auto-annotation for each instrumentation type. +type MonitorConfig struct { + MonitorAllServices bool `json:"monitorAllServices"` + Languages instrumentation.TypeSet `json:"languages,omitempty"` + RestartPods bool `json:"restartPods"` + Exclude AnnotationConfig `json:"exclude,omitempty"` + CustomSelector AnnotationConfig `json:"customSelector,omitempty"` +} diff --git a/pkg/instrumentation/auto/monitor_test.go b/pkg/instrumentation/auto/monitor_test.go new file mode 100644 index 000000000..19b57f8af --- /dev/null +++ b/pkg/instrumentation/auto/monitor_test.go @@ -0,0 +1,1025 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package auto + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/go-logr/logr/testr" + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/controller-runtime/pkg/client" + fake2 "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" +) + +const defaultNs = "default" + +var workloadTypes = []struct { + name string + create func(name, namespace string, labels, annotations map[string]string) client.Object + get func(clientset kubernetes.Interface, namespace, name string) (client.Object, error) + getWithClient func(c client.Reader, ns, name string) (client.Object, error) +}{ + { + name: "Deployment", + create: func(name, ns string, labels, annotations map[string]string) client.Object { + return newTestDeployment(name, ns, labels, annotations) + }, + get: func(c kubernetes.Interface, ns, name string) (client.Object, error) { + return c.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{}) + }, + getWithClient: func(c client.Reader, ns, name string) (client.Object, error) { + obj := &appsv1.Deployment{} + err := c.Get(context.TODO(), client.ObjectKey{Namespace: ns, Name: name}, obj) + return obj, err + }, + }, + { + name: "StatefulSet", + create: func(name, ns string, labels, annotations map[string]string) client.Object { + return newTestStatefulSet(name, ns, labels, annotations) + }, + get: func(c kubernetes.Interface, ns, name string) (client.Object, error) { + return c.AppsV1().StatefulSets(ns).Get(context.TODO(), name, metav1.GetOptions{}) + }, + getWithClient: func(c client.Reader, ns, name string) (client.Object, error) { + obj := &appsv1.StatefulSet{} + err := c.Get(context.TODO(), client.ObjectKey{Namespace: ns, Name: name}, obj) + return obj, err + }, + }, + { + name: "DaemonSet", + create: func(name, ns string, labels, annotations map[string]string) client.Object { + return newTestDaemonSet(name, ns, labels, annotations) + }, + get: func(c kubernetes.Interface, ns, name string) (client.Object, error) { + return c.AppsV1().DaemonSets(ns).Get(context.TODO(), name, metav1.GetOptions{}) + }, + getWithClient: func(c client.Reader, ns, name string) (client.Object, error) { + obj := &appsv1.DaemonSet{} + err := c.Get(context.TODO(), client.ObjectKey{Namespace: ns, Name: name}, obj) + return obj, err + }, + }, +} + +func TestUnmarshal(t *testing.T) { + j := []byte(`["java", "nodejs", "python"]`) + set := instrumentation.TypeSet{} + err := set.UnmarshalJSON(j) + assert.NoError(t, err) + assert.Equal(t, instrumentation.TypeSet{instrumentation.TypeNodeJS: nil, instrumentation.TypeJava: nil, instrumentation.TypePython: nil}, set) +} + +func TestMarshal(t *testing.T) { + types := instrumentation.TypeSet{instrumentation.TypeNodeJS: nil, instrumentation.TypePython: nil} + res, err := types.MarshalJSON() + assert.NoError(t, err) + // todo test irregardless of order + var s []string + err = json.Unmarshal(res, &s) + assert.NoError(t, err) + + assert.ElementsMatch(t, []string{"nodejs", "python"}, s) +} + +func Test_safeToMutate(t *testing.T) { + deploy := &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Image: "nginx:1"}}, + }, + }, + }, + } + namespace := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-namespace", + }, + } + mutatedDeploy := appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Image: "nginx:2"}}, + }, + }, + }, + } + tests := []struct { + name string + oldObject client.Object + object client.Object + restartPods bool + restartPodsCustomSelectors bool + want bool + }{ + {"identical deployments", deploy.DeepCopy(), deploy.DeepCopy(), false, false, false}, + {"identical deployments, auto restart", deploy.DeepCopy(), deploy.DeepCopy(), true, false, true}, //should try and mutate in case deployment should no longer have annotations and mutators need to run to remove annotations + {"changed pod template", deploy.DeepCopy(), &mutatedDeploy, false, false, true}, + {"non-workload", &corev1.ConfigMap{}, &corev1.ConfigMap{}, false, false, false}, + {"non-workload, auto restart", &corev1.ConfigMap{}, &corev1.ConfigMap{Data: map[string]string{"test": "test"}}, true, false, false}, + {"create (oldObject nil)", nil, deploy.DeepCopy(), false, false, true}, + {"namespace, auto restart false", nil, &namespace, false, false, true}, + {"namespace, auto restart true", nil, &namespace, true, false, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + //noinspection GoDeprecation + assert.Equal(t, tt.want, safeToMutate(tt.oldObject, tt.object, tt.restartPods)) + }) + } +} + +func Test_getPodTemplate(t *testing.T) { + template := corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Image: "nginx"}}, + }, + } + + tests := []struct { + name string + obj client.Object + want *corev1.PodTemplateSpec + }{ + {"deployment", &appsv1.Deployment{Spec: appsv1.DeploymentSpec{Template: template}}, &template}, + {"statefulset", &appsv1.StatefulSet{Spec: appsv1.StatefulSetSpec{Template: template}}, &template}, + {"daemonset", &appsv1.DaemonSet{Spec: appsv1.DaemonSetSpec{Template: template}}, &template}, + {"other", &corev1.Pod{}, nil}, + {"nil", nil, nil}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, getPodTemplate(tt.obj)) + }) + } +} + +type MutateObjectTest struct { + name string + config MonitorConfig + deploymentNs string + serviceNs string + deploymentSelector map[string]string + serviceSelector map[string]string + expectedWorkloadAnnotations map[string]string +} + +var none = AnnotationConfig{} + +func TestMonitor_MutateObject(t *testing.T) { + annotated := buildAnnotations(instrumentation.TypeJava) + tests := []MutateObjectTest{ + { + name: "same namespace, same selector, monitorallservices true, not excluded", + config: simpleConfig(true, false, none, none), + deploymentNs: "namespace-1", + serviceNs: "namespace-1", + deploymentSelector: map[string]string{"app": "same"}, + serviceSelector: map[string]string{"app": "same"}, + expectedWorkloadAnnotations: annotated, + }, + { + name: "different namespace, same selector, monitorallservices true, not excluded", + config: simpleConfig(true, false, none, none), + deploymentNs: "namespace-2", + serviceNs: "namespace-1", + deploymentSelector: map[string]string{"app": "same"}, + serviceSelector: map[string]string{"app": "same"}, + expectedWorkloadAnnotations: map[string]string{}, + }, + { + name: "same namespace, different selector, monitorallservices true, not excluded", + config: simpleConfig(true, false, none, none), + deploymentNs: "namespace-1", + serviceNs: "namespace-1", + deploymentSelector: map[string]string{"app": "different-1"}, + serviceSelector: map[string]string{"app": "different-2"}, + expectedWorkloadAnnotations: map[string]string{}, + }, + { + name: "same namespace, same selector, monitorallservices false, not excluded", + config: simpleConfig(false, false, none, none), + deploymentNs: "namespace-1", + serviceNs: "namespace-1", + deploymentSelector: map[string]string{"app": "different-1"}, + serviceSelector: map[string]string{"app": "different-1"}, + expectedWorkloadAnnotations: map[string]string{}, + }, + { + name: "same namespace, same selector, monitorallservices true, excluded namespace", + config: simpleConfig(true, false, none, + AnnotationConfig{Java: AnnotationResources{Namespaces: []string{"namespace-1"}}}), + deploymentNs: "namespace-1", + serviceNs: "namespace-1", + deploymentSelector: map[string]string{"app": "different-1"}, + serviceSelector: map[string]string{"app": "different-1"}, + expectedWorkloadAnnotations: map[string]string{}, + }, + { + name: "same namespace, same selector, monitorallservices true, excluded service", + config: simpleConfig(true, false, none, AnnotationConfig{Java: AnnotationResources{ + Deployments: []string{"namespace-1/workload"}, + DaemonSets: []string{"namespace-1/workload"}, + StatefulSets: []string{"namespace-1/workload"}, + }}), + deploymentNs: "namespace-1", + serviceNs: "namespace-1", + deploymentSelector: map[string]string{"app": "different-1"}, + serviceSelector: map[string]string{"app": "different-1"}, + expectedWorkloadAnnotations: map[string]string{}, + }, + { + name: "different namespace, different selector, monitorallservices false, custom selected workload", + config: simpleConfig(true, false, AnnotationConfig{Java: AnnotationResources{ + Namespaces: nil, + Deployments: []string{"namespace-1/workload"}, + DaemonSets: []string{"namespace-1/workload"}, + StatefulSets: []string{"namespace-1/workload"}, + }}, none), + deploymentNs: "namespace-1", + serviceNs: "namespace-2", + deploymentSelector: map[string]string{"app": "different-1"}, + serviceSelector: map[string]string{"app": "different-2"}, + expectedWorkloadAnnotations: annotated, + }, + { + name: "different namespace, different selector, monitorallservices false, custom selected namespace of workload", + config: simpleConfig(true, false, AnnotationConfig{Java: AnnotationResources{ + Namespaces: []string{"namespace-1"}, + Deployments: nil, + DaemonSets: nil, + StatefulSets: nil, + }}, none), + deploymentNs: "namespace-1", + serviceNs: "namespace-2", + deploymentSelector: map[string]string{"app": "different-1"}, + serviceSelector: map[string]string{"app": "different-2"}, + expectedWorkloadAnnotations: map[string]string{}, // empty because even though it should be custom selected, it is modified on the pod level for namespaces, so the pod template is not updated + }, + { + name: "different namespace, different selector, monitorallservices false, custom selected namespace of service, not workload", + config: simpleConfig(true, false, AnnotationConfig{Java: AnnotationResources{ + Namespaces: []string{"namespace-2"}, + Deployments: nil, + DaemonSets: nil, + StatefulSets: nil, + }}, none), + deploymentNs: "namespace-1", + serviceNs: "namespace-2", + deploymentSelector: map[string]string{"app": "different-1"}, + serviceSelector: map[string]string{"app": "different-2"}, + expectedWorkloadAnnotations: map[string]string{}, // empty because even though it should be custom selected, it is modified on the pod level for namespaces, so the pod template is not updated + }, + } + + workloadTypes := []struct { + name string + create func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) + }{ + { + name: "Deployment", + create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { + deployment := newTestDeployment("workload", ns, selector, nil) + return clientset.AppsV1().Deployments(ns).Create(ctx, deployment, metav1.CreateOptions{}) + }, + }, + { + name: "StatefulSet", + create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { + statefulset := newTestStatefulSet("workload", ns, selector, nil) + return clientset.AppsV1().StatefulSets(ns).Create(ctx, statefulset, metav1.CreateOptions{}) + }, + }, + { + name: "DaemonSet", + create: func(clientset *fake.Clientset, ctx context.Context, ns string, selector map[string]string) (client.Object, error) { + daemonset := newTestDaemonSet("workload", ns, selector, nil) + return clientset.AppsV1().DaemonSets(ns).Create(ctx, daemonset, metav1.CreateOptions{}) + }, + }, + } + + for _, workload := range workloadTypes { + t.Run(workload.name, func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + //t.Parallel() + // Setup fresh clients for each workload test + fakeClient := fake2.NewFakeClient() + clientset := fake.NewSimpleClientset() + ctx := context.TODO() + + logger := testr.New(t) + monitor := NewMonitor(ctx, tt.config, clientset, fakeClient, fakeClient, logger) + + // Create service + service := newTestService("svc", tt.serviceNs, tt.serviceSelector) + + // Setup test environment + serviceNamespace := createNamespace(t, clientset, ctx, service.Namespace) + if tt.deploymentNs != serviceNamespace.Name { + createNamespace(t, clientset, ctx, tt.deploymentNs) + } + + // Create service + _, err := monitor.k8sInterface.CoreV1().Services(service.Namespace).Create(ctx, service, metav1.CreateOptions{}) + assert.NoError(t, err) + + // Create workload + workloadObj, err := workload.create(clientset, ctx, tt.deploymentNs, tt.deploymentSelector) + assert.NoError(t, err) + // need to wait until service informer is updated + err = waitForInformerUpdate(monitor, func(numKeys int) bool { return numKeys > 0 }) + assert.NoError(t, err) + + // Test + mutatedAnnotations := monitor.MutateObject(nil, workloadObj) + assert.Equal(t, tt.expectedWorkloadAnnotations, mutatedAnnotations) + }) + } + }) + } +} + +func TestSystemNamespaceExclusion(t *testing.T) { + tests := []struct { + name string + namespace string + config MonitorConfig + expectedWorkloadAnnotations map[string]string + }{ + { + name: "kube-system namespace is excluded by default", + namespace: "kube-system", + config: MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + RestartPods: true, + }, + expectedWorkloadAnnotations: map[string]string{}, + }, + { + name: "amazon-cloudwatch namespace is excluded by default", + namespace: "amazon-cloudwatch", + config: MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + RestartPods: true, + }, + expectedWorkloadAnnotations: map[string]string{}, + }, + { + name: "kube-system can be included via customSelector", + namespace: "kube-system", + config: MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + RestartPods: true, + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + DaemonSets: []string{"kube-system/workload"}, + Deployments: []string{"kube-system/workload"}, + StatefulSets: []string{"kube-system/workload"}, + }, + }, + }, + expectedWorkloadAnnotations: buildAnnotations(instrumentation.TypeJava), + }, + { + name: "amazon-cloudwatch can be included via customSelector", + namespace: "amazon-cloudwatch", + config: MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + RestartPods: true, + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + DaemonSets: []string{"amazon-cloudwatch/workload"}, + Deployments: []string{"amazon-cloudwatch/workload"}, + StatefulSets: []string{"amazon-cloudwatch/workload"}, + }, + }, + }, + expectedWorkloadAnnotations: buildAnnotations(instrumentation.TypeJava), + }, + { + name: "regular namespace is not excluded by default", + namespace: "test-namespace", + config: MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + RestartPods: true, + }, + expectedWorkloadAnnotations: buildAnnotations(instrumentation.TypeJava), + }, + } + + for _, workloadType := range workloadTypes { + t.Run(workloadType.name, func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup fresh clients for each test + clientset := fake.NewSimpleClientset() + fakeClient := fake2.NewFakeClient() + ctx := context.TODO() + logger := testr.New(t) + + // Create namespace + namespace := createNamespace(t, clientset, ctx, tt.namespace) + + // Create a monitor + monitor := NewMonitor(ctx, tt.config, clientset, fakeClient, fakeClient, logger) + + // Create service + labels := map[string]string{"app": "test"} + service := newTestService("service", namespace.Name, labels) + _, err := clientset.CoreV1().Services(namespace.Name).Create(ctx, service, metav1.CreateOptions{}) + assert.NoError(t, err) + + // Create workload object + workloadObj := workloadType.create("workload", namespace.Name, labels, nil) + + // Add workload to clientset based on its type + switch obj := workloadObj.(type) { + case *appsv1.Deployment: + _, err = clientset.AppsV1().Deployments(namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + case *appsv1.StatefulSet: + _, err = clientset.AppsV1().StatefulSets(namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + case *appsv1.DaemonSet: + _, err = clientset.AppsV1().DaemonSets(namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + } + assert.NoError(t, err) + + // Wait for service informer to be updated + err = waitForInformerUpdate(monitor, func(numKeys int) bool { return numKeys > 0 }) + assert.NoError(t, err) + + // Test mutation + mutatedAnnotations := monitor.MutateObject(nil, workloadObj) + assert.Equal(t, tt.expectedWorkloadAnnotations, mutatedAnnotations) + }) + } + }) + } +} + +func TestMonitor_MutateObject_Namespace(t *testing.T) { + tests := []struct { + name string + config MonitorConfig + namespace string + existingAnnotations map[string]string + expectedNamespaceAnnotations map[string]string + expectedMutated map[string]string + }{ + { + name: "namespace included in custom selector java", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + }, + namespace: "test-namespace", + existingAnnotations: nil, + expectedNamespaceAnnotations: buildAnnotations(instrumentation.TypeJava), + expectedMutated: buildAnnotations(instrumentation.TypeJava), + }, + { + name: "namespace included in custom selector python", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypePython), + CustomSelector: AnnotationConfig{ + Python: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + }, + namespace: "test-namespace", + existingAnnotations: nil, + expectedNamespaceAnnotations: buildAnnotations(instrumentation.TypePython), + expectedMutated: buildAnnotations(instrumentation.TypePython), + }, + { + name: "namespace excluded", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + Exclude: AnnotationConfig{ + Java: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + }, + namespace: "test-namespace", + existingAnnotations: nil, + expectedNamespaceAnnotations: map[string]string{}, + expectedMutated: map[string]string{}, + }, + { + name: "namespace not in any selection", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + }, + namespace: "test-namespace", + existingAnnotations: nil, + expectedNamespaceAnnotations: map[string]string{}, + expectedMutated: map[string]string{}, + }, + { + name: "multiple languages in custom selector", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava, instrumentation.TypePython), + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + Python: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + }, + namespace: "test-namespace", + existingAnnotations: nil, + expectedNamespaceAnnotations: mergeMaps( + buildAnnotations(instrumentation.TypeJava), + buildAnnotations(instrumentation.TypePython), + ), + expectedMutated: mergeMaps( + buildAnnotations(instrumentation.TypeJava), + buildAnnotations(instrumentation.TypePython), + ), + }, + { + name: "remove java when no longer selected", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + }, + namespace: "test-namespace", + existingAnnotations: buildAnnotations(instrumentation.TypeJava), + expectedNamespaceAnnotations: map[string]string{}, + expectedMutated: buildAnnotations(instrumentation.TypeJava), + }, + { + name: "preserve existing non-instrumentation annotations", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + }, + namespace: "test-namespace", + existingAnnotations: map[string]string{"custom-key": "custom-value"}, + expectedNamespaceAnnotations: mergeMaps( + buildAnnotations(instrumentation.TypeJava), + map[string]string{"custom-key": "custom-value"}, + ), + expectedMutated: buildAnnotations(instrumentation.TypeJava), + }, + { + name: "restart pods has no effect on namespace selection", + config: MonitorConfig{ + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + RestartPods: true, + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + }, + namespace: "test-namespace", + existingAnnotations: nil, + expectedNamespaceAnnotations: buildAnnotations(instrumentation.TypeJava), + expectedMutated: buildAnnotations(instrumentation.TypeJava), + }, + { + name: "add annotation to namespace with existing annotations", + config: MonitorConfig{ + CustomSelector: AnnotationConfig{ + Java: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + Python: AnnotationResources{ + Namespaces: []string{"test-namespace"}, + }, + }, + }, + namespace: "test-namespace", + existingAnnotations: buildAnnotations(instrumentation.TypePython), + expectedNamespaceAnnotations: mergeMaps( + buildAnnotations(instrumentation.TypeJava), + buildAnnotations(instrumentation.TypePython), + ), + expectedMutated: buildAnnotations(instrumentation.TypeJava), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup test environment + clientset := fake.NewSimpleClientset() + fakeClient := fake2.NewFakeClient() + ctx := context.TODO() + + // Create namespace with existing annotations if any + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: tt.namespace, + Annotations: tt.existingAnnotations, + }, + } + + // Create monitor + logger := testr.New(t) + monitor := NewMonitor(ctx, tt.config, clientset, fakeClient, fakeClient, logger) + + // Test + mutatedAnnotations := monitor.MutateObject(nil, namespace) + assert.Equal(t, tt.expectedMutated, mutatedAnnotations, "Mutated annotations don't match expected") + + assert.Equal(t, tt.expectedNamespaceAnnotations, namespace.GetAnnotations(), "Namespace annotations don't match expected") + }) + } +} + +func waitForInformerUpdate(monitor *Monitor, isValid func(int) bool) error { + return wait.PollUntilContextTimeout( + context.TODO(), // parent context + 1*time.Millisecond, // interval between polls + 5*time.Millisecond, // timeout + false, // immediate (set to false to match PollImmediate behavior) + func(ctx context.Context) (bool, error) { + return isValid(len(monitor.serviceInformer.GetStore().ListKeys())), nil + }) +} + +func Test_OptOutByRemovingService(t *testing.T) { + for _, wt := range workloadTypes { + t.Run(wt.name, func(t *testing.T) { + t.Run("auto restart true, delete and then restart operator", func(t *testing.T) { + userAnnotations := map[string]string{"test": "blah"} + annotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) + workload := wt.create("deployment", defaultNs, nil, annotations) + + clientset := fake.NewSimpleClientset(workload) + c := fake2.NewFakeClient(workload) + var config MonitorConfig = simpleConfig(true, true, none, none) + var k8sInterface kubernetes.Interface = clientset + var logger logr.Logger = testr.New(t) + monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) + monitor.MutateAndPatchAll(context.TODO()) + updatedWorkload, err := wt.getWithClient(c, defaultNs, workload.GetName()) + assert.NoError(t, err) + assert.Equal(t, userAnnotations, getPodTemplate(updatedWorkload).GetAnnotations()) + }) + + t.Run("auto restart true, delete while operator running", func(t *testing.T) { + userAnnotations := map[string]string{"test": "blah"} + annotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) + labels := map[string]string{"app": "test"} + service := newTestService("service", defaultNs, labels) + workload := wt.create("workload", defaultNs, labels, annotations) + + clientset := fake.NewSimpleClientset(service, workload) + c := fake2.NewFakeClient(service, workload) + var config MonitorConfig = simpleConfig(true, true, none, none) + var k8sInterface kubernetes.Interface = clientset + var logger logr.Logger = testr.New(t) + monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) + monitor.MutateAndPatchAll(context.TODO()) + + err := clientset.CoreV1().Services(defaultNs).Delete(context.TODO(), service.Name, metav1.DeleteOptions{}) + assert.NoError(t, err) + err = waitForInformerUpdate(monitor, func(numKeys int) bool { return numKeys == 0 }) + assert.NoError(t, err) + + updatedWorkload, err := wt.get(clientset, defaultNs, workload.GetName()) + assert.NoError(t, err) + assert.Equal(t, userAnnotations, getPodTemplate(updatedWorkload).GetAnnotations()) + }) + + t.Run("auto restart false, delete and then restart operator", func(t *testing.T) { + userAnnotations := map[string]string{"test": "blah"} + originalAnnotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) + workload := wt.create("workload", defaultNs, nil, originalAnnotations) + + clientset := fake.NewSimpleClientset(workload) + c := fake2.NewFakeClient(workload) + var config MonitorConfig = simpleConfig(true, false, none, none) + var k8sInterface kubernetes.Interface = clientset + var logger logr.Logger = testr.New(t) + monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) + monitor.MutateAndPatchAll(context.TODO()) + + updatedWorkload, err := wt.get(clientset, defaultNs, workload.GetName()) + assert.NoError(t, err) + assert.Equal(t, originalAnnotations, getPodTemplate(updatedWorkload).GetAnnotations()) + }) + + t.Run("auto restart false, delete while operator running", func(t *testing.T) { + userAnnotations := map[string]string{"test": "blah"} + originalAnnotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) + labels := map[string]string{"app": "test"} + service := newTestService("service", defaultNs, labels) + workload := wt.create("workload", defaultNs, labels, originalAnnotations) + + clientset := fake.NewSimpleClientset(service, workload) + c := fake2.NewFakeClient(service, workload) + var config MonitorConfig = simpleConfig(true, false, none, none) + var k8sInterface kubernetes.Interface = clientset + var logger logr.Logger = testr.New(t) + monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) + monitor.MutateAndPatchAll(context.TODO()) + + err := clientset.CoreV1().Services(defaultNs).Delete(context.TODO(), service.Name, metav1.DeleteOptions{}) + assert.NoError(t, err) + err = waitForInformerUpdate(monitor, func(numKeys int) bool { return numKeys == 0 }) + assert.NoError(t, err) + + updatedWorkload, err := wt.get(clientset, defaultNs, workload.GetName()) + assert.NoError(t, err) + assert.Equal(t, originalAnnotations, getPodTemplate(updatedWorkload).GetAnnotations()) + }) + }) + } +} + +func Test_OptOutByDisablingMonitorAllServices(t *testing.T) { + for _, wt := range workloadTypes { + t.Run(wt.name, func(t *testing.T) { + t.Run("auto restart true", func(t *testing.T) { + userAnnotations := map[string]string{"test": "blah"} + annotations := mergeMaps(buildAnnotations(instrumentation.TypeJava), userAnnotations) + labels := map[string]string{"app": "test"} + service := newTestService("service", defaultNs, labels) + workload := wt.create("workload", defaultNs, labels, annotations) + + clientset := fake.NewSimpleClientset(service, workload) + c := fake2.NewFakeClient(service, workload) + var config MonitorConfig = simpleConfig(false, true, none, none) + var k8sInterface kubernetes.Interface = clientset + var logger logr.Logger = testr.New(t) + monitor := NewMonitor(context.TODO(), config, k8sInterface, c, c, logger) + monitor.MutateAndPatchAll(context.TODO()) + + updatedWorkload, err := wt.get(clientset, defaultNs, workload.GetName()) + assert.NoError(t, err) + assert.Equal(t, userAnnotations, getPodTemplate(updatedWorkload).GetAnnotations()) + + }) + }) + } +} + +func Test_mutate(t *testing.T) { + tests := []struct { + name string + podAnnotations map[string]string + languagesToMonitor instrumentation.TypeSet + wantObjAnnotations map[string]string + wantMutated map[string]string + }{ + { + name: "java only", + podAnnotations: nil, + languagesToMonitor: instrumentation.TypeSet{"java": struct{}{}}, + wantObjAnnotations: buildAnnotations("java"), + wantMutated: buildAnnotations("java"), + }, + { + name: "java and python", + podAnnotations: nil, + languagesToMonitor: instrumentation.TypeSet{ + "java": struct{}{}, + "python": struct{}{}, + }, + wantObjAnnotations: mergeMaps(buildAnnotations("java"), buildAnnotations("python")), + wantMutated: mergeMaps(buildAnnotations("java"), buildAnnotations("python")), + }, + { + name: "remove python instrumentation", + podAnnotations: buildAnnotations("python"), + languagesToMonitor: instrumentation.TypeSet{}, + wantObjAnnotations: map[string]string{}, + wantMutated: buildAnnotations("python"), + }, + { + name: "remove one of two languages", + podAnnotations: mergeMaps(buildAnnotations("python"), buildAnnotations("java")), + languagesToMonitor: instrumentation.TypeSet{"java": struct{}{}}, + wantObjAnnotations: buildAnnotations("java"), + wantMutated: buildAnnotations("python"), + }, + { + name: "manually specified annotation is not touched", + podAnnotations: map[string]string{instrumentation.InjectAnnotationKey(instrumentation.TypeJava): defaultAnnotationValue}, + languagesToMonitor: instrumentation.TypeSet{}, + wantObjAnnotations: map[string]string{instrumentation.InjectAnnotationKey(instrumentation.TypeJava): defaultAnnotationValue}, + wantMutated: map[string]string{}, + }, + { + name: "remove all", + podAnnotations: buildAnnotations("java"), + languagesToMonitor: instrumentation.TypeSet{}, + wantObjAnnotations: map[string]string{}, + wantMutated: buildAnnotations("java"), + }, + { + name: "remove only language annotations", + podAnnotations: mergeAnnotations(buildAnnotations("java"), map[string]string{"test": "test"}), + languagesToMonitor: instrumentation.TypeSet{}, + wantObjAnnotations: map[string]string{"test": "test"}, + wantMutated: buildAnnotations("java"), + }, + } + + for _, workload := range workloadTypes { + t.Run(workload.name, func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + obj := workload.create("workload", "default", nil, tt.podAnnotations).DeepCopyObject().(client.Object) + // TODO test different isWorkloadAutoMonitored values + gotMutated := mutate(obj, tt.languagesToMonitor) + assert.Equal(t, tt.wantObjAnnotations, getPodTemplate(obj).GetAnnotations()) + assert.Equal(t, tt.wantMutated, gotMutated) + }) + } + }) + } +} + +func Test_StartupRestartPods(t *testing.T) { + service := newTestService("service-1", defaultNs, map[string]string{"test": "test"}) + matchingDeployment := newTestDeployment("deployment-1", defaultNs, map[string]string{"test": "test"}, nil) + nonMatchingDeployment := newTestDeployment("deployment-2", defaultNs, map[string]string{}, nil) + customSelectedDeployment := newTestDeployment("deployment-3", defaultNs, map[string]string{}, nil) + config := MonitorConfig{ + MonitorAllServices: true, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + RestartPods: true, + CustomSelector: AnnotationConfig{ + AnnotationResources{}, AnnotationResources{Deployments: []string{namespacedName(customSelectedDeployment)}}, + AnnotationResources{}, AnnotationResources{}, + }, + } + objs := []runtime.Object{service, matchingDeployment, nonMatchingDeployment, customSelectedDeployment} + clientset := fake.NewSimpleClientset(objs...) + fakeClient := fake2.NewFakeClient(objs...) + var k8sInterface kubernetes.Interface = clientset + var logger logr.Logger = testr.New(t) + m := NewMonitor(context.TODO(), config, k8sInterface, fakeClient, fakeClient, logger) + m.MutateAndPatchAll(context.TODO()) + updatedMatchingDeployment, err := m.k8sInterface.AppsV1().Deployments(defaultNs).Get(context.TODO(), matchingDeployment.Name, metav1.GetOptions{}) + assert.NoError(t, err) + assert.Equal(t, buildAnnotations(instrumentation.TypeJava), updatedMatchingDeployment.Spec.Template.GetAnnotations()) + updatedNonMatchingDeployment, err := m.k8sInterface.AppsV1().Deployments(defaultNs).Get(context.TODO(), nonMatchingDeployment.Name, metav1.GetOptions{}) + assert.NoError(t, err) + assert.Empty(t, updatedNonMatchingDeployment.Spec.Template.GetAnnotations()) + err = fakeClient.Get(context.TODO(), client.ObjectKeyFromObject(customSelectedDeployment), customSelectedDeployment) + assert.NoError(t, err) + assert.Equal(t, buildAnnotations(instrumentation.TypePython), customSelectedDeployment.Spec.Template.GetAnnotations()) +} + +func Test_listServiceDeployments(t *testing.T) { + testService := newTestService("service-1", defaultNs, map[string]string{"test": "test"}) + testDeployment := newTestDeployment("deployment-1", defaultNs, map[string]string{"test": "test"}, nil) + notMatchingService := newTestService("service-2", defaultNs, map[string]string{"test2": "test2"}) + clientset := fake.NewSimpleClientset(testService, testDeployment, notMatchingService) + m := Monitor{k8sInterface: clientset, logger: testr.New(t)} + matchingServiceDeployments := m.listServiceDeployments(context.TODO(), testService) + assert.Len(t, matchingServiceDeployments, 1) + notMatchingServiceDeployments := m.listServiceDeployments(context.TODO(), notMatchingService) + assert.Len(t, notMatchingServiceDeployments, 0) +} + +// Helper functions + +func createNamespace(t *testing.T, clientset *fake.Clientset, ctx context.Context, namespaceName string) *corev1.Namespace { + namespace := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespaceName}} + serviceNamespace, err := clientset.CoreV1().Namespaces().Create(ctx, &namespace, metav1.CreateOptions{}) + assert.NoError(t, err) + return serviceNamespace +} + +func newTestService(name string, namespace string, selector map[string]string) *corev1.Service { + service := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.ServiceSpec{ + Selector: selector, + }, + } + return service.DeepCopy() +} + +func newTestDeployment(name string, namespace string, labels map[string]string, annotations map[string]string) *appsv1.Deployment { + deployment := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: annotations, + }, + }, + }, + } + return deployment.DeepCopy() +} + +func newTestStatefulSet(name string, namespace string, labels map[string]string, annotations map[string]string) *appsv1.StatefulSet { + statefulSet := appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: annotations, + }, + }, + }, + } + return statefulSet.DeepCopy() +} + +func newTestDaemonSet(name string, namespace string, labels map[string]string, annotations map[string]string) *appsv1.DaemonSet { + daemonSet := appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: annotations, + }, + }, + }, + } + return daemonSet.DeepCopy() +} + +func mergeMaps(maps ...map[string]string) map[string]string { + result := make(map[string]string) + for _, m := range maps { + for k, v := range m { + result[k] = v + } + } + return result +} + +func simpleConfig(monitorAll bool, restartPods bool, customSelector AnnotationConfig, excluded AnnotationConfig) MonitorConfig { + return MonitorConfig{ + MonitorAllServices: monitorAll, + Languages: instrumentation.NewTypeSet(instrumentation.TypeJava), + RestartPods: restartPods, + Exclude: excluded, + CustomSelector: customSelector, + } +} diff --git a/pkg/instrumentation/auto/util.go b/pkg/instrumentation/auto/util.go new file mode 100644 index 000000000..846b9bf4e --- /dev/null +++ b/pkg/instrumentation/auto/util.go @@ -0,0 +1,101 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package auto + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/go-logr/logr" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation" +) + +// configureAutoAnnotation handles the auto annotation configuration logic +func configureAutoAnnotation(autoAnnotationConfigStr string, client client.Client, reader client.Reader, setupLog logr.Logger) (InstrumentationAnnotator, error) { + // Check environment variables first + if os.Getenv("DISABLE_AUTO_ANNOTATION") == "true" { + setupLog.Info("detected DISABLE_AUTO_ANNOTATION environment variable, disabling AutoAnnotation") + return nil, nil + } + + if autoAnnotationConfigStr == "" { + return nil, fmt.Errorf("auto-annotation configuration not provided, disabling AutoAnnotation") + } + + var autoAnnotationConfig AnnotationConfig + if err := json.Unmarshal([]byte(autoAnnotationConfigStr), &autoAnnotationConfig); err != nil { + return nil, fmt.Errorf("unable to unmarshal auto-annotation config, disabling AutoAnnotation: %w", err) + } + + if autoAnnotationConfig.Empty() { + return nil, fmt.Errorf("AutoAnnotation configuration is empty, disabling AutoAnnotation") + } + + setupLog.Info("W! Using deprecated autoAnnotateAutoInstrumentation config, Disabling AutoMonitor. Please upgrade to AutoMonitor. autoAnnotateAutoInstrumentation will be removed in a future release.") + return NewAnnotationMutators( + client, + reader, + setupLog, + autoAnnotationConfig, + instrumentation.SupportedTypes, + ), nil +} + +// configureAutoMonitor handles the auto monitor configuration logic +func configureAutoMonitor(ctx context.Context, autoMonitorConfigStr string, clientSet kubernetes.Interface, client client.Client, reader client.Reader, setupLog logr.Logger) (*Monitor, error) { + // If auto-annotation is not configured or failed, try auto-monitor + if os.Getenv("DISABLE_AUTO_MONITOR") == "true" { + setupLog.Info("W! auto-monitor is disabled due to DISABLE_AUTO_MONITOR environment variable") + return nil, nil + } + + var autoMonitorConfig *MonitorConfig + if err := json.Unmarshal([]byte(autoMonitorConfigStr), &autoMonitorConfig); err != nil { + return nil, fmt.Errorf("unable to unmarshal auto-monitor config: %w", err) + } + + logger := ctrl.Log.WithName("auto_monitor") + return NewMonitor(ctx, *autoMonitorConfig, clientSet, client, reader, logger), nil +} + +// CreateInstrumentationAnnotator creates an instrumentationAnnotator based on config and environment. Returns the InstrumentationAnnotator and whether AutoMonitor is enabled. +func CreateInstrumentationAnnotator(autoMonitorConfigStr string, autoAnnotationConfigStr string, ctx context.Context, client client.Client, reader client.Reader, setupLog logr.Logger) (InstrumentationAnnotator, bool) { + k8sConfig, err := rest.InClusterConfig() + if err != nil { + setupLog.Error(err, "unable to create in-cluster config") + } + + clientSet, err := kubernetes.NewForConfig(k8sConfig) + if err != nil { + setupLog.Error(err, "unable to create clientset") + } + return createInstrumentationAnnotatorWithClientset(autoMonitorConfigStr, autoAnnotationConfigStr, ctx, clientSet, client, reader, setupLog) +} + +// for testing +func createInstrumentationAnnotatorWithClientset(autoMonitorConfigStr string, autoAnnotationConfigStr string, ctx context.Context, clientSet kubernetes.Interface, client client.Client, reader client.Reader, setupLog logr.Logger) (InstrumentationAnnotator, bool) { + autoAnnotation, err := configureAutoAnnotation(autoAnnotationConfigStr, client, reader, setupLog) + if err != nil { + setupLog.Error(err, "Failed to configure auto-annotation, trying AutoMonitor") + } else if autoAnnotation != nil { + return autoAnnotation, false + } + + monitor, err := configureAutoMonitor(ctx, autoMonitorConfigStr, clientSet, client, reader, setupLog) + if err != nil { + setupLog.Error(err, "Failed to configure auto-monitor") + return nil, false + } else if monitor != nil { + return monitor, monitor.config.MonitorAllServices + } + + return nil, false +} diff --git a/pkg/instrumentation/auto/util_test.go b/pkg/instrumentation/auto/util_test.go new file mode 100644 index 000000000..b09e3a04c --- /dev/null +++ b/pkg/instrumentation/auto/util_test.go @@ -0,0 +1,154 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package auto + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/go-logr/logr/testr" + "github.com/stretchr/testify/assert" + "k8s.io/client-go/kubernetes/fake" + + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestCreateInstrumentationAnnotator(t *testing.T) { + // Setup + fakeClient := fakeclient.NewClientBuilder().Build() + ctx := context.Background() + logger := testr.New(t) + + tests := []struct { + name string + envDisableAnnotation bool + envDisableMonitor bool + autoAnnotationConfig string + autoMonitorConfig string + expectNilAnnotator bool + expectedMonitorAll bool + expectedType string + }{ + { + name: "Both annotation and monitor disabled", + envDisableAnnotation: true, + envDisableMonitor: true, + autoAnnotationConfig: `{"java":{"deployments":["default/myapp"]}}`, + autoMonitorConfig: `{"monitorAllServices":true}`, + expectNilAnnotator: true, + expectedMonitorAll: false, + expectedType: "", + }, + { + name: "Annotation enabled, valid config", + envDisableAnnotation: false, + envDisableMonitor: false, + autoAnnotationConfig: `{"java":{"deployments":["default/myapp"]}}`, + autoMonitorConfig: `{"monitorAllServices":true}`, + expectNilAnnotator: false, + expectedMonitorAll: false, + expectedType: "*auto.AnnotationMutators", + }, + { + name: "Annotation disabled, Monitor enabled with monitorAllServices=true", + envDisableAnnotation: true, + envDisableMonitor: false, + autoAnnotationConfig: `{"java":{"deployments":["default/myapp"]}}`, + autoMonitorConfig: `{"monitorAllServices":true}`, + expectNilAnnotator: false, + expectedMonitorAll: true, + expectedType: "*auto.Monitor", + }, + { + name: "Annotation disabled, Monitor enabled with monitorAllServices=false", + envDisableAnnotation: true, + envDisableMonitor: false, + autoAnnotationConfig: `{"java":{"deployments":["default/myapp"]}}`, + autoMonitorConfig: `{"monitorAllServices":false}`, + expectNilAnnotator: false, + expectedMonitorAll: false, + expectedType: "*auto.Monitor", + }, + { + name: "Invalid annotation config, valid monitor config", + envDisableAnnotation: false, + envDisableMonitor: false, + autoAnnotationConfig: `{invalid-json}`, + autoMonitorConfig: `{"monitorAllServices":true}`, + expectNilAnnotator: false, + expectedMonitorAll: true, + expectedType: "*auto.Monitor", + }, + { + name: "Empty annotation config, valid monitor config", + envDisableAnnotation: false, + envDisableMonitor: false, + autoAnnotationConfig: `{}`, + autoMonitorConfig: `{"monitorAllServices":true}`, + expectNilAnnotator: false, + expectedMonitorAll: true, + expectedType: "*auto.Monitor", + }, + { + name: "Valid annotation config, invalid monitor config", + envDisableAnnotation: false, + envDisableMonitor: false, + autoAnnotationConfig: `{"java":{"deployments":["default/myapp"]}}`, + autoMonitorConfig: `{invalid-json}`, + expectNilAnnotator: false, + expectedMonitorAll: false, + expectedType: "*auto.AnnotationMutators", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set environment variables + if tt.envDisableAnnotation { + os.Setenv("DISABLE_AUTO_ANNOTATION", "true") + } else { + os.Unsetenv("DISABLE_AUTO_ANNOTATION") + } + + if tt.envDisableMonitor { + os.Setenv("DISABLE_AUTO_MONITOR", "true") + } else { + os.Unsetenv("DISABLE_AUTO_MONITOR") + } + + // Call the function + annotator, monitorAll := createInstrumentationAnnotatorWithClientset(tt.autoMonitorConfig, tt.autoAnnotationConfig, ctx, fake.NewSimpleClientset(), fakeClient, fakeClient, logger) + + // Check results + if tt.expectNilAnnotator { + assert.Nil(t, annotator, "Expected nil annotator") + } else { + assert.NotNil(t, annotator, "Expected non-nil annotator") + + // Check type using type assertion + actualType := fmt.Sprintf("%T", annotator) + assert.Equal(t, tt.expectedType, actualType, "Unexpected annotator type") + + // Specific type assertions + switch tt.expectedType { + case "*auto.AnnotationMutators": + _, ok := annotator.(*AnnotationMutators) + assert.True(t, ok, "Expected annotator to be of type *AnnotationMutators") + case "*auto.Monitor": + monitor, ok := annotator.(*Monitor) + assert.True(t, ok, "Expected annotator to be of type *Monitor") + if tt.expectedMonitorAll { + assert.True(t, monitor.config.MonitorAllServices, "Expected MonitorAllServices to be true") + } else { + assert.False(t, monitor.config.MonitorAllServices, "Expected MonitorAllServices to be false") + } + } + } + + assert.Equal(t, tt.expectedMonitorAll, monitorAll, "Unexpected monitorAll value") + }) + } +} diff --git a/pkg/instrumentation/podmutator.go b/pkg/instrumentation/podmutator.go index 16ea40e94..994434c6d 100644 --- a/pkg/instrumentation/podmutator.go +++ b/pkg/instrumentation/podmutator.go @@ -32,10 +32,11 @@ var ( ) type instPodMutator struct { - Client client.Client - sdkInjector *sdkInjector - Logger logr.Logger - Recorder record.EventRecorder + Client client.Client + sdkInjector *sdkInjector + Logger logr.Logger + Recorder record.EventRecorder + overrideEnabledMultiInstrumentation bool } type instrumentationWithContainers struct { @@ -192,7 +193,7 @@ func (langInsts *languageInstrumentations) setInstrumentationLanguageContainers( var _ podmutation.PodMutator = (*instPodMutator)(nil) -func NewMutator(logger logr.Logger, client client.Client, recorder record.EventRecorder) *instPodMutator { +func NewMutator(logger logr.Logger, client client.Client, recorder record.EventRecorder, overrideEnabledMultiInstrumentation bool) *instPodMutator { return &instPodMutator{ Logger: logger, Client: client, @@ -200,7 +201,8 @@ func NewMutator(logger logr.Logger, client client.Client, recorder record.EventR logger: logger, client: client, }, - Recorder: recorder, + Recorder: recorder, + overrideEnabledMultiInstrumentation: overrideEnabledMultiInstrumentation, } } @@ -322,7 +324,7 @@ func (pm *instPodMutator) Mutate(ctx context.Context, ns corev1.Namespace, pod c } // We retrieve the annotation for podname - if featuregate.EnableMultiInstrumentationSupport.IsEnabled() { + if featuregate.EnableMultiInstrumentationSupport.IsEnabled() || pm.overrideEnabledMultiInstrumentation { // We use annotations specific for instrumentation language insts.Java.Containers = annotationValue(ns.ObjectMeta, pod.ObjectMeta, annotationInjectJavaContainersName) insts.NodeJS.Containers = annotationValue(ns.ObjectMeta, pod.ObjectMeta, annotationInjectNodeJSContainersName) @@ -335,7 +337,7 @@ func (pm *instPodMutator) Mutate(ctx context.Context, ns corev1.Namespace, pod c // We check if provided annotations and instrumentations are valid ok, msg := insts.areContainerNamesConfiguredForMultipleInstrumentations() - if !ok { + if !ok && !pm.overrideEnabledMultiInstrumentation { logger.V(1).Error(msg, "skipping instrumentation injection") return pod, nil } diff --git a/pkg/instrumentation/podmutator_test.go b/pkg/instrumentation/podmutator_test.go index bb7cccaa8..c7db4b8a7 100644 --- a/pkg/instrumentation/podmutator_test.go +++ b/pkg/instrumentation/podmutator_test.go @@ -105,7 +105,7 @@ func TestGetInstrumentationInstanceJMX(t *testing.T) { } func TestMutatePod(t *testing.T) { - mutator := NewMutator(logr.Discard(), k8sClient, record.NewFakeRecorder(100)) + mutator := NewMutator(logr.Discard(), k8sClient, record.NewFakeRecorder(100), false) require.NotNil(t, mutator) true := true