[PFMENG-3039] add cw disable option in fluent-bit config (#168)

* [PFMENG-3039] add cw disable option in fluent-bit config

* [PFMENG-3039] add cw disable option in fluent-bit config and improve the variable names

* [PFMENG-3039] add cw disable option in fluent-bit config and improve the variable names

Author: zodilib

modules/essentials/data.tf

@@ -41,7 +41,7 @@ data "aws_iam_policy_document" "fluent_bit" {
 
 data "aws_iam_policy_document" "fluent_bit_cw_and_s3" {
 
-  for_each = var.fluent_bit_s3_bucket_enable ? { "enabled" = 1 } : {}
+  for_each = var.fluent_bit_enable_s3_output ? { "enabled" = 1 } : {}
 
   statement {
     sid       = "PutLogEvents"

modules/essentials/fluent_bit.tf

@@ -38,7 +38,8 @@ locals {
     tolerations          = jsonencode(var.fluent_bit_tolerations),
     affinity             = jsonencode(local.affinity),
     excluded_namespaces  = var.fluent_bit_excluded_namespaces,
-    s3_bucket_name       = var.fluent_bit_s3_bucket_enable ? module.fluentbit_s3_bucket[0].s3_bucket_id : null,
+    s3_bucket_name       = var.fluent_bit_enable_s3_output ? module.fluentbit_s3_bucket[0].s3_bucket_id : null,
+    cw_enable            = var.fluent_bit_enable_cw_output
   })
 
   fluent_bit_helm_config = merge(
@@ -93,7 +94,7 @@ resource "aws_iam_policy" "fluent_bit_irsa" {
 
   name        = "${var.cluster_name}-fluentbit"
   description = "IAM Policy for AWS for FluentBit IRSA"
-  policy      = var.fluent_bit_s3_bucket_enable ? data.aws_iam_policy_document.fluent_bit_cw_and_s3["enabled"].json : data.aws_iam_policy_document.fluent_bit.json
+  policy      = var.fluent_bit_enable_s3_output ? data.aws_iam_policy_document.fluent_bit_cw_and_s3["enabled"].json : data.aws_iam_policy_document.fluent_bit.json
 }
 
 moved {

modules/essentials/fluent_bit_s3.tf

@@ -1,5 +1,5 @@
 module "fluentbit_s3_bucket" {
-  count = var.fluent_bit_s3_bucket_enable ? 1 : 0
+  count = var.fluent_bit_enable_s3_output ? 1 : 0
 
   source  = "terraform-aws-modules/s3-bucket/aws"
   version = "~> 4.6.1"

modules/essentials/templates/fluent_bit.yaml

@@ -135,6 +135,7 @@ config:
 
   ## https://docs.fluentbit.io/manual/pipeline/outputs
   outputs: |
+%{if cw_enable == true}
     [OUTPUT]
         Name cloudwatch_logs
         Match kube.*
@@ -143,6 +144,7 @@ config:
         log_stream_template $kubernetes['namespace_name'].$kubernetes['pod_name'].$kubernetes['container_name']
         log_stream_prefix fluentbit-
         auto_create_group false
+%{ endif }
 
 %{if s3_bucket_name != null}
     [OUTPUT]

modules/essentials/variables.tf

@@ -1565,8 +1565,14 @@ variable "fluent_bit_excluded_namespaces" {
   default     = []
 }
 
-variable "fluent_bit_s3_bucket_enable" {
-  description = "S3 bucket name to store fluentbit logs"
+variable "fluent_bit_enable_s3_output" {
+  description = "Enable S3 output logging"
   type        = bool
   default     = false
 }
+
+variable "fluent_bit_enable_cw_output" {
+  description = "Enable cloudwatch logging"
+  type        = bool
+  default     = true
+}