Skip to content

Commit 25345ca

Browse files
zodilibzodilib
authored
[PFMENG-3035] enable s3 as an output in fluentbit config (#167)
* [PFMENG-3035] enable s3 as an output in fluentbit config * [PFMENG-3035] enable s3 as an output in fluentbit config * [PFMENG-3035] enable s3 as an output in fluentbit config * [PFMENG-3035] enable s3 as an output in fluentbit config * [PFMENG-3035] enable s3 as an output in fluentbit config * [PFMENG-3035] enable s3 as an output in fluentbit config * [PFMENG-3035] enable s3 as an output in fluentbit config * [PFMENG-3035] enable s3 as an output in fluentbit config * [PFMENG-3035] enable s3 as an output in fluentbit config * [PFMENG-3035] enable s3 as an output in fluentbit config
1 parent 42d8b81 commit 25345ca

File tree

6 files changed

+127
-2
lines changed

6 files changed

+127
-2
lines changed

‎modules/essentials/data.tf

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,35 @@ data "aws_iam_policy_document" "fluent_bit" {
3838
]
3939
}
4040
}
41+
42+
data "aws_iam_policy_document" "fluent_bit_cw_and_s3" {
43+
44+
for_each = var.fluent_bit_s3_bucket_enable ? { "enabled" = 1 } : {}
45+
46+
statement {
47+
sid = "PutLogEvents"
48+
effect = "Allow"
49+
resources = ["arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:*:log-stream:*"]
50+
actions = ["logs:PutLogEvents"]
51+
}
52+
53+
statement {
54+
sid = "CreateCWLogs"
55+
effect = "Allow"
56+
resources = ["arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:*"]
57+
58+
actions = [
59+
"logs:CreateLogGroup",
60+
"logs:CreateLogStream",
61+
"logs:DescribeLogGroups",
62+
"logs:DescribeLogStreams",
63+
"logs:PutRetentionPolicy",
64+
]
65+
}
66+
statement {
67+
sid = "S3"
68+
effect = "Allow"
69+
resources = ["${module.fluentbit_s3_bucket[0].s3_bucket_arn}/*"]
70+
actions = ["s3:PutObject"]
71+
}
72+
}

‎modules/essentials/fluent_bit.tf

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ locals {
3737
resources = jsonencode(var.fluent_bit_resources),
3838
tolerations = jsonencode(var.fluent_bit_tolerations),
3939
affinity = jsonencode(local.affinity),
40-
excluded_namespaces = var.fluent_bit_excluded_namespaces
40+
excluded_namespaces = var.fluent_bit_excluded_namespaces,
41+
s3_bucket_name = var.fluent_bit_s3_bucket_enable ? module.fluentbit_s3_bucket[0].s3_bucket_id : null,
4142
})
4243

4344
fluent_bit_helm_config = merge(
@@ -92,7 +93,7 @@ resource "aws_iam_policy" "fluent_bit_irsa" {
9293

9394
name = "${var.cluster_name}-fluentbit"
9495
description = "IAM Policy for AWS for FluentBit IRSA"
95-
policy = data.aws_iam_policy_document.fluent_bit.json
96+
policy = var.fluent_bit_s3_bucket_enable ? data.aws_iam_policy_document.fluent_bit_cw_and_s3["enabled"].json : data.aws_iam_policy_document.fluent_bit.json
9697
}
9798

9899
moved {

‎modules/essentials/fluent_bit_s3.tf

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
module "fluentbit_s3_bucket" {
2+
count = var.fluent_bit_s3_bucket_enable ? 1 : 0
3+
4+
source = "terraform-aws-modules/s3-bucket/aws"
5+
version = "~> 4.6.1"
6+
7+
bucket = "fluentbit-log-bucket-${random_string.s3_suffix.result}"
8+
9+
versioning = {
10+
enabled = false
11+
}
12+
13+
lifecycle_rule = [
14+
{
15+
id = "log-expiration"
16+
enabled = true
17+
abort_incomplete_multipart_upload_days = 7
18+
19+
expiration = {
20+
days = 90
21+
}
22+
23+
transitions = [
24+
{
25+
days = 30
26+
storage_class = "STANDARD_IA"
27+
},
28+
{
29+
days = 60
30+
storage_class = "ONEZONE_IA"
31+
}
32+
]
33+
}
34+
]
35+
}
36+
37+
resource "random_string" "s3_suffix" {
38+
length = 8
39+
special = false
40+
upper = false
41+
}

‎modules/essentials/templates/fluent_bit.yaml

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,18 @@ config:
7575
Mem_Buf_Limit 5MB
7676
Skip_Long_Lines On
7777
78+
%{if s3_bucket_name != null}
79+
[INPUT]
80+
Name tail
81+
Tag kube_s3.*
82+
Path /var/log/containers/*.log
83+
DB /var/log/flb_kube_s3.db
84+
multiline.parser docker, cri
85+
Docker_Mode On
86+
Mem_Buf_Limit 5MB
87+
Skip_Long_Lines On
88+
%{ endif }
89+
7890
## https://docs.fluentbit.io/manual/pipeline/filters
7991
filters: |
8092
[FILTER]
@@ -103,6 +115,24 @@ config:
103115
%{ endfor ~}
104116
%{ endif }
105117

118+
%{if s3_bucket_name != null}
119+
[FILTER]
120+
Name kubernetes
121+
Match kube_s3.*
122+
Kube_URL https://kubernetes.default.svc.cluster.local:443
123+
Merge_Log On
124+
Keep_Log Off
125+
K8S-Logging.Parser On
126+
Buffer_Size 31k
127+
128+
[FILTER]
129+
Name parser
130+
Match kube_s3.*
131+
Key_Name log
132+
Parser custom_apache
133+
Reserve_Data true
134+
%{ endif }
135+
106136
## https://docs.fluentbit.io/manual/pipeline/outputs
107137
outputs: |
108138
[OUTPUT]
@@ -114,6 +144,17 @@ config:
114144
log_stream_prefix fluentbit-
115145
auto_create_group false
116146
147+
%{if s3_bucket_name != null}
148+
[OUTPUT]
149+
Name s3
150+
Match kube_s3.*
151+
region ap-southeast-1
152+
bucket ${s3_bucket_name}
153+
compression gzip
154+
storage_class REDUCED_REDUNDANCY
155+
retry_limit 2
156+
%{ endif }
157+
117158
# extraFiles: {}
118159
# upstream.conf: |
119160
# [UPSTREAM]

‎modules/essentials/variables.tf

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1564,3 +1564,9 @@ variable "fluent_bit_excluded_namespaces" {
15641564
type = list(string)
15651565
default = []
15661566
}
1567+
1568+
variable "fluent_bit_s3_bucket_enable" {
1569+
description = "S3 bucket name to store fluentbit logs"
1570+
type = bool
1571+
default = false
1572+
}

‎modules/essentials/versions.tf

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,9 @@ terraform {
1414
source = "hashicorp/kubernetes"
1515
version = ">= 2.33"
1616
}
17+
random = {
18+
source = "hashicorp/random"
19+
version = ">= 3.5"
20+
}
1721
}
1822
}

0 commit comments

Comments
 (0)