Skip to content

Commit 284a54e

Browse files
bbgu1bbgu1
authored
mwaa v5 migration (#248)
1 parent 33119d2 commit 284a54e

File tree

14 files changed

+534
-424
lines changed

14 files changed

+534
-424
lines changed

‎schedulers/terraform/managed-airflow-mwaa/README.md

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Amazon Managed Workflows for Apache Airflow (MWAA)
2-
Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/job-schedulers/aws-managed-airflow) to deploy this pattern and run sample tests.
2+
Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/blueprints/job-schedulers/aws-managed-airflow) to deploy this pattern and run sample tests.
33

44
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
55
## Requirements
@@ -24,8 +24,10 @@ Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/
2424

2525
| Name | Source | Version |
2626
|------|--------|---------|
27-
| <a name="module_eks_blueprints"></a> [eks\_blueprints](#module\_eks\_blueprints) | github.com/aws-ia/terraform-aws-eks-blueprints | v4.32.1 |
28-
| <a name="module_eks_blueprints_addons"></a> [eks\_blueprints\_addons](#module\_eks\_blueprints\_addons) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons | v4.32.1 |
27+
| <a name="module_ebs_csi_driver_irsa"></a> [ebs\_csi\_driver\_irsa](#module\_ebs\_csi\_driver\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.14 |
28+
| <a name="module_eks"></a> [eks](#module\_eks) | terraform-aws-modules/eks/aws | ~> 19.15 |
29+
| <a name="module_eks_blueprints_addons"></a> [eks\_blueprints\_addons](#module\_eks\_blueprints\_addons) | aws-ia/eks-blueprints-addons/aws | ~> 1.2 |
30+
| <a name="module_emr_containers"></a> [emr\_containers](#module\_emr\_containers) | ../../../workshop/modules/emr-eks-containers | n/a |
2931
| <a name="module_mwaa"></a> [mwaa](#module\_mwaa) | aws-ia/mwaa/aws | 0.0.4 |
3032
| <a name="module_s3_bucket"></a> [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 3.0 |
3133
| <a name="module_vpc"></a> [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 |

‎schedulers/terraform/managed-airflow-mwaa/addons.tf

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#------------------------------------------------------------------------
2+
# EKS Addons
3+
#------------------------------------------------------------------------
4+
module "eks_blueprints_addons" {
5+
# Users should pin the version to the latest available release
6+
# tflint-ignore: terraform_module_pinned_source
7+
source = "aws-ia/eks-blueprints-addons/aws"
8+
version = "~> 1.2"
9+
10+
cluster_name = module.eks.cluster_name
11+
cluster_endpoint = module.eks.cluster_endpoint
12+
cluster_version = module.eks.cluster_version
13+
oidc_provider_arn = module.eks.oidc_provider_arn
14+
15+
#---------------------------------------
16+
# Amazon EKS Managed Add-ons
17+
#---------------------------------------
18+
eks_addons = {
19+
aws-ebs-csi-driver = {
20+
service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn
21+
}
22+
coredns = {
23+
preserve = true
24+
}
25+
vpc-cni = {
26+
preserve = true
27+
}
28+
kube-proxy = {
29+
preserve = true
30+
}
31+
}
32+
33+
enable_metrics_server = true
34+
enable_cluster_autoscaler = true
35+
36+
tags = local.tags
37+
}
38+
39+
#---------------------------------------------------------------
40+
# IRSA for EBS CSI Driver
41+
#---------------------------------------------------------------
42+
module "ebs_csi_driver_irsa" {
43+
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
44+
version = "~> 5.20"
45+
role_name = format("%s-%s", local.name, "ebs-csi-driver")
46+
attach_ebs_csi_policy = true
47+
oidc_providers = {
48+
main = {
49+
provider_arn = module.eks.oidc_provider_arn
50+
namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"]
51+
}
52+
}
53+
tags = local.tags
54+
}

‎schedulers/terraform/managed-airflow-mwaa/cleanup.sh

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/bin/bash
2+
set -o errexit
3+
set -o pipefail
4+
5+
targets=(
6+
"module.emr_containers"
7+
"module.eks_blueprints_addons"
8+
"module.eks"
9+
"module.mwaa"
10+
)
11+
12+
#-------------------------------------------
13+
# Helpful to delete the stuck in "Terminating" namespaces
14+
# Rerun the cleanup.sh script to detect and delete the stuck resources
15+
#-------------------------------------------
16+
terminating_namespaces=$(kubectl get namespaces --field-selector status.phase=Terminating -o json | jq -r '.items[].metadata.name')
17+
18+
# If there are no terminating namespaces, exit the script
19+
if [[ -z $terminating_namespaces ]]; then
20+
echo "No terminating namespaces found"
21+
fi
22+
23+
for ns in $terminating_namespaces; do
24+
echo "Terminating namespace: $ns"
25+
kubectl get namespace $ns -o json | sed 's/"kubernetes"//' | kubectl replace --raw "/api/v1/namespaces/$ns/finalize" -f -
26+
done
27+
28+
#-------------------------------------------
29+
# Terraform destroy per module target
30+
#-------------------------------------------
31+
for target in "${targets[@]}"
32+
do
33+
terraform destroy -auto-approve
34+
destroy_output=$(terraform destroy -auto-approve 2>&1)
35+
if [[ $? -eq 0 && $destroy_output == *"Destroy complete!"* ]]; then
36+
echo "SUCCESS: Terraform destroy of $target completed successfully"
37+
else
38+
echo "FAILED: Terraform destroy of $target failed"
39+
exit 1
40+
fi
41+
done
42+
43+
#-------------------------------------------
44+
# Terraform destroy full
45+
#-------------------------------------------
46+
terraform destroy -auto-approve
47+
destroy_output=$(terraform destroy -auto-approve 2>&1)
48+
if [[ $? -eq 0 && $destroy_output == *"Destroy complete!"* ]]; then
49+
echo "SUCCESS: Terraform destroy of all targets completed successfully"
50+
else
51+
echo "FAILED: Terraform destroy of all targets failed"
52+
exit 1
53+
fi

‎schedulers/terraform/managed-airflow-mwaa/data.tf

Lines changed: 1 addition & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
data "aws_eks_cluster_auth" "this" {
2-
name = module.eks_blueprints.eks_cluster_id
2+
name = module.eks.cluster_name
33
}
44

55
data "aws_availability_zones" "available" {}
@@ -10,36 +10,6 @@ data "aws_region" "current" {}
1010

1111
data "aws_partition" "current" {}
1212

13-
data "aws_iam_policy_document" "emr_on_eks" {
14-
statement {
15-
sid = ""
16-
effect = "Allow"
17-
resources = ["arn:${data.aws_partition.current.partition}:s3:::*"]
18-
19-
actions = [
20-
"s3:DeleteObject",
21-
"s3:DeleteObjectVersion",
22-
"s3:GetObject",
23-
"s3:ListBucket",
24-
"s3:PutObject",
25-
]
26-
}
27-
28-
statement {
29-
sid = ""
30-
effect = "Allow"
31-
resources = ["arn:${data.aws_partition.current.partition}:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:*"]
32-
33-
actions = [
34-
"logs:CreateLogGroup",
35-
"logs:CreateLogStream",
36-
"logs:DescribeLogGroups",
37-
"logs:DescribeLogStreams",
38-
"logs:PutLogEvents",
39-
]
40-
}
41-
}
42-
4313
data "aws_iam_policy_document" "mwaa_emrjob" {
4414
statement {
4515
actions = [

0 commit comments

Comments
 (0)