Skip to content

Commit ccf55bf

Browse files
garutilorenzogarutilorenzo
committed
Install nginx ingress controller instead of Traefik
1 parent 73d4f90 commit ccf55bf

File tree

6 files changed

+226
-23
lines changed

6 files changed

+226
-23
lines changed

‎README.md

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ edit the main.tf files and set the following variables:
4343
| `default_instance_profile_name` | `no` | Instance profile name. Default: AWSEC2K3SInstanceProfile |
4444
| `default_iam_role` | `no` | IAM role name. Default: AWSEC2K3SRole |
4545
| `create_extlb` | `no` | Boolean value true/false, specify true for deploy an external LB pointing to k3s worker nodes. Default: false |
46+
| `install_nginx_ingress` | `no` | Boolean value, install kubernetes nginx ingress controller instead of Traefik. Default: true. For more information see [Nginx ingress controller](#nginx-ingress-controller) |
47+
| `install_certmanager` | `no` | Boolean value, install [cert manager](https://cert-manager.io/) "Cloud native certificate management". Default: true |
48+
| `certmanager_release` | `no` | Cert manager release. Default: v1.8.2 |
49+
| `certmanager_email_address` | `no` | Email address used for signing https certificates. Defaul: changeme@example.com |
4650
| `extlb_http_port` | `no` | http port used by the external LB. Default: 80 |
4751
| `extlb_https_port` | `no` | https port used by the external LB. Default: 443 |
4852
| `PATH_TO_PUBLIC_KEY` | `no` | Path to your public ssh key (Default: "~/.ssh/id_rsa.pub) |
@@ -135,6 +139,62 @@ Notes about the auoscaling group:
135139

136140
You can change this setting by editing the value of on_demand_percentage_above_base_capacity in asg.tf. You can require that all the EC2 will be launced using on-demand instances setting on_demand_percentage_above_base_capacity to 100. More details [here](https://docs.aws.amazon.com/autoscaling/ec2/APIReference/API_InstancesDistribution.html)
137141

142+
### Nginx ingress controller
143+
144+
In this environment [Nginx ingress controller](https://kubernetes.github.io/ingress-nginx/) is used instead of the standard [Traefik](https://traefik.io/) ingress controller.
145+
146+
The installation is the [bare metal](https://kubernetes.github.io/ingress-nginx/deploy/#bare-metal-clusters) installation, the ingress controller then is exposed via K3s LoadBalancer Service.
147+
148+
```yaml
149+
---
150+
apiVersion: v1
151+
kind: Service
152+
metadata:
153+
name: ingress-nginx-controller-loadbalancer
154+
namespace: ingress-nginx
155+
spec:
156+
selector:
157+
app.kubernetes.io/component: controller
158+
app.kubernetes.io/instance: ingress-nginx
159+
app.kubernetes.io/name: ingress-nginx
160+
ports:
161+
- name: http
162+
port: 80
163+
protocol: TCP
164+
targetPort: 80
165+
- name: https
166+
port: 443
167+
protocol: TCP
168+
targetPort: 443
169+
type: LoadBalancer
170+
```
171+
172+
To get the real ip address of the clients using a public L4 load balancer we need to use the proxy protocol feature of nginx ingress controller:
173+
174+
```yaml
175+
---
176+
apiVersion: v1
177+
data:
178+
allow-snippet-annotations: "true"
179+
enable-real-ip: "true"
180+
proxy-real-ip-cidr: "0.0.0.0/0"
181+
proxy-body-size: "20m"
182+
use-proxy-protocol: "true"
183+
kind: ConfigMap
184+
metadata:
185+
labels:
186+
app.kubernetes.io/component: controller
187+
app.kubernetes.io/instance: ingress-nginx
188+
app.kubernetes.io/managed-by: Helm
189+
app.kubernetes.io/name: ingress-nginx
190+
app.kubernetes.io/part-of: ingress-nginx
191+
app.kubernetes.io/version: 1.1.1
192+
helm.sh/chart: ingress-nginx-4.0.16
193+
name: ingress-nginx-controller
194+
namespace: ingress-nginx
195+
```
196+
197+
and enable the proxy protocol on the load balancer target group, *proxy_protocol_v2* set to true.
138198
139199
## Instances used
140200

‎k3s_cluster/data.tf

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,16 @@ data "template_cloudinit_config" "k3s_server" {
4545

4646
part {
4747
content_type = "text/x-shellscript"
48-
content = templatefile("${path.module}/files/k3s-install-server.sh", { k3s_token = var.k3s_token, is_k3s_server = true, k3s_url = aws_lb.k3s-server-lb.dns_name, k3s_tls_san = aws_lb.k3s-server-lb.dns_name })
48+
content = templatefile("${path.module}/files/k3s-install-server.sh", {
49+
k3s_token = var.k3s_token,
50+
is_k3s_server = true,
51+
install_nginx_ingress = var.install_nginx_ingress,
52+
install_certmanager = var.install_certmanager,
53+
certmanager_release = var.certmanager_release,
54+
certmanager_email_address = var.certmanager_email_address,
55+
k3s_url = aws_lb.k3s-server-lb.dns_name,
56+
k3s_tls_san = aws_lb.k3s-server-lb.dns_name
57+
})
4958
}
5059
}
5160

@@ -62,6 +71,11 @@ data "template_cloudinit_config" "k3s_agent" {
6271

6372
part {
6473
content_type = "text/x-shellscript"
65-
content = templatefile("${path.module}/files/k3s-install-agent.sh", { k3s_token = var.k3s_token, is_k3s_server = false, k3s_url = aws_lb.k3s-server-lb.dns_name, k3s_tls_san = aws_lb.k3s-server-lb.dns_name })
74+
content = templatefile("${path.module}/files/k3s-install-agent.sh", {
75+
k3s_token = var.k3s_token,
76+
is_k3s_server = false,
77+
k3s_url = aws_lb.k3s-server-lb.dns_name,
78+
k3s_tls_san = aws_lb.k3s-server-lb.dns_name
79+
})
6680
}
6781
}

‎k3s_cluster/extlb.tf

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,11 @@ resource "aws_lb_listener" "external-lb-listener-http" {
3434
}
3535

3636
resource "aws_lb_target_group" "external-lb-tg-http" {
37-
count = var.create_extlb ? 1 : 0
38-
port = var.extlb_http_port
39-
protocol = "TCP"
40-
vpc_id = var.vpc_id
41-
37+
count = var.create_extlb ? 1 : 0
38+
port = var.extlb_http_port
39+
protocol = "TCP"
40+
vpc_id = var.vpc_id
41+
proxy_protocol_v2 = true
4242

4343
depends_on = [
4444
aws_lb.external-lb
@@ -90,11 +90,11 @@ resource "aws_lb_listener" "external-lb-listener-https" {
9090
}
9191

9292
resource "aws_lb_target_group" "external-lb-tg-https" {
93-
count = var.create_extlb ? 1 : 0
94-
port = var.extlb_https_port
95-
protocol = "TCP"
96-
vpc_id = var.vpc_id
97-
93+
count = var.create_extlb ? 1 : 0
94+
port = var.extlb_https_port
95+
protocol = "TCP"
96+
vpc_id = var.vpc_id
97+
proxy_protocol_v2 = true
9898

9999
depends_on = [
100100
aws_lb.external-lb

‎k3s_cluster/files/k3s-install-server.sh

Lines changed: 109 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,20 @@ first_instance=$(aws ec2 describe-instances --filters Name=tag-value,Values=k3s-
1616
instance_id=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
1717
first_last="last"
1818

19+
%{ if install_nginx_ingress }
20+
disable_traefik="--disable traefik"
21+
%{ endif }
22+
1923
if [[ "$first_instance" == "$instance_id" ]]; then
2024
echo "I'm the first yeeee: Cluster init!"
2125
first_last="first"
22-
until (curl -sfL https://get.k3s.io | K3S_TOKEN=${k3s_token} sh -s - --cluster-init --node-ip $local_ip --advertise-address $local_ip --flannel-iface $flannel_iface --tls-san ${k3s_tls_san} --kubelet-arg="provider-id=aws:///$provider_id"); do
26+
until (curl -sfL https://get.k3s.io | K3S_TOKEN=${k3s_token} sh -s - --cluster-init $disable_traefik --node-ip $local_ip --advertise-address $local_ip --flannel-iface $flannel_iface --tls-san ${k3s_tls_san} --kubelet-arg="provider-id=aws:///$provider_id"); do
2327
echo 'k3s did not install correctly'
2428
sleep 2
2529
done
2630
else
2731
echo "I'm like England :( Cluster join"
28-
until (curl -sfL https://get.k3s.io | K3S_TOKEN=${k3s_token} sh -s - --server https://${k3s_url}:6443 --node-ip $local_ip --advertise-address $local_ip --flannel-iface $flannel_iface --tls-san ${k3s_tls_san} --kubelet-arg="provider-id=aws:///$provider_id" ); do
32+
until (curl -sfL https://get.k3s.io | K3S_TOKEN=${k3s_token} sh -s - --server https://${k3s_url}:6443 $disable_traefik --node-ip $local_ip --advertise-address $local_ip --flannel-iface $flannel_iface --tls-san ${k3s_tls_san} --kubelet-arg="provider-id=aws:///$provider_id" ); do
2933
echo 'k3s did not install correctly'
3034
sleep 2
3135
done
@@ -42,6 +46,109 @@ if [[ "$first_last" == "first" ]]; then
4246
echo 'Install node termination handler'
4347
kubectl apply -f https://github.com/aws/aws-node-termination-handler/releases/download/v1.13.3/all-resources.yaml
4448
fi
49+
50+
%{ if install_nginx_ingress }
51+
if [[ "$first_last" == "first" ]]; then
52+
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/baremetal/deploy.yaml
53+
cat << 'EOF' > /root/nginx-ingress-resources.yaml
54+
---
55+
apiVersion: v1
56+
kind: Service
57+
metadata:
58+
name: ingress-nginx-controller-loadbalancer
59+
namespace: ingress-nginx
60+
spec:
61+
selector:
62+
app.kubernetes.io/component: controller
63+
app.kubernetes.io/instance: ingress-nginx
64+
app.kubernetes.io/name: ingress-nginx
65+
ports:
66+
- name: http
67+
port: 80
68+
protocol: TCP
69+
targetPort: 80
70+
- name: https
71+
port: 443
72+
protocol: TCP
73+
targetPort: 443
74+
type: LoadBalancer
75+
---
76+
apiVersion: v1
77+
data:
78+
allow-snippet-annotations: "true"
79+
enable-real-ip: "true"
80+
proxy-real-ip-cidr: "0.0.0.0/0"
81+
proxy-body-size: "20m"
82+
use-proxy-protocol: "true"
83+
kind: ConfigMap
84+
metadata:
85+
labels:
86+
app.kubernetes.io/component: controller
87+
app.kubernetes.io/instance: ingress-nginx
88+
app.kubernetes.io/managed-by: Helm
89+
app.kubernetes.io/name: ingress-nginx
90+
app.kubernetes.io/part-of: ingress-nginx
91+
app.kubernetes.io/version: 1.1.1
92+
helm.sh/chart: ingress-nginx-4.0.16
93+
name: ingress-nginx-controller
94+
namespace: ingress-nginx
95+
EOF
96+
kubectl apply -f /root/nginx-ingress-resources.yaml
97+
fi
98+
%{ endif }
99+
100+
%{ if install_certmanager }
101+
if [[ "$first_last" == "first" ]]; then
102+
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/${certmanager_release}/cert-manager.yaml
103+
104+
cat << 'EOF' > /root/staging_issuer.yaml
105+
apiVersion: cert-manager.io/v1
106+
kind: ClusterIssuer
107+
metadata:
108+
name: letsencrypt-staging
109+
namespace: cert-manager
110+
spec:
111+
acme:
112+
# The ACME server URL
113+
server: https://acme-staging-v02.api.letsencrypt.org/directory
114+
# Email address used for ACME registration
115+
email: ${certmanager_email_address}
116+
# Name of a secret used to store the ACME account private key
117+
privateKeySecretRef:
118+
name: letsencrypt-staging
119+
# Enable the HTTP-01 challenge provider
120+
solvers:
121+
- http01:
122+
ingress:
123+
class: nginx
124+
EOF
125+
126+
cat << 'EOF' > /root/prod_issuer.yaml
127+
apiVersion: cert-manager.io/v1
128+
kind: ClusterIssuer
129+
metadata:
130+
name: letsencrypt-prod
131+
namespace: cert-manager
132+
spec:
133+
acme:
134+
# The ACME server URL
135+
server: https://acme-v02.api.letsencrypt.org/directory
136+
# Email address used for ACME registration
137+
email: ${certmanager_email_address}
138+
# Name of a secret used to store the ACME account private key
139+
privateKeySecretRef:
140+
name: letsencrypt-prod
141+
# Enable the HTTP-01 challenge provider
142+
solvers:
143+
- http01:
144+
ingress:
145+
class: nginx
146+
EOF
147+
kubectl create -f prod_issuer.yaml
148+
kubectl create -f staging_issuer.yaml
149+
fi
150+
%{ endif }
151+
45152
%{ endif }
46153

47154
#kubectl get pods -n kube-system | grep aws-node | wc -l

‎k3s_cluster/vars.tf

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,26 @@ variable "my_public_ip_cidr" {
3737
description = "My public ip CIDR"
3838
}
3939

40+
variable "install_nginx_ingress" {
41+
type = bool
42+
default = true
43+
}
44+
45+
variable "install_certmanager" {
46+
type = bool
47+
default = true
48+
}
49+
50+
variable "certmanager_release" {
51+
type = string
52+
default = "v1.8.2"
53+
}
54+
55+
variable "certmanager_email_address" {
56+
type = string
57+
default = "changeme@example.com"
58+
}
59+
4060
variable "vpc_subnet_cidr" {
4161
type = string
4262
description = "VPC subnet CIDR"

‎main.tf

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,17 @@ variable "AWS_REGION" {
1313
}
1414

1515
module "k3s_cluster" {
16-
AWS_REGION = "<change_me>"
17-
environment = "staging"
18-
my_public_ip_cidr = "<change_me>"
19-
vpc_id = "<change_me>"
20-
vpc_subnets = ["<change_me>", "<change_me>", "<change_me>"]
21-
vpc_subnet_cidr = "<change_me>"
22-
cluster_name = "k3s-cluster"
23-
k3s_token = "<change_me>"
24-
source = "./k3s_cluster/"
16+
AWS_REGION = "<change_me>"
17+
environment = "staging"
18+
my_public_ip_cidr = "<change_me>"
19+
vpc_id = "<change_me>"
20+
vpc_subnets = ["<change_me>", "<change_me>", "<change_me>"]
21+
vpc_subnet_cidr = "<change_me>"
22+
cluster_name = "k3s-cluster"
23+
k3s_token = "<change_me>"
24+
create_extlb = true
25+
certmanager_email_address = "<change_me>"
26+
source = "./k3s_cluster/"
2527
}
2628

2729
output "elb_dns_name" {

0 commit comments

Comments
 (0)