Skip to content

Commit 5f3e3d1

Browse files
hyderhyder
authored
Support for VCN Native Pod Networking (#542)
* feat: added support for Native Pod Networking Signed-off-by: Ali Mukadam <ali.mukadam@oracle.com> * fix: incorrect value for flannel Signed-off-by: Ali Mukadam <ali.mukadam@oracle.com> * feat: added support for VCN Native pod networking. Closes #539 Signed-off-by: Ali Mukadam <ali.mukadam@oracle.com> * fix: removed commented code as requested by @Djelibeybi Signed-off-by: Ali Mukadam <ali.mukadam@oracle.com>
1 parent 14ac652 commit 5f3e3d1

File tree

12 files changed

+269
-21
lines changed

12 files changed

+269
-21
lines changed

‎docs/terraformoptions.adoc

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,8 @@ EOT
223223
int_lb = { netnum = 16, newbits = 11 }
224224
pub_lb = { netnum = 17, newbits = 11 }
225225
workers = { netnum = 1, newbits = 2 }
226+
pods = { netnum = 2, newbits = 2 }
227+
fss = { netnum = 18, newbits = 11 }
226228
}
227229
|
228230
subnets = {
@@ -232,6 +234,8 @@ EOT
232234
int_lb = { netnum = 16, newbits = 11 }
233235
pub_lb = { netnum = 17, newbits = 11 }
234236
workers = { netnum = 1, newbits = 2 }
237+
pods = { netnum = 2, newbits = 2 }
238+
fss = { netnum = 18, newbits = 11 }
235239
}
236240

237241

@@ -251,6 +255,7 @@ EOT
251255
|oke-vcn
252256

253257
|===
258+
254259
== Reuse VCN
255260

256261
[stripes=odd,cols="1m,4d,2m,2m", options=header,width="100%"]
@@ -596,6 +601,11 @@ EOT
596601
|true/false
597602
|false
598603

604+
|allow_pod_internet_access
605+
|Allow pods to egress to internet. Required if the pods are invoking Internet services.
606+
|true/false
607+
|true
608+
599609
|allow_worker_internet_access
600610
|Whether to allow access to NodePorts when worker nodes are deployed in public mode..
601611
|true/false
@@ -611,6 +621,11 @@ EOT
611621
|
612622
|oke
613623

624+
|cni_type
625+
|The CNI for the cluster. Choose between flannel or npn (Native Pod Networking).
626+
|flannel/npn
627+
|flannel
628+
614629
|control_plane_type
615630
|Whether to allow public or private access to the control plane endpoint.
616631
|public/private
@@ -710,6 +725,11 @@ EOT
710725
|one,all,none
711726
|none
712727

728+
|max_pods_per_node
729+
|The maximum number of pods to deploy per node. Absolute maximum is 110. Applies only when CNI type is `npn`.
730+
|
731+
|31
732+
713733
|node_pools
714734
|The number, shape of node pools and node_pool_size to create. Each key and tuple pair corresponds to 1 node pool.
715735

‎main.tf

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,12 +192,14 @@ module "network" {
192192

193193

194194
# control plane endpoint parameters
195+
cni_type = var.cni_type
195196
control_plane_type = var.control_plane_type
196197
control_plane_allowed_cidrs = var.control_plane_allowed_cidrs
197198

198199
# oke worker network parameters
199200
allow_node_port_access = var.allow_node_port_access
200201
allow_worker_internet_access = var.allow_worker_internet_access
202+
allow_pod_internet_access = var.allow_pod_internet_access
201203
allow_worker_ssh_access = var.allow_worker_ssh_access
202204
worker_type = var.worker_type
203205

@@ -247,6 +249,7 @@ module "oke" {
247249
cluster_options_kubernetes_network_config_pods_cidr = var.pods_cidr
248250
cluster_options_kubernetes_network_config_services_cidr = var.services_cidr
249251
cluster_subnets = module.network.subnet_ids
252+
cni_type = var.cni_type
250253
vcn_id = local.vcn_id
251254
use_cluster_encryption = var.use_cluster_encryption
252255
cluster_kms_key_id = var.cluster_kms_key_id
@@ -256,6 +259,7 @@ module "oke" {
256259
admission_controller_options = var.admission_controller_options
257260

258261
# oke node pool parameters
262+
max_pods_per_node = var.max_pods_per_node
259263
node_pools = var.node_pools
260264
node_pool_name_prefix = var.node_pool_name_prefix
261265
node_pool_image_id = var.node_pool_image_id
@@ -272,7 +276,8 @@ module "oke" {
272276
# oke load balancer parameters
273277
preferred_load_balancer = var.preferred_load_balancer
274278

275-
# worker nsgs
279+
# nsgs
280+
pod_nsgs = concat(module.network.pod_nsg_id)
276281
worker_nsgs = concat(var.worker_nsgs, [module.network.worker_nsg_id])
277282

278283
freeform_tags = var.freeform_tags["oke"]

‎modules/network/locals.tf

Lines changed: 62 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ locals {
2020

2121
workers_subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["workers"], "newbits"), lookup(var.subnets["workers"], "netnum"))
2222

23+
pods_subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["pods"], "newbits"), lookup(var.subnets["pods"], "netnum"))
24+
2325
fss_subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["fss"], "newbits"), lookup(var.subnets["fss"], "netnum"))
2426

2527
anywhere = "0.0.0.0/0"
@@ -91,15 +93,15 @@ locals {
9193
destination = local.osn,
9294
destination_type = "SERVICE_CIDR_BLOCK",
9395
protocol = local.tcp_protocol,
94-
port = 443,
96+
port = -1,
9597
stateless = false
9698
},
9799
{
98-
description = "Allow all TCP traffic from control plane to worker nodes",
100+
description = "Allow Kubernetes Control plane to communicate with worker nodes",
99101
destination = local.workers_subnet,
100102
destination_type = "CIDR_BLOCK",
101103
protocol = local.tcp_protocol,
102-
port = -1,
104+
port = 10250,
103105
stateless = false
104106
},
105107
{
@@ -149,17 +151,9 @@ locals {
149151

150152
# workers
151153
workers_egress = [
152-
{
153-
description = "Allow egress for all traffic to allow pods to communicate between each other on different worker nodes on the worker subnet",
154-
destination = local.workers_subnet,
155-
destination_type = "CIDR_BLOCK",
156-
protocol = local.all_protocols,
157-
port = -1,
158-
stateless = false
159-
},
160154
{
161155
description = "Allow ICMP traffic for path discovery",
162-
destination = local.workers_subnet
156+
destination = local.anywhere
163157
destination_type = "CIDR_BLOCK",
164158
protocol = local.icmp_protocol,
165159
port = -1,
@@ -203,7 +197,7 @@ locals {
203197
{
204198
description = "Allow control plane to communicate with worker nodes",
205199
protocol = local.tcp_protocol,
206-
port = -1,
200+
port = 10250,
207201
source = local.cp_subnet,
208202
source_type = "CIDR_BLOCK",
209203
stateless = false
@@ -219,6 +213,60 @@ locals {
219213
}
220214
]
221215

216+
pods_egress = [
217+
{
218+
description = "Allow pods to communicate with other pods.",
219+
destination = local.pods_subnet,
220+
destination_type = "CIDR_BLOCK",
221+
protocol = local.all_protocols,
222+
port = -1,
223+
stateless = false
224+
},
225+
{
226+
description = "Allow ICMP traffic for path discovery",
227+
destination = local.osn,
228+
destination_type = "SERVICE_CIDR_BLOCK",
229+
protocol = local.icmp_protocol,
230+
port = -1,
231+
stateless = false
232+
},
233+
{
234+
description = "Allow pods to communicate with OCI Services",
235+
destination = local.osn,
236+
destination_type = "SERVICE_CIDR_BLOCK",
237+
protocol = local.tcp_protocol,
238+
port = -1,
239+
stateless = false
240+
},
241+
]
242+
243+
pods_ingress = [
244+
{
245+
description = "Allow worker nodes to access pods.",
246+
protocol = local.all_protocols,
247+
port = -1,
248+
source = local.cp_subnet,
249+
source_type = "CIDR_BLOCK",
250+
stateless = false
251+
},
252+
{
253+
description = "Allow Kubernetes Control Plane to communicate with pods.",
254+
protocol = local.all_protocols,
255+
port = -1,
256+
source = local.workers_subnet,
257+
source_type = "CIDR_BLOCK",
258+
stateless = false
259+
},
260+
{
261+
description = "Allow pods to communicate with each other.",
262+
protocol = local.all_protocols,
263+
port = -1,
264+
source = local.pods_subnet,
265+
source_type = "CIDR_BLOCK",
266+
stateless = false
267+
},
268+
]
269+
222270
int_lb_egress = [
223271
{
224272
description = "Allow stateful egress to workers. Required for NodePorts",
@@ -247,7 +295,7 @@ locals {
247295
]
248296

249297
# Combine supplied allow list and the public load balancer subnet
250-
internal_lb_allowed_cidrs = var.load_balancers == "both"? concat(var.internal_lb_allowed_cidrs, tolist([local.pub_lb_subnet])) : var.internal_lb_allowed_cidrs
298+
internal_lb_allowed_cidrs = var.load_balancers == "both" ? concat(var.internal_lb_allowed_cidrs, tolist([local.pub_lb_subnet])) : var.internal_lb_allowed_cidrs
251299

252300
# Create a Cartesian product of allowed cidrs and ports
253301
internal_lb_allowed_cidrs_and_ports = setproduct(local.internal_lb_allowed_cidrs, var.internal_lb_allowed_ports)

‎modules/network/nsgs.tf

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,19 @@ resource "oci_core_network_security_group_security_rule" "cp_egress" {
3737
}
3838

3939
count = length(local.cp_egress)
40+
}
41+
42+
resource "oci_core_network_security_group_security_rule" "cp_egress_npn" {
43+
network_security_group_id = oci_core_network_security_group.cp.id
44+
description = "Allow Kubernetes Control plane to communicate with pods"
45+
destination = local.pods_subnet
46+
destination_type = "CIDR_BLOCK"
47+
direction = "EGRESS"
48+
protocol = local.all_protocols
49+
50+
stateless = false
51+
52+
count = var.cni_type == "npn" ? 1 :0
4053

4154
}
4255

@@ -134,7 +147,32 @@ resource "oci_core_network_security_group_security_rule" "workers_egress" {
134147
}
135148

136149
count = length(local.workers_egress)
150+
}
137151

152+
resource "oci_core_network_security_group_security_rule" "workers_egress_flannel" {
153+
network_security_group_id = oci_core_network_security_group.workers.id
154+
description = "Allow egress for all traffic to allow pods to communicate between each other on different worker nodes on the worker subnet"
155+
destination = local.workers_subnet
156+
destination_type = "CIDR_BLOCK"
157+
direction = "EGRESS"
158+
protocol = local.all_protocols
159+
160+
stateless = false
161+
162+
count = var.cni_type == "flannel" ? 1: 0
163+
}
164+
165+
resource "oci_core_network_security_group_security_rule" "workers_egress_npn" {
166+
network_security_group_id = oci_core_network_security_group.workers.id
167+
description = "Allow worker nodes access to pods"
168+
destination = local.pods_subnet
169+
destination_type = "CIDR_BLOCK"
170+
direction = "EGRESS"
171+
protocol = local.all_protocols
172+
173+
stateless = false
174+
175+
count = var.cni_type == "npn" ? 1: 0
138176
}
139177

140178
# add this rule separately so it can be controlled independently
@@ -290,6 +328,58 @@ resource "oci_core_network_security_group_security_rule" "workers_ssh_ingress_fr
290328

291329
}
292330

331+
# pod nsg and rules
332+
resource "oci_core_network_security_group" "pods" {
333+
compartment_id = var.compartment_id
334+
display_name = var.label_prefix == "none" ? "pods" : "${var.label_prefix}-pods"
335+
vcn_id = var.vcn_id
336+
}
337+
338+
resource "oci_core_network_security_group_security_rule" "pods_egress" {
339+
network_security_group_id = oci_core_network_security_group.pods.id
340+
description = local.pods_egress[count.index].description
341+
destination = local.pods_egress[count.index].destination
342+
destination_type = local.pods_egress[count.index].destination_type
343+
direction = "EGRESS"
344+
protocol = local.pods_egress[count.index].protocol
345+
346+
stateless = false
347+
348+
dynamic "tcp_options" {
349+
for_each = local.pods_egress[count.index].protocol == local.tcp_protocol && local.pods_egress[count.index].port != -1 ? [1] : []
350+
content {
351+
destination_port_range {
352+
min = local.pods_egress[count.index].port
353+
max = local.pods_egress[count.index].port
354+
}
355+
}
356+
}
357+
358+
dynamic "icmp_options" {
359+
for_each = local.pods_egress[count.index].protocol == local.icmp_protocol ? [1] : []
360+
content {
361+
type = 3
362+
code = 4
363+
}
364+
}
365+
366+
count = var.cni_type =="npn" ? length(local.pods_egress) : 0
367+
}
368+
369+
# add this rule separately so it can be controlled independently
370+
resource "oci_core_network_security_group_security_rule" "pods_egress_internet" {
371+
network_security_group_id = oci_core_network_security_group.pods.id
372+
description = "Allow pods access to Internet"
373+
destination = local.anywhere
374+
destination_type = "CIDR_BLOCK"
375+
direction = "EGRESS"
376+
protocol = local.tcp_protocol
377+
378+
stateless = false
379+
count = (var.cni_type =="npn" && var.allow_pod_internet_access == true) ? 1 : 0
380+
381+
}
382+
293383
# internal lb nsg and rules
294384
resource "oci_core_network_security_group" "int_lb" {
295385
compartment_id = var.compartment_id

‎modules/network/outputs.tf

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
output "subnet_ids" {
55
value = {
66
"cp" = join(",", oci_core_subnet.cp[*].id)
7+
"pods" = join(",", oci_core_subnet.pods[*].id)
78
"workers" = join(",", oci_core_subnet.workers[*].id)
89
"int_lb" = join(",", oci_core_subnet.int_lb[*].id)
910
"pub_lb" = join(",", oci_core_subnet.pub_lb[*].id)
@@ -22,6 +23,10 @@ output "pub_lb" {
2223
value = var.load_balancers == "public" || var.load_balancers == "both" ? oci_core_network_security_group.pub_lb[0].id : ""
2324
}
2425

26+
output "pod_nsg_id" {
27+
value = tolist([oci_core_network_security_group.pods.id])
28+
}
29+
2530
output "worker_nsg_id" {
2631
value = oci_core_network_security_group.workers.id
2732
}

‎modules/network/subnets.tf

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,18 @@ resource "oci_core_subnet" "workers" {
2222
vcn_id = var.vcn_id
2323
}
2424

25+
resource "oci_core_subnet" "pods" {
26+
cidr_block = local.pods_subnet
27+
compartment_id = var.compartment_id
28+
display_name = var.label_prefix == "none" ? "pods" : "${var.label_prefix}-pods"
29+
dns_label = "pods"
30+
prohibit_public_ip_on_vnic = true
31+
route_table_id = var.nat_route_id
32+
vcn_id = var.vcn_id
33+
34+
count = var.cni_type == "npn" ? 1 : 0
35+
}
36+
2537
resource "oci_core_subnet" "int_lb" {
2638
cidr_block = local.int_lb_subnet
2739
compartment_id = var.compartment_id

‎modules/network/variables.tf

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ variable "vcn_id" {}
1919

2020
# cluster endpoint
2121

22+
variable "cni_type" {}
23+
2224
variable "control_plane_type" {
2325
type = string
2426
}
@@ -37,6 +39,10 @@ variable "allow_worker_internet_access" {
3739
type = bool
3840
}
3941

42+
variable "allow_pod_internet_access" {
43+
type = bool
44+
}
45+
4046
variable "allow_worker_ssh_access" {
4147
type = bool
4248
}

0 commit comments

Comments
 (0)