fix: Add missing NSG rules for VCN-Native Pod Networking (#563)

Add missing NSG rules for VCN-Native Pod Networking

Changed to reflect https://docs.oracle.com/en-us/iaas/Content/ContEng/Concepts/contengnetworkconfig.htm#securitylistconfig:
* Moved `oci_core_network_security_group_security_rule.workers_egress_flannel` to CNI-agnostic `local.workers_egress` for node to node.
* Added `oci_core_network_security_group_security_rule.workers_ingress_npn` for pod to worker.
* Added `oci_core_network_security_group_security_rule.pods_ingress"` for CP to pod, worker to pod, and pod to pod (`local.pods_ingress` existed for them but was not yet referenced).
* Added to TCP 6443 to `local.pods_egress` for pod to CP.
* Updated `oci_core_network_security_group_security_rule.pods_egress_internet` from tcp -> all protocols.
* Updated some NSG rule descriptions to clarify usage.
* Added NPN conditional to `oci_core_network_security_group.pods`.

Signed-off-by: Devon Crouse <devon.crouse@oracle.com>

Author: devoncrouse

modules/network/locals.tf

@@ -2,6 +2,8 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
 
 locals {
+  # VCN subnet configuration
+  # See https://docs.oracle.com/en-us/iaas/Content/ContEng/Concepts/contengnetworkconfig.htm#vcnconfig
 
   # first vcn cidr
   # pick the first cidr block in the list as this is where we will create the oke subnets
@@ -54,9 +56,11 @@ locals {
     waf_subnet.cidr
   ] : []
 
+  # Security configuration
+  # See https://docs.oracle.com/en-us/iaas/Content/ContEng/Concepts/contengnetworkconfig.htm#securitylistconfig
   # if port = -1, allow all ports
 
-  #control plane seclist
+  # Security List rules for control plane subnet (Flannel & VCN-Native Pod networking)
   cp_egress_seclist = [
     {
       description      = "Allow Bastion service to communicate to the control plane endpoint. Required for when using OCI Bastion service.",
@@ -78,7 +82,8 @@ locals {
       stateless   = false
     }
   ]
-  # control plane
+
+  # Network Security Group egress rules for control plane subnet (Flannel & VCN-Native Pod networking)
   cp_egress = [
     {
       description      = "Allow Kubernetes Control plane to communicate to the control plane subnet. Required for when using OCI Bastion service.",
@@ -114,6 +119,7 @@ locals {
     },
   ]
 
+  # Network Security Group ingress rules for control plane subnet (Flannel & VCN-Native Pod networking)
   cp_ingress = [
     {
       description = "Allow worker nodes to control plane API endpoint communication"
@@ -149,8 +155,16 @@ locals {
     },
   ]
 
-  # workers
+  # Network Security Group egress rules for workers subnet (Flannel & VCN-Native Pod networking)
   workers_egress = [
+    {
+      description      = "Allows communication from (or to) worker nodes.",
+      destination      = local.workers_subnet
+      destination_type = "CIDR_BLOCK",
+      protocol         = local.all_protocols,
+      port             = -1,
+      stateless        = false
+    },
     {
       description      = "Allow ICMP traffic for path discovery",
       destination      = local.anywhere
@@ -185,6 +199,7 @@ locals {
     }
   ]
 
+  # Network Security Group ingress rules for workers subnet (Flannel & VCN-Native Pod networking)
   workers_ingress = [
     {
       description = "Allow ingress for all traffic to allow pods to communicate between each other on different worker nodes on the worker subnet",
@@ -213,6 +228,7 @@ locals {
     }
   ]
 
+  # Network Security Group egress rules for pods subnet (VCN-Native Pod networking only)
   pods_egress = [
     {
       description      = "Allow pods to communicate with other pods.",
@@ -238,19 +254,28 @@ locals {
       port             = -1,
       stateless        = false
     },
+    {
+      description      = "Allow pods to communicate with Kubernetes API server",
+      destination      = local.cp_subnet,
+      destination_type = "CIDR_BLOCK",
+      protocol         = local.tcp_protocol,
+      port             = 6443,
+      stateless        = false
+    }
   ]
 
+  # Network Security Group ingress rules for pods subnet (VCN-Native Pod networking only)
   pods_ingress = [
     {
-      description = "Allow worker nodes to access pods.",
+      description = "Allow Kubernetes control plane to communicate with webhooks served by pods",
       protocol    = local.all_protocols,
       port        = -1,
       source      = local.cp_subnet,
       source_type = "CIDR_BLOCK",
       stateless   = false
     },
     {
-      description = "Allow Kubernetes Control Plane to communicate with pods.",
+      description = "Allow cross-node pod communication when using NodePorts or hostNetwork: true",
       protocol    = local.all_protocols,
       port        = -1,
       source      = local.workers_subnet,
@@ -267,6 +292,7 @@ locals {
     },
   ]
 
+  # Network Security Group rules for load balancer subnet
   int_lb_egress = [
     {
       description      = "Allow stateful egress to workers. Required for NodePorts",

modules/network/nsgs.tf

@@ -149,19 +149,6 @@ resource "oci_core_network_security_group_security_rule" "workers_egress" {
   count = length(local.workers_egress)
 }
 
-resource "oci_core_network_security_group_security_rule" "workers_egress_flannel" {
-  network_security_group_id = oci_core_network_security_group.workers.id
-  description               = "Allow egress for all traffic to allow pods to communicate between each other on different worker nodes on the worker subnet"
-  destination               = local.workers_subnet
-  destination_type          = "CIDR_BLOCK"
-  direction                 = "EGRESS"
-  protocol                  = local.all_protocols
-
-  stateless = false
-
-  count = var.cni_type == "flannel" ? 1: 0
-}
-
 resource "oci_core_network_security_group_security_rule" "workers_egress_npn" {
   network_security_group_id = oci_core_network_security_group.workers.id
   description               = "Allow worker nodes access to pods"
@@ -307,6 +294,17 @@ resource "oci_core_network_security_group_security_rule" "workers_healthcheck_in
 
 }
 
+resource "oci_core_network_security_group_security_rule" "workers_ingress_npn" {
+  network_security_group_id = oci_core_network_security_group.workers.id
+  description               = "Allow cross-node pod communication when using NodePorts or hostNetwork: true"
+  direction                 = "INGRESS"
+  protocol                  = local.all_protocols
+  source                    = local.pods_subnet
+  source_type               = "CIDR_BLOCK"
+  stateless                 = false
+  count                     = var.cni_type == "npn" ? 1 : 0
+}
+
 resource "oci_core_network_security_group_security_rule" "workers_ssh_ingress_from_bastion" {
   network_security_group_id = oci_core_network_security_group.workers.id
   description               = "Allow ssh access to workers via Bastion host"
@@ -333,6 +331,7 @@ resource "oci_core_network_security_group" "pods" {
   compartment_id = var.compartment_id
   display_name   = var.label_prefix == "none" ? "pods" : "${var.label_prefix}-pods"
   vcn_id         = var.vcn_id
+  count          = var.cni_type == "npn" ? 1 : 0
 }
 
 resource "oci_core_network_security_group_security_rule" "pods_egress" {
@@ -366,14 +365,25 @@ resource "oci_core_network_security_group_security_rule" "pods_egress" {
   count = var.cni_type =="npn" ? length(local.pods_egress) : 0
 }
 
+resource "oci_core_network_security_group_security_rule" "pods_ingress" {
+  network_security_group_id = oci_core_network_security_group.pods.id
+  description               = local.pods_ingress[count.index].description
+  source                    = local.pods_ingress[count.index].source
+  source_type               = local.pods_ingress[count.index].source_type
+  protocol                  = local.pods_ingress[count.index].protocol
+  direction                 = "INGRESS"
+  stateless                 = false
+  count                     = var.cni_type =="npn" ? length(local.pods_ingress) : 0
+}
+
 # add this rule separately so it can be controlled independently
 resource "oci_core_network_security_group_security_rule" "pods_egress_internet" {
   network_security_group_id = oci_core_network_security_group.pods.id
   description               = "Allow pods access to Internet"
   destination               = local.anywhere
   destination_type          = "CIDR_BLOCK"
   direction                 = "EGRESS"
-  protocol                  = local.tcp_protocol
+  protocol                  = local.all_protocols
 
   stateless = false
   count = (var.cni_type =="npn" && var.allow_pod_internet_access == true) ? 1 : 0