feat: added support for OKE images for faster worker node provisioning (#529)

* feat: added supported for OKE images for faster worker node
      provisioning

Signed-off-by: Ali Mukadam <ali.mukadam@oracle.com>

* fix: added default freeform tags for cluster and nodepools

Signed-off-by: Ali Mukadam <ali.mukadam@oracle.com>

* fix: removed `node_source_details` from lifecycle

Signed-off-by: Ali Mukadam <ali.mukadam@oracle.com>

* added separate dynamic blocks for platform, custom and oke images

Signed-off-by: Ali Mukadam <ali.mukadam@oracle.com>

Author: hyder

docs/terraformoptions.adoc

@@ -688,6 +688,11 @@ node_pools = {
 |
 |"none"
 
+|node_pool_image_type
+|Whether to use a Platform, OKE or custom image. When custom is set, the node_pool_image_id *must* be specified. Using an OKE image minimizes the time it takes to provision worker nodes at runtime when compared to platform images and custom images. OKE images are optimized for use as worker node base images, with all the necessary configurations and required software. The use of OKE images reduces worker node provisioning time by more than half when compared to platform images. OKE images are provided by Oracle and built on top of platform images.
+| "custom","oke","platform"
+|"oke"
+
 |node_pool_os
 |The name of the Operating System image to use to provision the worker nodes.
 |

main.tf

@@ -236,6 +236,7 @@ module "oke" {
   node_pools                      = var.node_pools
   node_pool_name_prefix           = var.node_pool_name_prefix
   node_pool_image_id              = var.node_pool_image_id
+  node_pool_image_type            = var.node_pool_image_type
   node_pool_os                    = var.node_pool_os
   node_pool_os_version            = var.node_pool_os_version
   node_pool_timezone              = var.node_pool_timezone

modules/oke/locals.tf

@@ -28,4 +28,8 @@ locals {
   # 3. if more than 1 image found for this version, pick the latest
   node_pool_image_ids = data.oci_containerengine_node_pool_option.node_pool_options.sources
 
+  # kubernetes string version length
+  k8s_version_length = length(var.cluster_kubernetes_version)
+  k8s_version_only = substr(var.cluster_kubernetes_version,1,local.k8s_version_length)
+
 }

modules/oke/nodepools.tf

@@ -11,9 +11,10 @@ resource "oci_containerengine_node_pool" "nodepools" {
   name               = var.label_prefix == "none" ? each.key : "${var.label_prefix}-${each.key}"
 
   freeform_tags = var.freeform_tags["node_pool"]
-  
-  node_config_details {
 
+  node_config_details {
+    
+    # iterating over ADs
     dynamic "placement_configs" {
       iterator = ad_iterator
       for_each = local.ad_names
@@ -25,10 +26,12 @@ resource "oci_containerengine_node_pool" "nodepools" {
     nsg_ids                             = var.worker_nsgs
     is_pv_encryption_in_transit_enabled = var.enable_pv_encryption_in_transit
     kms_key_id                          = var.node_pool_volume_kms_key_id
+
     # allow zero-sized node pools
     size = max(0, lookup(each.value, "node_pool_size", 0))
   }
 
+  # setting shape
   dynamic "node_shape_config" {
     for_each = length(regexall("Flex", lookup(each.value, "shape", "VM.Standard.E4.Flex"))) > 0 ? [1] : []
     content {
@@ -37,25 +40,55 @@ resource "oci_containerengine_node_pool" "nodepools" {
     }
   }
 
+  # cloud-init
   node_metadata = {
     user_data = var.cloudinit_nodepool_common == "" && lookup(var.cloudinit_nodepool, each.key, null) == null ? data.cloudinit_config.worker.rendered : lookup(var.cloudinit_nodepool, each.key, null) != null ? filebase64(lookup(var.cloudinit_nodepool, each.key, null)) : filebase64(var.cloudinit_nodepool_common)
   }
 
-  node_source_details {
-    boot_volume_size_in_gbs = lookup(each.value, "boot_volume_size", 50)
-    # check is done for GPU,A1 and other shapes.In future if some other shapes or images added we need to modify
-    image_id    = (var.node_pool_image_id == "none" && length(regexall("GPU|A1", lookup(each.value, "shape"))) == 0) ? (element([for source in local.node_pool_image_ids : source.image_id if length(regexall("Oracle-Linux-${var.node_pool_os_version}-20[0-9]*.*", source.source_name)) > 0], 0)) : (var.node_pool_image_id == "none" && length(regexall("GPU", lookup(each.value, "shape"))) > 0) ? (element([for source in local.node_pool_image_ids : source.image_id if length(regexall("Oracle-Linux-${var.node_pool_os_version}-Gen[0-9]-GPU-20[0-9]*.*", source.source_name)) > 0], 0)) : (var.node_pool_image_id == "none" && length(regexall("A1", lookup(each.value, "shape"))) > 0) ? (element([for source in local.node_pool_image_ids : source.image_id if length(regexall("Oracle-Linux-${var.node_pool_os_version}-aarch64-20[0-9]*.*", source.source_name)) > 0], 0)) : var.node_pool_image_id
-    source_type = data.oci_containerengine_node_pool_option.node_pool_options.sources[0].source_type
+  # optimized OKE images
+  dynamic "node_source_details" {
+    for_each = var.node_pool_image_type == "oke" ? [1] : []
+    content {
+      boot_volume_size_in_gbs = lookup(each.value, "boot_volume_size", 50)
+      # check for GPU,A1 and other shapes. In future, if some other shapes or images are added, we need to modify
+      image_id = (var.node_pool_image_type == "oke" && length(regexall("GPU|A1", lookup(each.value, "shape"))) == 0) ? (element([for source in local.node_pool_image_ids : source.image_id if length(regexall("Oracle-Linux-${var.node_pool_os_version}-20[0-9]*.*-OKE-${local.k8s_version_only}", source.source_name)) > 0], 0)) : (var.node_pool_image_type == "oke" && length(regexall("GPU", lookup(each.value, "shape"))) > 0) ? (element([for source in local.node_pool_image_ids : source.image_id if length(regexall("Oracle-Linux-${var.node_pool_os_version}-Gen[0-9]-GPU-20[0-9]*.*-OKE-${local.k8s_version_only}", source.source_name)) > 0], 0)) : (var.node_pool_image_type == "oke" && length(regexall("A1", lookup(each.value, "shape"))) > 0) ? (element([for source in local.node_pool_image_ids : source.image_id if length(regexall("Oracle-Linux-${var.node_pool_os_version}-aarch64-20[0-9]*.*-OKE-${local.k8s_version_only}", source.source_name)) > 0], 0)) : null
+
+      source_type = data.oci_containerengine_node_pool_option.node_pool_options.sources[0].source_type
+    }
+  }
+
+  # OCI platform images
+  dynamic "node_source_details" {
+    for_each = var.node_pool_image_type == "platform" ? [1] : []
+    content {
+      boot_volume_size_in_gbs = lookup(each.value, "boot_volume_size", 50)
+      # check for GPU,A1 and other shapes. In future, if some other shapes or images are added, we need to modify
+      image_id = (var.node_pool_image_type == "platform" && length(regexall("GPU|A1", lookup(each.value, "shape"))) == 0) ? (element([for source in local.node_pool_image_ids : source.image_id if length(regexall("^(Oracle-Linux-${var.node_pool_os_version}-\\d{4}.\\d{2}.\\d{2}-[0-9]*)$", source.source_name)) > 0], 0)) : (var.node_pool_image_type == "platform" && length(regexall("GPU", lookup(each.value, "shape"))) > 0) ? (element([for source in local.node_pool_image_ids : source.image_id if length(regexall("^(Oracle-Linux-${var.node_pool_os_version}-Gen[0-9]-GPU-\\d{4}.\\d{2}.\\d{2}-[0-9]*)$", source.source_name)) > 0], 0)) : (var.node_pool_image_type == "platform" && length(regexall("A1", lookup(each.value, "shape"))) > 0) ? (element([for source in local.node_pool_image_ids : source.image_id if length(regexall("^(Oracle-Linux-${var.node_pool_os_version}-aarch64-\\d{4}.\\d{2}.\\d{2}-[0-9]*)$", source.source_name)) > 0], 0)) : null
+
+      source_type = data.oci_containerengine_node_pool_option.node_pool_options.sources[0].source_type
+    }
   }
 
+  # custom images 
+  dynamic "node_source_details" {
+    for_each = var.node_pool_image_type == "custom" ? [1] : []
+    content {
+      boot_volume_size_in_gbs = lookup(each.value, "boot_volume_size", 50)
+      image_id    = var.node_pool_image_id
+      source_type = data.oci_containerengine_node_pool_option.node_pool_options.sources[0].source_type
+    }
+  }
   node_shape = lookup(each.value, "shape", "VM.Standard.E4.Flex")
 
   ssh_public_key = (var.ssh_public_key != "") ? var.ssh_public_key : (var.ssh_public_key_path != "none") ? file(var.ssh_public_key_path) : ""
 
   # do not destroy the node pool if the kubernetes version has changed as part of the upgrade
+
   lifecycle {
     ignore_changes = [kubernetes_version]
   }
+
+  # initial node labels for the different node pools
   dynamic "initial_node_labels" {
     for_each = lookup(each.value, "label", "") != "" ? each.value.label : {}
     content {

modules/oke/variables.tf

@@ -91,6 +91,8 @@ variable "node_pool_name_prefix" {}
 
 variable "node_pool_image_id" {}
 
+variable "node_pool_image_type" {}
+
 variable "node_pool_os" {}
 
 variable "node_pool_os_version" {}

terraform.tfvars.example

@@ -149,7 +149,7 @@ pods_cidr                    = "10.244.0.0/16"
 services_cidr                = "10.96.0.0/16"
 
 ## oke cluster kms integration
-use_encryption  = false
+use_cluster_encryption  = false
 cluster_kms_key_id      = ""
 create_policies = true
 
@@ -177,6 +177,7 @@ node_pools = {
   # np10 = {shape="BM.HPC2.36	", node_pool_size=5}
 }
 node_pool_image_id    = "none"
+node_pool_image_type  = "oke"
 node_pool_name_prefix = "np"
 node_pool_os          = "Oracle Linux"
 node_pool_os_version  = "7.9"

variables.tf

@@ -612,6 +612,16 @@ variable "node_pool_image_id" {
   type        = string
 }
 
+variable "node_pool_image_type" {
+  default     = "oke"
+  description = "Whether to use a Platform, OKE or custom image. When custom is set, the node_pool_image_id must be specified."
+  type        = string
+  validation {
+    condition     = contains(["custom", "oke", "platform"], var.node_pool_image_type)
+    error_message = "Accepted values are custom, oke, platform."
+  }  
+}
+
 variable "node_pool_name_prefix" {
   default     = "np"
   description = "The prefix of the node pool name."
@@ -899,6 +909,10 @@ variable "freeform_tags" {
         environment = "dev"
         role        = "load balancer"
       }
+      cluster = {
+        environment = "dev"
+      }
+      node_pool = {}
     }
   }
   description = "Tags to apply to different resources."