Add missing toleration to aws_nvidia_installer DaemonSet to support tainted GPU nodes (#3075)

Author: marcelovilla

src/_nebari/stages/kubernetes_initialize/__init__.py

@@ -89,7 +89,9 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
                 for node_group in self.config.amazon_web_services.node_groups.values()
             )
             input_vars.gpu_node_group_names = [
-                group for group in self.config.amazon_web_services.node_groups.keys()
+                group
+                for group in self.config.amazon_web_services.node_groups.keys()
+                if self.config.amazon_web_services.node_groups[group].gpu
             ]
             input_vars.aws_region = self.config.amazon_web_services.region
 

src/_nebari/stages/kubernetes_initialize/template/modules/nvidia-installer/aws-nvidia-installer.tf

@@ -68,6 +68,13 @@ resource "kubernetes_daemonset" "aws_nvidia_installer" {
           operator = "Exists"
           effect   = "NoSchedule"
         }
+
+        toleration {
+          key      = "dedicated"
+          operator = "Equal"
+          value    = "nebari"
+          effect   = "NoSchedule"
+        }
       }
     }