feat: EMR on EKS with Karpenter Managed Endpoint (#261)

* adding managed endpoint script

* added doc

* removed output json

* adding optional assignment for cloudwatch logs and streams prefix

Author: youngjeong46

analytics/terraform/emr-eks-karpenter/examples/managed-endpoints/create-managed-endpoint.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+read -p "Enter EMR Virtual Cluster Id: " EMR_VIRTUAL_CLUSTER_ID
+read -p "Provide your EMR on EKS team (emr-data-team-a or emr-data-team-b): " EMR_EKS_TEAM
+read -p "Enter your AWS Region: " AWS_REGION
+read -p "Enter a name for your endpoint: " EMR_EKS_MANAGED_ENDPOINT
+read -p "Provide an S3 bucket location for logging (i.e. s3://my-bucket/logging/): " S3_BUCKET
+read -p "Provide CloudWatch Logs Group Name: " CLOUDWATCH_LOGS_GROUP_NAME
+read -p "Provide CloudWatch Logs Prefix: " CLOUDWATCH_LOGS_PREFIX
+read -p "Enter the EMR Execution Role ARN (i.e. arn:aws:00000000000000000:role/EMR-Execution-Role): " EMR_EXECUTION_ROLE_ARN
+read -p "Enter the release label (i.e. emr-6.9.0-latest): " EMR_EKS_RELEASE_LABEL
+
+#-------------------------------------------------------
+# Set Managed Endpoint JSON file with provided variables
+#-------------------------------------------------------
+export EMR_VIRTUAL_CLUSTER_ID=$EMR_VIRTUAL_CLUSTER_ID
+export EMR_EKS_TEAM=$EMR_EKS_TEAM
+export AWS_REGION=$AWS_REGION
+export EMR_EKS_MANAGED_ENDPOINT=$EMR_EKS_MANAGED_ENDPOINT
+export S3_BUCKET=$S3_BUCKET
+export EMR_EXECUTION_ROLE_ARN=$EMR_EXECUTION_ROLE_ARN
+export EMR_EKS_RELEASE_LABEL=$EMR_EKS_RELEASE_LABEL
+export CLOUDWATCH_LOGS_GROUP_NAME=$CLOUDWATCH_LOGS_GROUP_NAME
+export CLOUDWATCH_LOGS_PREFIX=$CLOUDWATCH_LOGS_PREFIX
+
+envsubst < managed-endpoint.json > managed-endpoint-final.json
+
+#------------------------------------------------------------------------------
+# Create managed endpoint and assign Load Balancer information as env variables
+#------------------------------------------------------------------------------
+
+# Creating managed endpoint and saving the endpoint ID and Tag needed to identify the right Load Balancer
+MANAGED_ENDPOINT_ID=$(aws emr-containers create-managed-endpoint --cli-input-json file://./managed-endpoint-final.json --query id --output text)
+echo -e "The Managed Endpoint ID is $MANAGED_ENDPOINT_ID. \n"
+
+TAG_VALUE=${EMR_EKS_TEAM}/ingress-${MANAGED_ENDPOINT_ID}
+
+echo -e "Waiting for the managed endpoint load balancer to be active...\n"
+
+sleep 30
+
+# Saving Load Balancer ARN for the endpoint ingress
+for i in $(aws elbv2 describe-load-balancers | jq -r '.LoadBalancers[].LoadBalancerArn'); do aws elbv2 describe-tags --resource-arns "$i" | jq -ce --arg TAG_VALUE $TAG_VALUE '.TagDescriptions[].Tags[] | select( .Key == "ingress.k8s.aws/stack" and .Value == $TAG_VALUE)' && ARN=$i ;done
+echo -e "The Load Balancer ARN is $ARN. \n"
+
+aws elbv2 wait load-balancer-available --load-balancer-arns $ARN
+
+echo -e "The load balancer is in service. \n"
+
+#------------------------------------------------------------------------
+# Revise to add the Karpenter Security Group to the Load Balancer created
+#------------------------------------------------------------------------
+
+echo "Setting Security Groups for Jupyter Notebook and Karpenter..."
+# Security Group for Endpoint with the Jupyter Notebook
+NOTEBOOK_SG=$(aws ec2 describe-security-groups \
+    --filters Name=group-name,Values="emr-containers-lb-$EMR_VIRTUAL_CLUSTER_ID-$MANAGED_ENDPOINT_ID" \
+    --query "SecurityGroups[*].{ID:GroupId}" --output text)
+
+# Karpenter Security Group
+KARPENTER_SG=$(aws ec2 describe-security-groups \
+    --filters Name=group-name,Values="emr-eks-karpenter-node-*" \
+    --query "SecurityGroups[*].{ID:GroupId}" --output text)
+
+echo "Adding the security groups to the load balancer..."
+# Add these two Security Groups to the Load Balancer
+RESULT=$(aws elbv2 set-security-groups --load-balancer-arn $ARN --security-groups $NOTEBOOK_SG $KARPENTER_SG)
+
+# Wait for 30 seconds before the managed endpoint is ready.
+sleep 30
+echo "The managed endpoint has been created."

analytics/terraform/emr-eks-karpenter/examples/managed-endpoints/managed-endpoint.json

@@ -0,0 +1,33 @@
+{
+    "name": "$EMR_EKS_MANAGED_ENDPOINT",
+    "virtualClusterId": "$EMR_VIRTUAL_CLUSTER_ID",
+    "type": "JUPYTER_ENTERPRISE_GATEWAY",
+    "releaseLabel": "$EMR_EKS_RELEASE_LABEL",
+    "executionRoleArn": "$EMR_EXECUTION_ROLE_ARN",
+    "configurationOverrides":
+    {
+        "applicationConfiguration":
+        [
+            {
+                "classification": "spark-defaults",
+                "properties":
+                {
+                    "spark.driver.memory": "8G"
+                }
+            }
+        ],
+        "monitoringConfiguration":
+        {
+            "persistentAppUI": "ENABLED",
+            "cloudWatchMonitoringConfiguration":
+            {
+                "logGroupName": "$CLOUDWATCH_LOGS_GROUP_NAME",
+                "logStreamNamePrefix": "$CLOUDWATCH_LOGS_PREFIX"
+            },
+            "s3MonitoringConfiguration":
+            {
+                "logUri": "$S3_BUCKET"
+            }
+        }
+    }
+}

website/docs/blueprints/amazon-emr-on-eks/emr-eks-karpenter.md

@@ -590,6 +590,47 @@ cd analytics/terraform/emr-eks-karpenter/examples/nvme-ssd/deltalake
 
 </CollapsibleContent>
 
+## Run Interactive Workload with Managed Endpoint
+
+Managed endpoint is a gateway that provides connectivity from EMR Studio to EMR on EKS so that you can run interactive workloads. You can find out more information about it [here](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/how-it-works.html).
+
+### Creating a managed endpoint
+
+In this example, we will create a managed endpoint under one of the data teams.
+
+```bash
+Navigate to folder and execute script:
+
+cd analytics/terraform/emr-eks-karpenter/examples/managed-endpoints
+./create-managed-endpoint.sh
+```
+```
+Enter the EMR Virtual Cluster Id: 4ucrncg6z4nd19vh1lidna2b3
+Provide your EMR on EKS team (emr-data-team-a or emr-data-team-b): emr-eks-data-team-a
+Enter your AWS Region: us-west-2
+Enter a name for your endpoint: emr-eks-team-a-endpoint
+Provide an S3 bucket location for logging (i.e. s3://my-bucket/logging/): s3://<bucket-name>/logs
+Enter the EMR Execution Role ARN (i.e. arn:aws:00000000000000000:role/EMR-Execution-Role): arn:aws:iam::181460066119:role/emr-eks-karpenter-emr-data-team-a
+```
+
+The script will provide the following:
+- JSON configuration file for the Managed Endpoint
+- Configuration settings:
+  - Default 8G Spark Driver
+  - CloudWatch monitoring, with logs stored in the S3 bucket provided
+- Proper endpoint creation with appropriate security group to allow using Karpenter
+- Outputs: Managed Endpoint ID and Load Balancer ARN. 
+
+Once you have created a managed endpoint, you can follow the instructions [here](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-studio-configure.html) to configure EMR Studio and associate the Managed endpoint to a workspace.
+
+### Cleanup of Endpoint resources
+
+To delete the managed endpoint, simply run the following command:
+
+```bash
+aws emr-containers delete-managed-endpoint --id <Managed Endpoint ID> --virtual-cluster-id <Virtual Cluster ID>
+```
+
 ## Cleanup
 <CollapsibleContent header={<h2><span>Cleanup</span></h2>}>
 This script will cleanup the environment using `-target` option to ensure all the resources are deleted in correct order.