fix: built api route image (#108)

* fix: built api route image

* fix: code clean up

* fix: fixed stream output log level

* fix: code cleanup

Author: yanbasic

pyproject.toml

@@ -22,7 +22,7 @@ python = "^3.9"
 typer = {extras = ["all"], version = "^0.13.0"}
 rich = "^13.9.4"
 boto3 = "^1.35.0"
-questionary = "^2.0.1"
+questionary = "^2.1.0"
 requests = "^2.26"
 pydantic = "^2.9.2"
 sagemaker = "^2.237.0"

src/emd/cfn/ecs/template.yaml

@@ -18,8 +18,10 @@ Parameters:
     Description: The listener to be used for the ECS Endpoint
   PublicLoadBalancerSecurityGroup:
     Type: String
-    Default: "sg-0d059e1ba522921fe"
     Description: The security group to be used for the ECS Endpoint
+  APIRouterSecurityGroup:
+    Type: String
+    Description: The security group to be used for the API Router
   LambdaDeploymentHelperArn:
     Type: String
     Description: The ARN of the Lambda function for capacity provider association
@@ -35,23 +37,19 @@ Parameters:
     Description: The instance type to be used for the ECS Endpoint
   ModelId:
     Type: String
-    Default: "qwen2-5-7b"
     Description: The emd model ID to be used for the ECS Endpoint
   ModelTag:
     Type: String
     Default: "dev"
     Description: The model tag to be used for the ECS Endpoint
   FrameWorkType:
     Type: String
-    Default: "awq"
     Description: The framework type to be used for the ECS Endpoint
   ServiceType:
     Type: String
-    Default: "instruct"
     Description: The service type to be used for the ECS Endpoint
   EngineType:
     Type: String
-    Default: "qwen2-5-72b"
     Description: The engine type to be used for the ECS Endpoint
   DesiredCapacity:
     Type: Number
@@ -171,6 +169,12 @@ Resources:
     Type: Custom::GetPriorityNumber
     Properties:
       ServiceToken: !Ref LambdaDeploymentHelperArn
+  ConvertDnsName:
+    Type: Custom::ConvertDnsName
+    Properties:
+      ServiceToken: !Ref LambdaDeploymentHelperArn
+      ModelName: !Join ['', [!Ref ModelId, '/', !Ref ModelTag]]
+
   ContainerHostSecurityGroup:
     Type: AWS::EC2::SecurityGroup
     Properties:
@@ -249,6 +253,8 @@ Resources:
           PortMappings:
             - ContainerPort: !Ref ContainerPort
               HostPort: !Ref ContainerPort
+              Name: model-port
+              AppProtocol: http
             - ContainerPort: 80
               HostPort: 80
           # Command: [--gpus, all]
@@ -265,7 +271,7 @@ Resources:
             interval: 120
             retries: 10
             timeout: 60
-            startPeriod: 120
+            startPeriod: 300
           LogConfiguration:
             LogDriver: awslogs
             Options:
@@ -310,11 +316,31 @@ Resources:
         - ContainerName: !Sub '${AWS::StackName}'
           ContainerPort: !Ref ContainerPort
           TargetGroupArn: !Ref ServiceTargetGroup
+      ServiceConnectConfiguration:
+        Enabled: true
+        Namespace: emd-service-connect-namespace
+        Services:
+          - PortName: model-port
+            DiscoveryName: model-service
+            ClientAliases:
+              - Port: !Ref ContainerPort
+                DnsName: !GetAtt ConvertDnsName.DnsName
   ServiceSecurityGroup:
     Type: 'AWS::EC2::SecurityGroup'
     Properties:
       GroupDescription: Security group for service
       VpcId: !Ref VPCID
+
+  ServiceSecurityGroupIngress:
+    Type: AWS::EC2::SecurityGroupIngress
+    Properties:
+      GroupId: !GetAtt ServiceSecurityGroup.GroupId
+      IpProtocol: tcp
+      FromPort: !Ref ContainerPort
+      ToPort: !Ref ContainerPort
+      SourceSecurityGroupId: !Ref APIRouterSecurityGroup
+      Description: Allow traffic from API router service to model service
+
   ServiceTargetGroup:
     Type: 'AWS::ElasticLoadBalancingV2::TargetGroup'
     Properties:
@@ -362,8 +388,11 @@ Resources:
 
 Outputs:
   Model:
-    Description: Model ID used to generate the response
+    Description: Model ID used to generate the response.
     Value: !Join ['', [!Ref ModelId, '/', !Ref ModelTag]]
-  PublicLoadBalancerDNSName:
-    Description: The DNS name of the public load balancer. To use HTTPS, create an SSL certificate in AWS Certificate Manager and attach it to the load balancer.
-    Value: !Join ['', ['http://', !Ref DNSName, '/', !Ref ModelId, '/', !Ref ModelTag]]
+  BaseURL:
+    Description: Please use this URL for the OpenAI-compatible model API, like /v1/chat/completions. To use HTTPS, create an SSL certificate in AWS Certificate Manager and attach it to the load balancer.
+    Value: !Join ['', ['http://', !Ref DNSName]]
+  ECSServiceConnect:
+    Description: Service Connect allows for service-to-service communications with automatic discovery using short names and standard ports.
+    Value: !Join ['', ['http://', !GetAtt ConvertDnsName.DnsName, ':', !Ref ContainerPort]]

src/emd/cfn/sagemaker_realtime/template.yaml

@@ -38,6 +38,9 @@ Parameters:
     Type: String
     Description: The name of the SageMaker Endpoint
     Default: "Auto-generate"
+  DNSName:
+    Type: String
+    Description: The DNS name of the public load balancer. To use HTTPS, create an SSL certificate in AWS Certificate Manager and attach it to the load balancer.
 
 Conditions:
   UseDefaultEndpointName: !Equals [!Ref SageMakerEndpointName, "Auto-generate"]
@@ -149,8 +152,11 @@ Resources:
 
 Outputs:
   Model:
-    Description: Model ID used to generate the response
+    Description: Model ID used to generate the response.
     Value: !Join ['', [!Ref ModelId, '/', !Ref ModelTag]]
+  BaseURL:
+    Description: Please use this URL for the OpenAI-compatible model API, like /v1/chat/completions. To use HTTPS, create an SSL certificate in AWS Certificate Manager and attach it to the load balancer.
+    Value: !Join ['', ['http://', !Ref DNSName]]
   SageMakerEndpointName:
-    Description: The name of the SageMaker Endpoint
+    Description: You can use invoke the endpoint by using the SageMaker runtime API.
     Value: !GetAtt SageMakerEndpoint.EndpointName

src/emd/cfn/shared/ecs_cluster.py

@@ -170,7 +170,7 @@ def deploy_vpc_template(region):
     return vpc_id, subnets
 
 
-def deploy_ecs_cluster_template(region, vpc_id, subnets, api_router_uri, use_spot):
+def deploy_ecs_cluster_template(region, vpc_id, subnets, use_spot):
     client = boto3.client("cloudformation", region_name=region)
     stack_name = "EMD-ECS-Cluster"
     template_path = f"{CFN_ROOT_PATH}/shared/ecs_cluster.yaml"
@@ -187,10 +187,6 @@ def deploy_ecs_cluster_template(region, vpc_id, subnets, api_router_uri, use_spo
                 "ParameterKey": "Subnets",
                 "ParameterValue": subnets,
             },
-            {
-                "ParameterKey": "APIRouterImageURI",
-                "ParameterValue": api_router_uri,
-            },
             {
                 "ParameterKey": "UseSpot",
                 "ParameterValue": "yes" if use_spot else "no",
@@ -222,10 +218,10 @@ def deploy_ecs_cluster(region, vpc_id=None, subnets=None, use_spot=False):
     update_parameters_file("parameters.json", {"VPCID": vpc_id, "Subnets": subnets})
 
     # Build and push Fargate image to ECR as the OpenAI compatible API router
-    api_router_uri = build_router_image(region)
+    # api_router_uri = build_router_image(region)
 
     # Deploy the ECS cluster
-    deploy_ecs_cluster_template(region, vpc_id, subnets, api_router_uri, use_spot)
+    deploy_ecs_cluster_template(region, vpc_id, subnets, use_spot)
 
 if __name__ == "__main__":
     deploy_ecs_cluster("us-east-1")

src/emd/cfn/shared/ecs_cluster.yaml

@@ -10,7 +10,7 @@ Parameters:
   APIRouterImageURI:
     Type: String
     Description: The URI of OpenAI compatbile API router image. If provided, the router will be deployed.
-    Default: ""
+    Default: "public.ecr.aws/aws-gcr-solutions/dmaa/api-router:latest"
   UseSpot:
     Type: String
     Description: Use Fargate Spot capacity?
@@ -28,9 +28,11 @@ Resources:
   ECSCluster:
     Type: AWS::ECS::Cluster
     Properties:
+      ServiceConnectDefaults:
+        Namespace: emd-service-connect-namespace
       ClusterSettings:
         - Name: containerInsights
-          Value: enabled
+          Value: disabled
   PublicLoadBalancerSecurityGroup:
     Type: AWS::EC2::SecurityGroup
     Properties:
@@ -219,6 +221,25 @@ Resources:
               # Send response back to CloudFormation
               cfnresponse.send(event, context, response_status, response_data)
 
+          def convert_dns_name(event, context):
+              """
+              Converts a DNS name to comply with AWS Service Connect naming rules.
+              DNS names can only contain lowercase letters and numbers.
+              """
+              import re
+
+              # Get the input DNS name from the event
+              dns_name = event['ResourceProperties'].get('ModelName', '')
+
+              # Step 1: Convert to lowercase
+              dns_name = dns_name.lower()
+
+              # Step 2: Replace any character that is not a lowercase letter or number with a hyphen
+              dns_name = re.sub(r'[^a-z0-9]', '-', dns_name)
+
+              response_data = {'DnsName': dns_name}
+              cfnresponse.send(event, context, cfnresponse.SUCCESS, response_data)
+
           def handler(event, context):
               print(event)
               print(context)
@@ -371,6 +392,9 @@ Resources:
             Weight: 1
           - CapacityProvider: FARGATE
             Weight: 1
+      ServiceConnectConfiguration:
+        Enabled: true
+        Namespace: emd-service-connect-namespace
 
   APIRouterSecurityGroup:
     Type: AWS::EC2::SecurityGroup
@@ -472,3 +496,6 @@ Outputs:
   APIRouterTargetGroupArn:
     Description: The ARN of the OpenAI compatbile API outer Target Group
     Value: !If [DeployAPIRouterCondition, !Ref APIRouterTargetGroup, ""]
+  APIRouterSecurityGroup:
+    Description: The security group for the OpenAI compatbile API router
+    Value: !GetAtt APIRouterSecurityGroup.GroupId