Added auto validation for load balancer.

Author: lu-ohai

ads/model/deployment/model_deployment.py

@@ -8,6 +8,7 @@
 import collections
 import copy
 import datetime
+import sys
 import oci
 import warnings
 import time
@@ -70,6 +71,9 @@
 MODEL_DEPLOYMENT_INSTANCE_COUNT = 1
 MODEL_DEPLOYMENT_BANDWIDTH_MBPS = 10
 
+TIME_FRAME = 60
+MAXIMUM_PAYLOAD_SIZE = 10 * 1024 * 1024 # bytes
+
 MODEL_DEPLOYMENT_RUNTIMES = {
     ModelDeploymentRuntimeType.CONDA: ModelDeploymentCondaRuntime,
     ModelDeploymentRuntimeType.CONTAINER: ModelDeploymentContainerRuntime,
@@ -253,6 +257,10 @@ class ModelDeployment(Builder):
         CONST_TIME_CREATED: "time_created",
     }
 
+    count_start_time = 0
+    request_counter = 0
+    estimate_request_per_second = 100
+
     initialize_spec_attributes = [
         "display_name",
         "description",
@@ -911,6 +919,8 @@ def predict(
             raise AttributeError(
                 "`data` and `json_input` are both provided. You can only use one of them."
             )
+        
+        self._validate_bandwidth(data or json_input)
 
         if auto_serialize_data:
             data = data or json_input
@@ -1794,6 +1804,45 @@ def _extract_spec_kwargs(self, **kwargs) -> Dict:
             if attribute in kwargs:
                 spec_kwargs[attribute] = kwargs[attribute]
         return spec_kwargs
+    
+    def _validate_bandwidth(self, data: Any):
+        """Validates payload size and load balancer bandwidth.
+
+        Parameters
+        ----------
+        data: Any
+            Data or JSON payload for the prediction.
+        """
+        payload_size = sys.getsizeof(data)
+        if payload_size > MAXIMUM_PAYLOAD_SIZE:
+            raise ValueError(
+                f"Payload size exceeds the maximum allowed {MAXIMUM_PAYLOAD_SIZE} bytes. Size down the payload."
+            )
+                        
+        time_now = int(time.time())
+        if self.count_start_time == 0:
+            self.count_start_time = time_now
+        if time_now - self.count_start_time < TIME_FRAME:
+            self.request_counter += 1
+        else:
+            self.estimate_request_per_second = (int)(self.request_counter / TIME_FRAME)
+            self.request_counter = 0
+            self.count_start_time = 0
+
+        if not self.infrastructure or not self.runtime:
+            raise ValueError("Missing parameter infrastructure or runtime. Try reruning it after parameters are fully configured.")
+        
+        # load balancer bandwidth is only needed for HTTPS mode.
+        if self.runtime.deployment_mode == ModelDeploymentMode.HTTPS:
+            bandwidth_mbps = self.infrastructure.bandwidth_mbps or MODEL_DEPLOYMENT_BANDWIDTH_MBPS
+            # formula: (payload size in KB) * (estimated requests per second) * 8 / 1024
+            # 20% extra for estimation errors and sporadic peak traffic
+            payload_size_in_kb = payload_size / 1024
+            if (payload_size_in_kb * self.estimate_request_per_second * 8 * 1.2) / 1024 > bandwidth_mbps:
+                raise ValueError(
+                    f"Load balancer bandwidth exceeds the allocated {bandwidth_mbps} Mbps."
+                    "Try sizing down the payload, slowing down the request rate or increasing bandwidth."
+                )
 
     def build(self) -> "ModelDeployment":
         """Load default values from the environment for the job infrastructure."""

tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py

@@ -6,10 +6,12 @@
 
 import copy
 from datetime import datetime
+import time
 import oci
 import pytest
 import unittest
 import pandas
+import sys
 from unittest.mock import MagicMock, patch
 from ads.common.oci_datascience import OCIDataScienceMixin
 from ads.common.oci_logging import ConsolidatedLog, OCILog
@@ -19,6 +21,7 @@
 from ads.model.datascience_model import DataScienceModel
 
 from ads.model.deployment.model_deployment import (
+    MAXIMUM_PAYLOAD_SIZE,
     ModelDeployment,
     ModelDeploymentLogType,
     ModelDeploymentFailedError,
@@ -1480,3 +1483,33 @@ def test_model_deployment_with_large_size_artifact(
         )
         mock_create_model_deployment.assert_called_with(create_model_deployment_details)
         mock_sync.assert_called()
+
+    @patch.object(sys, "getsizeof")
+    def test_validate_bandwidth(self, mock_get_size_of):
+        model_deployment = self.initialize_model_deployment()
+
+        mock_get_size_of.return_value = 11 * 1024 * 1024
+        with pytest.raises(
+            ValueError,
+            match=f"Payload size exceeds the maximum allowed {MAXIMUM_PAYLOAD_SIZE} bytes. Size down the payload."
+        ):
+            model_deployment._validate_bandwidth("test")
+            mock_get_size_of.assert_called()
+
+        mock_get_size_of.return_value = 9 * 1024 * 1024
+        with pytest.raises(
+            ValueError,
+            match=f"Load balancer bandwidth exceeds the allocated {model_deployment.infrastructure.bandwidth_mbps} Mbps."
+                    "Try sizing down the payload, slowing down the request rate or increasing bandwidth."
+        ):
+            model_deployment._validate_bandwidth("test")
+            mock_get_size_of.assert_called()
+
+        mock_get_size_of.return_value = 5
+        model_deployment._validate_bandwidth("test")
+        mock_get_size_of.assert_called()
+
+        model_deployment.count_start_time = (int)(time.time()) - 700
+        model_deployment._validate_bandwidth("test")
+        mock_get_size_of.assert_called()
+