Set default FT configs for unverified models and update validation API (#863)

Author: VipulMascarenhas

ads/aqua/config/config.py

@@ -2,3 +2,17 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+# TODO: move this to global config.json in object storage
+def get_finetuning_config_defaults():
+    """Generate and return the fine-tuning default configuration dictionary."""
+    return {
+        "shape": {
+            "VM.GPU.A10.1": {"batch_size": 1, "replica": "1-10"},
+            "VM.GPU.A10.2": {"batch_size": 1, "replica": "1-10"},
+            "BM.GPU.A10.4": {"batch_size": 1, "replica": 1},
+            "BM.GPU4.8": {"batch_size": 4, "replica": 1},
+            "BM.GPU.A100-v2.8": {"batch_size": 6, "replica": 1},
+        }
+    }

ads/aqua/finetuning/finetuning.py

@@ -5,7 +5,7 @@
 
 import json
 import os
-from dataclasses import asdict, fields
+from dataclasses import asdict, fields, MISSING
 from typing import Dict
 
 from oci.data_science.models import (
@@ -21,7 +21,6 @@
 from ads.aqua.common.utils import (
     get_container_image,
     upload_local_to_os,
-    get_params_dict,
 )
 from ads.aqua.constants import (
     DEFAULT_FT_BATCH_SIZE,
@@ -32,6 +31,7 @@
     UNKNOWN,
     UNKNOWN_DICT,
 )
+from ads.aqua.config.config import get_finetuning_config_defaults
 from ads.aqua.data import AquaResourceIdentifier
 from ads.aqua.finetuning.constants import *
 from ads.aqua.finetuning.entities import *
@@ -553,7 +553,11 @@ def get_finetuning_config(self, model_id: str) -> Dict:
             A dict of allowed finetuning configs.
         """
 
-        return self.get_config(model_id, AQUA_MODEL_FINETUNING_CONFIG)
+        config = self.get_config(model_id, AQUA_MODEL_FINETUNING_CONFIG)
+        if not config:
+            logger.info(f"Fetching default fine-tuning config for model: {model_id}")
+            config = get_finetuning_config_defaults()
+        return config
 
     @telemetry(
         entry_point="plugin=finetuning&action=get_finetuning_default_params",
@@ -586,31 +590,32 @@ def get_finetuning_default_params(self, model_id: str) -> Dict:
 
         return default_params
 
-    def validate_finetuning_params(self, params: List[str] = None) -> Dict:
+    def validate_finetuning_params(self, params: Dict = None) -> Dict:
         """Validate if the fine-tuning parameters passed by the user can be overridden. Parameter values are not
         validated, only param keys are validated.
 
         Parameters
         ----------
-        params : List[str], optional
+        params :Dict, optional
             Params passed by the user.
 
         Returns
         -------
             Return a list of restricted params.
         """
-        restricted_params = []
-        if params:
-            dataclass_fields = {field.name for field in fields(AquaFineTuningParams)}
-            params_dict = get_params_dict(params)
-            for key, items in params_dict.items():
-                key = key.lstrip("--")
-                if key not in dataclass_fields:
-                    restricted_params.append(key)
-
-        if restricted_params:
+        try:
+            AquaFineTuningParams(
+                **params,
+            )
+        except Exception as e:
+            logger.debug(str(e))
+            allowed_fine_tuning_parameters = ", ".join(
+                f"{field.name} (required)" if field.default is MISSING else field.name
+                for field in fields(AquaFineTuningParams)
+            ).rstrip()
             raise AquaValueError(
-                f"Parameters {restricted_params} are set by Aqua "
-                f"and cannot be overridden or are invalid."
+                f"Invalid fine tuning parameters. Allowable parameters are: "
+                f"{allowed_fine_tuning_parameters}."
             )
+
         return dict(valid=True)

tests/unitary/with_extras/aqua/test_finetuning.py

@@ -26,6 +26,7 @@
 from ads.model.datascience_model import DataScienceModel
 from ads.model.model_metadata import ModelCustomMetadata
 from ads.aqua.common.errors import AquaValueError
+from ads.aqua.config.config import get_finetuning_config_defaults
 
 
 class FineTuningTestCase(TestCase):
@@ -234,6 +235,21 @@ def test_build_oci_launch_cmd(self):
             == f"--training_data {dataset_path} --output_dir {report_path} --val_set_size {val_set_size} --num_epochs {parameters.epochs} --learning_rate {parameters.learning_rate} --sample_packing {parameters.sample_packing} --micro_batch_size {parameters.batch_size} --sequence_len {parameters.sequence_len} --lora_target_modules q_proj,k_proj {finetuning_params}"
         )
 
+    def test_get_finetuning_config(self):
+        """Test for fetching config details for a given model to be finetuned."""
+
+        config_json = os.path.join(self.curr_dir, "test_data/finetuning/ft_config.json")
+        with open(config_json, "r") as _file:
+            config = json.load(_file)
+
+        self.app.get_config = MagicMock(return_value=config)
+        result = self.app.get_finetuning_config(model_id="test-model-id")
+        assert result == config
+
+        self.app.get_config = MagicMock(return_value=None)
+        result = self.app.get_finetuning_config(model_id="test-model-id")
+        assert result == get_finetuning_config_defaults()
+
     def test_get_finetuning_default_params(self):
         """Test for fetching finetuning config params for a given model."""
 
@@ -267,28 +283,28 @@ def test_get_finetuning_default_params(self):
     @parameterized.expand(
         [
             (
-                [
-                    "--batch_size 1",
-                    "--sequence_len 2048",
-                    "--sample_packing true",
-                    "--pad_to_sequence_len true",
-                    "--learning_rate 0.0002",
-                    "--lora_r 32",
-                    "--lora_alpha 16",
-                    "--lora_dropout 0.05",
-                    "--lora_target_linear true",
-                    "--lora_target_modules q_proj,k_proj",
-                ],
+                {
+                    "epochs": 1,
+                    "learning_rate": 0.0002,
+                    "batch_size": 1,
+                    "sequence_len": 2048,
+                    "sample_packing": True,
+                    "pad_to_sequence_len": True,
+                    "lora_alpha": 16,
+                    "lora_dropout": 0.05,
+                    "lora_target_linear": True,
+                    "lora_target_modules": ["q_proj", " k_proj"],
+                },
                 True,
             ),
             (
-                [
-                    "--micro_batch_size 1",
-                    "--max_sequence_len 2048",
-                    "--flash_attention true",
-                    "--pad_to_sequence_len true",
-                    "--lr_scheduler cosine",
-                ],
+                {
+                    "micro_batch_size": 1,
+                    "max_sequence_len": 2048,
+                    "flash_attention": True,
+                    "pad_to_sequence_len": True,
+                    "lr_scheduler": "cosine",
+                },
                 False,
             ),
         ]