20
20
from ads .common import utils
21
21
from ads .common .extended_enum import ExtendedEnumMeta
22
22
from ads .common .object_storage_details import ObjectStorageDetails
23
- from ads .config import (
24
- AQUA_SERVICE_MODELS_BUCKET as SERVICE_MODELS_BUCKET ,
25
- )
26
23
from ads .config import (
27
24
COMPARTMENT_OCID ,
28
25
PROJECT_OCID ,
26
+ AQUA_SERVICE_MODELS_BUCKET as SERVICE_MODELS_BUCKET ,
29
27
)
30
28
from ads .feature_engineering .schema import Schema
31
29
from ads .jobs .builders .base import Builder
48
46
49
47
logger = logging .getLogger (__name__ )
50
48
49
+
51
50
_MAX_ARTIFACT_SIZE_IN_BYTES = 2147483648 # 2GB
52
51
MODEL_BY_REFERENCE_VERSION = "1.0"
53
52
MODEL_BY_REFERENCE_JSON_FILE_NAME = "model_description.json"
@@ -65,8 +64,8 @@ def __init__(self, max_artifact_size: str):
65
64
66
65
class BucketNotVersionedError (Exception ): # pragma: no cover
67
66
def __init__ (
68
- self ,
69
- msg = "Model artifact bucket is not versioned. Enable versioning on the bucket to proceed with model creation by reference." ,
67
+ self ,
68
+ msg = "Model artifact bucket is not versioned. Enable versioning on the bucket to proceed with model creation by reference." ,
70
69
):
71
70
super ().__init__ (msg )
72
71
@@ -527,6 +526,7 @@ class DataScienceModel(Builder):
527
526
Sets path details for models created by reference. Input can be either a dict, string or json file and
528
527
the schema is dictated by model_file_description_schema.json
529
528
529
+
530
530
Examples
531
531
--------
532
532
>>> ds_model = (DataScienceModel()
@@ -796,7 +796,7 @@ def defined_tags(self) -> Dict[str, Dict[str, object]]:
796
796
return self .get_spec (self .CONST_DEFINED_TAG )
797
797
798
798
def with_defined_tags (
799
- self , ** kwargs : Dict [str , Dict [str , object ]]
799
+ self , ** kwargs : Dict [str , Dict [str , object ]]
800
800
) -> "DataScienceModel" :
801
801
"""Sets defined tags.
802
802
@@ -877,7 +877,7 @@ def defined_metadata_list(self) -> ModelTaxonomyMetadata:
877
877
return self .get_spec (self .CONST_DEFINED_METADATA )
878
878
879
879
def with_defined_metadata_list (
880
- self , metadata : Union [ModelTaxonomyMetadata , Dict ]
880
+ self , metadata : Union [ModelTaxonomyMetadata , Dict ]
881
881
) -> "DataScienceModel" :
882
882
"""Sets model taxonomy (defined) metadata.
883
883
@@ -901,7 +901,7 @@ def custom_metadata_list(self) -> ModelCustomMetadata:
901
901
return self .get_spec (self .CONST_CUSTOM_METADATA )
902
902
903
903
def with_custom_metadata_list (
904
- self , metadata : Union [ModelCustomMetadata , Dict ]
904
+ self , metadata : Union [ModelCustomMetadata , Dict ]
905
905
) -> "DataScienceModel" :
906
906
"""Sets model custom metadata.
907
907
@@ -925,7 +925,7 @@ def provenance_metadata(self) -> ModelProvenanceMetadata:
925
925
return self .get_spec (self .CONST_PROVENANCE_METADATA )
926
926
927
927
def with_provenance_metadata (
928
- self , metadata : Union [ModelProvenanceMetadata , Dict ]
928
+ self , metadata : Union [ModelProvenanceMetadata , Dict ]
929
929
) -> "DataScienceModel" :
930
930
"""Sets model provenance metadata.
931
931
@@ -1018,7 +1018,7 @@ def model_file_description(self) -> dict:
1018
1018
return self .get_spec (self .CONST_MODEL_FILE_DESCRIPTION )
1019
1019
1020
1020
def with_model_file_description (
1021
- self , json_dict : dict = None , json_string : str = None , json_uri : str = None
1021
+ self , json_dict : dict = None , json_string : str = None , json_uri : str = None
1022
1022
):
1023
1023
"""Sets the json file description for model passed by reference
1024
1024
Parameters
@@ -1041,7 +1041,7 @@ def with_model_file_description(
1041
1041
elif json_string :
1042
1042
json_data = json .loads (json_string )
1043
1043
elif json_uri :
1044
- with open (json_uri ) as json_file :
1044
+ with open (json_uri , "r" ) as json_file :
1045
1045
json_data = json .load (json_file )
1046
1046
else :
1047
1047
raise ValueError ("Must provide either a valid json string or URI location." )
@@ -1256,15 +1256,15 @@ def create(self, **kwargs) -> "DataScienceModel":
1256
1256
return self
1257
1257
1258
1258
def upload_artifact (
1259
- self ,
1260
- bucket_uri : Optional [str ] = None ,
1261
- auth : Optional [Dict ] = None ,
1262
- region : Optional [str ] = None ,
1263
- overwrite_existing_artifact : Optional [bool ] = True ,
1264
- remove_existing_artifact : Optional [bool ] = True ,
1265
- timeout : Optional [int ] = None ,
1266
- parallel_process_count : int = utils .DEFAULT_PARALLEL_PROCESS_COUNT ,
1267
- model_by_reference : Optional [bool ] = False ,
1259
+ self ,
1260
+ bucket_uri : Optional [str ] = None ,
1261
+ auth : Optional [Dict ] = None ,
1262
+ region : Optional [str ] = None ,
1263
+ overwrite_existing_artifact : Optional [bool ] = True ,
1264
+ remove_existing_artifact : Optional [bool ] = True ,
1265
+ timeout : Optional [int ] = None ,
1266
+ parallel_process_count : int = utils .DEFAULT_PARALLEL_PROCESS_COUNT ,
1267
+ model_by_reference : Optional [bool ] = False ,
1268
1268
) -> None :
1269
1269
"""Uploads model artifacts to the model catalog.
1270
1270
@@ -1334,7 +1334,7 @@ def upload_artifact(
1334
1334
bucket_uri = self .artifact
1335
1335
1336
1336
if not model_by_reference and (
1337
- bucket_uri or utils .folder_size (self .artifact ) > _MAX_ARTIFACT_SIZE_IN_BYTES
1337
+ bucket_uri or utils .folder_size (self .artifact ) > _MAX_ARTIFACT_SIZE_IN_BYTES
1338
1338
):
1339
1339
if not bucket_uri :
1340
1340
raise ModelArtifactSizeError (
@@ -1405,15 +1405,15 @@ def restore_model(
1405
1405
)
1406
1406
1407
1407
def download_artifact (
1408
- self ,
1409
- target_dir : str ,
1410
- auth : Optional [Dict ] = None ,
1411
- force_overwrite : Optional [bool ] = False ,
1412
- bucket_uri : Optional [str ] = None ,
1413
- region : Optional [str ] = None ,
1414
- overwrite_existing_artifact : Optional [bool ] = True ,
1415
- remove_existing_artifact : Optional [bool ] = True ,
1416
- timeout : Optional [int ] = None ,
1408
+ self ,
1409
+ target_dir : str ,
1410
+ auth : Optional [Dict ] = None ,
1411
+ force_overwrite : Optional [bool ] = False ,
1412
+ bucket_uri : Optional [str ] = None ,
1413
+ region : Optional [str ] = None ,
1414
+ overwrite_existing_artifact : Optional [bool ] = True ,
1415
+ remove_existing_artifact : Optional [bool ] = True ,
1416
+ timeout : Optional [int ] = None ,
1417
1417
):
1418
1418
"""Downloads model artifacts from the model catalog.
1419
1419
@@ -1488,9 +1488,9 @@ def download_artifact(
1488
1488
)
1489
1489
1490
1490
if (
1491
- artifact_size > _MAX_ARTIFACT_SIZE_IN_BYTES
1492
- or bucket_uri
1493
- or model_by_reference
1491
+ artifact_size > _MAX_ARTIFACT_SIZE_IN_BYTES
1492
+ or bucket_uri
1493
+ or model_by_reference
1494
1494
):
1495
1495
artifact_downloader = LargeArtifactDownloader (
1496
1496
dsc_model = self .dsc_model ,
@@ -1536,22 +1536,21 @@ def update(self, **kwargs) -> "DataScienceModel":
1536
1536
self .dsc_model = self ._to_oci_dsc_model (** kwargs ).update ()
1537
1537
1538
1538
logger .debug (f"Updating a model provenance metadata { self .provenance_metadata } " )
1539
- if self .provenance_metadata :
1540
- try :
1541
- self .dsc_model .get_model_provenance ()
1542
- self .dsc_model .update_model_provenance (
1543
- self .provenance_metadata ._to_oci_metadata ()
1544
- )
1545
- except ModelProvenanceNotFoundError :
1546
- self .dsc_model .create_model_provenance (
1547
- self .provenance_metadata ._to_oci_metadata ()
1548
- )
1539
+ try :
1540
+ self .dsc_model .get_model_provenance ()
1541
+ self .dsc_model .update_model_provenance (
1542
+ self .provenance_metadata ._to_oci_metadata ()
1543
+ )
1544
+ except ModelProvenanceNotFoundError :
1545
+ self .dsc_model .create_model_provenance (
1546
+ self .provenance_metadata ._to_oci_metadata ()
1547
+ )
1549
1548
1550
1549
return self .sync ()
1551
1550
1552
1551
def delete (
1553
- self ,
1554
- delete_associated_model_deployment : Optional [bool ] = False ,
1552
+ self ,
1553
+ delete_associated_model_deployment : Optional [bool ] = False ,
1555
1554
) -> "DataScienceModel" :
1556
1555
"""Removes model from the model catalog.
1557
1556
@@ -1570,7 +1569,7 @@ def delete(
1570
1569
1571
1570
@classmethod
1572
1571
def list (
1573
- cls , compartment_id : str = None , project_id : str = None , ** kwargs
1572
+ cls , compartment_id : str = None , project_id : str = None , ** kwargs
1574
1573
) -> List ["DataScienceModel" ]:
1575
1574
"""Lists datascience models in a given compartment.
1576
1575
@@ -1597,7 +1596,7 @@ def list(
1597
1596
1598
1597
@classmethod
1599
1598
def list_df (
1600
- cls , compartment_id : str = None , project_id : str = None , ** kwargs
1599
+ cls , compartment_id : str = None , project_id : str = None , ** kwargs
1601
1600
) -> "pandas.DataFrame" :
1602
1601
"""Lists datascience models in a given compartment.
1603
1602
@@ -1617,7 +1616,7 @@ def list_df(
1617
1616
"""
1618
1617
records = []
1619
1618
for model in OCIDataScienceModel .list_resource (
1620
- compartment_id , project_id = project_id , ** kwargs
1619
+ compartment_id , project_id = project_id , ** kwargs
1621
1620
):
1622
1621
records .append (
1623
1622
{
@@ -1660,8 +1659,6 @@ def _init_complex_attributes(self):
1660
1659
self .with_provenance_metadata (self .provenance_metadata )
1661
1660
self .with_input_schema (self .input_schema )
1662
1661
self .with_output_schema (self .output_schema )
1663
- # self.with_backup_setting(self.backup_setting)
1664
- # self.with_retention_setting(self.retention_setting)
1665
1662
1666
1663
def _to_oci_dsc_model (self , ** kwargs ):
1667
1664
"""Creates an `OCIDataScienceModel` instance from the `DataScienceModel`.
@@ -1700,7 +1697,7 @@ def _to_oci_dsc_model(self, **kwargs):
1700
1697
return OCIDataScienceModel (** dsc_spec )
1701
1698
1702
1699
def _update_from_oci_dsc_model (
1703
- self , dsc_model : OCIDataScienceModel
1700
+ self , dsc_model : OCIDataScienceModel
1704
1701
) -> "DataScienceModel" :
1705
1702
"""Update the properties from an OCIDataScienceModel object.
1706
1703
@@ -1973,12 +1970,12 @@ def _download_file_description_artifact(self) -> Tuple[Union[str, List[str]], in
1973
1970
return bucket_uri [0 ] if len (bucket_uri ) == 1 else bucket_uri , artifact_size
1974
1971
1975
1972
def add_artifact (
1976
- self ,
1977
- uri : Optional [str ] = None ,
1978
- namespace : Optional [str ] = None ,
1979
- bucket : Optional [str ] = None ,
1980
- prefix : Optional [str ] = None ,
1981
- files : Optional [List [str ]] = None ,
1973
+ self ,
1974
+ uri : Optional [str ] = None ,
1975
+ namespace : Optional [str ] = None ,
1976
+ bucket : Optional [str ] = None ,
1977
+ prefix : Optional [str ] = None ,
1978
+ files : Optional [List [str ]] = None ,
1982
1979
):
1983
1980
"""
1984
1981
Adds information about objects in a specified bucket to the model description JSON.
@@ -2127,11 +2124,11 @@ def list_obj_versions_unpaginated():
2127
2124
self .set_spec (self .CONST_MODEL_FILE_DESCRIPTION , tmp_model_file_description )
2128
2125
2129
2126
def remove_artifact (
2130
- self ,
2131
- uri : Optional [str ] = None ,
2132
- namespace : Optional [str ] = None ,
2133
- bucket : Optional [str ] = None ,
2134
- prefix : Optional [str ] = None ,
2127
+ self ,
2128
+ uri : Optional [str ] = None ,
2129
+ namespace : Optional [str ] = None ,
2130
+ bucket : Optional [str ] = None ,
2131
+ prefix : Optional [str ] = None ,
2135
2132
):
2136
2133
"""
2137
2134
Removes information about objects in a specified bucket or using a specified URI from the model description JSON.
0 commit comments