8
8
import ads
9
9
from ads .common import logger
10
10
11
+
11
12
class ModelDescription :
12
13
13
14
empty_json = {
14
15
"version" : "1.0" ,
15
16
"type" : "modelOSSReferenceDescription" ,
16
- "models" : []
17
+ "models" : [],
17
18
}
18
-
19
- def auth (self ):
19
+
20
+ def auth (self ):
20
21
authData = ads .common .auth .default_signer ()
21
- signer = authData [' signer' ]
22
- self .region = authData [' config' ][ ' region' ]
22
+ signer = authData [" signer" ]
23
+ self .region = authData [" config" ][ " region" ]
23
24
24
25
# data science client
25
- self .data_science_client = oci .data_science .DataScienceClient ({'region' : self .region }, signer = signer )
26
+ self .data_science_client = oci .data_science .DataScienceClient (
27
+ {"region" : self .region }, signer = signer
28
+ )
26
29
# oss client
27
- self .object_storage_client = oci .object_storage .ObjectStorageClient ({'region' : self .region }, signer = signer )
28
-
30
+ self .object_storage_client = oci .object_storage .ObjectStorageClient (
31
+ {"region" : self .region }, signer = signer
32
+ )
33
+
29
34
def __init__ (self , model_ocid = None ):
30
35
31
- self .region = ''
36
+ self .region = ""
32
37
self .auth ()
33
38
34
- if model_ocid == None :
39
+ if model_ocid == None :
35
40
# if no model given then start from scratch
36
41
self .modelDescriptionJson = self .empty_json
37
42
else :
38
43
# if model given then get that as the starting reference point
39
44
logger .info ("Getting model details from backend" )
40
45
try :
41
- get_model_artifact_content_response = self .data_science_client .get_model_artifact_content (
42
- model_id = model_ocid ,
46
+ get_model_artifact_content_response = (
47
+ self .data_science_client .get_model_artifact_content (
48
+ model_id = model_ocid ,
49
+ )
43
50
)
44
51
content = get_model_artifact_content_response .data .content
45
52
self .modelDescriptionJson = json .loads (content )
@@ -57,16 +64,18 @@ def add(self, namespace, bucket, prefix=None, files=None):
57
64
def checkIfFileExists (fileName ):
58
65
isExists = False
59
66
try :
60
- headResponse = self .object_storage_client .head_object (namespace , bucket , object_name = fileName )
67
+ headResponse = self .object_storage_client .head_object (
68
+ namespace , bucket , object_name = fileName
69
+ )
61
70
if headResponse .status == 200 :
62
71
isExists = True
63
72
except Exception as e :
64
- if hasattr (e , ' status' ) and e .status == 404 :
73
+ if hasattr (e , " status" ) and e .status == 404 :
65
74
logger .error (f"File not found in bucket: { fileName } " )
66
75
else :
67
76
logger .error (f"An error occured: { e } " )
68
77
return isExists
69
-
78
+
70
79
# Function to un-paginate the api call with while loop
71
80
def listObjectVersionsUnpaginated ():
72
81
objectStorageList = []
@@ -77,8 +86,8 @@ def listObjectVersionsUnpaginated():
77
86
bucket_name = bucket ,
78
87
prefix = prefix ,
79
88
fields = "name,size" ,
80
- page = opc_next_page
81
- )
89
+ page = opc_next_page ,
90
+ )
82
91
objectStorageList .extend (response .data .items )
83
92
has_next_page = response .has_next_page
84
93
opc_next_page = response .next_page
@@ -91,38 +100,46 @@ def listObjectVersionsUnpaginated():
91
100
else :
92
101
for fileName in files :
93
102
if checkIfFileExists (fileName = fileName ):
94
- objectStorageList .append (self .object_storage_client .list_object_versions (
95
- namespace_name = namespace ,
96
- bucket_name = bucket ,
97
- prefix = fileName ,
98
- fields = "name,size" ,
99
- ).data .items [0 ])
100
-
101
- objects = [{
102
- "name" : obj .name ,
103
- "version" : obj .version_id ,
104
- "sizeInBytes" : obj .size
105
- } for obj in objectStorageList if obj .size > 0 ]
106
-
103
+ objectStorageList .append (
104
+ self .object_storage_client .list_object_versions (
105
+ namespace_name = namespace ,
106
+ bucket_name = bucket ,
107
+ prefix = fileName ,
108
+ fields = "name,size" ,
109
+ ).data .items [0 ]
110
+ )
111
+
112
+ objects = [
113
+ {"name" : obj .name , "version" : obj .version_id , "sizeInBytes" : obj .size }
114
+ for obj in objectStorageList
115
+ if obj .size > 0
116
+ ]
117
+
107
118
if len (objects ) == 0 :
108
119
error_message = (
109
120
f"No files to add in the bucket: { bucket } with namespace: { namespace } "
110
121
f"and prefix: { prefix } . File names: { files } "
111
122
)
112
123
logger .error (error_message )
113
124
raise ValueError (error_message )
114
-
115
- self .modelDescriptionJson ['models' ].append ({
116
- "namespace" : namespace ,
117
- "bucketName" : bucket ,
118
- "prefix" : prefix ,
119
- "objects" : objects
120
- })
121
-
125
+
126
+ self .modelDescriptionJson ["models" ].append (
127
+ {
128
+ "namespace" : namespace ,
129
+ "bucketName" : bucket ,
130
+ "prefix" : prefix ,
131
+ "objects" : objects ,
132
+ }
133
+ )
134
+
122
135
def remove (self , namespace , bucket , prefix = None ):
123
136
def findModelIdx ():
124
- for idx , model in enumerate (self .modelDescriptionJson ['models' ]):
125
- if (model ['namespace' ], model ['bucketName' ], (model ['prefix' ] if ('prefix' in model ) else None ) ) == (namespace , bucket , prefix ):
137
+ for idx , model in enumerate (self .modelDescriptionJson ["models" ]):
138
+ if (
139
+ model ["namespace" ],
140
+ model ["bucketName" ],
141
+ (model ["prefix" ] if ("prefix" in model ) else None ),
142
+ ) == (namespace , bucket , prefix ):
126
143
return idx
127
144
return - 1
128
145
@@ -131,7 +148,7 @@ def findModelIdx():
131
148
return
132
149
else :
133
150
# model found case
134
- self .modelDescriptionJson [' models' ].pop (modelSearchIdx )
151
+ self .modelDescriptionJson [" models" ].pop (modelSearchIdx )
135
152
136
153
def show (self ):
137
154
logger .info (json .dumps (self .modelDescriptionJson , indent = 4 ))
@@ -143,30 +160,37 @@ def build(self):
143
160
with open (file_path , "w" ) as json_file :
144
161
json .dump (self .modelDescriptionJson , json_file , indent = 2 )
145
162
except IOError as e :
146
- logger .error (f"Error writing to file '{ file_path } ': { e } " ) # Handle the exception accordingly, e.g., log the error, retry writing, etc.
163
+ logger .error (
164
+ f"Error writing to file '{ file_path } ': { e } "
165
+ ) # Handle the exception accordingly, e.g., log the error, retry writing, etc.
147
166
except Exception as e :
148
- logger .error (f"An unexpected error occurred: { e } " ) # Handle other unexpected exceptions
167
+ logger .error (
168
+ f"An unexpected error occurred: { e } "
169
+ ) # Handle other unexpected exceptions
149
170
logger .info ("Model Artifact stored at location: 'resultModelDescription.json'" )
150
171
return os .path .abspath (file_path )
151
-
172
+
152
173
def save (self , project_ocid , compartment_ocid , display_name = None ):
153
- display_name = 'Created by MMS SDK on ' + datetime .datetime .now (pytz .utc ).strftime ('%Y-%m-%d %H:%M:%S %Z' ) if (display_name == None ) else display_name
154
- customMetadataList = [
155
- Metadata (key = "modelDescription" , value = "true" )
156
- ]
174
+ display_name = (
175
+ "Created by MMS SDK on "
176
+ + datetime .datetime .now (pytz .utc ).strftime ("%Y-%m-%d %H:%M:%S %Z" )
177
+ if (display_name == None )
178
+ else display_name
179
+ )
180
+ customMetadataList = [Metadata (key = "modelDescription" , value = "true" )]
157
181
model_details = oci .data_science .models .CreateModelDetails (
158
- compartment_id = compartment_ocid ,
159
- project_id = project_ocid ,
160
- display_name = display_name ,
161
- custom_metadata_list = customMetadataList
182
+ compartment_id = compartment_ocid ,
183
+ project_id = project_ocid ,
184
+ display_name = display_name ,
185
+ custom_metadata_list = customMetadataList ,
162
186
)
163
187
logger .info ("Created model details" )
164
188
model = self .data_science_client .create_model (model_details )
165
189
logger .info ("Created model" )
166
190
self .data_science_client .create_model_artifact (
167
191
model .data .id ,
168
192
json .dumps (self .modelDescriptionJson ),
169
- content_disposition = 'attachment; filename="modelDescription.json"'
193
+ content_disposition = 'attachment; filename="modelDescription.json"' ,
170
194
)
171
- logger .info (' Successfully created model with OCID: ' , model .data .id )
172
- return model .data .id
195
+ logger .info (" Successfully created model with OCID: " , model .data .id )
196
+ return model .data .id
0 commit comments