Skip to content

Commit 474b011

Browse files
lu-ohailu-ohai
committed
Updated pr.
1 parent 809a519 commit 474b011

File tree

6 files changed

+284
-386
lines changed

6 files changed

+284
-386
lines changed

‎ads/common/dsc_file_system.py

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
#!/usr/bin/env python
2+
# -*- coding: utf-8; -*-
3+
4+
# Copyright (c) 2023 Oracle and/or its affiliates.
5+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
6+
import ads
7+
import oci
8+
import ipaddress
9+
10+
from dataclasses import dataclass
11+
12+
FILE_STORAGE_TYPE = "FILE_STORAGE"
13+
14+
15+
@dataclass
16+
class DSCFileSystem:
17+
18+
src: str = None
19+
dest: str = None
20+
storage_type: str = None
21+
destination_directory_name: str = None
22+
23+
def update_to_dsc_model(self) -> dict:
24+
"""Updates arguments to dsc model.
25+
26+
Returns
27+
-------
28+
dict:
29+
A dictionary of arguments.
30+
"""
31+
pass
32+
33+
@classmethod
34+
def update_from_dsc_model(cls, dsc_model) -> dict:
35+
"""Updates arguments from dsc model.
36+
37+
Parameters
38+
----------
39+
dsc_model: oci.data_science.models.JobStorageMountConfigurationDetails
40+
An instance of oci.data_science.models.JobStorageMountConfigurationDetails.
41+
42+
Returns
43+
-------
44+
dict
45+
A dictionary of arguments.
46+
"""
47+
pass
48+
49+
50+
@dataclass
51+
class OCIFileStorage(DSCFileSystem):
52+
53+
mount_target_id: str = None
54+
mount_target: str = None
55+
export_id: str = None
56+
export_path: str = None
57+
storage_type: str = FILE_STORAGE_TYPE
58+
59+
def update_to_dsc_model(self) -> dict:
60+
"""Updates arguments to dsc model.
61+
62+
Returns
63+
-------
64+
dict:
65+
A dictionary of arguments.
66+
"""
67+
arguments = {
68+
"destinationDirectoryName" : self.dest,
69+
"storageType" : self.storage_type
70+
}
71+
72+
self._get_mount_target_and_export_ids(arguments)
73+
74+
return arguments
75+
76+
def _get_mount_target_and_export_ids(self, arguments: dict):
77+
"""Gets the mount target id and export id from src.
78+
79+
Parameters
80+
----------
81+
arguments: dict
82+
A dictionary of arguments.
83+
"""
84+
resource_client = oci.resource_search.ResourceSearchClient(**ads.auth.default_signer())
85+
src_list = self.src.split(":")
86+
first_segment = src_list[0]
87+
second_segment = src_list[1]
88+
89+
if first_segment.startswith("ocid") and "mounttarget" in first_segment:
90+
arguments["mountTargetId"] = first_segment
91+
else:
92+
ip_resource = resource_client.search_resources(
93+
search_details=oci.resource_search.models.FreeTextSearchDetails(
94+
text=first_segment,
95+
matching_context_type="NONE"
96+
)
97+
).data.items
98+
99+
ip_resource = sorted(ip_resource, key=lambda resource_summary: resource_summary.time_created)
100+
101+
if not ip_resource or not hasattr(ip_resource[-1], "identifier"):
102+
raise ValueError(f"Can't find the identifier from ip {first_segment}. Specify a valid `src`.")
103+
104+
mount_target_resource = resource_client.search_resources(
105+
search_details=oci.resource_search.models.FreeTextSearchDetails(
106+
text=ip_resource[-1].identifier,
107+
matching_context_type="NONE"
108+
)
109+
).data.items
110+
111+
mount_targets = [
112+
mount_target.identifier
113+
for mount_target in mount_target_resource
114+
if mount_target.resource_type == "MountTarget"
115+
]
116+
if len(mount_targets) == 0:
117+
raise ValueError(
118+
f"No `mount_target_id` found under ip {first_segment}. Specify a valid `src`."
119+
)
120+
if len(mount_targets) > 1:
121+
raise ValueError(
122+
f"Multiple `mount_target_id` found under ip {first_segment}. Specify the `mount_target_id` in `src` instead."
123+
)
124+
125+
arguments["mountTargetId"] = mount_targets[0]
126+
127+
if second_segment.startswith("ocid") and "export" in second_segment:
128+
arguments["exportId"] = second_segment
129+
else:
130+
export_resource = resource_client.search_resources(
131+
search_details=oci.resource_search.models.FreeTextSearchDetails(
132+
text=second_segment,
133+
matching_context_type="NONE"
134+
)
135+
).data.items
136+
137+
exports = [
138+
export.identifier
139+
for export in export_resource
140+
if export.resource_type == "Export"
141+
]
142+
if len(exports) == 0:
143+
raise ValueError(
144+
f"No `export_id` found with `export_path` {second_segment}. Specify a valid `src`."
145+
)
146+
if len(exports) > 1:
147+
raise ValueError(
148+
f"Multiple `export_id` found with `export_path` {second_segment}. Specify the `export_id` in `src` instead."
149+
)
150+
151+
arguments["exportId"] = exports[0]
152+
153+
@classmethod
154+
def update_from_dsc_model(cls, dsc_model) -> dict:
155+
"""Updates arguments from dsc model.
156+
157+
Parameters
158+
----------
159+
dsc_model: oci.data_science.models.JobStorageMountConfigurationDetails
160+
An instance of oci.data_science.models.JobStorageMountConfigurationDetails.
161+
162+
Returns
163+
-------
164+
dict
165+
A dictionary of arguments.
166+
"""
167+
argument = {
168+
"storageType": dsc_model.storage_type,
169+
"mountTargetId": dsc_model.mount_target_id,
170+
"exportId": dsc_model.export_id,
171+
"destinationDirectoryName": dsc_model.destination_directory_name
172+
}
173+
174+
file_storage_client = oci.file_storage.FileStorageClient(
175+
**ads.auth.default_signer()
176+
)
177+
if not dsc_model.mount_target_id:
178+
raise ValueError(
179+
"Missing parameter `mount_target_id` from service. Check service log to see the error."
180+
)
181+
argument["mountTarget"] = file_storage_client.get_mount_target(
182+
mount_target_id=dsc_model.mount_target_id
183+
).data.display_name
184+
if not dsc_model.export_id:
185+
raise ValueError(
186+
"Missing parameter `export_id` from service. Check service log to see the error."
187+
)
188+
argument["exportPath"] = file_storage_client.get_export(
189+
export_id=dsc_model.export_id
190+
).data.path
191+
192+
return argument
193+
194+
195+
class DSCFileSystemManager:
196+
197+
@classmethod
198+
def initialize(cls, arguments: dict) -> dict:
199+
"""Initialize and update arguments to dsc model.
200+
201+
Parameters
202+
----------
203+
arguments: dict
204+
A dictionary of arguments.
205+
"""
206+
if "src" not in arguments:
207+
raise ValueError(
208+
"Parameter `src` is required for mounting file storage system."
209+
)
210+
211+
if "dest" not in arguments:
212+
raise ValueError(
213+
"Parameter `dest` is required for mounting file storage system."
214+
)
215+
216+
first_segment = arguments["src"].split(":")[0]
217+
# case <mount_target_id>:<export_id>
218+
if first_segment.startswith("ocid"):
219+
return OCIFileStorage(**arguments).update_to_dsc_model()
220+
221+
# case <ip_address>:<export_path>
222+
try:
223+
ipaddress.IPv4Network(first_segment)
224+
return OCIFileStorage(**arguments).update_to_dsc_model()
225+
except:
226+
pass
227+
228+
raise ValueError("Invalid dict for mounting file systems. Specify a valid one.")

‎ads/jobs/__init__.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/usr/bin/env python
22
# -*- coding: utf-8; -*-
33

4-
# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
4+
# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
55
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
66

77

@@ -23,7 +23,6 @@
2323
from ads.jobs.builders.infrastructure.dsc_job import (
2424
DataScienceJob,
2525
DataScienceJobRun,
26-
OCIFileStorage,
2726
)
2827
from ads.jobs.builders.infrastructure.dataflow import DataFlow, DataFlowRun
2928
except AttributeError as e:
@@ -49,5 +48,4 @@
4948
"DataFlowRun",
5049
"DataFlowRuntime",
5150
"DataFlowNotebookRuntime",
52-
"OCIFileStorage",
5351
]

‎ads/jobs/builders/infrastructure/__init__.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env python
22
# -*- coding: utf-8; -*-
33

4-
# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
4+
# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
55
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
6-
from .dsc_job import DataScienceJob, OCIFileStorage
6+
from .dsc_job import DataScienceJob

0 commit comments

Comments
 (0)