Skip to content

Commit 708ce2f

Browse files
z7yez7ye
committed
Merge branch 'develop' of github.com:oracle/accelerated-data-science into ODSC-39392/triton
2 parents 3c32175 + 9378978 commit 708ce2f

28 files changed

+2950
-528
lines changed

‎ads/config.py

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
OCI_ODSC_SERVICE_ENDPOINT = os.environ.get("OCI_ODSC_SERVICE_ENDPOINT")
1515
OCI_IDENTITY_SERVICE_ENDPOINT = os.environ.get("OCI_IDENTITY_SERVICE_ENDPOINT")
1616
NB_SESSION_COMPARTMENT_OCID = os.environ.get("NB_SESSION_COMPARTMENT_OCID")
17-
PROJECT_OCID = os.environ.get("PROJECT_OCID")
17+
PROJECT_OCID = os.environ.get("PROJECT_OCID") or os.environ.get("PIPELINE_PROJECT_OCID")
1818
NB_SESSION_OCID = os.environ.get("NB_SESSION_OCID")
1919
USER_OCID = os.environ.get("USER_OCID")
2020
OCI_RESOURCE_PRINCIPAL_VERSION = os.environ.get("OCI_RESOURCE_PRINCIPAL_VERSION")
@@ -24,17 +24,26 @@
2424
OCI_REGION_METADATA = os.environ.get("OCI_REGION_METADATA")
2525
JOB_RUN_OCID = os.environ.get("JOB_RUN_OCID")
2626
JOB_RUN_COMPARTMENT_OCID = os.environ.get("JOB_RUN_COMPARTMENT_OCID")
27+
PIPELINE_RUN_OCID = os.environ.get("PIPELINE_RUN_OCID")
28+
PIPELINE_RUN_COMPARTMENT_OCID = os.environ.get("PIPELINE_RUN_COMPARTMENT_OCID")
29+
PIPELINE_COMPARTMENT_OCID = os.environ.get("PIPELINE_COMPARTMENT_OCID")
30+
2731
CONDA_BUCKET_NAME = os.environ.get("CONDA_BUCKET_NAME", "service-conda-packs")
2832
CONDA_BUCKET_NS = os.environ.get("CONDA_BUCKET_NS", "id19sfcrra6z")
2933
OCI_RESOURCE_PRINCIPAL_RPT_ENDPOINT = os.environ.get(
3034
"OCI_RESOURCE_PRINCIPAL_RPT_ENDPOINT"
3135
)
32-
COMPARTMENT_OCID = NB_SESSION_COMPARTMENT_OCID or JOB_RUN_COMPARTMENT_OCID
36+
COMPARTMENT_OCID = (
37+
NB_SESSION_COMPARTMENT_OCID
38+
or JOB_RUN_COMPARTMENT_OCID
39+
or PIPELINE_RUN_COMPARTMENT_OCID
40+
)
3341
MD_OCID = os.environ.get("MD_OCID")
3442
DATAFLOW_RUN_OCID = os.environ.get("DATAFLOW_RUN_ID")
43+
3544
RESOURCE_OCID = (
36-
NB_SESSION_OCID or JOB_RUN_OCID or MD_OCID
37-
) # We can add DATAFLOW_RUN_OCID here. Needs impact analysis
45+
NB_SESSION_OCID or JOB_RUN_OCID or MD_OCID or PIPELINE_RUN_OCID or DATAFLOW_RUN_OCID
46+
)
3847
NO_CONTAINER = os.environ.get("NO_CONTAINER")
3948

4049

‎ads/telemetry/telemetry.py

Lines changed: 36 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,26 @@
11
#!/usr/bin/env python
2-
# -*- coding: utf-8; -*-
2+
# -*- coding: utf-8 -*--
33

4-
# Copyright (c) 2022 Oracle and/or its affiliates.
4+
# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
55
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
6-
from ads import __version__
6+
7+
import os
78
from enum import Enum, auto
9+
from typing import Any, Dict, Optional
810

911
import ads.config
12+
from ads import __version__
13+
from ads.common import logger
1014

15+
LIBRARY = "Oracle-ads"
1116
EXTRA_USER_AGENT_INFO = "EXTRA_USER_AGENT_INFO"
1217
USER_AGENT_KEY = "additional_user_agent"
13-
ENV_MD_OCID = "MD_OCID"
1418
UNKNOWN = "UNKNOWN"
1519

1620

1721
class Surface(Enum):
1822
"""
19-
An Enum class for labeling the surface where ADS is being used
23+
An Enum class for labeling the surface where ADS is being used.
2024
"""
2125

2226
WORKSTATION = auto()
@@ -25,6 +29,7 @@ class Surface(Enum):
2529
DATASCIENCE_MODEL_DEPLOYMENT = auto()
2630
DATAFLOW = auto()
2731
OCI_SERVICE = auto()
32+
DATASCIENCE_PIPELINE = auto()
2833

2934
@classmethod
3035
def surface(cls):
@@ -43,12 +48,33 @@ def surface(cls):
4348
surface = cls.DATASCIENCE_MODEL_DEPLOYMENT
4449
elif ads.config.DATAFLOW_RUN_OCID:
4550
surface = cls.DATAFLOW
51+
elif ads.config.PIPELINE_RUN_OCID:
52+
surface = cls.DATASCIENCE_PIPELINE
4653
return surface
4754

4855

49-
def update_oci_client_config(config={}):
50-
if not config.get(USER_AGENT_KEY):
51-
config[
52-
USER_AGENT_KEY
53-
] = f"Oracle-ads/version={__version__}/surface={Surface.surface().name}" # To be enabled in future - /api={os.environ.get(EXTRA_USER_AGENT_INFO,UNKNOWN)}"
56+
def update_oci_client_config(config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
57+
"""Adds user agent information to the config if it is not setup yet.
58+
59+
Returns
60+
-------
61+
Dict
62+
The updated configuration.
63+
"""
64+
65+
try:
66+
config = config or {}
67+
if not config.get(USER_AGENT_KEY):
68+
config.update(
69+
{
70+
USER_AGENT_KEY: (
71+
f"{LIBRARY}/version={__version__}/"
72+
f"surface={Surface.surface().name}/"
73+
f"api={os.environ.get(EXTRA_USER_AGENT_INFO,UNKNOWN) or UNKNOWN}"
74+
)
75+
}
76+
)
77+
except Exception as ex:
78+
logger.debug(ex)
79+
5480
return config

‎docs/source/user_guide/jobs/index.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,4 @@ Data Science Jobs
1818
../cli/opctl/_template/jobs
1919
../cli/opctl/_template/monitoring
2020
../cli/opctl/localdev/local_jobs
21+
yaml_schema

‎docs/source/user_guide/jobs/yaml_schema.rst

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
YAML Schema
2+
***********
3+
4+
ADS Job
5+
=======
6+
7+
.. raw:: html
8+
:file: ../../yaml_schema/jobs/job.html
9+
10+
|
11+
12+
Following is the YAML schema for validating the YAML using `Cerberus <https://docs.python-cerberus.org/en/stable/>`_:
13+
14+
.. literalinclude:: ../../yaml_schema/jobs/job.yaml
15+
:language: yaml
16+
:linenos:
17+
18+
19+
Data Science Job
20+
================
21+
22+
.. raw:: html
23+
:file: ../../yaml_schema/jobs/dataScienceJob.html
24+
25+
|
26+
27+
Following is the YAML schema for validating the YAML using `Cerberus <https://docs.python-cerberus.org/en/stable/>`_:
28+
29+
.. literalinclude:: ../../yaml_schema/jobs/dataScienceJob.yaml
30+
:language: yaml
31+
:linenos:
32+
33+
34+
DataFlow
35+
========
36+
37+
.. raw:: html
38+
:file: ../../yaml_schema/jobs/dataFlow.html
39+
40+
|
41+
42+
Following is the YAML schema for validating the YAML using `Cerberus <https://docs.python-cerberus.org/en/stable/>`_:
43+
44+
.. literalinclude:: ../../yaml_schema/jobs/dataFlow.yaml
45+
:language: yaml
46+
:linenos:
47+
48+
49+
Job Runtime Schemas
50+
===================
51+
52+
The runtime of a job can be one of the following.
53+
54+
Python Runtime
55+
--------------
56+
57+
.. raw:: html
58+
:file: ../../yaml_schema/jobs/pythonRuntime.html
59+
60+
|
61+
62+
Following is the YAML schema for validating the YAML using `Cerberus <https://docs.python-cerberus.org/en/stable/>`_:
63+
64+
.. literalinclude:: ../../yaml_schema/jobs/pythonRuntime.yaml
65+
:language: yaml
66+
:linenos:
67+
68+
69+
Notebook Runtime
70+
----------------
71+
72+
.. raw:: html
73+
:file: ../../yaml_schema/jobs/notebookRuntime.html
74+
75+
|
76+
77+
Following is the YAML schema for validating the YAML using `Cerberus <https://docs.python-cerberus.org/en/stable/>`_:
78+
79+
.. literalinclude:: ../../yaml_schema/jobs/notebookRuntime.yaml
80+
:language: yaml
81+
:linenos:
82+
83+
84+
GitPython Runtime
85+
-----------------
86+
87+
.. raw:: html
88+
:file: ../../yaml_schema/jobs/gitPythonRuntime.html
89+
90+
|
91+
92+
Following is the YAML schema for validating the YAML using `Cerberus <https://docs.python-cerberus.org/en/stable/>`_:
93+
94+
.. literalinclude:: ../../yaml_schema/jobs/gitPythonRuntime.yaml
95+
:language: yaml
96+
:linenos:
97+
98+
99+
Container Runtime
100+
-----------------
101+
102+
.. raw:: html
103+
:file: ../../yaml_schema/jobs/containerRuntime.html
104+
105+
|
106+
107+
Following is the YAML schema for validating the YAML using `Cerberus <https://docs.python-cerberus.org/en/stable/>`_:
108+
109+
.. literalinclude:: ../../yaml_schema/jobs/containerRuntime.yaml
110+
:language: yaml
111+
:linenos:
112+
113+
114+
Script Runtime
115+
--------------
116+
117+
.. raw:: html
118+
:file: ../../yaml_schema/jobs/scriptRuntime.html
119+
120+
|
121+
122+
Following is the YAML schema for validating the YAML using `Cerberus <https://docs.python-cerberus.org/en/stable/>`_:
123+
124+
.. literalinclude:: ../../yaml_schema/jobs/scriptRuntime.yaml
125+
:language: yaml
126+
:linenos:
127+
128+
129+
DataFlow Runtime
130+
--------------
131+
132+
.. raw:: html
133+
:file: ../../yaml_schema/jobs/dataFlowRuntime.html
134+
135+
|
136+
137+
Following is the YAML schema for validating the YAML using `Cerberus <https://docs.python-cerberus.org/en/stable/>`_:
138+
139+
.. literalinclude:: ../../yaml_schema/jobs/dataFlowRuntime.yaml
140+
:language: yaml
141+
:linenos:

0 commit comments

Comments
 (0)