Merge branch 'main' into spavlusieva-patch-1

Author: darenr

.github/workflows/run-unittests.yml

@@ -0,0 +1,69 @@
+name: Unit Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - 'release/**'
+      - develop
+    paths:
+      - '!docs/**'
+
+  pull_request:
+
+# Cancel in progress workflows on pull_requests.
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: ${{ matrix.tests-type }}, python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.7","3.8","3.9","3.10"]
+        tests-type: ["DefaultSetup"]
+
+    steps:
+      - uses: actions/checkout@v3
+#      - uses: actions/cache@v3
+#        with:
+#          path: ~/.cache/pip
+#          key: ${{ runner.os }}-pip-${{ hashFiles('**/test-requirements.txt') }}
+#          restore-keys: |
+#            ${{ runner.os }}-pip-
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: "Setup test env"
+        run: |
+          pip install coverage pytest-codecov tox==4.2.8
+
+      - name: "Run unit tests"
+        timeout-minutes: 45
+        shell: bash
+        run: |
+          set -x # print commands that are executed
+#          coverage erase
+#          ./scripts/runtox.sh "${{ matrix.python-version }}-${{ matrix.tests-type }}" --cov --cov-report=
+#          coverage combine .coverage-*
+#          coverage html -i
+
+      # Uploading test artifacts
+      # https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts#uploading-build-and-test-artifacts
+#      - name: "Upload artifact"
+#        uses: actions/upload-artifact@v3
+#        with:
+#          name: code-coverage-report
+#          path: htmlcov/
+#          retention-days: 10

README.md

@@ -23,95 +23,95 @@ You have various options when installing ADS.
 ### Installing the oracle-ads base package
 
 ```bash
-  $ python3 -m pip install oracle-ads
+  python3 -m pip install oracle-ads
 ```
 
 ### Installing extras libraries
 
-The `all-optional` module will install all optional dependencies.
+The `all-optional` module will install all optional dependencies. Note the single quotes around installation of extra libraries.
 
 ```bash
-  $ python3 -m pip install oracle-ads[all-optional]
+  python3 -m pip install 'oracle-ads[all-optional]'
 ```
 
 To work with gradient boosting models, install the `boosted` module. This module includes XGBoost and LightGBM model classes.
 
 ```bash
-  $ python3 -m pip install oracle-ads[boosted]
+  python3 -m pip install 'oracle-ads[boosted]'
 ```
 
 For big data use cases using Oracle Big Data Service (BDS), install the `bds` module. It includes the following libraries, `ibis-framework[impala]`, `hdfs[kerberos]` and `sqlalchemy`.
 
 ```bash
-  $ python3 -m pip install oracle-ads[bds]
+  python3 -m pip install 'oracle-ads[bds]'
 ```
 
 To work with a broad set of data formats (for example, Excel, Avro, etc.) install the `data` module. It includes the `fastavro`, `openpyxl`, `pandavro`, `asteval`, `datefinder`, `htmllistparse`, and `sqlalchemy` libraries.
 
 ```bash
-  $ python3 -m pip install oracle-ads[data]
+  python3 -m pip install 'oracle-ads[data]'
 ```
 
 To work with geospatial data install the `geo` module. It includes the `geopandas` and libraries from the `viz` module.
 
 ```bash
-  $ python3 -m pip install oracle-ads[geo]
+  python3 -m pip install 'oracle-ads[geo]'
 ```
 
 Install the `notebook` module to use ADS within a OCI Data Science service [notebook session](https://docs.oracle.com/en-us/iaas/data-science/using/manage-notebook-sessions.htm). This module installs `ipywidgets` and `ipython` libraries.
 
 ```bash
-  $ python3 -m pip install oracle-ads[notebook]
+  python3 -m pip install 'oracle-ads[notebook]'
 ```
 
 To work with ONNX-compatible run times and libraries designed to maximize performance and model portability, install the `onnx` module. It includes the following libraries, `onnx`, `onnxruntime`, `onnxmltools`, `skl2onnx`, `xgboost`, `lightgbm` and libraries from the `viz` module.
 
 ```bash
-  $ python3 -m pip install oracle-ads[onnx]
+  python3 -m pip install 'oracle-ads[onnx]'
 ```
 
 For infrastructure tasks, install the `opctl` module. It includes the following libraries, `oci-cli`, `docker`, `conda-pack`, `nbconvert`, `nbformat`, and `inflection`.
 
 ```bash
-  $ python3 -m pip install oracle-ads[opctl]
+  python3 -m pip install 'oracle-ads[opctl]'
 ```
 
 For hyperparameter optimization tasks install the `optuna` module. It includes the `optuna` and libraries from the `viz` module.
 
 ```bash
-  $ python3 -m pip install oracle-ads[optuna]
+  python3 -m pip install 'oracle-ads[optuna]'
 ```
 
 Install the `tensorflow` module to include `tensorflow` and libraries from the `viz` module.
 
 ```bash
-  $ python3 -m pip install oracle-ads[tensorflow]
+  python3 -m pip install 'oracle-ads[tensorflow]'
 ```
 
 For text related tasks, install the `text` module. This will include the `wordcloud`, `spacy` libraries.
 
 ```bash
-  $ python3 -m pip install oracle-ads[text]
+  python3 -m pip install 'oracle-ads[text]'
 ```
 
 Install the `torch` module to include `pytorch` and libraries from the `viz` module.
 
 ```bash
-  $ python3 -m pip install oracle-ads[torch]
+  python3 -m pip install 'oracle-ads[torch]'
 ```
 
 Install the `viz` module to include libraries for visualization tasks. Some of the key packages are `bokeh`, `folium`, `seaborn` and related packages.
 
 ```bash
-  $ python3 -m pip install oracle-ads[viz]
+  python3 -m pip install 'oracle-ads[viz]'
 ```
 
 **Note**
 
 Multiple extra dependencies can be installed together. For example:
 
 ```bash
-  $ python3 -m pip install  oracle-ads[notebook,viz,text]
+  python3 -m pip install  'oracle-ads[notebook,viz,text]'
 ```
 
 ## Documentation
@@ -166,7 +166,7 @@ This example uses SQL injection safe binding variables.
 
 ## Contributing
 
-This project welcomes contributions from the community. Before submitting a pull request, please review our contribution guide [CONTRIBUTING.md](https://github.com/oracle/accelerated-data-science/blob/main/CONTRIBUTING.md).
+This project welcomes contributions from the community. Before submitting a pull request, please [review our contribution guide](./CONTRIBUTING.md)
 
 Find Getting Started instructions for developers in [README-development.md](https://github.com/oracle/accelerated-data-science/blob/main/README-development.md)
 

SECURITY.md

@@ -6,11 +6,10 @@ and privacy of all our users.
 
 Please do NOT raise a GitHub Issue to report a security vulnerability. If you
 believe you have found a security vulnerability, please submit a report to
-[secalert_us@oracle.com](mailto:secalert_us@oracle.com) preferably with a proof of concept. 
-Please review some additional information on 
-[how to report security vulnerabilities to Oracle](https://www.oracle.com/corporate/security-practices/assurance/vulnerability/reporting.html).
+[secalert_us@oracle.com][1] preferably with a proof of concept. Please review
+some additional information on [how to report security vulnerabilities to Oracle][2].
 We encourage people who contact Oracle Security to use email encryption using
-[our encryption key](https://www.oracle.com/security-alerts/encryptionkey.html).
+[our encryption key][3].
 
 We ask that you do not use other channels or contact the project maintainers
 directly.
@@ -22,15 +21,18 @@ security features are welcome on GitHub Issues.
 
 Security updates will be released on a regular cadence. Many of our projects
 will typically release security fixes in conjunction with the
-[Oracle Critical Patch Update](https://www.oracle.com/security-alerts/encryptionkey.html) program. 
-Security updates are released on the Tuesday closest to the 17th day of January, April, July and October. 
-A pre-release announcement will be published on the Thursday preceding each release. Additional
-information, including past advisories, is available on our 
-[security alerts](https://www.oracle.com/security-alerts/) page.
+[Oracle Critical Patch Update][3] program. Additional
+information, including past advisories, is available on our [security alerts][4]
+page.
 
 ## Security-related information
 
 We will provide security related information such as a threat model, considerations
 for secure use, or any known security issues in our documentation. Please note
 that labs and sample code are intended to demonstrate a concept and may not be
 sufficiently hardened for production use.
+
+[1]: mailto:secalert_us@oracle.com
+[2]: https://www.oracle.com/corporate/security-practices/assurance/vulnerability/reporting.html
+[3]: https://www.oracle.com/security-alerts/encryptionkey.html
+[4]: https://www.oracle.com/security-alerts/

ads/ads_version.json

@@ -1,3 +1,3 @@
 {
-  "version": "2.7.2"
+  "version": "2.8.1"
 }

ads/catalog/model.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2020, 2022 Oracle and/or its affiliates.
+# Copyright (c) 2020, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import warnings
@@ -51,6 +51,7 @@
 )
 from ads.dataset.progress import DummyProgressBar, TqdmProgressBar
 from ads.feature_engineering.schema import Schema
+from ads.model.model_version_set import ModelVersionSet, _extract_model_version_set_id
 from ads.model.deployment.model_deployer import ModelDeployer
 from oci.data_science.data_science_client import DataScienceClient
 from oci.data_science.models import (
@@ -72,6 +73,8 @@
     "description",
     "freeform_tags",
     "defined_tags",
+    "model_version_set_id",
+    "version_label",
 ]
 _MODEL_PROVENANCE_ATTRIBUTES = ModelProvenance().swagger_types.keys()
 _ETAG_KEY = "ETag"
@@ -1284,6 +1287,8 @@ def upload_model(
         bucket_uri: Optional[str] = None,
         remove_existing_artifact: Optional[bool] = True,
         overwrite_existing_artifact: Optional[bool] = True,
+        model_version_set: Optional[Union[str, ModelVersionSet]] = None,
+        version_label: Optional[str] = None,
     ):
         """
         Uploads the model artifact to cloud storage.
@@ -1315,6 +1320,10 @@ def upload_model(
             Whether artifacts uploaded to object storage bucket need to be removed or not.
         overwrite_existing_artifact: (bool, optional). Defaults to `True`.
             Overwrite target bucket artifact if exists.
+        model_version_set: (Union[str, ModelVersionSet], optional). Defaults to None.
+            The Model version set OCID, or name, or `ModelVersionSet` instance.
+        version_label: (str, optional). Defaults to None.
+            The model version label.
 
         Returns
         -------
@@ -1340,6 +1349,9 @@ def upload_model(
                 )
             copy_artifact_to_os = True
 
+        # extract model_version_set_id from model_version_set attribute or environment
+        # variables in case of saving model in context of model version set.
+        model_version_set_id = _extract_model_version_set_id(model_version_set)
         # Set default display_name if not specified - randomly generated easy to remember name generated
         display_name = display_name or utils.get_random_name_for_resource()
 
@@ -1373,6 +1385,8 @@ def upload_model(
                 else '{"schema": []}',
                 freeform_tags=freeform_tags,
                 defined_tags=defined_tags,
+                model_version_set_id=model_version_set_id,
+                version_label=version_label,
             )
             model = self.ds_client.create_model(create_model_details)
 
@@ -1545,15 +1559,21 @@ def _wait_for_work_request(
                 work_request_logs = self.ds_client.list_work_request_logs(
                     work_request_id
                 ).data
-                new_work_request_logs = work_request_logs[i:]
+                if work_request_logs:
+                    new_work_request_logs = work_request_logs[i:]
 
                 for wr_item in new_work_request_logs:
                     progress.update(wr_item.message)
                     i += 1
 
                 if work_request.data.status in STOP_STATE:
                     if work_request.data.status != WorkRequest.STATUS_SUCCEEDED:
-                        raise Exception(work_request_logs[-1].message)
+                        if work_request_logs:
+                            raise Exception(work_request_logs[-1].message)
+                        else:
+                            raise Exception(
+                                "An error occurred in attempt to perform the operation. Check the service logs to get more details."
+                            )
                     else:
                         break
         return work_request

ads/catalog/summary.py

@@ -68,6 +68,10 @@ def __init__(self, entity_list, datetime_format=utils.date_format):
             self.df["compartment_id"] = "..." + self.df["compartment_id"].str[-6:]
             if "project_id" in ordered_columns:
                 self.df["project_id"] = "..." + self.df["project_id"].str[-6:]
+            if "model_version_set_id" in ordered_columns:
+                self.df["model_version_set_id"] = (
+                    "..." + self.df["model_version_set_id"].str[-6:]
+                )
             self.df["time_created"] = pd.to_datetime(
                 self.df["time_created"]
             ).dt.strftime(datetime_format)

ads/cli.py

@@ -7,13 +7,16 @@
     import click
     import ads.opctl.cli
     import ads.jobs.cli
+    import ads.pipeline.cli
     import os
     import json
 except:
     print(
         "Please run `pip install oracle-ads[opctl]` to install the required dependencies for ADS CLI"
     )
     exit()
+
+
 with open(
     os.path.join(os.path.dirname(os.path.abspath(__file__)), "ads_version.json")
 ) as version_file:
@@ -29,6 +32,7 @@ def cli():
 
 cli.add_command(ads.opctl.cli.commands)
 cli.add_command(ads.jobs.cli.commands)
+cli.add_command(ads.pipeline.cli.commands)
 
 
 if __name__ == "__main__":