upgrade to latest sagemaker-experiments (#286)

Co-authored-by: TusharKanekiDey <dey.tushar@yahoo.com>
Co-authored-by: Zhuo Weng <wenzhuo@amazon.com>

Author: 3 people

docker/1.15.2/py3/Dockerfile.cpu

@@ -101,7 +101,7 @@ RUN pip install --no-cache-dir -U \
     requests==2.22.0 \
     smdebug==0.7.0 \
     sagemaker==1.50.17 \
-    sagemaker-experiments==0.1.3 \
+    sagemaker-experiments==0.1.7 \
     mpi4py==3.0.2 \
     "cryptography>=2.3" \
     "sagemaker-tensorflow>=1.15,<1.16" \

docker/1.15.2/py3/Dockerfile.gpu

@@ -139,7 +139,7 @@ RUN pip install --no-cache-dir -U \
     keras==2.3.1 \
     smdebug==0.7.0 \
     sagemaker==1.50.17 \
-    sagemaker-experiments==0.1.3 \
+    sagemaker-experiments==0.1.7 \
     mpi4py==3.0.2 \
     "cryptography>=2.3" \
     "sagemaker-tensorflow>=1.15,<1.16" \

setup.py

@@ -60,6 +60,7 @@ def read_version():
                  'sagemaker==1.50.1', 'tensorflow<2.0', 'docker-compose', 'boto3==1.10.50',
                  'six==1.13.0', 'python-dateutil>=2.1,<2.8.1', 'botocore==1.13.50',
                  'requests-mock', 'awscli==1.16.314'],
-        'benchmark': ['click']
+        'benchmark': ['click'],
+        ':python_version=="3.6"': ['sagemaker-experiments==0.1.7']
     },
 )

test/integration/conftest.py

@@ -92,6 +92,20 @@ def instance_type(request, processor):
     return provided_instance_type if provided_instance_type is not None else default_instance_type
 
 
+@pytest.fixture()
+def py_version():
+    if 'TEST_PY_VERSIONS' in os.environ:
+        return os.environ['TEST_PY_VERSIONS'].split(',')
+    return None
+
+
+@pytest.fixture()
+def processor():
+    if 'TEST_PROCESSORS' in os.environ:
+        return os.environ['TEST_PROCESSORS'].split(',')
+    return None
+
+
 @pytest.fixture(autouse=True)
 def skip_by_device_type(request, processor):
     is_gpu = (processor == 'gpu')

test/integration/sagemaker/test_experiments.py

@@ -0,0 +1,97 @@
+# Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+from __future__ import absolute_import
+
+import os
+import time
+
+import pytest
+from sagemaker import utils
+from sagemaker.tensorflow import TensorFlow
+
+from test.integration import RESOURCE_PATH
+from timeout import timeout
+
+DATA_PATH = os.path.join(RESOURCE_PATH, "mnist")
+SCRIPT_PATH = os.path.join(DATA_PATH, "mnist_gluon_basic_hook_demo.py")
+
+
+@pytest.mark.skip_py2_containers
+def test_training(sagemaker_session, ecr_image, instance_type, framework_version, py_version):
+
+    if py_version is None or '2' in py_version:
+        pytest.skip('Skipping python2 {}'.format(py_version))
+        return
+
+    from smexperiments.experiment import Experiment
+    from smexperiments.trial import Trial
+    from smexperiments.trial_component import TrialComponent
+
+    sm_client = sagemaker_session.sagemaker_client
+
+    experiment_name = "tf-container-integ-test-{}".format(int(time.time()))
+
+    experiment = Experiment.create(
+        experiment_name=experiment_name,
+        description="Integration test experiment from sagemaker-tf-container",
+        sagemaker_boto_client=sm_client,
+    )
+
+    trial_name = "tf-container-integ-test-{}".format(int(time.time()))
+    trial = Trial.create(
+        experiment_name=experiment_name, trial_name=trial_name, sagemaker_boto_client=sm_client
+    )
+
+    training_job_name = utils.unique_name_from_base("test-tf-experiments-mnist")
+
+    # create a training job and wait for it to complete
+    with timeout(minutes=15):
+        resource_path = os.path.join(os.path.dirname(__file__), "..", "..", "resources")
+        script = os.path.join(resource_path, "mnist", "mnist.py")
+        estimator = TensorFlow(
+            entry_point=script,
+            role="SageMakerRole",
+            train_instance_type=instance_type,
+            train_instance_count=1,
+            sagemaker_session=sagemaker_session,
+            image_name=ecr_image,
+            framework_version=framework_version,
+            script_mode=True,
+        )
+        inputs = estimator.sagemaker_session.upload_data(
+            path=os.path.join(resource_path, "mnist", "data"), key_prefix="scriptmode/mnist"
+        )
+        estimator.fit(inputs, job_name=training_job_name)
+
+    training_job = sm_client.describe_training_job(TrainingJobName=training_job_name)
+    training_job_arn = training_job["TrainingJobArn"]
+
+    # verify trial component auto created from the training job
+    trial_components = list(
+        TrialComponent.list(source_arn=training_job_arn, sagemaker_boto_client=sm_client)
+    )
+
+    trial_component_summary = trial_components[0]
+    trial_component = TrialComponent.load(
+        trial_component_name=trial_component_summary.trial_component_name,
+        sagemaker_boto_client=sm_client,
+    )
+
+    # associate the trial component with the trial
+    trial.add_trial_component(trial_component)
+
+    # cleanup
+    trial.remove_trial_component(trial_component_summary.trial_component_name)
+    trial_component.delete()
+    trial.delete()
+    experiment.delete()