feature: install sagemaker-tensorflow-toolkit from PyPI. (#334)

Author: nadiaya

README.rst

@@ -114,17 +114,6 @@ Before building "final" images:
 Build your "base" image. Make sure it is named and tagged in accordance with your "final"
 Dockerfile. Skip this step if you want to build image of Tensorflow Version 1.9.0 and above.
 
-Then prepare the SageMaker TensorFlow Container python package in the image folder like below:
-
-::
-
-    # Create the SageMaker TensorFlow Container Python package.
-    cd sagemaker-tensorflow-containers
-    python setup.py sdist
-
-    #. Copy your Python package to "final" Dockerfile directory that you are building.
-    cp dist/sagemaker_tensorflow_container-<package_version>.tar.gz docker/<tensorflow_version>/final/py2
-
 If you want to build "final" Docker images, for versions 1.6 and above, you will first need to download the appropriate tensorflow pip wheel, then pass in its location as a build argument. These can be obtained from pypi. For example, the files for 1.6.0 are here:
 
 https://pypi.org/project/tensorflow/1.6.0/#files

buildspec.yml

@@ -2,7 +2,7 @@ version: 0.2
 
 env:
   variables:
-    FRAMEWORK_VERSION: '1.15.0'
+    FRAMEWORK_VERSION: '2.1.0'
     ECR_REPO: 'sagemaker-test'
     GITHUB_REPO: 'sagemaker-tensorflow-container'
     SETUP_FILE: 'setup_cmds.sh'
@@ -20,66 +20,65 @@ phases:
   build:
     commands:
       # install
-      - pip3 install -U -e .
       - pip3 install -U -e .[test]
 
       # run flake8
       - tox -e flake8,twine
 
-      # run unit tests
-      - tox -e py36,py27 test/unit
-
-      # Create pip archive
       - root_dir=$(pwd)
       - build_id="$(echo $CODEBUILD_BUILD_ID | sed -e 's/:/-/g')"
-      - python3 setup.py sdist
-      - tar_name=$(ls dist)
 
       # Find build artifacts
-      - build_artifacts=$root_dir/docker/artifacts
+      - build_artifacts=$root_dir/docker/build_artifacts
 
       # build py2 images
 
       # prepare build context
       - build_dir="$root_dir/docker/$FRAMEWORK_VERSION/py2"
-      - cp $root_dir/dist/$tar_name $build_dir
       - cp $build_artifacts/* $build_dir/
       - cd $build_dir
 
       # build cpu image
       - cpu_dockerfile="Dockerfile.cpu"
       - CPU_TAG_PY2="$FRAMEWORK_VERSION-cpu-py2-$build_id"
-      - docker build -f $cpu_dockerfile -t $PREPROD_IMAGE:$CPU_TAG_PY2 .
+      - build_cmd="docker build -f $cpu_dockerfile -t $PREPROD_IMAGE:$CPU_TAG_PY2 . "
+      - execute-command-if-has-matching-changes "$build_cmd" "test/" "docker/*" "buildspec.yml"
 
       # build gpu image
       - gpu_dockerfile="Dockerfile.gpu"
       - GPU_TAG_PY2="$FRAMEWORK_VERSION-gpu-py2-$build_id"
-      - docker build -f $gpu_dockerfile -t $PREPROD_IMAGE:$GPU_TAG_PY2 .
+      - build_cmd="docker build -f $gpu_dockerfile -t $PREPROD_IMAGE:$GPU_TAG_PY2 . "
+      - execute-command-if-has-matching-changes "$build_cmd" "test/" "docker/*" "buildspec.yml"
 
       # build py3 images
 
       # prepare build context
       - build_dir="$root_dir/docker/$FRAMEWORK_VERSION/py3"
-      - cp $root_dir/dist/$tar_name $build_dir
       - cp $build_artifacts/* $build_dir/
       - cd $build_dir
 
       # build cpu image
       - cpu_dockerfile="Dockerfile.cpu"
       - CPU_TAG_PY3="$FRAMEWORK_VERSION-cpu-py3-$build_id"
-      - docker build -f $cpu_dockerfile -t $PREPROD_IMAGE:$CPU_TAG_PY3 .
+      - build_cmd="docker build -f $cpu_dockerfile -t $PREPROD_IMAGE:$CPU_TAG_PY3 . "
+      - execute-command-if-has-matching-changes "$build_cmd" "test/" "docker/*" "buildspec.yml"
 
       # build gpu image
       - gpu_dockerfile="Dockerfile.gpu"
       - GPU_TAG_PY3="$FRAMEWORK_VERSION-gpu-py3-$build_id"
-      - docker build -f $gpu_dockerfile -t $PREPROD_IMAGE:$GPU_TAG_PY3 .
+      - build_cmd="docker build -f $gpu_dockerfile -t $PREPROD_IMAGE:$GPU_TAG_PY3 . "
+      - execute-command-if-has-matching-changes "$build_cmd" "test/" "docker/*" "buildspec.yml"
 
       # push images to ecr
       - $(aws ecr get-login --registry-ids $ACCOUNT --no-include-email --region $AWS_DEFAULT_REGION)
-      - docker push $PREPROD_IMAGE:$CPU_TAG_PY2
-      - docker push $PREPROD_IMAGE:$GPU_TAG_PY2
-      - docker push $PREPROD_IMAGE:$CPU_TAG_PY3
-      - docker push $PREPROD_IMAGE:$GPU_TAG_PY3
+      - push_cmd="docker push $PREPROD_IMAGE:$CPU_TAG_PY2"
+      - execute-command-if-has-matching-changes "$push_cmd" "test/" "docker/*" "buildspec.yml"
+      - push_cmd="docker push $PREPROD_IMAGE:$GPU_TAG_PY2"
+      - execute-command-if-has-matching-changes "$push_cmd" "test/" "docker/*" "buildspec.yml"
+      - push_cmd="docker push $PREPROD_IMAGE:$CPU_TAG_PY3"
+      - execute-command-if-has-matching-changes "$push_cmd" "test/" "docker/*" "buildspec.yml"
+      - push_cmd="docker push $PREPROD_IMAGE:$GPU_TAG_PY3"
+      - execute-command-if-has-matching-changes "$push_cmd" "test/" "docker/*" "buildspec.yml"
 
       # launch remote gpu instance
       - instance_type='p2.xlarge'
@@ -89,36 +88,40 @@ phases:
       # run cpu integration tests
       - py3_cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $CPU_TAG_PY2 --framework-version $FRAMEWORK_VERSION --py-version 2 --processor cpu"
       - py2_cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $CPU_TAG_PY3 --framework-version $FRAMEWORK_VERSION --py-version 3 --processor cpu"
-      - execute-command-if-has-matching-changes "$py3_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
-      - execute-command-if-has-matching-changes "$py2_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$py2_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run gpu integration tests
       - printf "$SETUP_CMDS" > $SETUP_FILE
       - cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $GPU_TAG_PY2 --framework-version $FRAMEWORK_VERSION --py-version 2 --processor gpu"
       - py3_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\""
-      - execute-command-if-has-matching-changes "$py3_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$py3_cmd" "test/" "docker/*" "buildspec.yml"
 
       - cmd="pytest test/integration/local --region $AWS_DEFAULT_REGION --docker-base-name $PREPROD_IMAGE --tag $GPU_TAG_PY3 --framework-version $FRAMEWORK_VERSION --py-version 3 --processor gpu"
       - py2_cmd="remote-test --github-repo $GITHUB_REPO --test-cmd \"$cmd\" --setup-file $SETUP_FILE --pr-number \"$PR_NUM\""
-      - execute-command-if-has-matching-changes "$py2_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$py2_cmd" "test/" "docker/*" "buildspec.yml"
 
       # run sagemaker tests
       - test_cmd="pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG_PY2 --py-version 2 --processor cpu"
-      - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$test_cmd" "test/" "docker/*" "buildspec.yml"
       - test_cmd="pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG_PY2 --py-version 2 --processor gpu"
-      - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$test_cmd" "test/" "docker/*" "buildspec.yml"
       - test_cmd="pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $CPU_TAG_PY3 --py-version 3 --processor cpu"
-      - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$test_cmd" "test/" "docker/*" "buildspec.yml"
       - test_cmd="pytest test/integration/sagemaker -n 8 --region $AWS_DEFAULT_REGION --docker-base-name $ECR_REPO --account-id $ACCOUNT --tag $GPU_TAG_PY3 --py-version 3 --processor gpu"
-      - execute-command-if-has-matching-changes "$test_cmd" "test/" "src/*.py" "setup.py" "docker/*" "buildspec.yml"
+      - execute-command-if-has-matching-changes "$test_cmd" "test/" "docker/*" "buildspec.yml"
 
     finally:
       # shut down remote gpu instance
       - cleanup-gpu-instances
       - cleanup-key-pairs
 
       # remove ecr image
-      - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG_PY2
-      - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG_PY2
-      - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG_PY3
-      - aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG_PY3
+      - delete_cmd="aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG_PY2"
+      - execute-command-if-has-matching-changes "$delete_cmd" "test/" "docker/*" "buildspec.yml"
+      - delete_cmd="aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG_PY2"
+      - execute-command-if-has-matching-changes "$delete_cmd" "test/" "docker/*" "buildspec.yml"
+      - delete_cmd="aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$CPU_TAG_PY3"
+      - execute-command-if-has-matching-changes "$delete_cmd" "test/" "docker/*" "buildspec.yml"
+      - delete_cmd="aws ecr batch-delete-image --repository-name $ECR_REPO --region $AWS_DEFAULT_REGION --image-ids imageTag=$GPU_TAG_PY3"
+      - execute-command-if-has-matching-changes "$delete_cmd" "test/" "docker/*" "buildspec.yml"

docker/2.1.0/py2/Dockerfile.cpu

@@ -20,7 +20,6 @@ ENV PYTHONIOENCODING=UTF-8
 ENV LANG=C.UTF-8 
 ENV LC_ALL=C.UTF-8
 
-ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_training*.tar.gz
 ARG TF_URL=https://tensorflow-aws.s3-us-west-2.amazonaws.com/2.1/AmazonLinux/cpu/final/tensorflow-2.1.0-cp27-cp27mu-manylinux2010_x86_64.whl
 
 ARG PYTHON=python
@@ -82,8 +81,6 @@ RUN apt-get update && apt-get install -y \
     ${PYTHON} \
     ${PYTHON_PIP}
 
-COPY $FRAMEWORK_SUPPORT_INSTALLABLE .
-
 RUN ${PIP} --no-cache-dir install --upgrade \
     pip \
     setuptools
@@ -111,13 +108,11 @@ RUN ${PIP} install --no-cache-dir -U \
     opencv-python==4.2.0.32 \
     "cryptography>=2.3" \
     "sagemaker-tensorflow>=2.1,<2.2" \
+    "sagemaker-tensorflow-training>2,<4" \
     # Let's install TensorFlow separately in the end to avoid
     # the library version to be overwritten
  && ${PIP} install --no-cache-dir -U \
     ${TF_URL} \
- && ${PIP} install --no-cache-dir -U \
-    $FRAMEWORK_SUPPORT_INSTALLABLE \
- && rm -f $FRAMEWORK_SUPPORT_INSTALLABLE \
  && ${PIP} install --no-cache-dir -U \
     horovod==0.18.2 
 

docker/2.1.0/py2/Dockerfile.gpu

@@ -13,7 +13,6 @@ ENV PYTHONIOENCODING=UTF-8
 ENV LANG=C.UTF-8 
 ENV LC_ALL=C.UTF-8
 
-ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_training*.tar.gz
 ARG TF_URL=https://tensorflow-aws.s3-us-west-2.amazonaws.com/2.1/AmazonLinux/gpu/final/tensorflow_gpu-2.1.0-cp27-cp27mu-manylinux2010_x86_64.whl
 
 ARG PYTHON=python
@@ -122,8 +121,6 @@ RUN ${PIP} --no-cache-dir install --upgrade \
 # Some TF tools expect a "python" binary
 RUN ln -s $(which ${PYTHON}) /usr/local/bin/python
 
-COPY $FRAMEWORK_SUPPORT_INSTALLABLE .
-
 # install PyYAML==5.1.2 to avoid conflict with latest awscli
 # # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli
 RUN ${PIP} install --no-cache-dir -U \
@@ -144,13 +141,11 @@ RUN ${PIP} install --no-cache-dir -U \
     opencv-python==4.2.0.32 \
     "cryptography>=2.3" \
     "sagemaker-tensorflow>=2.1,<2.2" \
+    "sagemaker-tensorflow-training>2,<4" \
     # Let's install TensorFlow separately in the end to avoid
     # the library version to be overwritten
  && ${PIP} install --no-cache-dir -U \
-    ${TF_URL} \
- && ${PIP} install --no-cache-dir -U \
-    $FRAMEWORK_SUPPORT_INSTALLABLE \
- && rm -f $FRAMEWORK_SUPPORT_INSTALLABLE 
+    ${TF_URL}
 
 # Install Horovod, temporarily using CUDA stubs
 RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs \

docker/2.1.0/py3/Dockerfile.cpu

@@ -24,7 +24,6 @@ ARG PYTHON=python3
 ARG PYTHON_PIP=python3-pip
 ARG PIP=pip3
 
-ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_training*.tar.gz
 ARG TF_URL=https://tensorflow-aws.s3-us-west-2.amazonaws.com/2.1/AmazonLinux/cpu/final/tensorflow-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -81,8 +80,6 @@ RUN mkdir -p /root/.ssh/ \
 
 WORKDIR /
 
-COPY $FRAMEWORK_SUPPORT_INSTALLABLE .
-
 RUN ${PIP} --no-cache-dir install --upgrade \
     pip \
     setuptools
@@ -113,14 +110,12 @@ RUN ${PIP} install --no-cache-dir -U \
     sagemaker==1.50.17 \
     sagemaker-experiments==0.1.7 \
     "sagemaker-tensorflow>=2.1,<2.2" \
+    "sagemaker-tensorflow-training>2,<4" \
     # Let's install TensorFlow separately in the end to avoid
     # the library version to be overwritten
  && ${PIP} install --no-cache-dir -U \
     ${TF_URL} \
-    horovod==0.18.2 \
- && ${PIP} install --no-cache-dir -U \
-    $FRAMEWORK_SUPPORT_INSTALLABLE \
- && rm -f $FRAMEWORK_SUPPORT_INSTALLABLE
+    horovod==0.18.2
 
 ADD https://raw.githubusercontent.com/aws/aws-deep-learning-containers-utils/master/deep_learning_container.py /usr/local/bin/deep_learning_container.py
 

docker/2.1.0/py3/Dockerfile.gpu

@@ -24,7 +24,6 @@ ARG PYTHON=python3
 ARG PYTHON_PIP=python3-pip
 ARG PIP=pip3
 
-ARG FRAMEWORK_SUPPORT_INSTALLABLE=sagemaker_tensorflow_training*.tar.gz
 ARG TF_URL=https://tensorflow-aws.s3-us-west-2.amazonaws.com/2.1/AmazonLinux/gpu/final/tensorflow_gpu-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl
 
 RUN apt-get update && apt-get install -y --no-install-recommends --allow-unauthenticated \
@@ -133,8 +132,6 @@ RUN ${PIP} --no-cache-dir install --upgrade \
 RUN ln -s $(which ${PYTHON}) /usr/local/bin/python \
  && ln -s $(which ${PIP}) /usr/bin/pip
 
-COPY $FRAMEWORK_SUPPORT_INSTALLABLE .
-
 # install PyYAML==5.1.2 to avoid conflict with latest awscli
 # # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli
 RUN ${PIP} install --no-cache-dir -U \
@@ -157,13 +154,11 @@ RUN ${PIP} install --no-cache-dir -U \
     sagemaker==1.50.17 \
     sagemaker-experiments==0.1.7 \
     "sagemaker-tensorflow>=2.1,<2.2" \
+    "sagemaker-tensorflow-training>2,<4" \
     # Let's install TensorFlow separately in the end to avoid
     # the library version to be overwritten
  && ${PIP} install --no-cache-dir -U \
-    ${TF_URL} \
- && ${PIP} install --no-cache-dir -U \
-    $FRAMEWORK_SUPPORT_INSTALLABLE \
- && rm -f $FRAMEWORK_SUPPORT_INSTALLABLE
+    ${TF_URL}
 
 # Install Horovod, temporarily using CUDA stubs
 RUN ldconfig /usr/local/cuda-10.1/targets/x86_64-linux/lib/stubs \