Upgrading Dockerfile packages, and increasing throttling from 0 to 1 second (#258)

* Upgrading awscli, pinning PYYaml and python-dateutil, keeping markdown

* Line ending change

* Increasing throttling from 0 to 1, to alleviate eventual consistency with s3

Author: sboshin

docker/2.0.0/py2/Dockerfile.cpu

@@ -90,6 +90,8 @@ RUN ${PIP} --no-cache-dir install --upgrade \
 # Some TF tools expect a "python" binary
 RUN ln -s $(which ${PYTHON}) /usr/local/bin/python
 
+# install PyYAML==5.1.2 to avoid conflict with latest awscli
+# # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli
 RUN ${PIP} install --no-cache-dir -U \
     numpy==1.16.5 \
     scipy==1.2.2 \
@@ -101,20 +103,22 @@ RUN ${PIP} install --no-cache-dir -U \
     keras_preprocessing==1.1.0 \
     requests==2.22.0 \
     keras==2.3.1 \
-    awscli \
+    python-dateutil==2.8.0 \
+    PyYAML==5.1.2 \
+    awscli==1.16.303 \
     mpi4py==3.0.2 \
+    "cryptography>=2.3" \
+    "sagemaker-tensorflow>=2.0,<2.1" \
     # Let's install TensorFlow separately in the end to avoid
     # the library version to be overwritten
-    # ${PIP} install --force-reinstall --no-cache-dir -U ${TF_URL} \
- && ${PIP} install --force-reinstall --no-cache-dir -U \
+    # ${PIP} install --no-cache-dir -U ${TF_URL} \
+ && ${PIP} install --no-cache-dir -U \
     $TENSORFLOW_WHL \
  && ${PIP} install --no-cache-dir -U \
     $FRAMEWORK_SUPPORT_INSTALLABLE \
  && rm -f $FRAMEWORK_SUPPORT_INSTALLABLE \
  && ${PIP} install --no-cache-dir -U \
-    horovod==0.18.2 \
- && ${PIP} uninstall -y --no-cache-dir \
-    markdown
+    horovod==0.18.2 
 
 COPY dockerd-entrypoint.py /usr/local/bin/dockerd-entrypoint.py
 COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py

docker/2.0.0/py2/Dockerfile.gpu

@@ -118,6 +118,8 @@ RUN ln -s $(which ${PYTHON}) /usr/local/bin/python
 COPY $FRAMEWORK_SUPPORT_INSTALLABLE .
 COPY $TENSORFLOW_WHL .
 
+# install PyYAML==5.1.2 to avoid conflict with latest awscli
+# # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli
 RUN ${PIP} install --no-cache-dir -U \
     numpy==1.16.5 \
     scipy==1.2.2 \
@@ -129,18 +131,20 @@ RUN ${PIP} install --no-cache-dir -U \
     keras_preprocessing==1.1.0 \
     requests==2.22.0 \
     keras==2.3.1 \
-    awscli \
+    python-dateutil==2.8.0 \
+    PyYAML==5.1.2 \
+    awscli==1.16.303 \
     mpi4py==3.0.2 \
+    "cryptography>=2.3" \
+    "sagemaker-tensorflow>=2.0,<2.1" \
     # Let's install TensorFlow separately in the end to avoid
     # the library version to be overwritten
-    # ${PIP} install --force-reinstall --no-cache-dir -U ${TF_URL} \
- && ${PIP} install --force-reinstall --no-cache-dir -U \
+    # ${PIP} install --no-cache-dir -U ${TF_URL} \
+ && ${PIP} install --no-cache-dir -U \
     $TENSORFLOW_WHL \
  && ${PIP} install --no-cache-dir -U \
     $FRAMEWORK_SUPPORT_INSTALLABLE \
- && rm -f $FRAMEWORK_SUPPORT_INSTALLABLE \
- && ${PIP} uninstall -y --no-cache-dir \
-    markdown
+ && rm -f $FRAMEWORK_SUPPORT_INSTALLABLE 
 
 # Install Horovod, temporarily using CUDA stubs
 RUN ldconfig /usr/local/cuda/targets/x86_64-linux/lib/stubs \

docker/2.0.0/py3/Dockerfile.cpu

@@ -93,6 +93,8 @@ RUN ${PIP} --no-cache-dir install --upgrade \
 RUN ln -s $(which ${PYTHON}) /usr/local/bin/python \
  && ln -s $(which ${PIP}) /usr/bin/pip
 
+# install PyYAML==5.1.2 to avoid conflict with latest awscli
+# # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli
 RUN ${PIP} install --no-cache-dir -U \
     numpy==1.17.4 \
     scipy==1.2.2 \
@@ -103,20 +105,21 @@ RUN ${PIP} install --no-cache-dir -U \
     keras_applications==1.0.8 \
     keras_preprocessing==1.1.0 \
     keras==2.3.1 \
+    python-dateutil==2.8.0 \
+    PyYAML==5.1.2 \
     requests==2.22.0 \
-    awscli==1.16.196 \
+    awscli==1.16.303 \
     mpi4py==3.0.2 \
     "sagemaker-tensorflow>=2.0,<2.1" \
     # Let's install TensorFlow separately in the end to avoid
     # the library version to be overwritten
- && ${PIP} install --force-reinstall --no-cache-dir -U ${TF_URL} \
+    # ${PIP} install --no-cache-dir -U ${TF_URL} \
+ && ${PIP} install --no-cache-dir -U \
     $TENSORFLOW_WHL \
     horovod==0.18.2 \
  && ${PIP} install --no-cache-dir -U \
     $FRAMEWORK_SUPPORT_INSTALLABLE \
- && rm -f $FRAMEWORK_SUPPORT_INSTALLABLE \
- && ${PIP} uninstall -y --no-cache-dir \
-    markdown
+ && rm -f $FRAMEWORK_SUPPORT_INSTALLABLE 
 
 COPY dockerd-entrypoint.py /usr/local/bin/dockerd-entrypoint.py
 COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py

docker/2.0.0/py3/Dockerfile.gpu

@@ -132,6 +132,8 @@ RUN ln -s $(which ${PYTHON}) /usr/local/bin/python \
 COPY $FRAMEWORK_SUPPORT_INSTALLABLE .
 COPY $TENSORFLOW_WHL .
 
+# install PyYAML==5.1.2 to avoid conflict with latest awscli
+# # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli
 RUN ${PIP} install --no-cache-dir -U \
     numpy==1.17.4 \
     scipy==1.2.2 \
@@ -143,19 +145,19 @@ RUN ${PIP} install --no-cache-dir -U \
     keras_preprocessing==1.1.0 \
     requests==2.22.0 \
     keras==2.3.1 \
-    awscli==1.16.196 \
+    python-dateutil==2.8.0 \
+    PyYAML==5.1.2 \
+    awscli==1.16.303 \
     mpi4py==3.0.2 \
     "sagemaker-tensorflow>=2.0,<2.1" \
     # Let's install TensorFlow separately in the end to avoid
     # the library version to be overwritten
-    # ${PIP} install --force-reinstall --no-cache-dir -U ${TF_URL} \
- && ${PIP} install --force-reinstall --no-cache-dir -U \
+    # ${PIP} install --no-cache-dir -U ${TF_URL} \
+ && ${PIP} install --no-cache-dir -U \
     $TENSORFLOW_WHL \
  && ${PIP} install --no-cache-dir -U \
     $FRAMEWORK_SUPPORT_INSTALLABLE \
- && rm -f $FRAMEWORK_SUPPORT_INSTALLABLE \
- && ${PIP} uninstall -y --no-cache-dir \
-    markdown
+ && rm -f $FRAMEWORK_SUPPORT_INSTALLABLE 
 
 # Install Horovod, temporarily using CUDA stubs
 RUN ldconfig /usr/local/cuda-10.0/targets/x86_64-linux/lib/stubs \

test/integration/sagemaker/test_mnist.py

@@ -89,8 +89,8 @@ def test_s3_plugin(sagemaker_session, ecr_image, instance_type, region, framewor
                            hyperparameters={
                                # Saving a checkpoint after every 5 steps to hammer the S3 plugin
                                'save-checkpoint-steps': 10,
-                               # Disable throttling for checkpoint and model saving
-                               'throttle-secs': 0,
+                               # Reducing throttling for checkpoint and model saving
+                               'throttle-secs': 1,
                                # Without the patch training jobs would fail around 100th to
                                # 150th step
                                'max-steps': 200,