Merge pull request #31 from kenny-ezirim/release-sparkml33

upgrading sparkml from v2.4 to v3.3

Author: kenny-ezirim

Dockerfile

@@ -5,6 +5,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true
 
 RUN apt-get update \
+ && apt-get -y upgrade \
  && apt-get -y install apt-utils \
     net-tools \
     apt-transport-https \
@@ -15,14 +16,13 @@ RUN apt-get update \
     maven \
     make \
     gcc \
+    # openssl \
     zlib1g-dev
 
-RUN apt -y update
-
-ARG OPENSSL_VERSION=1.1.1l
+ARG OPENSSL_VERSION=1.1.1q
 ARG PYTHON=python3
 ARG PIP=pip3
-ARG PYTHON_VERSION=3.6.13
+ARG PYTHON_VERSION=3.10.7
 
 # Open-SSL
 RUN wget -q -c https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \
@@ -34,7 +34,7 @@ RUN wget -q -c https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
  && rmdir /usr/local/ssl/certs \
  && ln -s /etc/ssl/certs /usr/local/ssl/certs
 
-# Install Python-3.6.13 from source
+# Install Python-3.10.7 from source
 RUN wget -q https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz \
  && tar -xzf Python-$PYTHON_VERSION.tgz \
  && cd Python-$PYTHON_VERSION \
@@ -54,8 +54,36 @@ WORKDIR /sagemaker-sparkml-model-server
 
 RUN mvn clean package
 
-RUN cp ./target/sparkml-serving-2.4.jar /usr/local/lib/sparkml-serving-2.4.jar
-RUN cp ./serve.sh /usr/local/bin/serve.sh
+RUN cp ./target/sparkml-serving-3.3.jar /usr/local/lib/sparkml-serving-3.3.jar \
+ && cp ./serve.sh /usr/local/bin/serve.sh \
+ && chmod a+x /usr/local/bin/serve.sh
+
+# remove the maven-shared-utils packages - older versions create vulnerabilities
+RUN find / -depth -name maven-shared-utils -type d -exec rm -r "{}" \; \
+ # remove the surefire packages - contains versions of maven-shared-utils that create vulnerabilities
+ && find / -depth -name surefire -type d -exec rm -r "{}" \; \
+ # remove maven-shared-utils jar file with vulnerabilities
+ # comment out if need to use maven utilities
+ && rm /usr/share/java/maven-shared-utils.jar \
+ # remove wagon-http-shaded jar file with vulnerabilities associated with org.jsoup:jsoup
+ && rm /usr/share/java/wagon-http-shaded-3.3.4.jar \
+ # remove plexus-utils directory because plexus-utils has vulnerabilities
+ # comment out if need to use maven utilities
+ && find / -depth -name plexus-utils -type d -exec rm -r "{}" \; \
+ # remove old version of commons-compress with vulnerability
+ && find / -depth -name commons-compress -type d -exec rm -r "{}/1.20" \; \
+ # remove jar files from common-io v2.5 and 2.6 both have vulnerabilities
+ && find / -name commons-io*2.5.jar -type f -exec rm "{}" \; \
+ && find / -name commons-io*2.6.jar -type f -exec rm "{}" \; \
+ # remove jackson-databind
+ && find / -name jackson-databind -type d -exec rm -r "{}/2.13.3" \; \
+ # remove junit-4.12.jar
+ && find / -name junit-4.12.jar -type f -exec rm "{}" \; \
+ # remove maven-compiler-plugin jar from maven repo
+ && find / -name maven-compiler-plugin*.jar -type f -exec rm "{}" \; \
+ # remove guava jar files
+ && rm /usr/share/java/guava.jar && rm /root/.m2/repository/com/google/guava/guava/10.0.1/guava-10.0.1.jar \
+ # remove commons-codec jar
+ && find / -name commons-codec-1.11.jar -type f -exec rm "{}" \;
 
-RUN chmod a+x /usr/local/bin/serve.sh
-ENTRYPOINT ["/usr/local/bin/serve.sh"]
+ENTRYPOINT ["/usr/local/bin/serve.sh"]

README.md

@@ -22,7 +22,7 @@ SageMaker SparkML Serving Container is primarily built on the underlying Spring
 Supported Spark/MLeap version
 =============================
 
-Currently SageMaker SparkML Serving is powered by MLeap 0.13.0 and it is tested with Spark major version - 2.3.
+Currently SageMaker SparkML Serving is powered by MLeap 0.20.0 and it is tested with Spark major version - 3.3.
 
 Table of Contents
 =================
@@ -223,20 +223,20 @@ Calling `CreateModel` is required for creating a `Model` in SageMaker with this
 SageMaker works with Docker images stored in [Amazon ECR](https://aws.amazon.com/ecr/). SageMaker team has prepared and uploaded the Docker images for SageMaker SparkML Serving Container in all regions where SageMaker operates. 
 Region to ECR container URL mapping can be found below. For a mapping from Region to Region Name, please see [here](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html).
 
-* us-west-1 = 746614075791.dkr.ecr.us-west-1.amazonaws.com/sagemaker-sparkml-serving:2.4
-* us-west-2 = 246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-sparkml-serving:2.4
-* us-east-1 = 683313688378.dkr.ecr.us-east-1.amazonaws.com/sagemaker-sparkml-serving:2.4
-* us-east-2 = 257758044811.dkr.ecr.us-east-2.amazonaws.com/sagemaker-sparkml-serving:2.4
-* ap-northeast-1 = 354813040037.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-sparkml-serving:2.4
-* ap-northeast-2 = 366743142698.dkr.ecr.ap-northeast-2.amazonaws.com/sagemaker-sparkml-serving:2.4
-* ap-southeast-1 = 121021644041.dkr.ecr.ap-southeast-1.amazonaws.com/sagemaker-sparkml-serving:2.4
-* ap-southeast-2 = 783357654285.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-sparkml-serving:2.4
-* ap-south-1 = 720646828776.dkr.ecr.ap-south-1.amazonaws.com/sagemaker-sparkml-serving:2.4
-* eu-west-1 = 141502667606.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-sparkml-serving:2.4
-* eu-west-2 = 764974769150.dkr.ecr.eu-west-2.amazonaws.com/sagemaker-sparkml-serving:2.4
-* eu-central-1 = 492215442770.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-sparkml-serving:2.4
-* ca-central-1 = 341280168497.dkr.ecr.ca-central-1.amazonaws.com/sagemaker-sparkml-serving:2.4
-* us-gov-west-1 = 414596584902.dkr.ecr.us-gov-west-1.amazonaws.com/sagemaker-sparkml-serving:2.4
+* us-west-1 = 746614075791.dkr.ecr.us-west-1.amazonaws.com/sagemaker-sparkml-serving:3.3
+* us-west-2 = 246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-sparkml-serving:3.3
+* us-east-1 = 683313688378.dkr.ecr.us-east-1.amazonaws.com/sagemaker-sparkml-serving:3.3
+* us-east-2 = 257758044811.dkr.ecr.us-east-2.amazonaws.com/sagemaker-sparkml-serving:3.3
+* ap-northeast-1 = 354813040037.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-sparkml-serving:3.3
+* ap-northeast-2 = 366743142698.dkr.ecr.ap-northeast-2.amazonaws.com/sagemaker-sparkml-serving:3.3
+* ap-southeast-1 = 121021644041.dkr.ecr.ap-southeast-1.amazonaws.com/sagemaker-sparkml-serving:3.3
+* ap-southeast-2 = 783357654285.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-sparkml-serving:3.3
+* ap-south-1 = 720646828776.dkr.ecr.ap-south-1.amazonaws.com/sagemaker-sparkml-serving:3.3
+* eu-west-1 = 141502667606.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-sparkml-serving:3.3
+* eu-west-2 = 764974769150.dkr.ecr.eu-west-2.amazonaws.com/sagemaker-sparkml-serving:3.3
+* eu-central-1 = 492215442770.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-sparkml-serving:3.3
+* ca-central-1 = 341280168497.dkr.ecr.ca-central-1.amazonaws.com/sagemaker-sparkml-serving:3.3
+* us-gov-west-1 = 414596584902.dkr.ecr.us-gov-west-1.amazonaws.com/sagemaker-sparkml-serving:3.3
 
 With [SageMaker Python SDK](https://github.com/aws/sagemaker-python-sdk)
 ------------------------------------------------------------------------
@@ -263,7 +263,7 @@ First you need to ensure that have installed [Docker](https://www.docker.com/) o
 In order to build the Docker image, you need to run a single Docker command:
 
 ```
-docker build -t sagemaker-sparkml-serving:2.4 .
+docker build -t sagemaker-sparkml-serving:3.3 .
 ```
 
 #### Running the image locally
@@ -272,7 +272,7 @@ In order to run the Docker image, you need to run the following command. Please
 The command will start the server on port 8080 and will also pass the schema as an environment variable to the Docker container. Alternatively, you can edit the `Dockerfile` to add `ENV SAGEMAKER_SPARKML_SCHEMA=schema` as well before building the Docker image.
 
 ```
-docker run -p 8080:8080 -e SAGEMAKER_SPARKML_SCHEMA=schema -v /tmp/model:/opt/ml/model sagemaker-sparkml-serving:2.4 serve
+docker run -p 8080:8080 -e SAGEMAKER_SPARKML_SCHEMA=schema -v /tmp/model:/opt/ml/model sagemaker-sparkml-serving:3.3 serve
 ```
 
 #### Invoking with a payload
@@ -287,7 +287,7 @@ or
 curl -i -H "content-type:application/json" -d "{\"data\":[feature_1,\"feature_2\",feature_3]}" http://localhost:8080/invocations
 ```
 
-The `Dockerfile` can be found at the root directory of the package. SageMaker SparkML Serving Container tags the Docker images using the Spark major version it is compatible with. Right now, it only supports Spark 2.4 and as a result, the Docker image is tagged with 2.4. 
+The `Dockerfile` can be found at the root directory of the package. SageMaker SparkML Serving Container tags the Docker images using the Spark major version it is compatible with. Right now, it only supports Spark 3.3.0 and as a result, the Docker image is tagged with 3.3. 
 
 In order to save the effort of building the Docker image everytime you are making a code change, you can also install [Maven](http://maven.apache.org/) and run `mvn clean package` at your project root to verify if the code is compiling fine and unit tests are running without any issue. 
 
@@ -310,7 +310,7 @@ aws ecr get-login --region us-west-2 --registry-ids 246618743249 --no-include-em
 * Download the Docker image with the following command:
 
 ```
-docker pull 246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-sparkml-serving:2.4
+docker pull 246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-sparkml-serving:3.3
 ```
 
 For running the Docker image, please see the Running the image locally section from above.

ci/buildspec.yml

@@ -1,18 +1,41 @@
 version: 0.2
-
+env:
+  variables:
+    IS_GENERIC_IMAGE: "True"
+    CODEBUILD_RESOLVED_SOURCE_VERSION: "sparkml-v33"
+    TGT_IMAGE: "515193369038.dkr.ecr.us-west-2.amazonaws.com/sagemaker-sparkml-serving:3.3"
+    DLC_IMAGES: "515193369038.dkr.ecr.us-west-2.amazonaws.com/sagemaker-sparkml-serving:3.3-pre-scan"
 phases:
+  install:
+    runtime-versions:
+      python: latest
+    commands:
+      - pip3 install pytest
+      - pip3 install -r $CODEBUILD_SRC_DIR_Source2/src/requirements.txt
+      - pip3 install -r $CODEBUILD_SRC_DIR_Source2/test/requirements.txt
   pre_build:
     commands:
     - echo Logging in to Amazon ECR...
-    - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION)
+    - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin 515193369038.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
   build:
     commands:
-    - echo Build started on `date`
-    - echo Building the Docker image...
-    - docker build -t sagemaker-sparkml-serving:2.4 .
-    - docker tag sagemaker-sparkml-serving:2.4 515193369038.dkr.ecr.us-west-2.amazonaws.com/sagemaker-sparkml-serving:2.4
+      - echo Build started on `date`
+      - echo Building the Docker image...
+      - docker build -t sagemaker-sparkml-serving:3.3 .
+      - echo Build completed on `date`
   post_build:
+    on-failure: ABORT
     commands:
-    - echo Build completed on `date`
-    - echo Pushing the Docker image...
-    - docker push 515193369038.dkr.ecr.us-west-2.amazonaws.com/sagemaker-sparkml-serving:2.4
+      - echo Tagging pre-scan image...
+      - docker tag sagemaker-sparkml-serving:3.3 $DLC_IMAGES
+      - docker push $DLC_IMAGES
+      - cd $CODEBUILD_SRC_DIR_Source2
+      - export PYTHONPATH=$(pwd)/src
+      - cd test/dlc_tests
+      - echo Running enhanced ecr image scan
+      - pytest -s sanity/test_ecr_scan.py::test_ecr_enhanced_scan
+      - echo Tagging image for final push
+      - docker tag sagemaker-sparkml-serving:3.3 $TGT_IMAGE
+      - docker push $TGT_IMAGE
+      - echo $TGT_IMAGE pushed to ECR
+      - echo Push completed successfully on `date`