Merge pull request #73 from developmentseed/develop

release 1.2.0rc1

Author: matthewhanson

.circleci/config.yml

@@ -29,35 +29,46 @@ jobs:
       - run:
           name: Build application Docker image
           command: |
-            VERSION=$(cat VERSION)
-            docker build . -t developmentseed/geolambda:${VERSION}
-            # publish docker images
+            # update known hosts
             mkdir -p ~/.ssh
             ssh-keyscan github.com >> ~/.ssh/known_hosts
-            #git tag -d ${VERSION}
+            # build and publish geolambda
+            VERSION=$(cat VERSION)
+            docker build . -t developmentseed/geolambda:${VERSION}
+            docker run --rm -v ${PWD}:/home/geolambda --entrypoint package.sh -it developmentseed/geolambda:${VERSION}
             git tag ${VERSION}
             git push origin ${VERSION}
             docker login -u $DOCKER_USER -p $DOCKER_PASS
             docker tag developmentseed/geolambda:${VERSION} developmentseed/geolambda:latest
             docker push developmentseed/geolambda:latest
             docker push developmentseed/geolambda:${VERSION}
+            # build and publish geolambda-python docker image
             cd python
-            docker build --build-arg VERSION=${VERSION} --build-arg PYVERSION=3.6.1 . -t developmentseed/geolambda:${VERSION}-python36
-            docker push developmentseed/geolambda:${VERSION}-python36
+            docker build --build-arg VERSION=${VERSION} . -t developmentseed/geolambda:${VERSION}-python
+            docker run --rm -v ${PWD}:/home/geolambda --entrypoint package-python.sh -it developmentseed/geolambda:${VERSION}-python
+            docker push developmentseed/geolambda:${VERSION}-python
             cd ..
             # deploy public Lambda layers
             pip install awscli
-            docker run --rm -v $PWD:/home/geolambda --entrypoint package.sh -it developmentseed/geolambda:${VERSION}
             for region in us-east-1 us-west-2 eu-central-1
             do
               LVERSION="$(aws lambda publish-layer-version --region ${region} \
-                --layer-name geolambda --license-info 'MIT' \
+                --layer-name geolambda-dev --license-info 'MIT' \
                 --description 'Native geospatial libaries for all runtimes' \
                 --zip-file fileb://lambda-deploy.zip | jq '.Version')"
               aws lambda add-layer-version-permission --region ${region} \
-                --layer-name geolambda --action lambda:GetLayerVersion \
+                --layer-name geolambda-dev --action lambda:GetLayerVersion \
+                --statement-id public --version-number ${LVERSION} --principal '*'
+              LVERSION="$(aws lambda publish-layer-version --region ${region} \
+                --layer-name geolambda-dev-python --license-info 'MIT' \
+                --description 'Geospatial Python libraries' \
+                --zip-file fileb://lambda-deploy.zip | jq '.Version')
+                --compatible-runtimes python3.7"
+              aws lambda add-layer-version-permission --region ${region} \
+                --layer-name geolambda-dev-python --action lambda:GetLayerVersion \
                 --statement-id public --version-number ${LVERSION} --principal '*'
             done
+            # deploy Python Lambda layers
 
 
 workflows:

.gitignore

@@ -1,2 +1,4 @@
 .env
 lambda-deploy.zip
+lambda-layer-deploy.zip
+.DS_Store

CHANGELOG.md

@@ -6,6 +6,31 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [v1.2.0] - 2019-10-21
+
+### Added
+- A Python GeoLambda Layer is now published, along with the base Lambda Layer. The base layer *must* be included in any Lambda that uses the Python GeoLambda layer. It includes the Python libraries GDAL, rasterio, pyproj, and shapely.
+
+### Changed
+- The python diectory, and the new Lambda layer, now uses Python 3.7.4
+
+### Package versions
+- GDAL_VERSION=2.4.2
+- PROJ_VERSION=5.2.0
+- GEOS_VERSION=3.8.0
+- GEOTIFF_VERSION=1.4.3
+- HDF4_VERSION=4.2.14
+- HDF5_VERSION=1.10.5
+- NETCDF_VERSION=4.7.1
+- NGHTTP2_VERSION=1.39.2
+- OPENJPEG_VERSION=2.3.1
+- CURL_VERSION=7.66.0
+- LIBJPEG_TURBO_VERSION=2.0.3
+- PKGCONFIG_VERSION=0.29.2
+- SZIP_VERSION=2.1.1
+- WEBP_VERSION=1.0.3
+- ZSTD_VERSION=1.4.3
+
 ## [v1.1.0] - 2018-03-22
 
 ### Added
@@ -22,27 +47,27 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Removed
 - Removed Python codes to make geolambda system libraries only
 
-### Versions:
-- CURL_VERSION=7.59.0
+### Package versions:
+- GDAL_VERSION=2.4.1
+- PROJ_VERSION=5.2.0
 - GEOS_VERSION=3.7.1
 - GEOTIFF_VERSION=1.4.3
-- GDAL_VERSION=2.4.1
+- OPENJPEG_VERSION=2.3.0
+- LIBJPEG_TURBO_VERSION=2.0.1
 - HDF4_VERSION=4.2.14
 - HDF5_VERSION=1.10.5
 - NETCDF_VERSION=4.6.2
+- CURL_VERSION=7.59.0
 - NGHTTP2_VERSION=1.35.1
-- OPENJPEG_VERSION=2.3.0
-- LIBJPEG_TURBO_VERSION=2.0.1
 - PKGCONFIG_VERSION=0.29.2
-- PROJ_VERSION=5.2.0
 - SZIP_VERSION=2.1.1
 - WEBP_VERSION=1.0.1
 - ZSTD_VERSION=1.3.8
 
 ## [v1.0.0] - 2018-07-27
 
 #### Versions:
-- PROJ_VERSION=5.1.0
+- PROJ_VERSION=5.2.0
 - GEOS_VERSION=3.6.2
 - HDF4_VERSION=4.2.12
 - SZIP_VERSION=2.1.1

Dockerfile

@@ -13,29 +13,29 @@ RUN \
 
 # versions of packages
 ENV \
-    CURL_VERSION=7.59.0 \
-    GEOS_VERSION=3.7.1 \
+    GDAL_VERSION=2.4.2 \
+    PROJ_VERSION=5.2.0 \
+    GEOS_VERSION=3.8.0 \
     GEOTIFF_VERSION=1.4.3 \
-	GDAL_VERSION=2.4.1 \
     HDF4_VERSION=4.2.14 \
-	HDF5_VERSION=1.10.5 \
-    NETCDF_VERSION=4.6.2 \
-    NGHTTP2_VERSION=1.35.1 \
-	OPENJPEG_VERSION=2.3.0 \
-    LIBJPEG_TURBO_VERSION=2.0.1 \
+    HDF5_VERSION=1.10.5 \
+    NETCDF_VERSION=4.7.1 \
+    NGHTTP2_VERSION=1.39.2 \
+    OPENJPEG_VERSION=2.3.1 \
+    CURL_VERSION=7.66.0 \
+    LIBJPEG_TURBO_VERSION=2.0.3 \
     PKGCONFIG_VERSION=0.29.2 \
-    PROJ_VERSION=5.2.0 \
     SZIP_VERSION=2.1.1 \
-    WEBP_VERSION=1.0.1 \
-    ZSTD_VERSION=1.3.8
+    WEBP_VERSION=1.0.3 \
+    ZSTD_VERSION=1.4.3
 
 # Paths to things
 ENV \
-	BUILD=/build \
+    BUILD=/build \
     NPROC=4 \
-	PREFIX=/usr/local \
-	GDAL_CONFIG=/usr/local/bin/gdal-config \
-	LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 \
+    PREFIX=/usr/local \
+    GDAL_CONFIG=/usr/local/bin/gdal-config \
+    LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 \
     GDAL_DATA=/usr/local/share/gdal
 
 # switch to a build directory
@@ -79,11 +79,11 @@ RUN \
 # GEOS
 RUN \
     mkdir geos; \
-	wget -qO- http://download.osgeo.org/geos/geos-$GEOS_VERSION.tar.bz2 \
+    wget -qO- http://download.osgeo.org/geos/geos-$GEOS_VERSION.tar.bz2 \
         | tar xvj -C geos --strip-components=1; cd geos; \
-	./configure --enable-python --prefix=$PREFIX CFLAGS="-O2 -Os"; \
-	make -j ${NPROC} install; \
-	cd ..; rm -rf geos
+    ./configure --enable-python --prefix=$PREFIX CFLAGS="-O2 -Os"; \
+    make -j ${NPROC} install; \
+    cd ..; rm -rf geos
 
 # szip (for hdf)
 RUN \
@@ -191,16 +191,26 @@ RUN \
         --with-zstd=${PREFIX} \
         --with-jpeg=${PREFIX} \
         --with-threads=yes \
-		--with-curl=${PREFIX}/bin/curl-config \
+        --with-curl=${PREFIX}/bin/curl-config \
         --without-python \
         --without-libtool \
         --with-geos=$PREFIX/bin/geos-config \
-		--with-hide-internal-symbols=yes \
+        --with-hide-internal-symbols=yes \
         CFLAGS="-O2 -Os" CXXFLAGS="-O2 -Os" \
         LDFLAGS="-Wl,-rpath,'\$\$ORIGIN'"; \
     make -j ${NPROC} install; \
     cd ${BUILD}; rm -rf gdal
-# 
+
+# Open SSL is needed for building Python so it's included here for ease
+RUN \
+    OPENSSL_VERSION=1.0.2; \
+    mkdir openssl; \
+    wget -qO- https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \
+        | tar xvz -C openssl --strip-components=1; cd openssl; \
+    ./config shared --prefix=${PREFIX}/openssl --openssldir=${PREFIX}/openssl; \
+    make depend; make install; cd ..; rm -rf openssl
+
+
 # Copy shell scripts and config files over
 COPY bin/* /usr/local/bin/
 

README.md

@@ -11,9 +11,10 @@ This repository also contains examples and pre-built images for specific runtime
 While GeoLambda was initially intended for AWS Lambda they are also useful as base geospatial Docker images. The version of GDAL included for different versions of GeoLambda is shown here:
 
 | geolambda | GDAL  |
-| -------- | ----  |
-| 1.0.0    | 2.3.1 |
-| 1.1.0  | 2.4.1 |
+| --------- | ----  |
+| 1.0.0     | 2.3.1 |
+| 1.1.0     | 2.4.1 |
+| 1.2.0     | 2.4.2 |
 
 To get the version numbers of all installed packages see the [CHANGELOG](CHANGELOG.md) or the Dockerfile for the specific version.
 
@@ -39,21 +40,29 @@ The GeoLambda image does not have an entrypoint defined, so a command must be pr
 
 If you just wish to use the publicly available Lambda layer you will need the ARN for the layer in same region as your Lambda function. Currently, GeoLambda layers are available in `us-east-1`, `us-west-2`, and `eu-central-1`. If you want to use it in another region please file an issue or you can also create your own layer using this repository (see instructions below on 'Create a new version').
 
-#### v1.1.0
+#### v1.2.0rc1
 
 | Region | ARN |
 | ------ | --- |
-| us-east-1 | arn:aws:lambda:us-east-1:552188055668:layer:geolambda:1 |
-| us-west-2 | arn:aws:lambda:us-west-2:552188055668:layer:geolambda:1 |
-| eu-central-1 | arn:aws:lambda:eu-central-1:552188055668:layer:geolambda:1 |
+| us-east-1 | arn:aws:lambda:us-east-1:552188055668:layer:geolambda-dev:5 |
+| us-west-2 | arn:aws:lambda:us-west-2:552188055668:layer:geolambda-dev:5 |
+| eu-central-1 | arn:aws:lambda:eu-central-1:552188055668:layer:geolambda-dev:5 |
+
+#### v1.2.0rc1-python
+
+| Region | ARN |
+| ------ | --- |
+| us-east-1 | arn:aws:lambda:us-east-1:552188055668:layer:geolambda-dev-python:1 |
+| us-west-2 | arn:aws:lambda:us-west-2:552188055668:layer:geolambda-dev-python:1 |
+| eu-central-1 | arn:aws:lambda:eu-central-1:552188055668:layer:geolambda-dev-python:1 |
 
-#### v1.1.0rc3
+#### v1.1.0
 
 | Region | ARN |
 | ------ | --- |
-| us-east-1 | arn:aws:lambda:us-east-1:552188055668:layer:geolambda-dev:3 |
-| us-west-2 | arn:aws:lambda:us-west-2:552188055668:layer:geolambda-dev:3 |
-| eu-central-1 | arn:aws:lambda:eu-central-1:552188055668:layer:geolambda-dev:3 |
+| us-east-1 | arn:aws:lambda:us-east-1:552188055668:layer:geolambda:1 |
+| us-west-2 | arn:aws:lambda:us-west-2:552188055668:layer:geolambda:1 |
+| eu-central-1 | arn:aws:lambda:eu-central-1:552188055668:layer:geolambda:1 |
 
 
 ## Development

VERSION

@@ -1 +1 @@
-1.1.0
+1.2.0rc1

python/Dockerfile

@@ -1,10 +1,10 @@
-ARG VERSION=latest
+ARG VERSION=1.2.0
 FROM developmentseed/geolambda:${VERSION}
 
 LABEL maintainer="Development Seed <info@developmentseed.org>"
 LABEL authors="Matthew Hanson  <matt.a.hanson@gmail.com>"
 
-ARG PYVERSION=3.6.1
+ARG PYVERSION=3.7.4
 
 # install Python
 ENV \
@@ -13,7 +13,9 @@ ENV \
 
 RUN \
     curl https://pyenv.run | bash; \
-    pyenv install ${PYVERSION}; \
+    CONFIGURE_OPTS="--with-openssl=${PREFIX}/openssl" \
+        LD_RUN_PATH="${PREFIX}/openssl/lib" \
+        pyenv install ${PYVERSION}; \
     pyenv global ${PYVERSION}; \
     pip install --upgrade pip
 
@@ -23,4 +25,4 @@ RUN \
     pip install -r requirements-pre.txt; \
     pip install -r requirements.txt
 
-COPY bin/* /usr/local/bin/
+COPY bin/* /usr/local/bin/