[WIP] gdal2.4.2-py3.6

.gitignore

@@ -0,0 +1 @@
+*.zip

Dockerfile

@@ -1,29 +1,38 @@
 FROM lambci/lambda:build-python3.6
 
+# Build GIS libs for AWS serverless solutions.  
+# See also the source builds for Debian packaging:
+# - https://salsa.debian.org/debian-gis-team
+#   - https://packages.ubuntu.com/source/eoan/gdal
+
 ENV \
   LANG=en_US.UTF-8 \
   LC_ALL=en_US.UTF-8
 
 RUN yum makecache fast
 RUN yum install -y automake16 libpng-devel nasm
 
-ENV PREFIX /var/task
+# labda runtime: /var/task
+# labda layer: /opt
+ARG prefix=/var/task
+ENV PREFIX=${prefix}
 
-# versions of packages
-ENV \
-  PKGCONFIG_VERSION=0.29.2 \
-  PROJ_VERSION=5.2.0 \
-  GEOS_VERSION=3.7.1 \
-  LIBPNG_VERSION=1.6.36 \
-  OPENJPEG_VERSION=2.3.1 \
-  LIBJPEG_TURBO_VERSION=2.0.1 \
-  WEBP_VERSION=1.0.2 \
-  ZSTD_VERSION=1.3.8 \
-  CURL_VERSION=7.59.0 \
-  NGHTTP2_VERSION=1.35.1 \
-  GDAL_VERSION=2.4.2
+ENV CPPFLAGS="-I${PREFIX}/include $CPPFLAGS"
+ENV LDFLAGS="-L${PREFIX}/lib $LDFLAGS"
+
+# pkg-config
+ENV PKGCONFIG_VERSION=0.29.2
+RUN mkdir /tmp/pkg-config \
+   && curl -sfL https://pkg-config.freedesktop.org/releases/pkg-config-${PKGCONFIG_VERSION}.tar.gz | tar zxf - -C /tmp/pkg-config --strip-components=1 \
+   && cd /tmp/pkg-config \
+   && CFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX \
+   && make -j $(nproc) --silent && make install && make clean \
+   && rm -rf /tmp/pkg-config
+
+ENV PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig/
 
 # nghttp2
+ENV NGHTTP2_VERSION=1.35.1
 RUN mkdir /tmp/nghttp2 \
   && curl -sfL https://github.com/nghttp2/nghttp2/releases/download/v${NGHTTP2_VERSION}/nghttp2-${NGHTTP2_VERSION}.tar.gz | tar zxf - -C /tmp/nghttp2 --strip-components=1 \
   && cd /tmp/nghttp2 \
@@ -32,55 +41,64 @@ RUN mkdir /tmp/nghttp2 \
   && rm -rf /tmp/nghttp2
 
 # libcurl
+ENV CURL_VERSION=7.59.0
 RUN mkdir /tmp/libcurl \
   && curl -sfL https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz | tar zxf - -C /tmp/libcurl --strip-components=1 \
   && cd /tmp/libcurl \
   && ./configure --disable-manual --disable-cookies --with-nghttp2=$PREFIX --prefix=$PREFIX \
   && make -j $(nproc) --silent && make install && make clean \
   && rm -rf /tmp/libcurl
 
-# pkg-config
-RUN mkdir /tmp/pkg-config \
-   && curl -sfL https://pkg-config.freedesktop.org/releases/pkg-config-$PKGCONFIG_VERSION.tar.gz | tar zxf - -C /tmp/pkg-config --strip-components=1 \
-   && cd /tmp/pkg-config \
-   && CFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX \
-   && make -j $(nproc) --silent && make install && make clean \
-   && rm -rf /tmp/pkg-config
-
-# proj
-RUN mkdir /tmp/proj \
-   && curl -sfL http://download.osgeo.org/proj/proj-$PROJ_VERSION.tar.gz | tar zxf - -C /tmp/proj --strip-components=1 \
-   && cd /tmp/proj \
-   && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX \
-   && make && make install && make clean \
-   && rm -rf /tmp/proj
+# sqlite3
+ENV SQLITE3_VERSION=3.29.0
+RUN mkdir /tmp/sqlite3 \
+    && curl -sfL https://www.sqlite.org/2019/sqlite-autoconf-3290000.tar.gz | tar zxf - -C /tmp/sqlite3 --strip-components=1 \
+    && cd /tmp/sqlite3 \
+    && ./configure --prefix=$PREFIX \
+    && make -j $(nproc) --silent && make install && make clean \
+    && rm -rf /tmp/sqlite3
 
 # geos
+ENV GEOS_VERSION=3.7.1
 RUN mkdir /tmp/geos \
-  && curl -sfL http://download.osgeo.org/geos/geos-$GEOS_VERSION.tar.bz2 | tar jxf - -C /tmp/geos --strip-components=1 \
+  && curl -sfL http://download.osgeo.org/geos/geos-${GEOS_VERSION}.tar.bz2 | tar jxf - -C /tmp/geos --strip-components=1 \
   && cd /tmp/geos \
   && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX \
   && make -j $(nproc) --silent && make install && make clean \
   && rm -rf /tmp/geos
 
+# proj
+ENV PROJ_VERSION=6.1.1
+RUN mkdir /tmp/proj \
+   && curl -sfL http://download.osgeo.org/proj/proj-${PROJ_VERSION}.tar.gz | tar zxf - -C /tmp/proj --strip-components=1 \
+   && cd /tmp/proj \
+   && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX \
+   && make && make install && make clean \
+   && rm -rf /tmp/proj
+
 # png
+ENV LIBPNG_VERSION=1.6.36
 RUN mkdir /tmp/png \
   && curl -sfL http://prdownloads.sourceforge.net/libpng/libpng-$LIBPNG_VERSION.tar.gz | tar zxf - -C /tmp/png --strip-components=1 \
   && cd /tmp/png \
   && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX \
   && make -j $(nproc) --silent && make install && make clean \
   && rm -rf /tmp/png
+ENV LIBPNG_CONFIG=${PREFIX}/bin/libpng-config
+RUN ls -l $LIBPNG_CONFIG
 
 # openjpeg
+ENV OPENJPEG_VERSION=2.3.1
 RUN mkdir /tmp/openjpeg \
-  && curl -sfL https://github.com/uclouvain/openjpeg/archive/v$OPENJPEG_VERSION.tar.gz | tar zxf - -C /tmp/openjpeg --strip-components=1 \
+  && curl -sfL https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz | tar zxf - -C /tmp/openjpeg --strip-components=1 \
   && cd /tmp/openjpeg \
   && mkdir build && cd build \
   && cmake .. -DBUILD_THIRDPARTY:BOOL=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$PREFIX \
   && make -j $(nproc) install && make clean \
   && rm -rf /tmp/openjpeg
 
 # jpeg_turbo
+ENV LIBJPEG_TURBO_VERSION=2.0.1
 RUN mkdir /tmp/jpeg \
   && curl -sfL https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${LIBJPEG_TURBO_VERSION}.tar.gz | tar zxf - -C /tmp/jpeg --strip-components=1 \
   && cd /tmp/jpeg \
@@ -89,6 +107,7 @@ RUN mkdir /tmp/jpeg \
   && rm -rf /tmp/jpeg
 
 # webp
+ENV WEBP_VERSION=1.0.2
 RUN mkdir /tmp/webp \
     && curl -sfL https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${WEBP_VERSION}.tar.gz | tar zxf - -C /tmp/webp --strip-components=1 \
     && cd /tmp/webp \
@@ -97,15 +116,28 @@ RUN mkdir /tmp/webp \
     && rm -rf /tmp/webp
 
 # zstd
+ENV ZSTD_VERSION=1.3.8
 RUN mkdir /tmp/zstd \
   && curl -sfL https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz | tar zxf - -C /tmp/zstd --strip-components=1 \
   && cd /tmp/zstd \
   && make -j $(nproc) PREFIX=$PREFIX ZSTD_LEGACY_SUPPORT=0 CFLAGS=-O1 --silent && make install PREFIX=$PREFIX ZSTD_LEGACY_SUPPORT=0 CFLAGS=-O1 && make clean \
   && rm -rf /tmp/zstd
 
-ENV PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig/
+## geotiff
+#ENV GEOTIFF_VERSION=1.5.1
+#RUN mkdir /tmp/geotiff \
+#    && archive="https://github.com/OSGeo/libgeotiff/releases/download/${GEOTIFF_VERSION}/libgeotiff-${GEOTIFF_VERSION}.tar.gz" \
+#    && curl -sfL ${archive} |  tar zxf - -C /tmp/geotiff --strip-components=1 \
+#    && cd /tmp/geotiff \
+#    && ./configure --prefix=${PREFIX} \
+#        --with-proj=${PREFIX} \
+#        --with-jpeg=${PREFIX} \
+#        --with-zip=yes \
+#    && make -j $(nproc) install && make clean \
+#    && rm -rf /tmp/geotiff
 
 # gdal
+ENV GDAL_VERSION=2.4.2
 RUN mkdir /tmp/gdal \
   && curl -sfL https://github.com/OSGeo/gdal/archive/v${GDAL_VERSION}.tar.gz | tar zxf - -C /tmp/gdal --strip-components=2
 
@@ -122,7 +154,9 @@ RUN cd /tmp/gdal \
       --with-webp=$PREFIX \
       --with-zstd=$PREFIX \
       --with-crypto \
+      #--with-geotiff=$PREFIX \
       --with-libtiff=internal \
+      --with-sqlite3 \
       --with-threads \
       --disable-debug \
       --with-hide-internal-symbols=yes \
@@ -161,7 +195,6 @@ RUN cd /tmp/gdal \
       --without-python \
       --without-qhull \
       --without-sde \
-      --without-sqlite3 \
       --without-xerces \
       --without-xml2
 
@@ -179,5 +212,25 @@ ENV \
 
 ENV PATH=$PREFIX/bin:$PATH
 
-RUN pip3 install pip -U
-RUN pip3 install cython numpy --no-binary numpy
+# numpy 1.17 requires an explicit c99 compiler option
+# - https://github.com/numpy/numpy/pull/12783/files
+ENV CFLAGS='-std=c99'
+
+# Using individual docker layers to cache a couple of
+# very common python packages that others depend on.
+RUN pip3 install -U pip
+RUN pip3 install -U cython numpy --no-compile --no-binary :all: 
+#RUN pip3 install "${PIP_OPTIONS}" numpy
+# Only install python packages that depend on the libs in this image.
+RUN pip3 install -U \
+    'gdal >=2.4,<3.0' \
+    pyproj \
+    rasterio \
+    fiona \
+    shapely \
+    pycrs \
+    --no-compile --no-binary :all: 
+
+# only for python 3.6; use built-in dataclasses for 3.7+
+RUN pip3 install -U dataclasses --no-compile --no-binary :all: 
+

Makefile

@@ -1,32 +1,66 @@
 SHELL = /bin/bash
-TAG = 2.4.2
-IMAGE := ${DOCKER_USERNAME}/amazonlinux-gdal:${TAG}
+PY_VERSION = 3.6
+GDAL_VERSION = 2.4.2
+TAG = gdal${GDAL_VERSION}-py${PY_VERSION}
+IMAGE := amazonlinux-gdal
+BUILD := ${IMAGE}:${TAG}
 
 all: build push
 
 build:
-	docker build -f Dockerfile -t amazonlinux-gdal:${TAG} .
+	docker build -f Dockerfile -t ${BUILD} .
 
-shell:
-	docker run --name amazonlinux --volume $(shell pwd)/:/data --rm  -it ${IMAGE} /bin/bash
+shell: build
+	docker run --name amazonlinux --volume $(shell pwd)/:/data --rm  -it ${BUILD} /bin/bash
 
 debug-gdal: build
-	docker run \
-		--name amazonlinux \
-		-itd amazonlinux-gdal:${TAG} /bin/bash
+	docker run --name amazonlinux \
+		-itd ${BUILD} /bin/bash
 	docker exec -it amazonlinux bash -c 'ldd /var/task/lib/libgdal.so'
 	docker exec -it amazonlinux bash -c 'readelf -d /var/task/lib/libgdal.so'
 	docker stop amazonlinux
 	docker rm amazonlinux
 
 test:
-	docker run amazonlinux-gdal:${TAG} bash -c "gdalinfo --version | grep '${TAG}'"
-	docker run amazonlinux-gdal:${TAG} bash -c "gdalinfo --formats | grep 'JP2OpenJPEG'"
+	docker run ${BUILD} bash -c "gdalinfo --version | grep '${GDAL_VERSION}'"
+	docker run ${BUILD} bash -c "gdalinfo --formats | grep 'JP2OpenJPEG'"
+	docker run ${BUILD} bash -c "python --version | grep '${PY_VERSION}'"
 
 push:
-	docker tag amazonlinux-gdal:${TAG} ${IMAGE}
-	docker push ${IMAGE}
+	docker push ${DOCKER_USERNAME}/${BUILD}
+
+container-clean:
+	docker stop amazonlinux > /dev/null 2>&1 || true
+	docker rm amazonlinux > /dev/null 2>&1 || true
+
+# ---
+# lambda layer build and package using /opt
+
+LAYER_BUILD = ${BUILD}-layer
+LAYER_PACKAGE := amazonlinux-${TAG}-layer
+
+lambda-layer-build:
+	docker build -f Dockerfile -t ${LAYER_BUILD} --build-arg prefix=/opt .
+
+lambda-layer-shell: lambda-layer-build container-clean
+	docker run --name amazonlinux --volume $(shell pwd)/:/data --rm -it ${LAYER_BUILD} /bin/bash
+
+lambda-layer-test: lambda-layer-build
+	docker run --volume $(shell pwd)/:/data --rm -it ${LAYER_BUILD} /bin/bash -c '/data/tests/test.sh'
+
+lambda-layer-package: lambda-layer-build container-clean
+	docker run --name amazonlinux \
+		-e PREFIX=${PREFIX} \
+		-e PY_VERSION=${PY_VERSION} \
+		-e LAYER_PACKAGE=${LAYER_PACKAGE} \
+		-itd ${LAYER_BUILD} /bin/bash
+	docker cp package_lambda_layer.sh amazonlinux:/tmp/package_lambda_layer.sh
+	docker exec -it amazonlinux bash -c '/tmp/package_lambda_layer.sh'
+	mkdir -p ./packages
+	docker cp amazonlinux:/tmp/${LAYER_PACKAGE}_libs.zip ./packages/
+	docker cp amazonlinux:/tmp/${LAYER_PACKAGE}_python.zip ./packages/
+	docker cp amazonlinux:/tmp/${LAYER_PACKAGE}_geopandas.zip ./packages/
+	docker cp amazonlinux:/tmp/${LAYER_PACKAGE}_pandas.zip ./packages/
+	docker cp amazonlinux:/tmp/${LAYER_PACKAGE}_pyarrow.zip ./packages/
+	docker stop amazonlinux && docker rm amazonlinux
 
-clean:
-	docker stop amazonlinux
-	docker rm amazonlinux

package_lambda_layer.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+prefix="${PREFIX:-/opt}"
+
+py_version="${PY_VERSION:-3.6}"
+
+layer_package="${LAYER_PACKAGE:-layer_package}"
+
+clean_python_packages () {
+    site=$1
+    find $site -type d -name '__pycache__' -exec rm -rf {} +
+    find $site -type d -name '*.dist-info' -exec rm -rf {} +
+    find $site -type d -name '*.egg-info' -exec rm -rf {} +
+    find $site -type d -name 'tests' -exec rm -rf {} +
+    find $site -type d -name 'datasets' -exec rm -rf {} +
+}
+
+src=/var/lang/lib/python${py_version}/site-packages
+dst=${prefix}/python/lib/python${py_version}/site-packages
+mkdir -p ${dst}
+
+clean_python_packages $src
+
+# Note: excluding many paths in the python libs crashed the layer runtime
+#       somehow, so the excludes are disabled for now.
+#rsync -a $src/ ${dst}/
+rsync -a \
+	--exclude 'boto3' \
+	--exclude 'botocore' \
+	--exclude 'awscli' \
+	--exclude 'samtranslator' \
+	--exclude 'docker' \
+	--exclude 'samcli' \
+	--exclude 'share/doc/*' \
+	--exclude 'share/man/man1/*' \
+	--exclude 'share/man/man3/*' \
+       $src/ ${dst}/
+
+
+# package the binary 'libs' and the python packages into two layers,
+# to reduce the size of each package and allow flexibility in using
+# just the binaries or the binaries plus the python bindings.
+cd ${prefix}
+zip -r9 --symlinks /tmp/${layer_package}_python.zip python
+zip -r9 --symlinks /tmp/${layer_package}_libs.zip bin lib/*.so* lib64/*.so* share
+
+
+PIP_OPTIONS="--no-compile --no-binary :all: -t ${dst}"
+
+# add a new python layer for pandas
+rm -rf "${dst}/*"
+pip3 install pandas "${PIP_OPTIONS}"
+clean_python_packages "${dst}"
+zip -r9 --symlinks /tmp/${layer_package}_pandas.zip python
+
+# add a new python layer for pyarrow
+rm -rf "${dst}/*"
+pip3 install pyarrow s3fs "${PIP_OPTIONS}"
+clean_python_packages "${dst}"
+zip -r9 --symlinks /tmp/${layer_package}_pyarrow.zip python
+
+# add a new python layer for geopandas
+rm -rf "${dst}/*"
+pip3 install geopandas "${PIP_OPTIONS}"
+clean_python_packages "${dst}"
+zip -r9 --symlinks /tmp/${layer_package}_geopandas.zip python
+

requirements.txt

@@ -0,0 +1,14 @@
+cython
+numpy >=1.17,<2.0
+gdal >=2.4,<3.0
+geopandas
+#pyarrow
+pycrs
+pyproj
+rasterio
+shapely
+
+# only for python 3.6
+# use built-in dataclasses for 3.7+
+dataclasses
+