Merge branch 'master' into ceos_master

.dockerignore

@@ -1,3 +1,5 @@
 .git
 **/*.tif
 odc_gee/config
+# created by Gitlab CI local runners
+builds

.gitignore

@@ -1,5 +1,6 @@
 notebooks/AMA
-notebooks/experimental
+notebooks/experimental/*
+!notebooks/experimental/voxel_visualizer
 notebooks/indexing
 notebooks/scalability/Dask/mydask.png
 notebooks/test/error_report.csv
@@ -27,3 +28,5 @@ __pycache__
 **/output
 **/.env
 odc_gee/config
+# created by Gitlab CI local runners
+builds

.gitlab-ci_bak.yml

@@ -0,0 +1,19 @@
+image: jcrattzama/odc-gitlab-runner
+
+stages:
+  - build-no-cache
+
+build:
+  stage: build
+  script:
+    - make dev-build
+
+build-no-cache:
+  stage: build-no-cache
+  script:
+    - make dev-build-no-cache
+
+push:
+  stage: push
+  script:
+    - make dev-push

Makefile

@@ -1,27 +1,36 @@
 SHELL:=/bin/bash
-docker_compose_dev = docker-compose --project-directory build/docker/dev -f build/docker/dev/docker-compose.yml
+# Set the project name to the path - making underscore the path separator.
+# Remove the leading slash and use lowercase since docker-compose will.
+project_name=$(shell PWD_var=$$(pwd); PWD_no_lead_slash=$${PWD_var:1}; echo $${PWD_no_lead_slash//\//_} | awk '{print tolower($$0)}' | cat)
+docker_compose_dev = docker-compose --project-directory build/docker/dev -f build/docker/dev/docker-compose.yml -p $(project_name)
+
 
 IMG_REPO?=jcrattzama/data_cube_notebooks
 IMG_VER?=
 ODC_VER?=1.8.3
 
 DEV_OUT_IMG?=${IMG_REPO}:odc${ODC_VER}${IMG_VER}
 
-COMMON_EXPRTS=export UID=$(id -u)
+export UID:=$(shell id -u)
+
+.PHONY: build
 
 ## Common ##
 
+dev-build-no-cache:
+	$(docker_compose_dev) build --no-cache
+
 dev-build:
-	(${COMMON_EXPRTS}; $(docker_compose_dev) build)
+	$(docker_compose_dev) build
 
 # Start the notebooks environment
 dev-up:
-	(${COMMON_EXPRTS}; $(docker_compose_dev) up -d --build)
+	$(docker_compose_dev) up -d --build
 
 # Start without rebuilding the Docker image
 # (use when dependencies have not changed for faster starts).
 dev-up-no-build:
-	(${COMMON_EXPRTS}; $(docker_compose_dev) up -d)
+	$(docker_compose_dev) up -d
 
 # Stop the notebooks environment
 dev-down:
@@ -144,24 +153,7 @@ sudo-ubuntu-install-docker:
 	sudo usermod -aG docker ${USER}
 ## End Docker Misc ##
 
-# Update S3 template (this is owned by Digital Earth Australia)
-upload-s3:
-	aws s3 cp cube-in-a-box-cloudformation.yml s3://opendatacube-cube-in-a-box/ --acl public-read
-
-# This section can be used to deploy onto CloudFormation instead of the 'magic link'
-create-infra:
-	aws cloudformation create-stack \
-		--region ap-southeast-2 \
-		--stack-name odc-test \
-		--template-body file://opendatacube-test.yml \
-		--parameter file://parameters.json \
-		--tags Key=Name,Value=OpenDataCube \
-		--capabilities CAPABILITY_NAMED_IAM
-
-update-infra:
-	aws cloudformation update-stack \
-		--stack-name odc-test \
-		--template-body file://opendatacube-test.yml \
-		--parameter file://parameters.json \
-		--tags Key=Name,Value=OpenDataCube \
-		--capabilities CAPABILITY_NAMED_IAM
+## CI ##
+test-ci-local:
+	gitlab-runner exec shell build-no-cache
+## End CI ##

README.ipynb

@@ -2,36 +2,30 @@
  "cells": [
   {
    "cell_type": "markdown",
-   "metadata": {},
    "source": [
     "# CEOS Open Data Cube Notebooks\n",
     "\n",
-    "The CEOS Data Cube Notebooks is a collection of [Jupyter Notebooks](https://jupyter.org/) using the [Open Data Cube](https://www.opendatacube.org/) to analyze satellite data.\n",
+    "The CEOS Data Cube Notebooks is a collection of Jupyter Notebooks in a [JupyterLabs](https://jupyter.org/) environment using the [Open Data Cube](https://www.opendatacube.org/) to analyze satellite data.\n",
     "\n",
     "For an overview of the available notebooks, see the [Notebooks Overview](docs/notebooks_overview.md) document.\n",
     "\n",
     "## Installation\n",
     "-------\n",
     "\n",
-    "First follow the instructions in the [Docker Installation Guide](https://github.com/ceos-seo/data_cube_ui/blob/master/docs/docker_install.md) if you do not have Docker installed yet.\n",
+    "First follow the instructions in the [Environment Setup Guide](https://ceos-odc.readthedocs.io/en/latest/modules/install_docs/environment_setup.html) if you have not yet.\n",
     "\n",
-    "Follow the instructions in the \n",
-    "[Open Data Cube Database Installation Guide](https://github.com/ceos-seo/data_cube_ui/blob/master/docs/odc_db_setup.md) to setup the Open Data Cube (ODC) database.\n",
+    "Follow the instructions in the [Open Data Cube Database Installation Guide](https://ceos-odc.readthedocs.io/en/latest/modules/install_docs/database_install.html) to setup the Open Data Cube (ODC) database.\n",
     "\n",
     "Follow the instructions in the [Operation Manual](docs/notebooks_operation_manual.md) to set up and operate the Jupyter Notebook environment.\n",
     "\n",
-    "## Obtaining Help\n",
-    "-------\n",
+    "Some notebooks make use of Google Earth Engine (GEE) data. For information on how to set up and use the [ODC-GEE integration package](https://github.com/ceos-seo/odc-gee) to load data from GEE, see [this documentation page](https://ceos-odc.readthedocs.io/en/latest/modules/odc_products_technical_docs/odc_gee_technical_doc.html).\n",
     "\n",
-    "If you encounter issues with the Open Data Cube or these notebooks that are not documented in the documents linked to above or the other files in the `docs` directory (remember to check the FAQ sections) and you are unable to diagnose and fix those issues on your own, follow these steps to obtain assistance:\n",
-    "1. Search for your question in the [GIS Stack Exchange](https://gis.stackexchange.com/) with the `open-data-cube` tag. You can find such tagged questions [here](https://gis.stackexchange.com/questions/tagged/open-data-cube). If a question similar to yours has already been asked, search for a suitable answer in that question's webpage. If you can not find a suitable answer, contine to step 2. If no such question exists or if the question is missing some information that you think may be important regarding your issue, create a new question with the `open-data-cube` tag.\n",
-    "2. State your problem in the [ODC Slack workspace](http://slack.opendatacube.org/) in the most appropriate channel (use that link to join the workspace if you have not already). Use the **#general** channel if no other channel is more appropriate for your question. Link to the question on GIS Stack Exchange. When you receive an answer, add that answer to the question page on GIS Stack Exchange if the answerer is not available to do so. This preserves the answer in a publicly searchable way - which is useful for remembering answers to one's own past questions and benefits the community.\n",
-    "\n",
-    "## More\n",
+    "## Obtaining Help\n",
     "-------\n",
     "\n",
-    "You may also consider running a webapp that uses the Open Data Cube. The CEOS ODC UI repository is [here](https://github.com/ceos-seo/data_cube_ui)."
-   ]
+    "If you need help with the Open Data Cube or derived products such as this one, see [this documentation page](https://ceos-odc.readthedocs.io/en/latest/modules/help/obtaining_help.html) for instructions on obtaining help.\n"
+   ],
+   "metadata": {}
   }
  ],
  "metadata": {

README.md

@@ -1,27 +1,21 @@
 # CEOS Open Data Cube Notebooks
 
-The CEOS Data Cube Notebooks is a collection of [Jupyter Notebooks](https://jupyter.org/) using the [Open Data Cube](https://www.opendatacube.org/) to analyze satellite data.
+The CEOS Data Cube Notebooks is a collection of Jupyter Notebooks in a [JupyterLabs](https://jupyter.org/) environment using the [Open Data Cube](https://www.opendatacube.org/) to analyze satellite data.
 
 For an overview of the available notebooks, see the [Notebooks Overview](docs/notebooks_overview.md) document.
 
 ## Installation
 -------
 
-First follow the instructions in the [Docker Installation Guide](https://github.com/ceos-seo/data_cube_ui/blob/master/docs/docker_install.md) if you do not have Docker installed yet.
+First follow the instructions in the [Environment Setup Guide](https://ceos-odc.readthedocs.io/en/latest/modules/install_docs/environment_setup.html) if you have not yet.
 
-Follow the instructions in the 
-[Open Data Cube Database Installation Guide](https://github.com/ceos-seo/data_cube_ui/blob/master/docs/odc_db_setup.md) to setup the Open Data Cube (ODC) database.
+Follow the instructions in the [Open Data Cube Database Installation Guide](https://ceos-odc.readthedocs.io/en/latest/modules/install_docs/database_install.html) to setup the Open Data Cube (ODC) database.
 
 Follow the instructions in the [Operation Manual](docs/notebooks_operation_manual.md) to set up and operate the Jupyter Notebook environment.
 
-## Obtaining Help
--------
+Some notebooks make use of Google Earth Engine (GEE) data. For information on how to set up and use the [ODC-GEE integration package](https://github.com/ceos-seo/odc-gee) to load data from GEE, see [this documentation page](https://ceos-odc.readthedocs.io/en/latest/modules/odc_products_technical_docs/odc_gee_technical_doc.html).
 
-If you encounter issues with the Open Data Cube or these notebooks that are not documented in the documents linked to above or the other files in the `docs` directory (remember to check the FAQ sections) and you are unable to diagnose and fix those issues on your own, follow these steps to obtain assistance:
-1. Search for your question in the [GIS Stack Exchange](https://gis.stackexchange.com/) with the `open-data-cube` tag. You can find such tagged questions [here](https://gis.stackexchange.com/questions/tagged/open-data-cube). If a question similar to yours has already been asked, search for a suitable answer in that question's webpage. If you can not find a suitable answer, contine to step 2. If no such question exists or if the question is missing some information that you think may be important regarding your issue, create a new question with the `open-data-cube` tag.
-2. State your problem in the [ODC Slack workspace](http://slack.opendatacube.org/) in the most appropriate channel (use that link to join the workspace if you have not already). Use the **#general** channel if no other channel is more appropriate for your question. Link to the question on GIS Stack Exchange. When you receive an answer, add that answer to the question page on GIS Stack Exchange if the answerer is not available to do so. This preserves the answer in a publicly searchable way - which is useful for remembering answers to one's own past questions and benefits the community.
-
-## More
+## Obtaining Help
 -------
 
-You may also consider running a webapp that uses the Open Data Cube. The CEOS ODC UI repository is [here](https://github.com/ceos-seo/data_cube_ui).
+If you need help with the Open Data Cube or derived products such as this one, see [this documentation page](https://ceos-odc.readthedocs.io/en/latest/modules/help/obtaining_help.html) for instructions on obtaining help.

build/docker/dev/.env

@@ -7,3 +7,4 @@ ODC_DB_HOSTNAME=odc-db
 ODC_DB_DATABASE=datacube
 ODC_DB_USER=dc_user
 ODC_DB_PASSWORD=localuser1234
+VOXEL_VISUALIZER_PORT=8100

build/docker/dev/Dockerfile

@@ -57,8 +57,10 @@ COPY build/reqs/11_reqs_analysis.txt reqs/11_reqs_analysis.txt
 RUN pip3 install -r reqs/11_reqs_analysis.txt
 COPY build/reqs/12_reqs_vis.txt reqs/12_reqs_vis.txt
 RUN pip3 install -r reqs/12_reqs_vis.txt
-COPY build/reqs/13_reqs_misc.txt reqs/13_reqs_misc.txt
-RUN pip3 install -r reqs/13_reqs_misc.txt
+COPY build/reqs/13_reqs_voxel_visualizer.txt reqs/13_reqs_voxel_visualizer.txt
+RUN pip3 install -r reqs/13_reqs_voxel_visualizer.txt
+COPY build/reqs/14_reqs_misc.txt reqs/14_reqs_misc.txt
+RUN pip3 install -r reqs/14_reqs_misc.txt
 
 # RUN pip3 install rasterio==1.1.8 && \
 #     mkdir -p /etc/pki/tls/certs && \
@@ -83,6 +85,13 @@ ENV ODC_DB_HOSTNAME \
     ODC_DB_PORT
 COPY build/native/odc_conf.sh native/odc_conf.sh
 
+# Install ODC-GEE.
+COPY odc_gee odc_gee
+RUN cd odc_gee; python3 setup.py build && python3 setup.py install
+# Set the recently changed or added packages' ownership to jovyan. 
+RUN find /env/lib/python3.6/site-packages -group root -mtime -1 -exec chown -R jovyan:users {} +
+
+# Create the main directory.
 ARG ROOT_DIR
 ARG NOTEBOOK_ROOT
 ENV ROOT_DIR=${ROOT_DIR} NOTEBOOK_ROOT=${NOTEBOOK_ROOT}

build/docker/dev/docker-compose.yml

@@ -11,6 +11,8 @@ services:
         - NOTEBOOK_ROOT=${NOTEBOOK_ROOT}
     environment:
       NBK_SERVER_PASSWORD: ${NBK_SERVER_PASSWORD}
+      VOXEL_VISUALIZER_PORT: ${VOXEL_VISUALIZER_PORT}
+      # ODC DB credentials
       ODC_DB_HOSTNAME: ${ODC_DB_HOSTNAME}
       ODC_DB_DATABASE: ${ODC_DB_DATABASE}
       ODC_DB_USER: ${ODC_DB_USER}
@@ -22,6 +24,7 @@ services:
     ports:
       # 8888 is the Jupyter notebook default port
       - ${HOST_PORT}:8888
+      - ${VOXEL_VISUALIZER_PORT}:${VOXEL_VISUALIZER_PORT}
     volumes:
       - ${DKR_CMP_CONTEXT}:${ROOT_DIR}
       - ${DKR_CMP_CONTEXT}/odc_gee/config:/home/jovyan/.config/odc-gee

build/reqs/13_reqs_voxel_visualizer.txt

@@ -0,0 +1,3 @@
+fastapi==0.68.0
+aiofiles==0.7.0
+uvicorn==0.14.0