Feature/stamen draft

.env.example

@@ -18,6 +18,10 @@ CKAN_SQLALCHEMY_URL=postgresql://ckandbuser:ckandbpassword@db/ckandb
 CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore
 CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore
 
+GITHUB_USER=stamen
+GITHUB_REPO=data.naturalcapitalproject.stanford.edu
+GITHUB_BRANCH=feature/stamen-draft
+
 # Test database connections
 TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test
 TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test
@@ -82,6 +86,11 @@ CKANEXT__SPATIAL__SEARCH_BACKEND="solr-spatial-field"
 CKANEXT__SPATIAL__COMMON_MAP__TYPE="Stadia.StamenTerrain"
 CKANEXT__SPATIAL__COMMON_MAP__APIKEY=""  # API key goes here.
 
+# Config for mappreview
+CKANEXT__MAPPREVIEW__MAPBOX_API_KEY=""
+CKANEXT__MAPPREVIEW__MAPBOX_STYLE=""
+CKANEXT__MAPPREVIEW__TITILER_URL=""
+
 # Theme configuration
 CKAN__SITE_LOGO=NatCapLogo.jpg
 CKAN__FEATURED_ORGS=natcap

.gitignore

@@ -10,4 +10,5 @@ local/*
 .env
 
 env310
-*~
+*~
+venv

api-scripts/sync-datasets.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python
+import json
+import os
+import requests
+import yaml
+
+SRC = os.environ.get('SYNC_SRC_URL', 'https://data.naturalcapitalproject.stanford.edu')
+DST = os.environ.get('SYNC_DST_URL', 'http://localhost:5000')
+DST_APIKEY = os.environ['SYNC_DST_CKAN_APIKEY']
+TITILER_URL = os.environ.get('TITILER_URL',
+                             'https://titiler-897938321824.us-west1.run.app')
+
+
+def get_dataset_metadata(dataset):
+    if dataset['resources']:
+        for resource in dataset['resources']:
+            if resource['description'] == 'Geometamaker YML':
+                r = requests.get(resource['url'])
+                return yaml.safe_load(r.text)
+    return None
+
+
+def get_dataset_sources(dataset_metadata):
+    return dataset_metadata.get('sources', None)
+
+
+def get_raster_info(url):
+    r = requests.get(TITILER_URL + '/cog/info', params={'url': url})
+    j = r.json()
+    return {
+        'bounds': j['bounds'],
+        'minzoom': j['minzoom'],
+        'maxzoom': j['maxzoom'],
+    }
+
+
+def get_raster_statistics(url):
+    statistics_response = requests.get(TITILER_URL + '/cog/statistics', params={'url': url})
+    stats = statistics_response.json()['b1']
+    return {
+        'min': stats['min'],
+        'max': stats['max'],
+        'percentile_2': stats['percentile_2'],
+        'percentile_98': stats['max'],
+    }
+
+
+def get_map_settings(layers):
+    return {
+        'minzoom': min([l['minzoom'] for l in layers]),
+        'maxzoom': max([l['maxzoom'] for l in layers]),
+        'bounds': [
+            min([l['bounds'][0] for l in layers]),
+            min([l['bounds'][1] for l in layers]),
+            max([l['bounds'][2] for l in layers]),
+            max([l['bounds'][3] for l in layers]),
+        ],
+    }
+
+def get_mappreview_metadata(dataset, zip_sources):
+    # TODO vectors
+    raster_resources = [r for r in dataset['resources'] if r['format'] == 'GeoTIFF']
+    layers = []
+
+    zip_resource = next((r for r in dataset['resources'] if r['format'] == 'ZIP'), None)
+
+    if zip_resource and zip_sources:
+        # Look at zip sources for a GeoTIFF, add
+
+        tif_source = next((s for s in zip_sources if s.endswith('tif')), None)
+
+        if tif_source:
+            path = tif_source.replace('\\', '/')
+            base = '/'.join(zip_resource['url'].split('/')[0:-1])
+            url = f'{base}/{path}'
+            name = path.split('/')[-1]
+
+            raster_resources.append({
+                'name': name,
+                'url': url,
+            })
+
+    for r in raster_resources:
+        # Does this GeoTIFF exist?
+        url = r['url']
+
+        # Avoid redirect from 'storage.cloud.google.com'
+        if url.startswith('https://storage.cloud.google.com/'):
+            url = url.replace('https://storage.cloud.google.com/', 'https://storage.googleapis.com/')
+
+        head_request = requests.head(url)
+        if head_request.status_code != 200 and 'retetion' in url:
+            print('Failed to access GeoTIFF', url)
+            print('Status code:', head_request.status_code)
+            continue
+
+        # If it exists, get all the info about it
+        info = get_raster_info(url)
+        stats = get_raster_statistics(url)
+
+        layers.append({
+            'name': r['name'],
+            'type': 'raster',
+            'url': url,
+            'pixel_min_value': stats['min'],
+            'pixel_max_value': stats['max'],
+            'pixel_percentile_2': stats['percentile_2'],
+            'pixel_percentile_98': stats['percentile_98'],
+            'bounds': info['bounds'],
+            'minzoom': info['minzoom'],
+            'maxzoom': info['maxzoom'],
+        })
+
+    if len(layers) > 0:
+        return {
+            'map': get_map_settings(layers),
+            'layers': layers,
+        }
+
+    return None
+
+
+def delete_datasets(dst, dst_apikey):
+    list_response = requests.get(dst + '/api/3/action/package_list')
+
+    for id in list_response.json()['result']:
+        print('Deleting ' + id)
+        delete_response = requests.post(dst + '/api/action/package_delete',
+                                        json={'id': id},
+                                        headers={'Authorization': dst_apikey})
+        purge_response = requests.post(dst + '/api/action/dataset_purge',
+                                        json={'id': id},
+                                        headers={'Authorization': dst_apikey})
+
+
+def sync_datasets(src, dst, dst_apikey):
+    list_response = requests.get(src + '/api/3/action/package_list')
+
+    for id in list_response.json()['result']:
+        print('Adding ' + id)
+        package_response = requests.get(src + '/api/3/action/package_show?id=' + id)
+
+        organization_id = None
+        organization_response = requests.get(dst + '/api/action/organization_show?id=' + package_response.json()['result']['owner_org'])
+
+        if organization_response.status_code == 404:
+            print('Creating org')
+            organization_post_response = requests.post(
+                dst + '/api/action/organization_create',
+                headers={'Authorization': dst_apikey},
+                json=package_response.json()['result']['organization']
+            )
+            organization_id = organization_post_response.json()['result']['id']
+
+        package = package_response.json()['result']
+        package['extras'] = [] # XXX skipping extras for now
+
+        # If dataset has metadata with sources in it, add those
+        metadata = get_dataset_metadata(package)
+        sources = get_dataset_sources(metadata)
+        if sources:
+            # TODO maybe better on the resource itself?
+            package['extras'].append({'key': 'sources', 'value': json.dumps(sources)})
+
+        mappreview_metadata = get_mappreview_metadata(package, sources)
+        if mappreview_metadata:
+            package['extras'].append({'key': 'mappreview', 'value': json.dumps(mappreview_metadata)})
+
+        post_response = requests.post(
+            dst + '/api/action/package_create',
+            headers={'Authorization': dst_apikey},
+            json=package
+        )
+
+        if (post_response.status_code != 200):
+            print(post_response.json()['error'])
+            break
+
+
+if __name__ == '__main__':
+    print('Deleting existing datasets...')
+    delete_datasets(DST, DST_APIKEY)
+    print('Done.')
+
+    print('Syncing datasets...')
+    sync_datasets(SRC, DST, DST_APIKEY)
+    print('Done.')

ckan/Dockerfile

@@ -1,5 +1,9 @@
 FROM ckan/ckan-base:2.10.4
 
+ARG GITHUB_USER
+ARG GITHUB_REPO
+ARG GITHUB_BRANCH
+
 # Set up environment variables
 ENV APP_DIR=/opt/ckan-catalog/data.naturalcapitalproject.stanford.edu
 ENV TZ=UTC
@@ -22,9 +26,23 @@ RUN pip3 install -e git+https://github.com/ckan/ckanext-spatial.git@$CKANEXT_SPA
 RUN pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@3af6056bbe16c3f8c6257f18cb2a0805370f85de#egg=ckanext-scheming'
 RUN pip3 install "numpy<2"
 
+RUN pip3 install -e 'git+https://github.com/keitaroinc/ckanext-suggest.git@main#egg=ckanext-suggest'
+RUN pip3 install -r https://raw.githubusercontent.com/keitaroinc/ckanext-suggest/main/requirements.txt
+
 # NOTE: requires that the build context includes ./src/
+# NOTE: Adding version.json to bust cache
+ADD https://api.github.com/repos/${GITHUB_USER}/${GITHUB_REPO}/git/refs/heads/${GITHUB_BRANCH} version.json
 COPY ./src/ckanext-natcap /tmp/ckanext-natcap
-RUN pip3 install -r /tmp/ckanext-natcap/requirements.txt && pip3 install /tmp/ckanext-natcap
+RUN pip3 install -r /tmp/ckanext-natcap/requirements.txt 
+RUN pip3 install /tmp/ckanext-natcap
+
+COPY ./src/ckanext-mappreview /tmp/ckanext-mappreview
+RUN pip3 install -r /tmp/ckanext-mappreview/requirements.txt 
+RUN pip3 install /tmp/ckanext-mappreview
+
+COPY ./src/ckanext-zipexpand /tmp/ckanext-zipexpand
+RUN pip3 install -r /tmp/ckanext-zipexpand/requirements.txt 
+RUN pip3 install /tmp/ckanext-zipexpand
 
 # Install any extensions needed by your CKAN instance
 # See Dockerfile.dev for more details and examples
@@ -34,7 +52,7 @@ COPY ./ckan/docker-entrypoint.d/* /docker-entrypoint.d/
 
 # Apply any patches needed to CKAN core or any of the built extensions (not the
 # runtime mounted ones)
-COPY patches ${APP_DIR}/patches
+COPY ./ckan/patches ${APP_DIR}/patches
 
 RUN for d in $APP_DIR/patches/*; do \
         if [ -d $d ]; then \

ckan/Dockerfile.dev

@@ -34,9 +34,6 @@ FROM ckan/ckan-dev:2.10.4
 # Clone the extension(s) your are writing for your own project in the `src` folder
 # to get them mounted in this image at runtime
 
-# Copy custom initialization scripts
-COPY docker-entrypoint.d/* /docker-entrypoint.d/
-
 # Apply any patches needed to CKAN core or any of the built extensions (not the
 # runtime mounted ones)
 
@@ -49,13 +46,23 @@ ENV CKANEXT_SPATIAL_GIT_REVISION=938308469892e4bcf7389cb4adee5ccdd5a0ccca
 RUN apk add --no-cache geos geos-dev proj proj-dev proj-util
 RUN pip3 install -e git+https://github.com/ckan/ckanext-spatial.git@$CKANEXT_SPATIAL_GIT_REVISION#egg=ckanext-spatial && \
     pip3 install -r https://raw.githubusercontent.com/ckan/ckanext-spatial/$CKANEXT_SPATIAL_GIT_REVISION/requirements.txt
-RUN pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@master#egg=ckanext-scheming'
+# RUN pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@master#egg=ckanext-scheming'
+RUN pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@3af6056bbe16c3f8c6257f18cb2a0805370f85de#egg=ckanext-scheming'
 RUN pip3 install "numpy<2"
 
+RUN pip3 install -e 'git+https://github.com/keitaroinc/ckanext-suggest.git@main#egg=ckanext-suggest'
+RUN pip3 install -r https://raw.githubusercontent.com/keitaroinc/ckanext-suggest/main/requirements.txt
+
 # Enable SQLAlchemy support by installing it
-RUN pip3 install Flask-SQLAlchemy "flask<2.4" "Werkzeug<=2.1.2"
+# RUN pip3 install Flask-SQLAlchemy "flask<2.4" "Werkzeug<=2.1.2"
+# RUN pip3 uninstall -y flask
+# RUN pip3 uninstall -y werkzeug
+# RUN pip3 install "werkzeug<=2.1.2" "Flask-SQLAlchemy==2.5.1"
+
+# Copy custom initialization scripts
+COPY ./ckan/docker-entrypoint.d/* /docker-entrypoint.d/
 
-COPY patches ${APP_DIR}/patches
+COPY ./ckan/patches ${APP_DIR}/patches
 
 RUN for d in $APP_DIR/patches/*; do \
         if [ -d $d ]; then \

docker-compose.dev.yml

@@ -10,8 +10,8 @@ services:
 
   ckan-dev:
     build:
-      context: ckan/
-      dockerfile: Dockerfile.dev
+      context: .
+      dockerfile: ckan/Dockerfile.dev
       args:
         - TZ=${TZ}
     env_file:
@@ -58,9 +58,15 @@ services:
       test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"]
 
   solr:
+    build:
+      context: .
+      dockerfile: solr/Dockerfile.dev
     image: ckan/ckan-solr:${SOLR_IMAGE_VERSION}
+    ports:
+     - "8983:8983"
     volumes:
       - solr_data:/var/solr
+      - ./solr/ckan-config:/opt/solr/server/solr/configsets/ckan/conf
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"]

docker-compose.yml

@@ -26,6 +26,9 @@ services:
       dockerfile: ckan/Dockerfile
       args:
         - TZ=${TZ}
+        - GITHUB_USER=${GITHUB_USER}
+        - GITHUB_REPO=${GITHUB_REPO}
+        - GITHUB_BRANCH=${GITHUB_BRANCH}
     networks:
       - ckannet
       - dbnet
@@ -84,6 +87,7 @@ services:
     image: ckan/ckan-solr:${SOLR_IMAGE_VERSION}
     volumes:
       - solr_data:/var/solr
+      - ./solr/ckan-config:/opt/solr/server/solr/configsets/ckan/conf
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"]

nginx/setup/default.conf

@@ -43,4 +43,4 @@ server {
       root /usr/share/nginx/html;
     }
 
-}
+}