From 32b56e9832689d3e4458acaba40e74d296a31366 Mon Sep 17 00:00:00 2001 From: Robbi Bishop-Taylor Date: Thu, 3 Nov 2022 04:06:28 +0000 Subject: [PATCH] Use latest version of dea-tools with new tide paths --- coastlines/raster.py | 4 +- docker-compose.yaml | 2 +- .../DEACoastlines_generation_raster.ipynb | 46 +++++++++---------- requirements.in | 2 +- requirements.txt | 2 +- 5 files changed, 26 insertions(+), 30 deletions(-) diff --git a/coastlines/raster.py b/coastlines/raster.py index 2db5a65..0841205 100644 --- a/coastlines/raster.py +++ b/coastlines/raster.py @@ -546,9 +546,7 @@ def generate_rasters( # Add this new data as a new variable in our satellite dataset to allow # each satellite pixel to be analysed and filtered/masked based on the # tide height at the exact moment of satellite image acquisition. - ds["tide_m"], tides_lowres = pixel_tides( - ds, resample=True, directory="/var/shared/tide_models" - ) + ds["tide_m"], tides_lowres = pixel_tides(ds, resample=True) log.info(f"Study area {study_area}: Finished modelling tide heights") # Based on the entire time-series of tide heights, compute the max diff --git a/docker-compose.yaml b/docker-compose.yaml index 935bd29..b5ed4c1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -31,6 +31,6 @@ services: volumes: - ./artifacts:/mnt/artifacts - ./docker/coastline/wait-for-db:/usr/local/bin/wait-for-db - - ./tide_models/:/var/shared/tide_models + - ./tide_models/:/var/share/tide_models restart: always command: tail -f /dev/null diff --git a/notebooks/DEACoastlines_generation_raster.ipynb b/notebooks/DEACoastlines_generation_raster.ipynb index 57b919d..bcbc6f6 100644 --- a/notebooks/DEACoastlines_generation_raster.ipynb +++ b/notebooks/DEACoastlines_generation_raster.ipynb @@ -82,7 +82,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "distributed.diskutils - INFO - Found stale lock file and directory '/home/jovyan/Robbi/dea-coastlines/dask-worker-space/worker-_63m6xr4', purging\n" + "distributed.diskutils - INFO - Found stale lock file and directory '/home/jovyan/Robbi/dea-coastlines/dask-worker-space/worker-iwg_jzml', purging\n" ] }, { @@ -92,7 +92,7 @@ "
\n", "
\n", "

Client

\n", - "

Client-e7395712-5b03-11ed-80f6-42456653a3e2

\n", + "

Client-906d639f-5b2c-11ed-8175-42456653a3e2

\n", " \n", "\n", " \n", @@ -121,7 +121,7 @@ " \n", "
\n", "

LocalCluster

\n", - "

fe92f201

\n", + "

43a0cb48

\n", "
\n", " \n", "
\n", @@ -158,11 +158,11 @@ "
\n", "
\n", "

Scheduler

\n", - "

Scheduler-08671d8a-d245-46be-826f-47cab0e40c2b

\n", + "

Scheduler-7a20eacb-31d9-4051-a3aa-daf68b805367

\n", " \n", " \n", " \n", "
\n", - " Comm: tcp://127.0.0.1:35497\n", + " Comm: tcp://127.0.0.1:46171\n", " \n", " Workers: 1\n", @@ -204,7 +204,7 @@ " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", @@ -253,7 +253,7 @@ "" ], "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -310,7 +310,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -333,7 +333,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -356,7 +356,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -384,7 +384,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -753,10 +753,10 @@ " ndwi (time, y, x) float32 dask.array<chunksize=(1, 1469, 1423), meta=np.ndarray>\n", "Attributes:\n", " crs: epsg:32653\n", - " grid_mapping: spatial_ref
\n", - " Comm: tcp://127.0.0.1:34807\n", + " Comm: tcp://127.0.0.1:36693\n", " \n", " Total threads: 2\n", @@ -212,7 +212,7 @@ "
\n", - " Dashboard: /user/robbi.bishoptaylor@ga.gov.au/proxy/44139/status\n", + " Dashboard: /user/robbi.bishoptaylor@ga.gov.au/proxy/34103/status\n", " \n", " Memory: 12.21 GiB\n", @@ -220,13 +220,13 @@ "
\n", - " Nanny: tcp://127.0.0.1:41333\n", + " Nanny: tcp://127.0.0.1:45839\n", "
\n", - " Local directory: /home/jovyan/Robbi/dea-coastlines/dask-worker-space/worker-k2bcbpmy\n", + " Local directory: /home/jovyan/Robbi/dea-coastlines/dask-worker-space/worker-le3sbb13\n", "
\n", + " dtype='datetime64[ns]')
  • y
    (y)
    float64
    -1.339e+06 ... -1.383e+06
    units :
    metre
    resolution :
    -30.0
    crs :
    epsg:32653
    array([-1338840., -1338870., -1338900., ..., -1382820., -1382850., -1382880.])
  • x
    (x)
    float64
    1.366e+05 1.366e+05 ... 1.792e+05
    units :
    metre
    resolution :
    30.0
    crs :
    epsg:32653
    array([136590., 136620., 136650., ..., 179190., 179220., 179250.])
  • spatial_ref
    ()
    int32
    32653
    spatial_ref :
    PROJCS["WGS 84 / UTM zone 53N",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",135],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["Easting",EAST],AXIS["Northing",NORTH],AUTHORITY["EPSG","32653"]]
    grid_mapping_name :
    transverse_mercator
    array(32653, dtype=int32)
    • mndwi
      (time, y, x)
      float32
      dask.array<chunksize=(1, 1469, 1423), meta=np.ndarray>
      crs :
      epsg:32653
      units :
      1
      nodata :
      nan
    • \n", " \n", "
    \n", " \n", @@ -872,7 +872,7 @@ "\n", " \n", " \n", - "
  • ndwi
    (time, y, x)
    float32
    dask.array<chunksize=(1, 1469, 1423), meta=np.ndarray>
    crs :
    epsg:32653
    units :
    1
    nodata :
    nan
    \n", + "
  • ndwi
    (time, y, x)
    float32
    dask.array<chunksize=(1, 1469, 1423), meta=np.ndarray>
    crs :
    epsg:32653
    units :
    1
    nodata :
    nan
    \n", " \n", "
    \n", " \n", @@ -988,7 +988,7 @@ "\n", " \n", " \n", - "
  • crs :
    epsg:32653
    grid_mapping :
    spatial_ref
    • " + "
  • crs :
    epsg:32653
    grid_mapping :
    spatial_ref
    • " ], "text/plain": [ "\n", @@ -1006,7 +1006,7 @@ " grid_mapping: spatial_ref" ] }, - "execution_count": 7, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -1042,7 +1042,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -1058,14 +1058,12 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 377/377 [00:28<00:00, 13.18it/s]\n" + "100%|██████████| 377/377 [00:28<00:00, 13.30it/s]\n" ] } ], "source": [ - "ds[\"tide_m\"], tides_lowres = pixel_tides(\n", - " ds, resample=True, directory=\"/var/share/tide_models\"\n", - ")" + "ds[\"tide_m\"], tides_lowres = pixel_tides(ds, resample=True)" ] }, { diff --git a/requirements.in b/requirements.in index 962f092..31c4f9c 100644 --- a/requirements.in +++ b/requirements.in @@ -4,7 +4,7 @@ affine>=2.3.0 botocore click>=8.0.1 datacube[s3,performance]>=1.8.6 -dea_tools>=0.2.8.dev17 +dea_tools>=0.2.8.dev21 Fiona>=1.8.20 geopandas>=0.10.2 matplotlib>=3.4.3 diff --git a/requirements.txt b/requirements.txt index 8da4452..00b8408 100644 --- a/requirements.txt +++ b/requirements.txt @@ -112,7 +112,7 @@ datacube[performance,s3]==1.8.6 # dea-tools # odc-algo # odc-ui -dea-tools==0.2.8.dev17 +dea-tools==0.2.8.dev21 # via -r requirements.in debugpy==1.5.1 # via ipykernel