Merge pull request #1215 from GeoscienceAustralia/develop

Merge latest changes from develop to stable

DEA_products/README.rst

@@ -10,6 +10,7 @@ Notebooks introducing DEA's satellite datasets and derived products, including h
    DEA_Sentinel2_Surface_Reflectance.ipynb
    DEA_Water_Observations.ipynb
    DEA_Fractional_Cover.ipynb
+   DEA_Intertidal.ipynb
    DEA_Land_Cover.ipynb
    DEA_High_and_Low_Tide_Imagery.ipynb
    DEA_Coastlines.ipynb

Tests/dea_tools/test_spatial.py

@@ -417,8 +417,8 @@ def test_xr_interpolate(dem_da, points_gdf, method):
             k=5,
         )
 
-    # Verify that error is raised if `gdf` doesn't overlap with `ds`
-    with pytest.raises(ValueError):
+    # Verify that warning is raised if `gdf` doesn't overlap with `ds`
+    with pytest.warns():
         xr_interpolate(
             dem_da,
             gdf=points_gdf.set_crs("EPSG:3577", allow_override=True),

Tests/setup_test_datacube.sh

@@ -20,17 +20,24 @@ for metadata_yaml in $(find ./dea-config/product_metadata -name '*.yaml'); do
 done
 
 # Index products we care about for dea-notebooks
-for prod_def_yaml in $(find ./dea-config/products -name '*.yaml' -regex '.*\(ga_ls7e_nbart_gm_cyear_3\|ga_ls8c_nbart_gm_cyear_3\|ga_ls_fc_3\|ga_ls_wo_3\|ga_ls_wo_fq_cyear_3\|ga_ls_landcover_class_cyear_2\|high_tide_comp_20p\|low_tide_comp_20p\|ga_s2am_ard_3\|ga_s2bm_ard_3\|ga_ls5t_ard_3\|ga_ls7e_ard_3\|ga_ls8c_ard_3\|ga_ls9c_ard_3\|ga_ls_mangrove_cover_cyear_3\).*'); do
+for prod_def_yaml in $(find ./dea-config/products -name '*.yaml' -regex '.*\(ga_ls7e_nbart_gm_cyear_3\|ga_ls8c_nbart_gm_cyear_3\|ga_ls_fc_3\|ga_ls_wo_3\|ga_ls_wo_fq_cyear_3\|ga_ls_landcover_class_cyear_2\|high_tide_comp_20p\|low_tide_comp_20p\|ga_s2am_ard_3\|ga_s2bm_ard_3\|ga_ls5t_ard_3\|ga_ls7e_ard_3\|ga_ls8c_ard_3\|ga_ls9c_ard_3\|ga_ls_mangrove_cover_cyear_3\|ga_s2ls_intertidal_cyear_3\).*'); do
         datacube product add $prod_def_yaml
 done
 
 datacube product list
 
+# Index DEA Intertidal
+s3-to-dc 's3://dea-public-data/derivative/ga_s2ls_intertidal_cyear_3/1-0-0/x080/y127/2018--P1Y/*.json' --no-sign-request --skip-lineage --stac 'ga_s2ls_intertidal_cyear_3'
+s3-to-dc 's3://dea-public-data/derivative/ga_s2ls_intertidal_cyear_3/1-0-0/x080/y127/2019--P1Y/*.json' --no-sign-request --skip-lineage --stac 'ga_s2ls_intertidal_cyear_3'
+s3-to-dc 's3://dea-public-data/derivative/ga_s2ls_intertidal_cyear_3/1-0-0/x080/y127/2020--P1Y/*.json' --no-sign-request --skip-lineage --stac 'ga_s2ls_intertidal_cyear_3'
+
 # Index Mangroves
 s3-to-dc 's3://dea-public-data/derivative/ga_ls_mangrove_cover_cyear_3/3-0-0/x33/y39/*/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls_mangrove_cover_cyear_3'
 
-
 # Index GeoMAD
+s3-to-dc 's3://dea-public-data/derivative/ga_ls8c_nbart_gm_cyear_3/3-0-0/x08/y27/2018--P1Y/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls8c_nbart_gm_cyear_3'
+s3-to-dc 's3://dea-public-data/derivative/ga_ls8c_nbart_gm_cyear_3/3-0-0/x08/y27/2019--P1Y/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls8c_nbart_gm_cyear_3'
+s3-to-dc 's3://dea-public-data/derivative/ga_ls8c_nbart_gm_cyear_3/3-0-0/x08/y27/2020--P1Y/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls8c_nbart_gm_cyear_3'
 s3-to-dc 's3://dea-public-data/derivative/ga_ls8c_nbart_gm_cyear_3/3-0-0/x49/y24/2017--P1Y/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls8c_nbart_gm_cyear_3'
 s3-to-dc 's3://dea-public-data/derivative/ga_ls7e_nbart_gm_cyear_3/3-0-0/x49/y23/2015--P1Y/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls7e_nbart_gm_cyear_3'
 s3-to-dc 's3://dea-public-data/derivative/ga_ls7e_nbart_gm_cyear_3/3-0-0/x49/y24/2015--P1Y/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls7e_nbart_gm_cyear_3'

Tools/dea_tools/spatial.py

@@ -601,7 +601,7 @@ def xr_interpolate(
         A dataset of spatial points including at least one numeric column.
         By default all numeric columns in this dataset will be spatially
         interpolated into the extent of `ds`; specific columns can be
-        selected using `columns`. An error will be raised if the points
+        selected using `columns`. An warning will be raised if the points
         in `gdf` do not overlap with the extent of `ds`.
     columns : list, optional
         An optional list of specific columns in gdf` to run the
@@ -646,10 +646,10 @@ def xr_interpolate(
     ds = add_geobox(ds, crs)
     y_dim, x_dim = ds.odc.spatial_dims
 
-    # Reproject to match input `ds`, and raise error if there are no overlaps
+    # Reproject to match input `ds`, and raise warning if there are no overlaps
     gdf = gdf.to_crs(ds.odc.crs)
     if not gdf.dissolve().intersects(ds.odc.geobox.extent.geom).item():
-        raise ValueError("The supplied `gdf` does not overlap spatially with `ds`.")
+        warnings.warn("The supplied `gdf` does not overlap spatially with `ds`.", stacklevel=2)
 
     # Select subset of numeric columns (non-numeric are not supported)
     numeric_gdf = gdf.select_dtypes("number")

Tools/dea_tools/validation.py

@@ -23,7 +23,6 @@
 import pandas as pd
 from sklearn.metrics import mean_squared_error
 from sklearn.metrics import mean_absolute_error
-from sklearn.metrics import r2_score
 from math import sqrt
 from scipy import stats
 
@@ -72,7 +71,7 @@ def eval_metrics(x, y, round=3, all_regress=False):
         "Correlation": xy_df.corr().iloc[0, 1],
         "RMSE": sqrt(mean_squared_error(xy_df.x, xy_df.y)),
         "MAE": mean_absolute_error(xy_df.x, xy_df.y),
-        "R-squared": r2_score(xy_df.x, xy_df.y),
+        "R-squared": lin_reg.rvalue**2,
         "Bias": (xy_df.y - xy_df.x).mean(),
         "Regression slope": lin_reg.slope,
     }
@@ -88,4 +87,4 @@ def eval_metrics(x, y, round=3, all_regress=False):
         )
 
     # Return as
-    return pd.Series(stats_dict).round(round)
+    return pd.Series(stats_dict).round(round)