Skip to content

Commit

Permalink
Update vector code to allow coastlines data to be written even if no …
Browse files Browse the repository at this point in the history 
…points are available
  • Loading branch information
@robbibt
robbibt committed Jan 31, 2023
1 parent f7af73f commit b178788
Show file tree
Hide file tree
Showing 3 changed files with 104 additions and 302 deletions.
23 changes: 10 additions & 13 deletions coastlines/vector.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1609,10 +1609,8 @@ def generate_vectors(
# Export stats #
################

# if points_gdf is not None and len(points_gdf) > 0:

# Clip stats to study area extent
points_gdf = points_gdf[points_gdf.intersects(gridcell_gdf.geometry.item())]
points_gdf_clipped = points_gdf.clip(gridcell_gdf)

# Set output path
stats_path = (
Expand All @@ -1623,23 +1621,22 @@ def generate_vectors(
try:

# Export to GeoJSON
points_gdf.to_crs("EPSG:4326").to_file(
points_gdf_clipped.to_crs("EPSG:4326").to_file(
f"{stats_path}.geojson",
driver="GeoJSON",
)

# Export as ESRI shapefiles
points_gdf.to_file(
points_gdf_clipped.to_file(
f"{stats_path}.shp",
schema={
"properties": vector_schema(points_gdf),
"properties": vector_schema(points_gdf_clipped),
"geometry": "Point",
},
)

except ValueError:
raise ValueError(
f"Study area {study_area}: No vector points data to export after clipping to study area extent"
log.warning(f"Study area {study_area}: No vector points data to export after clipping to study area extent"
)

else:
Expand All @@ -1653,7 +1650,7 @@ def generate_vectors(
contours_gdf = contour_certainty(contours_gdf, certainty_masks)

# Add tide datum details (this supports future addition of extra tide datums)
contours_gdf["tide_datum"] = "0 m AMSL"
contours_gdf["tide_datum"] = "0.0 m AMSL"

# Add region attributes
contours_gdf = region_atttributes(
Expand All @@ -1667,20 +1664,20 @@ def generate_vectors(
)

# Clip annual shoreline contours to study area extent
contours_gdf["geometry"] = contours_gdf.intersection(gridcell_gdf.geometry.item())
contours_gdf_clipped = contours_gdf.clip(gridcell_gdf)

try:

# Export to GeoJSON
contours_gdf.to_crs("EPSG:4326").to_file(
contours_gdf_clipped.to_crs("EPSG:4326").to_file(
f"{contour_path}.geojson", driver="GeoJSON"
)

# Export stats and contours as ESRI shapefiles
contours_gdf.to_file(
contours_gdf_clipped.to_file(
f"{contour_path}.shp",
schema={
"properties": vector_schema(contours_gdf),
"properties": vector_schema(contours_gdf_clipped),
"geometry": ["MultiLineString", "LineString"],
},
)
Expand Down
177 changes: 49 additions & 128 deletions notebooks/DEACoastlines_generation_CLI.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,22 +83,22 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 6,
"id": "4729d261-6b27-4a61-a937-366c5174ce05",
"metadata": {},
"outputs": [],
"source": [
"# study_area = 424\n",
"# raster_version = 'testing'\n",
"# vector_version = 'testing'\n",
"# continental_version = 'testing'\n",
"# config_path = 'configs/dea_coastlines_config.yaml'\n",
"study_area = 64\n",
"raster_version = 'testing'\n",
"vector_version = 'testing'\n",
"continental_version = 'testing'\n",
"config_path = 'configs/dea_coastlines_config.yaml'\n",
"\n",
"study_area = 9\n",
"raster_version = 'development'\n",
"vector_version = 'development'\n",
"continental_version = 'development'\n",
"config_path = 'configs/dea_coastlines_config_development.yaml'"
"# study_area = 9\n",
"# raster_version = 'development'\n",
"# vector_version = 'development'\n",
"# continental_version = 'development'\n",
"# config_path = 'configs/dea_coastlines_config_development.yaml'"
]
},
{
Expand All @@ -112,7 +112,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 3,
"id": "dc5632c4-f872-46e5-9762-80b8556f3622",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -198,28 +198,31 @@
"name": "stdout",
"output_type": "stream",
"text": [
"/env/lib/python3.8/site-packages/distributed/node.py:180: UserWarning: Port 8787 is already in use.\n",
"Perhaps you already have a cluster running?\n",
"Hosting the HTTP server on port 38341 instead\n",
" warnings.warn(\n",
"<Client: 'tcp://127.0.0.1:41371' processes=1 threads=7, memory=59.21 GiB>\n",
"2023-01-30 02:03:44 INFO Study area 9: Loaded study area grid\n",
"2023-01-30 02:04:00 INFO Study area 9: Loaded virtual product\n",
"<Client: 'tcp://127.0.0.1:40609' processes=1 threads=62, memory=477.21 GiB>\n",
"2023-01-30 06:17:55 INFO Study area 151: Loaded study area grid\n",
"2023-01-30 06:18:04 INFO Study area 151: Loaded virtual product\n",
"Creating reduced resolution tide modelling array\n",
"Modelling tides using FES2014 tide model\n",
"2023-01-30 02:04:00 ERROR Study area 9: Unable to access tide modelling files\n",
"Reprojecting tides into original array\n",
"100%|███████████████████████████████████████| 1021/1021 [00:30<00:00, 33.01it/s]\n",
"2023-01-30 06:19:44 INFO Study area 151: Finished modelling tide heights\n",
"2023-01-30 06:19:45 INFO Study area 151: Calculating low and high tide cutoffs for each pixel\n",
"2023-01-30 06:19:45 INFO Study area 151: Started exporting raster data\n",
"/env/lib/python3.8/site-packages/dask/core.py:119: RuntimeWarning: divide by zero encountered in true_divide\n",
" return func(*(_execute_task(a, cache) for a in args))\n",
"/env/lib/python3.8/site-packages/dask/core.py:119: RuntimeWarning: invalid value encountered in true_divide\n",
" return func(*(_execute_task(a, cache) for a in args))\n",
"2023-01-30 06:31:57 INFO Study area 151: Completed exporting raster data\n",
"distributed.nanny - WARNING - Worker process still alive after 3.999998474121094 seconds, killing\n",
"Exception in thread AsyncProcess Dask Worker process (from Nanny) watch process join:\n",
"Traceback (most recent call last):\n",
" File \"/home/jovyan/Robbi/dea-coastlines/coastlines/raster.py\", line 569, in generate_rasters\n",
" ds[\"tide_m\"], tides_lowres = pixel_tides(ds, resample=True, directory='blah')\n",
" File \"/env/lib/python3.8/site-packages/dea_tools/coastal.py\", line 592, in pixel_tides\n",
" tide_df = model_tides(\n",
" File \"/env/lib/python3.8/site-packages/dea_tools/coastal.py\", line 336, in model_tides\n",
" model = pyTMD.model(directory, format=\"netcdf\", compressed=False).elevation(model)\n",
" File \"/env/lib/python3.8/site-packages/pyTMD/model.py\", line 842, in elevation\n",
" self.model_file = self.pathfinder(model_files)\n",
" File \"/env/lib/python3.8/site-packages/pyTMD/model.py\", line 1425, in pathfinder\n",
" raise FileNotFoundError(output_file)\n",
"FileNotFoundError: ['blah/fes2014/ocean_tide/2n2.nc', 'blah/fes2014/ocean_tide/eps2.nc', 'blah/fes2014/ocean_tide/j1.nc', 'blah/fes2014/ocean_tide/k1.nc', 'blah/fes2014/ocean_tide/k2.nc', 'blah/fes2014/ocean_tide/l2.nc', 'blah/fes2014/ocean_tide/la2.nc', 'blah/fes2014/ocean_tide/m2.nc', 'blah/fes2014/ocean_tide/m3.nc', 'blah/fes2014/ocean_tide/m4.nc', 'blah/fes2014/ocean_tide/m6.nc', 'blah/fes2014/ocean_tide/m8.nc', 'blah/fes2014/ocean_tide/mf.nc', 'blah/fes2014/ocean_tide/mks2.nc', 'blah/fes2014/ocean_tide/mm.nc', 'blah/fes2014/ocean_tide/mn4.nc', 'blah/fes2014/ocean_tide/ms4.nc', 'blah/fes2014/ocean_tide/msf.nc', 'blah/fes2014/ocean_tide/msqm.nc', 'blah/fes2014/ocean_tide/mtm.nc', 'blah/fes2014/ocean_tide/mu2.nc', 'blah/fes2014/ocean_tide/n2.nc', 'blah/fes2014/ocean_tide/n4.nc', 'blah/fes2014/ocean_tide/nu2.nc', 'blah/fes2014/ocean_tide/o1.nc', 'blah/fes2014/ocean_tide/p1.nc', 'blah/fes2014/ocean_tide/q1.nc', 'blah/fes2014/ocean_tide/r2.nc', 'blah/fes2014/ocean_tide/s1.nc', 'blah/fes2014/ocean_tide/s2.nc', 'blah/fes2014/ocean_tide/s4.nc', 'blah/fes2014/ocean_tide/sa.nc', 'blah/fes2014/ocean_tide/ssa.nc', 'blah/fes2014/ocean_tide/t2.nc']\n"
" File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n",
" self.run()\n",
" File \"/usr/lib/python3.8/threading.py\", line 870, in run\n",
" self._target(*self._args, **self._kwargs)\n",
" File \"/env/lib/python3.8/site-packages/distributed/process.py\", line 218, in _watch_process\n",
" assert exitcode is not None\n",
"AssertionError\n"
]
}
],
Expand All @@ -237,7 +240,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 4,
"id": "0ed72c21-ba89-4fad-9521-f5493aad9d66",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -319,19 +322,17 @@
"name": "stdout",
"output_type": "stream",
"text": [
"2023-01-25 02:10:51 INFO Study area 424: Starting vector generation\n",
"2023-01-25 02:10:59 INFO Study area 424: Loaded rasters\n",
"2023-01-25 02:11:12 ERROR Study area 424: Failed to run process with error 'Dataset' object has no attribute 'land'\n",
"Traceback (most recent call last):\n",
" File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 1820, in generate_vectors_cli\n",
" generate_vectors(\n",
" File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 1443, in generate_vectors\n",
" masked_ds, certainty_masks = contours_preprocess(\n",
" File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 616, in contours_preprocess\n",
" geodata_da = dc.load(\n",
" File \"/env/lib/python3.8/site-packages/xarray/core/common.py\", line 239, in __getattr__\n",
" raise AttributeError(\n",
"AttributeError: 'Dataset' object has no attribute 'land'\n"
"2023-01-31 01:56:17 INFO Study area 64: Starting vector generation\n",
"2023-01-31 01:56:29 INFO Study area 64: Loaded rasters\n",
"Operating in single z-value, multiple arrays mode\n",
"2023-01-31 01:57:02 INFO Study area 64: Extracted shorelines from raster data\n",
"2023-01-31 01:57:02 INFO Study area 64: Extracted rates of change points\n",
"2023-01-31 01:57:26 INFO Study area 64: Calculated distances to each annual shoreline\n",
"2023-01-31 01:57:28 INFO Study area 64: Calculated rates of change regressions\n",
"2023-01-31 01:57:31 INFO Study area 64: Calculated all of time statistics\n",
"2023-01-31 01:57:31 INFO Study area 64: Calculated rate of change certainty flags\n",
"2023-01-31 01:57:32 INFO Study area 64: Added region attributes and geohash UIDs\n",
"2023-01-31 01:57:39 INFO Study area 64: Output vector files written to data/interim/vector/testing/64_testing\n"
]
}
],
Expand Down Expand Up @@ -386,7 +387,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": null,
"id": "6b804fec-39f4-48a9-8525-a78fbf06e084",
"metadata": {},
"outputs": [],
Expand All @@ -397,90 +398,10 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": null,
"id": "c39e6533-f4fd-4a3c-90a5-44f19825c2f5",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"193\n",
"<Client: 'tcp://127.0.0.1:45241' processes=1 threads=31, memory=237.21 GiB>\n",
"2023-01-25 02:11:18 INFO Study area 193: Loaded study area grid\n",
"2023-01-25 02:11:29 INFO Study area 193: Loaded virtual product\n",
"Creating reduced resolution tide modelling array\n",
"Modelling tides using FES2014 tide model\n",
"Reprojecting tides into original array\n",
"100%|███████████████████████████████████████| 1074/1074 [00:13<00:00, 76.98it/s]\n",
"2023-01-25 02:12:23 INFO Study area 193: Finished modelling tide heights\n",
"2023-01-25 02:12:23 INFO Study area 193: Calculating low and high tide cutoffs for each pixel\n",
"2023-01-25 02:12:23 INFO Study area 193: Started exporting raster data\n",
"2023-01-25 02:16:59 INFO Study area 193: Completed exporting raster data\n",
"distributed.nanny - WARNING - Worker process still alive after 3.9999988555908206 seconds, killing\n",
"Exception in thread AsyncProcess Dask Worker process (from Nanny) watch process join:\n",
"Traceback (most recent call last):\n",
" File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n",
" self.run()\n",
" File \"/usr/lib/python3.8/threading.py\", line 870, in run\n",
"2023-01-25 02:17:07 INFO Study area 193: Starting vector generation\n",
"2023-01-25 02:17:18 INFO Study area 193: Loaded rasters\n",
"Operating in single z-value, multiple arrays mode\n",
"Failed to generate contours: 1988, 1989, 1992, 1993\n",
"2023-01-25 02:17:47 INFO Study area 193: Extracted shorelines from raster data\n",
"2023-01-25 02:17:47 INFO Study area 193: Extracted rates of change points\n",
"2023-01-25 02:17:48 INFO Study area 193: Calculated distances to each annual shoreline\n",
"2023-01-25 02:17:48 INFO Study area 193: Calculated rates of change regressions\n",
"2023-01-25 02:17:48 INFO Study area 193: Calculated all of time statistics\n",
"2023-01-25 02:17:48 INFO Study area 193: Calculated rate of change certainty flags\n",
"2023-01-25 02:17:48 INFO Study area 193: Added region attributes and geohash UIDs\n",
"2023-01-25 02:17:48 ERROR Study area 193: Failed to run process with error Study area 193: No vector points data to export after clipping to study area extent\n",
"Traceback (most recent call last):\n",
" File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 1622, in generate_vectors\n",
" points_gdf.to_crs(\"EPSG:4326\").to_file(\n",
" File \"/env/lib/python3.8/site-packages/geopandas/geodataframe.py\", line 1114, in to_file\n",
" _to_file(self, filename, driver, schema, index, **kwargs)\n",
" File \"/env/lib/python3.8/site-packages/geopandas/io/file.py\", line 367, in _to_file\n",
" schema = infer_schema(df)\n",
" File \"/env/lib/python3.8/site-packages/geopandas/io/file.py\", line 428, in infer_schema\n",
" raise ValueError(\"Cannot write empty DataFrame to file.\")\n",
"ValueError: Cannot write empty DataFrame to file.\n",
"\n",
"During handling of the above exception, another exception occurred:\n",
"\n",
"Traceback (most recent call last):\n",
" File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 1820, in generate_vectors_cli\n",
" generate_vectors(\n",
" File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 1637, in generate_vectors\n",
" raise ValueError(\n",
"ValueError: Study area 193: No vector points data to export after clipping to study area extent\n",
"478\n",
"<Client: 'tcp://127.0.0.1:46233' processes=1 threads=31, memory=237.21 GiB>\n",
"2023-01-25 02:17:55 INFO Study area 478: Loaded study area grid\n",
"2023-01-25 02:18:01 INFO Study area 478: Loaded virtual product\n",
"Creating reduced resolution tide modelling array\n",
"Modelling tides using FES2014 tide model\n",
"Reprojecting tides into original array\n",
"100%|███████████████████████████████████████| 1405/1405 [00:17<00:00, 80.45it/s]\n",
"2023-01-25 02:19:01 INFO Study area 478: Finished modelling tide heights\n",
"2023-01-25 02:19:02 INFO Study area 478: Calculating low and high tide cutoffs for each pixel\n",
"2023-01-25 02:19:02 INFO Study area 478: Started exporting raster data\n",
"2023-01-25 02:24:52 INFO Study area 478: Completed exporting raster data\n",
"distributed.nanny - WARNING - Worker process still alive after 3.9999990463256836 seconds, killing\n",
"2023-01-25 02:24:59 INFO Study area 478: Starting vector generation\n",
"2023-01-25 02:25:10 INFO Study area 478: Loaded rasters\n",
"Operating in single z-value, multiple arrays mode\n",
"Failed to generate any valid contours; verify that values passed to `z_values` are valid and present in `da`\n",
"2023-01-25 02:25:39 ERROR Study area 478: Failed to run process with error Study area 478: Unable to extract any valid shorelines from raster data\n",
"Traceback (most recent call last):\n",
" File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 1820, in generate_vectors_cli\n",
" generate_vectors(\n",
" File \"/home/jovyan/Robbi/dea-coastlines/coastlines/vector.py\", line 1461, in generate_vectors\n",
" raise ValueError(\n",
"ValueError: Study area 478: Unable to extract any valid shorelines from raster data\n"
]
}
],
"outputs": [],
"source": [
"# Run raster and vector generation for each study area\n",
"for study_area in study_areas:\n",
Expand Down
Loading

0 comments on commit b178788

Please sign in to comment.