parenx & osmnx methods & data; evaluation WIP (#201)

* remove outdated parenx data sets

* gitignore parenx input files

* update parenx file structure

* update regex

* move previous osmnx nb to archive

* updated osmnx notebook

* add osmnx data

* move h3 nb to archive

* add read_osmnx func to utils

* draft eval nb

* update parenx nb

* parenx data

* update read-in parenx funcs

* update evaluation worklfow (WIP)

* add date to nb name in archive

* .explode in read in funcs and added func names to __all__

* add "option" to read_parenx docstring

* update parenx tests, add sgeop and osmnx tests

* fix

---------

Co-authored-by: Martin Fleischmann <[email protected]>

.gitignore

@@ -161,5 +161,5 @@ cython_debug/
 
 # Other stuff
 .DS_Store
-temp-parenx/*
+data/*/temp-parenx/*
 *.qgz

core/tests/test_utils.py

@@ -17,6 +17,10 @@ def test_read_sample_data():
 osm_records = [78_908, 60_364, 79_317, 84_819, 79_907, 50_917, 92_667]
 xnd2_records = [43_233, 12_439, 16_302, 30_552, 16_554, 13_468, 29_314]
 man_records = [38_772, 8_640, 14_170, 29_252, 13_508, 11_032, numpy.nan]
+parenx_voronoi_records = [42_302, 9_569, 16_844, 29_446, 15_516, 14_242, 35_694]
+parenx_skeletonize_records = [44_294, 9_561, 17_469, 30_641, 16_075, 14_784, 37_557]
+osmnx_records = [43_451, 15_770, 24_025, 37_954, 21_755, 17_704, 30_156]
+sgeop_records = [39_489, 8_229, 13_587, 29_028, 12_395, 11_059, 16_298]
 
 
 @pytest.mark.parametrize("city, n_records", zip(cities, osm_records, strict=True))
@@ -58,14 +62,59 @@ def test_read_manual(city, n_records):
     assert gdf_1.crs == gdf_2.crs == pytest.epsg_4326
 
 
+@pytest.mark.parametrize("city, n_records", zip(cities, osmnx_records, strict=True))
+def test_read_osmnx(city, n_records):
+    fua = core.utils.city_fua[city]
+
+    gdf_1 = core.utils.read_osmnx(fua, pytest.epsg_4326)
+    gdf_2 = core.utils.read_osmnx(core.utils.fua_city[fua], pytest.epsg_4326)
+
+    geopandas.testing.assert_geodataframe_equal(gdf_1, gdf_2)
+
+    assert gdf_1.shape[0] == gdf_2.shape[0] == n_records
+    assert gdf_1.crs == gdf_2.crs == pytest.epsg_4326
+
+
 @pytest.mark.parametrize(
-    "option, n_records", [["skeletonize", 32_029], ["voronoi", 31_251]]
+    "city, n_records", zip(cities, parenx_voronoi_records, strict=True)
 )
-def test_read_parenex(option, n_records):
-    fua = pytest.auckland
+def test_read_parenx_voronoi(city, n_records):
+    fua = core.utils.city_fua[city]
+    option = "voronoi"
+    gdf_1 = core.utils.read_parenx(fua, proj_crs=pytest.epsg_4326, option=option)
+    gdf_2 = core.utils.read_parenx(
+        core.utils.fua_city[fua], proj_crs=pytest.epsg_4326, option=option
+    )
+
+    geopandas.testing.assert_geodataframe_equal(gdf_1, gdf_2)
+
+    assert gdf_1.shape[0] == gdf_2.shape[0] == n_records
+    assert gdf_1.crs == gdf_2.crs == pytest.epsg_4326
+
+
+@pytest.mark.parametrize(
+    "city, n_records", zip(cities, parenx_skeletonize_records, strict=True)
+)
+def test_read_parenx_skeletonize(city, n_records):
+    fua = core.utils.city_fua[city]
+    option = "skeletonize"
+    gdf_1 = core.utils.read_parenx(fua, proj_crs=pytest.epsg_4326, option=option)
+    gdf_2 = core.utils.read_parenx(
+        core.utils.fua_city[fua], proj_crs=pytest.epsg_4326, option=option
+    )
+
+    geopandas.testing.assert_geodataframe_equal(gdf_1, gdf_2)
+
+    assert gdf_1.shape[0] == gdf_2.shape[0] == n_records
+    assert gdf_1.crs == gdf_2.crs == pytest.epsg_4326
+
+
+@pytest.mark.parametrize("city, n_records", zip(cities, sgeop_records, strict=True))
+def test_read_sgeop(city, n_records):
+    fua = core.utils.city_fua[city]
 
-    gdf_1 = core.utils.read_parenx(fua, option, pytest.epsg_4326)
-    gdf_2 = core.utils.read_parenx(core.utils.fua_city[fua], option, pytest.epsg_4326)
+    gdf_1 = core.utils.read_sgeop(fua, pytest.epsg_4326)
+    gdf_2 = core.utils.read_sgeop(core.utils.fua_city[fua], pytest.epsg_4326)
 
     geopandas.testing.assert_geodataframe_equal(gdf_1, gdf_2)
 

core/utils.py

@@ -14,7 +14,9 @@
     "read_original",
     "read_no_degree_2",
     "read_manual",
+    "read_osmnx",
     "read_parenx",
+    "read_sgeop",
     "graph_size",
     "load_usecases",
     "make_grid",
@@ -59,11 +61,11 @@ def _fua_code(fua: int | str) -> int:
     return fua
 
 
-def _fua_path(fua: int, dataset: str, option: None | str = None) -> pathlib.Path:
+def _fua_path(fua: int, dataset: str) -> pathlib.Path:
     """Helper for parsing input dataset paths."""
     fua = _fua_code(fua)
     dset = data_dir / pathlib.Path(f"{fua}", dataset)
-    return dset / f"{option}.{parq}" if option else dset / f"{fua}.{parq}"
+    return dset / f"{fua}.{parq}"
 
 
 def read_original(fua: int | str, geom_only: bool = True) -> geopandas.GeoDataFrame:
@@ -88,18 +90,42 @@ def read_manual(fua: int, proj_crs: str | int | pyproj.CRS) -> geopandas.GeoData
     )
 
 
+def read_osmnx(
+    fua: int | str, proj_crs: str | int | pyproj.CRS
+) -> geopandas.GeoDataFrame:
+    """Read OSM roads from parquet format; return bare columns."""
+    return (
+        geopandas.read_parquet(_fua_path(fua, "osmnx"))
+        .explode(ignore_index=True, index_parts=False)
+        .to_crs(proj_crs)
+    )
+
+
 def read_parenx(
     fua: int, option: str, proj_crs: str | int | pyproj.CRS
 ) -> geopandas.GeoDataFrame:
-    """Read in prepared parenx data."""
+    """
+    Read in prepared parenx data.
+    option is one of: voronoi, skeletonize
+    """
 
     return (
-        geopandas.read_parquet(_fua_path(fua, "parenx", option=option))
+        geopandas.read_parquet(_fua_path(fua, f"parenx-{option}"))
         .explode(ingore_index=True, index_parts=False)
         .to_crs(proj_crs)
     )
 
 
+def read_sgeop(fua: int, proj_crs: str | int | pyproj.CRS) -> geopandas.GeoDataFrame:
+    """Read in prepared sgeop data."""
+
+    return (
+        geopandas.read_parquet(_fua_path(fua, "sgeop"))
+        .explode(ignore_index=True, index_parts=False)
+        .to_crs(proj_crs)
+    )
+
+
 def graph_size(info: str, g: networkx.Graph) -> str:
     return f"{info}\n\t* {g}"