From e4fdcbdc3a21a2b3037702a4514014fd73fba5fa Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Wed, 29 Jan 2025 11:44:47 +0100 Subject: [PATCH 01/27] update backend to new delft fiat version --- flood_adapt/adapter/fiat_adapter.py | 32 +++++----- tests/test_adapter/test_fiat_adapter.py | 80 ++++++++++++------------- 2 files changed, 56 insertions(+), 56 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index 56c847b9..e0e7cc49 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -43,20 +43,20 @@ class FiatColumns: """Object with mapping of FIAT attributes to columns names.""" - object_id = "Object ID" - object_name = "Object Name" - primary_object_type = "Primary Object Type" - secondary_object_type = "Secondary Object Type" - extraction_method = "Extraction Method" - ground_floor_height = "Ground Floor Height" - ground_elevation = "Ground Elevation" - damage_function = "Damage Function: " - max_potential_damage = "Max Potential Damage: " - aggregation_label = "Aggregation Label: " - inundation_depth = "Inundation Depth" - damage = "Damage: " - total_damage = "Total Damage" - risk_ead = "Risk (EAD)" + object_id = "object_id" + object_name = "object_name" + primary_object_type = "primary_object_type" + secondary_object_type = "secondary_object_type" + extraction_method = "extract_method" + ground_floor_height = "ground_flht" + ground_elevation = "ground_elevtn" + damage_function = "fn_damage_" + max_potential_damage = "max_damage_" + aggregation_label = "aggregation_label:_" + inundation_depth = "inun_depth" + damage = "damage_" + total_damage = "total_damage" + risk_ead = "risk_ead" class FiatAdapter(IImpactAdapter): @@ -803,7 +803,7 @@ def apply_population_growth_new( percent_growth=population_growth, geom_file=Path(area_path), ground_floor_height=ground_floor_height, - damage_types=["Structure", "Content"], + damage_types=["structure", "content"], vulnerability=self._model.vulnerability, ground_elevation=ground_elevation, aggregation_area_fn=aggregation_areas, @@ -952,7 +952,7 @@ def floodproof_properties(self, floodproof: FloodProof) -> None: self._model.exposure.truncate_damage_function( objectids=objectids, floodproof_to=floodproof.attrs.elevation.value, - damage_function_types=["Structure", "Content"], + damage_function_types=["structure", "content"], vulnerability=self._model.vulnerability, ) diff --git a/tests/test_adapter/test_fiat_adapter.py b/tests/test_adapter/test_fiat_adapter.py index 71db5737..0920e8f8 100644 --- a/tests/test_adapter/test_fiat_adapter.py +++ b/tests/test_adapter/test_fiat_adapter.py @@ -68,12 +68,12 @@ def test_all_measures(self, run_scenario_all_measures): assert len(exposure_scenario) > len(exposure_template) # check if growth has been applied correctly - inds1 = exposure_scenario["Object ID"].isin(exposure_template["Object ID"]) & ( - exposure_scenario["Primary Object Type"] != "road" + inds1 = exposure_scenario["object_id"].isin(exposure_template["object_id"]) & ( + exposure_scenario["primary_object_type"] != "road" ) - exp1 = exposure_scenario.loc[inds1, "Max Potential Damage: Structure"] - inds0 = exposure_template["Primary Object Type"] != "road" - exp0 = exposure_template.loc[inds0, "Max Potential Damage: Structure"] + exp1 = exposure_scenario.loc[inds1, "max_damage_structure"] + inds0 = exposure_template["primary_object_type"] != "road" + exp0 = exposure_template.loc[inds0, "max_damage_structure"] eg = test_scenario.impacts.socio_economic_change.attrs.economic_growth pg = ( test_scenario.impacts.socio_economic_change.attrs.population_growth_existing @@ -84,13 +84,13 @@ def test_all_measures(self, run_scenario_all_measures): ) # check if new area max damage is implemented correctly - inds_new_area = ~exposure_scenario["Object ID"].isin( - exposure_template["Object ID"] + inds_new_area = ~exposure_scenario["object_id"].isin( + exposure_template["object_id"] ) assert ( pytest.approx( exposure_scenario.loc[ - inds_new_area, "Max Potential Damage: Structure" + inds_new_area, "max_damage_structure" ].sum() ) == ( @@ -101,7 +101,7 @@ def test_all_measures(self, run_scenario_all_measures): test_scenario.impacts.socio_economic_change.attrs.population_growth_new / 100 ) - * exposure_template.loc[:, "Max Potential Damage: Structure"].sum() + * exposure_template.loc[:, "max_damage_structure"].sum() ) # check if buildings are elevated correctly @@ -122,42 +122,42 @@ def test_all_measures(self, run_scenario_all_measures): bfes = pd.read_csv(db_path(TopLevelDir.static) / "bfe" / "bfe.csv") # Create a dataframe to save the initial object attributes - exposures = exposure_template.merge(bfes, on="Object ID")[ - ["Object ID", "bfe", "Ground Floor Height"] - ].rename(columns={"Ground Floor Height": "Ground Floor Height 1"}) + exposures = exposure_template.merge(bfes, on="object_id")[ + ["object_id", "bfe", "ground_flht_"] + ].rename(columns={"ground_flht_": "ground_flht_1"}) # Merge with the adapted fiat model exposure - exposures = exposures.merge(exposure_scenario, on="Object ID").rename( - columns={"Ground Floor Height": "Ground Floor Height 2"} + exposures = exposures.merge(exposure_scenario, on="object_id").rename( + columns={"ground_flht_": "ground_flht_2"} ) # Filter to only the objects affected by the measure exposures = exposures.loc[ - (exposure_scenario.loc[:, f"Aggregation Label: {aggr_label}"] == aggr_name) - & (exposure_scenario.loc[:, "Primary Object Type"] == build_type), + (exposure_scenario.loc[:, f"aggregation_label:_{aggr_label}"] == aggr_name) + & (exposure_scenario.loc[:, "primary_object_type"] == build_type), :, ] exposures = exposures[ [ - "Object ID", - "Ground Elevation", + "object_id", + "ground_elevtn", "bfe", - "Ground Floor Height 1", - "Ground Floor Height 2", + "ground_flht_1", + "ground_flht_2", ] ] # Check if elevation took place correctly at each object for i, row in exposures.iterrows(): # If the initial elevation is smaller than the required one it should have been elevated to than one if ( - row["Ground Elevation"] + row["Ground Floor Height 1"] + row["ground_elevtn"] + row["ground_flht_1"] < row["bfe"] + elevate_val ): assert ( - row["Ground Elevation"] + row["Ground Floor Height 2"] + row["ground_elevtn"] + row["ground_flht_2"] == row["bfe"] + elevate_val ) # if not it should have stated the same else: - assert row["Ground Floor Height 2"] == row["Ground Floor Height 1"] + assert row["ground_flht_2"] == row["ground_flht_1"] # check if buildings are bought-out aggr_label = test_scenario.impacts.impact_strategy.measures[ @@ -170,10 +170,10 @@ def test_all_measures(self, run_scenario_all_measures): 1 ].attrs.property_type inds = ( - exposure_scenario.loc[:, f"Aggregation Label: {aggr_label}"] == aggr_name - ) & (exposure_scenario.loc[:, "Primary Object Type"] == build_type) + exposure_scenario.loc[:, f"aggregation_label:_{aggr_label}"] == aggr_name + ) & (exposure_scenario.loc[:, "primary_object_type"] == build_type) - assert all(exposure_scenario.loc[inds, "Max Potential Damage: Structure"] == 0) + assert all(exposure_scenario.loc[inds, "max_damage_structure"] == 0) # check if buildings are flood-proofed aggr_label = test_scenario.impacts.impact_strategy.measures[ @@ -186,15 +186,15 @@ def test_all_measures(self, run_scenario_all_measures): 2 ].attrs.property_type inds1 = ( - exposure_template.loc[:, f"Aggregation Label: {aggr_label}"] == aggr_name - ) & (exposure_template.loc[:, "Primary Object Type"] == build_type) + exposure_template.loc[:, f"aggregation_label:_{aggr_label}"] == aggr_name + ) & (exposure_template.loc[:, "primary_object_type"] == build_type) inds2 = ( - exposure_scenario.loc[:, f"Aggregation Label: {aggr_label}"] == aggr_name - ) & (exposure_scenario.loc[:, "Primary Object Type"] == build_type) + exposure_scenario.loc[:, f"aggregation_label:_{aggr_label}"] == aggr_name + ) & (exposure_scenario.loc[:, "primary_object_type"] == build_type) assert all( - exposure_scenario.loc[inds2, "Damage Function: Structure"] - != exposure_template.loc[inds1, "Damage Function: Structure"] + exposure_scenario.loc[inds2, "fn_damage_structure"] + != exposure_template.loc[inds1, "fn_damage_structure"] ) def test_raise_datum(self, run_scenario_raise_datum): @@ -219,17 +219,17 @@ def test_raise_datum(self, run_scenario_raise_datum): 0 ].attrs.property_type inds1 = ( - exposure_template.loc[:, f"Aggregation Label: {aggr_label}"] == aggr_name - ) & (exposure_template.loc[:, "Primary Object Type"] == build_type) + exposure_template.loc[:, f"aggregation_label:_{aggr_label}"] == aggr_name + ) & (exposure_template.loc[:, "primary_object_type"] == build_type) inds2 = ( - exposure_scenario.loc[:, f"Aggregation Label: {aggr_label}"] == aggr_name - ) & (exposure_scenario.loc[:, "Primary Object Type"] == build_type) + exposure_scenario.loc[:, f"aggregation_label:_{aggr_label}"] == aggr_name + ) & (exposure_scenario.loc[:, "primary_object_type"] == build_type) assert all( elev1 <= elev2 for elev1, elev2 in zip( - exposure_template.loc[inds1, "Ground Floor Height"], - exposure_scenario.loc[inds2, "Ground Floor Height"], + exposure_template.loc[inds1, "ground_flht_"], + exposure_scenario.loc[inds2, "ground_flht_"], ) ) @@ -237,8 +237,8 @@ def test_raise_datum(self, run_scenario_raise_datum): height + elev >= test_scenario.impacts.impact_strategy.measures[0].attrs.elevation.value for height, elev in zip( - exposure_scenario.loc[inds2, "Ground Floor Height"], - exposure_scenario.loc[inds2, "Ground Elevation"], + exposure_scenario.loc[inds2, "ground_flht_"], + exposure_scenario.loc[inds2, "ground_elevtn"], ) ) From b0a304c120a2d6eb649b3f6407e7b98f63b2bdbc Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Wed, 29 Jan 2025 14:26:26 +0100 Subject: [PATCH 02/27] add fiat output class --- flood_adapt/adapter/fiat_adapter.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index e0e7cc49..6e1872d2 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -56,8 +56,24 @@ class FiatColumns: inundation_depth = "inun_depth" damage = "damage_" total_damage = "total_damage" - risk_ead = "risk_ead" + risk_ead = "ead_damage" +class FiatOutput: + """Object with mapping of FIAT attributes to columns names.""" + object_id = "object_id" + object_name = "object_name" + primary_object_type = "primary_object_type" + secondary_object_type = "secondary_object_type" + extraction_method = "extract_method" + ground_floor_height = "ground_flht" + ground_elevation = "ground_elevtn" + damage_function = "fn_damage_" + max_potential_damage = "max_damage_" + aggregation_label = "aggregation_label:_" + inundation_depth = "inun_depth" + damage = "damage_" + total_damage = "total_damage" + risk_ead = "ead_damage" class FiatAdapter(IImpactAdapter): """ From 054fce73416277ad60c8f679d44c94f0397482a4 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Wed, 29 Jan 2025 16:40:33 +0100 Subject: [PATCH 03/27] add fiat output class --- flood_adapt/adapter/fiat_adapter.py | 38 +++++++++++++++-------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index 6e1872d2..9a2aed19 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -58,22 +58,22 @@ class FiatColumns: total_damage = "total_damage" risk_ead = "ead_damage" class FiatOutput: - """Object with mapping of FIAT attributes to columns names.""" - - object_id = "object_id" - object_name = "object_name" - primary_object_type = "primary_object_type" - secondary_object_type = "secondary_object_type" - extraction_method = "extract_method" - ground_floor_height = "ground_flht" - ground_elevation = "ground_elevtn" - damage_function = "fn_damage_" - max_potential_damage = "max_damage_" - aggregation_label = "aggregation_label:_" - inundation_depth = "inun_depth" - damage = "damage_" - total_damage = "total_damage" - risk_ead = "ead_damage" + """Object with mapping of FIAT attributes to columns name for FloodAdapt GUI.""" + + object_id = "Object ID" + object_name = "Object Name" + primary_object_type = "Primary Object Type" + secondary_object_type = "Secondary Object Type" + extraction_method = "Extraction Method" + ground_floor_height = "Ground Floor Height" + ground_elevation = "Ground Elevation" + damage_function = "Damage Function: " + max_potential_damage = "Max Potential Damage: " + aggregation_label = "Aggregation Label: " + inundation_depth = "Inundation Depth" + damage = "Damage: " + total_damage = "Total Damage" + risk_ead = "Risk (EAD)" class FiatAdapter(IImpactAdapter): """ @@ -425,7 +425,7 @@ def postprocess(self, scenario): scenario_output_path = scenario.impacts.results_path impacts_output_path = scenario.impacts.impacts_path - # Add exceedance probabilities if needed (only for risk) + # Add exceedance probabilities if needed (only for risk) if mode == Mode.risk: # Get config path # TODO check where this configs should be read from @@ -444,6 +444,8 @@ def postprocess(self, scenario): fiat_results_path = impacts_output_path.joinpath( f"Impacts_detailed_{scenario.attrs.name}.csv" ) + + self.outputs["table"] = self.outputs["table"].map(lambda row: FiatOutput(**row)) self.outputs["table"].to_csv(fiat_results_path) # Create the infometrics files @@ -1403,4 +1405,4 @@ def save_roads(self, output_path: os.PathLike): on=FiatColumns.object_id, ) # Save as geopackage - roads.to_file(output_path, driver="GPKG") + roads.to_file(output_path, driver="GPKG") \ No newline at end of file From 62d9a69febdc9394a85385132e65de65cd06cd24 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 30 Jan 2025 09:18:08 +0100 Subject: [PATCH 04/27] update aggregation name --- flood_adapt/adapter/fiat_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index 9a2aed19..f00dcc01 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -52,7 +52,7 @@ class FiatColumns: ground_elevation = "ground_elevtn" damage_function = "fn_damage_" max_potential_damage = "max_damage_" - aggregation_label = "aggregation_label:_" + aggregation_label = "aggregation_label:" inundation_depth = "inun_depth" damage = "damage_" total_damage = "total_damage" From 59f1777a53286c3d6568c23c05a6172e942171e8 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 30 Jan 2025 10:08:32 +0100 Subject: [PATCH 05/27] update hydromt fiat version and remove output translations --- flood_adapt/adapter/fiat_adapter.py | 7 +------ pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index f00dcc01..63cbf569 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -54,9 +54,7 @@ class FiatColumns: max_potential_damage = "max_damage_" aggregation_label = "aggregation_label:" inundation_depth = "inun_depth" - damage = "damage_" - total_damage = "total_damage" - risk_ead = "ead_damage" + class FiatOutput: """Object with mapping of FIAT attributes to columns name for FloodAdapt GUI.""" @@ -71,9 +69,6 @@ class FiatOutput: max_potential_damage = "Max Potential Damage: " aggregation_label = "Aggregation Label: " inundation_depth = "Inundation Depth" - damage = "Damage: " - total_damage = "Total Damage" - risk_ead = "Risk (EAD)" class FiatAdapter(IImpactAdapter): """ diff --git a/pyproject.toml b/pyproject.toml index 8c8d1a8c..824c8961 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "geopandas", "jellyfish<1.0", # This is a dependency of us->hydromt-fiat, FloodAdapt doesnt use it directly. Remove when .whl files are available. # jellyfish v1.0 starts requiring rust as a dependency, which is fine if there are built .whl files. BUT THERE ARE NO BUILDS FOR WINDOWS YET. - "hydromt-fiat==0.4.2", + "hydromt-fiat==0.5.1", "hydromt-sfincs@ git+https://github.com/Deltares/hydromt_sfincs.git", "numpy < 2.0", "numpy-financial", From 725137ad24e041ba4b753f4bf3d0a4578ddd8888 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 30 Jan 2025 13:13:14 +0100 Subject: [PATCH 06/27] fix infometrics --- flood_adapt/adapter/fiat_adapter.py | 54 ++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index 63cbf569..dbe6a939 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -54,21 +54,17 @@ class FiatColumns: max_potential_damage = "max_damage_" aggregation_label = "aggregation_label:" inundation_depth = "inun_depth" - -class FiatOutput: - """Object with mapping of FIAT attributes to columns name for FloodAdapt GUI.""" - - object_id = "Object ID" - object_name = "Object Name" - primary_object_type = "Primary Object Type" - secondary_object_type = "Secondary Object Type" - extraction_method = "Extraction Method" - ground_floor_height = "Ground Floor Height" - ground_elevation = "Ground Elevation" - damage_function = "Damage Function: " - max_potential_damage = "Max Potential Damage: " - aggregation_label = "Aggregation Label: " - inundation_depth = "Inundation Depth" + damage_structure = "damage_structure" + damage_content = "damage_content" + damage_other = "damage_other" + total_damage = "total_damage" + risk_ead = "risk_ead" + segment_length = "segment_length" + fn_damage_structure= "fn_damage_structure" + fn_damage_content= "fn_damage_content" + fn_damage_other= "fn_damage_other" + max_damage_structure= "max_damage_structure" + max_damage_content= "max_damage_content" class FiatAdapter(IImpactAdapter): """ @@ -102,6 +98,28 @@ def __init__( self.config_base_path = config_base_path self.exe_path = exe_path self.delete_crashed_runs = delete_crashed_runs + self.fiat_output_mapping = {FiatColumns.object_id: "Object ID", + FiatColumns.object_name : "Object Name", + FiatColumns.primary_object_type : "Primary Object Type", + FiatColumns.secondary_object_type : "Secondary Object Type", + FiatColumns.extraction_method : "Extraction Method", + FiatColumns.ground_floor_height : "Ground Floor Height", + FiatColumns.ground_elevation : "Ground Elevation", + FiatColumns.damage_function : "Damage Function: ", + FiatColumns.max_potential_damage : "Max Potential Damage: ", + FiatColumns.aggregation_label : "Aggregation Label: ", + FiatColumns.inundation_depth : "Inundation Depth", + FiatColumns.damage_structure : "Damage: Structure", + FiatColumns.damage_content : "Damage: Content", + FiatColumns.damage_other : "Damage: Other", + FiatColumns.total_damage : "Total Damage", + FiatColumns.risk_ead : "Risk (EAD)", + FiatColumns.segment_length : "Segment Length", + FiatColumns.fn_damage_structure: "Damage Function: Structure", + FiatColumns.fn_damage_content: "Damage Function: Content", + FiatColumns.fn_damage_other: "Damage Function: Other", + FiatColumns.max_damage_structure: "Max Potential Damage: Structure", + FiatColumns.max_damage_content: "Max Potential Damage: Content"} self._model = FiatModel(root=str(model_root.resolve()), mode="r") self._model.read() @@ -440,7 +458,11 @@ def postprocess(self, scenario): f"Impacts_detailed_{scenario.attrs.name}.csv" ) - self.outputs["table"] = self.outputs["table"].map(lambda row: FiatOutput(**row)) + for aggregation in self.config.aggregation: + name = aggregation.name + self.fiat_output_mapping[f"aggregation_label:{name}"] = f"Aggregation Label: {name}" + + self.outputs["table"] = self.outputs["table"].rename(columns = self.fiat_output_mapping) self.outputs["table"].to_csv(fiat_results_path) # Create the infometrics files From 0e63ad71dbae9ff06851f6300691d017c0b3e8e6 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 30 Jan 2025 14:56:58 +0100 Subject: [PATCH 07/27] fix fiat adapter --- flood_adapt/adapter/fiat_adapter.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index dbe6a939..ae3f799d 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -42,7 +42,7 @@ class FiatColumns: """Object with mapping of FIAT attributes to columns names.""" - + object_id = "object_id" object_name = "object_name" primary_object_type = "primary_object_type" @@ -52,7 +52,6 @@ class FiatColumns: ground_elevation = "ground_elevtn" damage_function = "fn_damage_" max_potential_damage = "max_damage_" - aggregation_label = "aggregation_label:" inundation_depth = "inun_depth" damage_structure = "damage_structure" damage_content = "damage_content" @@ -107,7 +106,6 @@ def __init__( FiatColumns.ground_elevation : "Ground Elevation", FiatColumns.damage_function : "Damage Function: ", FiatColumns.max_potential_damage : "Max Potential Damage: ", - FiatColumns.aggregation_label : "Aggregation Label: ", FiatColumns.inundation_depth : "Inundation Depth", FiatColumns.damage_structure : "Damage: Structure", FiatColumns.damage_content : "Damage: Content", @@ -461,9 +459,8 @@ def postprocess(self, scenario): for aggregation in self.config.aggregation: name = aggregation.name self.fiat_output_mapping[f"aggregation_label:{name}"] = f"Aggregation Label: {name}" - - self.outputs["table"] = self.outputs["table"].rename(columns = self.fiat_output_mapping) - self.outputs["table"].to_csv(fiat_results_path) + metrics_exposure = self.outputs["table"].rename(columns = self.fiat_output_mapping) + metrics_exposure.to_csv(fiat_results_path) # Create the infometrics files if mode == Mode.risk: @@ -488,7 +485,7 @@ def postprocess(self, scenario): metrics_outputs_path = scenario_output_path.joinpath( f"Infometrics_{scenario.attrs.name}.csv" ) - self.create_infometrics(metric_config_paths, metrics_outputs_path) + self.create_infometrics(metric_config_paths, metrics_outputs_path, metrics_exposure) # Get paths of created aggregated infometrics aggr_metrics_paths = list( @@ -1135,7 +1132,7 @@ def add_exceedance_probability( return self.outputs["table"] def create_infometrics( - self, metric_config_paths: list[os.PathLike], metrics_output_path: os.PathLike + self, metric_config_paths: list[os.PathLike], metrics_output_path: os.PathLike, metrics_exposure: pd.DataFrame ) -> None: """ Create infometrics files based on the provided metric configuration paths. @@ -1162,13 +1159,13 @@ def create_infometrics( metrics_writer = MetricsFileWriter(metric_file) metrics_writer.parse_metrics_to_file( - df_results=self.outputs["table"], + df_results=metrics_exposure, metrics_path=metrics_output_path, write_aggregate=None, ) metrics_writer.parse_metrics_to_file( - df_results=self.outputs["table"], + df_results=metrics_exposure, metrics_path=metrics_output_path, write_aggregate="all", ) @@ -1409,7 +1406,7 @@ def save_roads(self, output_path: os.PathLike): aggr_cols = [ name for name in self.outputs["table"].columns - if FiatColumns.aggregation_label in name + if any(f"aggregation_label:{aggregation.name}" in name for aggregation in self.config.aggregation) ] inun_cols = [ name for name in roads.columns if FiatColumns.inundation_depth in name From 9c306c25a06d721dd75a4cb14832112497568dc4 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 30 Jan 2025 15:19:46 +0100 Subject: [PATCH 08/27] update to parse a dict for bf metrics --- flood_adapt/adapter/fiat_adapter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index ae3f799d..e203a553 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -97,6 +97,7 @@ def __init__( self.config_base_path = config_base_path self.exe_path = exe_path self.delete_crashed_runs = delete_crashed_runs + self.fiat_columns_dict = {k: v for k, v in FiatColumns.__dict__.items() if not k.startswith("__")} self.fiat_output_mapping = {FiatColumns.object_id: "Object ID", FiatColumns.object_name : "Object Name", FiatColumns.primary_object_type : "Primary Object Type", @@ -1382,8 +1383,8 @@ def save_building_footprints(self, output_path: os.PathLike): ) ) - footprints.aggregate(fiat_results_df) - footprints.calc_normalized_damages() + footprints.aggregate(fiat_results_df, self.fiat_columns_dict) + footprints.calc_normalized_damages(self.fiat_columns_dict) # Save footprint footprints.write(output_path) From 8b95156a69223173cb13bd479893e1f8064a1954 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Fri, 31 Jan 2025 10:33:14 +0100 Subject: [PATCH 09/27] add aggregation --- flood_adapt/adapter/fiat_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index e203a553..a4d03c42 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -1157,7 +1157,7 @@ def create_infometrics( # Check if type of metric configuration is available for metric_file in metric_config_paths: if metric_file.exists(): - metrics_writer = MetricsFileWriter(metric_file) + metrics_writer = MetricsFileWriter(metric_file, aggregation_label = "Aggregation Label: ") metrics_writer.parse_metrics_to_file( df_results=metrics_exposure, From 758a36a938a593ebfdf986d6c61c58e6b36c6904 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Fri, 31 Jan 2025 13:10:56 +0100 Subject: [PATCH 10/27] fix dict xposue columns for save_bf --- flood_adapt/adapter/fiat_adapter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index a4d03c42..fe81c6e5 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -1382,7 +1382,9 @@ def save_building_footprints(self, output_path: os.PathLike): how="left", ) ) - + # Add aggregation to fiat_columns + for aggregation in self.config.aggregation: + self.fiat_columns_dict[f"aggregation_label_{aggregation.name}"] = f"Aggregation Label :{aggregation.name}" footprints.aggregate(fiat_results_df, self.fiat_columns_dict) footprints.calc_normalized_damages(self.fiat_columns_dict) From c71cd4f915cf552ae3b9a8d9465269518eeda240 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Fri, 31 Jan 2025 15:48:23 +0100 Subject: [PATCH 11/27] add max damage other --- flood_adapt/adapter/fiat_adapter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index fe81c6e5..bd94090e 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -64,6 +64,7 @@ class FiatColumns: fn_damage_other= "fn_damage_other" max_damage_structure= "max_damage_structure" max_damage_content= "max_damage_content" + max_damage_other= "max_damage_other" class FiatAdapter(IImpactAdapter): """ @@ -118,7 +119,8 @@ def __init__( FiatColumns.fn_damage_content: "Damage Function: Content", FiatColumns.fn_damage_other: "Damage Function: Other", FiatColumns.max_damage_structure: "Max Potential Damage: Structure", - FiatColumns.max_damage_content: "Max Potential Damage: Content"} + FiatColumns.max_damage_content: "Max Potential Damage: Content", + FiatColumns.max_damage_other: "Max Potential Damage: Other"} self._model = FiatModel(root=str(model_root.resolve()), mode="r") self._model.read() @@ -1385,6 +1387,7 @@ def save_building_footprints(self, output_path: os.PathLike): # Add aggregation to fiat_columns for aggregation in self.config.aggregation: self.fiat_columns_dict[f"aggregation_label_{aggregation.name}"] = f"Aggregation Label :{aggregation.name}" + # Get damages footprints.aggregate(fiat_results_df, self.fiat_columns_dict) footprints.calc_normalized_damages(self.fiat_columns_dict) From fe8deac04f2739919b676a78e444b3eedb7bea2b Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Fri, 31 Jan 2025 17:26:32 +0100 Subject: [PATCH 12/27] change fiat_dict aggr --- flood_adapt/adapter/fiat_adapter.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index bd94090e..6003b5ba 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -50,8 +50,6 @@ class FiatColumns: extraction_method = "extract_method" ground_floor_height = "ground_flht" ground_elevation = "ground_elevtn" - damage_function = "fn_damage_" - max_potential_damage = "max_damage_" inundation_depth = "inun_depth" damage_structure = "damage_structure" damage_content = "damage_content" @@ -62,9 +60,9 @@ class FiatColumns: fn_damage_structure= "fn_damage_structure" fn_damage_content= "fn_damage_content" fn_damage_other= "fn_damage_other" - max_damage_structure= "max_damage_structure" - max_damage_content= "max_damage_content" - max_damage_other= "max_damage_other" + max_pot_damage_structure= "max_damage_structure" + max_pot_damage_content= "max_damage_content" + max_pot_damage_other= "max_damage_other" class FiatAdapter(IImpactAdapter): """ @@ -106,8 +104,6 @@ def __init__( FiatColumns.extraction_method : "Extraction Method", FiatColumns.ground_floor_height : "Ground Floor Height", FiatColumns.ground_elevation : "Ground Elevation", - FiatColumns.damage_function : "Damage Function: ", - FiatColumns.max_potential_damage : "Max Potential Damage: ", FiatColumns.inundation_depth : "Inundation Depth", FiatColumns.damage_structure : "Damage: Structure", FiatColumns.damage_content : "Damage: Content", @@ -118,9 +114,9 @@ def __init__( FiatColumns.fn_damage_structure: "Damage Function: Structure", FiatColumns.fn_damage_content: "Damage Function: Content", FiatColumns.fn_damage_other: "Damage Function: Other", - FiatColumns.max_damage_structure: "Max Potential Damage: Structure", - FiatColumns.max_damage_content: "Max Potential Damage: Content", - FiatColumns.max_damage_other: "Max Potential Damage: Other"} + FiatColumns.max_pot_damage_structure: "Max Potential Damage: Structure", + FiatColumns.max_pot_damage_content: "Max Potential Damage: Content", + FiatColumns.max_pot_damage_other: "Max Potential Damage: Other"} self._model = FiatModel(root=str(model_root.resolve()), mode="r") self._model.read() @@ -1386,7 +1382,7 @@ def save_building_footprints(self, output_path: os.PathLike): ) # Add aggregation to fiat_columns for aggregation in self.config.aggregation: - self.fiat_columns_dict[f"aggregation_label_{aggregation.name}"] = f"Aggregation Label :{aggregation.name}" + self.fiat_columns_dict[f"AGGregation_label_{aggregation.name}"] = f"aggregation_label_{aggregation.name}" # NOTE capital GG just for test # Get damages footprints.aggregate(fiat_results_df, self.fiat_columns_dict) footprints.calc_normalized_damages(self.fiat_columns_dict) From ddd0756b71cde4bcaee16659a1b0d3e75d448396 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Sun, 2 Feb 2025 21:33:10 +0100 Subject: [PATCH 13/27] update level to label --- flood_adapt/adapter/fiat_adapter.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index 6003b5ba..2dcd6618 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -63,6 +63,7 @@ class FiatColumns: max_pot_damage_structure= "max_damage_structure" max_pot_damage_content= "max_damage_content" max_pot_damage_other= "max_damage_other" + aggr_label_default= "aggregation_label" class FiatAdapter(IImpactAdapter): """ @@ -116,7 +117,8 @@ def __init__( FiatColumns.fn_damage_other: "Damage Function: Other", FiatColumns.max_pot_damage_structure: "Max Potential Damage: Structure", FiatColumns.max_pot_damage_content: "Max Potential Damage: Content", - FiatColumns.max_pot_damage_other: "Max Potential Damage: Other"} + FiatColumns.max_pot_damage_other: "Max Potential Damage: Other", + FiatColumns.aggr_label_default: "Aggregation level:"} self._model = FiatModel(root=str(model_root.resolve()), mode="r") self._model.read() @@ -457,7 +459,7 @@ def postprocess(self, scenario): for aggregation in self.config.aggregation: name = aggregation.name - self.fiat_output_mapping[f"aggregation_label:{name}"] = f"Aggregation Label: {name}" + self.fiat_output_mapping[f"{FiatColumns.aggr_label_default}:{name}"] = f"Aggregation Label: {name}" metrics_exposure = self.outputs["table"].rename(columns = self.fiat_output_mapping) metrics_exposure.to_csv(fiat_results_path) @@ -1382,7 +1384,7 @@ def save_building_footprints(self, output_path: os.PathLike): ) # Add aggregation to fiat_columns for aggregation in self.config.aggregation: - self.fiat_columns_dict[f"AGGregation_label_{aggregation.name}"] = f"aggregation_label_{aggregation.name}" # NOTE capital GG just for test + self.fiat_columns_dict[f"aggr_label_default:{aggregation.name}"] = f"{self.fiat_columns_dict['aggr_label_default']}:{aggregation.name}" # Get damages footprints.aggregate(fiat_results_df, self.fiat_columns_dict) footprints.calc_normalized_damages(self.fiat_columns_dict) From 39b5ee11ee8940fb868f25bc2ac2e3bf6de62afc Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Mon, 3 Feb 2025 11:57:47 +0100 Subject: [PATCH 14/27] convert risk metrics --- flood_adapt/adapter/fiat_adapter.py | 48 ++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index 2dcd6618..bc0029d1 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -55,11 +55,13 @@ class FiatColumns: damage_content = "damage_content" damage_other = "damage_other" total_damage = "total_damage" - risk_ead = "risk_ead" + ead_damage = "ead_damage" segment_length = "segment_length" + damage_default = "damage" fn_damage_structure= "fn_damage_structure" fn_damage_content= "fn_damage_content" fn_damage_other= "fn_damage_other" + max_pot_damage_default = "max_damage_" max_pot_damage_structure= "max_damage_structure" max_pot_damage_content= "max_damage_content" max_pot_damage_other= "max_damage_other" @@ -106,15 +108,17 @@ def __init__( FiatColumns.ground_floor_height : "Ground Floor Height", FiatColumns.ground_elevation : "Ground Elevation", FiatColumns.inundation_depth : "Inundation Depth", + FiatColumns.damage_default : "Damage: ", FiatColumns.damage_structure : "Damage: Structure", FiatColumns.damage_content : "Damage: Content", FiatColumns.damage_other : "Damage: Other", FiatColumns.total_damage : "Total Damage", - FiatColumns.risk_ead : "Risk (EAD)", + FiatColumns.ead_damage : "Risk (EAD)", FiatColumns.segment_length : "Segment Length", FiatColumns.fn_damage_structure: "Damage Function: Structure", FiatColumns.fn_damage_content: "Damage Function: Content", FiatColumns.fn_damage_other: "Damage Function: Other", + FiatColumns.max_pot_damage_default: "Max Potential Damage:", FiatColumns.max_pot_damage_structure: "Max Potential Damage: Structure", FiatColumns.max_pot_damage_content: "Max Potential Damage: Content", FiatColumns.max_pot_damage_other: "Max Potential Damage: Other", @@ -432,7 +436,13 @@ def postprocess(self, scenario): self.read_outputs() mode = scenario.event.attrs.mode - + + # Map exposure columns + for aggregation in self.config.aggregation: + name = aggregation.name + self.fiat_output_mapping[f"{FiatColumns.aggr_label_default}:{name}"] = f"Aggregation Label: {name}" + metrics_exposure = self.outputs["table"].rename(columns = self.fiat_output_mapping) + # Define scenario output path scenario_output_path = scenario.impacts.results_path impacts_output_path = scenario.impacts.impacts_path @@ -446,8 +456,26 @@ def postprocess(self, scenario): ) with open(config_path, mode="rb") as fp: config = tomli.load(fp)["flood_exceedance"] + for column in metrics_exposure: + if FiatColumns.inundation_depth in column: + new_column = column.split(FiatColumns.inundation_depth + "_")[-1] + new_column = str(self.fiat_output_mapping[FiatColumns.inundation_depth] + new_column) + metrics_exposure = metrics_exposure.rename(columns = {column: new_column}) + elif FiatColumns.damage_default in column: + if FiatColumns.total_damage in column: + damage_rp = column.split(FiatColumns.total_damage + "_")[-1] + rp = f"({damage_rp.split('.0y')[0]}Y)" + new_column = str(self.fiat_output_mapping[FiatColumns.total_damage] + " " + rp) + else: + damage_rp = column.split(FiatColumns.damage_default + "_")[-1] + damage = damage_rp.split("_")[0].capitalize() + rp = f"({damage_rp.split('_')[1].split('.0y')[0]}Y)" + new_column = str(self.fiat_output_mapping[FiatColumns.damage_default] + damage + " " + rp) + metrics_exposure= metrics_exposure.rename(columns = {column: new_column}) + self.add_exceedance_probability( - column=config["column"], + metrics_exposure = metrics_exposure, + column=self.fiat_output_mapping["inundation_depth"], threshold=config["threshold"], period=config["period"], ) @@ -456,11 +484,7 @@ def postprocess(self, scenario): fiat_results_path = impacts_output_path.joinpath( f"Impacts_detailed_{scenario.attrs.name}.csv" ) - - for aggregation in self.config.aggregation: - name = aggregation.name - self.fiat_output_mapping[f"{FiatColumns.aggr_label_default}:{name}"] = f"Aggregation Label: {name}" - metrics_exposure = self.outputs["table"].rename(columns = self.fiat_output_mapping) + metrics_exposure.to_csv(fiat_results_path) # Create the infometrics files @@ -1107,12 +1131,14 @@ def get_object_ids(self, measure: IMeasure) -> list[Any]: # POST-PROCESSING METHODS def add_exceedance_probability( - self, column: str, threshold: float, period: int + self, metrics_exposure: pd.DataFrame, column: str, threshold: float, period: int, ) -> pd.DataFrame: """Calculate exceedance probabilities and append them to the results table. Parameters ---------- + metric_exposure : pd.DataFrame + A DataFrame with the mapped exposure columns for metrics calculation. column : str The name of the column to calculate exceedance probabilities for. threshold : float @@ -1127,7 +1153,7 @@ def add_exceedance_probability( """ self.logger.info("Calculating exceedance probabilities") fiat_results_df = ExceedanceProbabilityCalculator(column).append_probability( - self.outputs["table"], threshold, period + metric_exposure, threshold, period, self.fiat_output_mapping["inundation_depth"] ) self.outputs["table"] = fiat_results_df return self.outputs["table"] From 6abd3bfee2cc4cc8157377ff541502326755d91e Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Mon, 3 Feb 2025 15:11:56 +0100 Subject: [PATCH 15/27] revert map risk before exceedance calculation to after --- flood_adapt/adapter/fiat_adapter.py | 56 ++++++++++++++++------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index bc0029d1..fffc0f99 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -454,28 +454,12 @@ def postprocess(self, scenario): config_path = scenario.database.static_path.joinpath( "templates", "infometrics", "metrics_additional_risk_configs.toml" ) + with open(config_path, mode="rb") as fp: config = tomli.load(fp)["flood_exceedance"] - for column in metrics_exposure: - if FiatColumns.inundation_depth in column: - new_column = column.split(FiatColumns.inundation_depth + "_")[-1] - new_column = str(self.fiat_output_mapping[FiatColumns.inundation_depth] + new_column) - metrics_exposure = metrics_exposure.rename(columns = {column: new_column}) - elif FiatColumns.damage_default in column: - if FiatColumns.total_damage in column: - damage_rp = column.split(FiatColumns.total_damage + "_")[-1] - rp = f"({damage_rp.split('.0y')[0]}Y)" - new_column = str(self.fiat_output_mapping[FiatColumns.total_damage] + " " + rp) - else: - damage_rp = column.split(FiatColumns.damage_default + "_")[-1] - damage = damage_rp.split("_")[0].capitalize() - rp = f"({damage_rp.split('_')[1].split('.0y')[0]}Y)" - new_column = str(self.fiat_output_mapping[FiatColumns.damage_default] + damage + " " + rp) - metrics_exposure= metrics_exposure.rename(columns = {column: new_column}) - self.add_exceedance_probability( - metrics_exposure = metrics_exposure, - column=self.fiat_output_mapping["inundation_depth"], + metrics_exposure = self.add_exceedance_probability( + column=FiatColumns.inundation_depth, threshold=config["threshold"], period=config["period"], ) @@ -568,6 +552,28 @@ def postprocess(self, scenario): if not self.config.save_simulation: self.delete_model() + def map_risk_metrics(self, metrics_exposure: pd.DataFrame): + for column in metrics_exposure: + if FiatColumns.inundation_depth in column: + inun_rp = column.split(FiatColumns.inundation_depth + "_")[-1] + rp = f"({inun_rp.split('.0y')[0]}Y)" + new_column = str(self.fiat_output_mapping[FiatColumns.inundation_depth] + " " + rp) + metrics_exposure = metrics_exposure.rename(columns = {column: new_column}) + elif FiatColumns.damage_default in column: + if FiatColumns.total_damage in column: + damage_rp = column.split(FiatColumns.total_damage + "_")[-1] + rp = f"({damage_rp.split('.0y')[0]}Y)" + new_column = str(self.fiat_output_mapping[FiatColumns.total_damage] + " " + rp) + else: + damage_rp = column.split(FiatColumns.damage_default + "_")[-1] + damage = damage_rp.split("_")[0].capitalize() + rp = f"({damage_rp.split('_')[1].split('.0y')[0]}Y)" + new_column = str(self.fiat_output_mapping[FiatColumns.damage_default] + damage + " " + rp) + metrics_exposure= metrics_exposure.rename(columns = {column: new_column}) + else: + metrics_exposure = metrics_exposure + + return metrics_exposure def add_measure(self, measure: IMeasure): """ Add and apply a specific impact measure to the properties of the FIAT model. @@ -1131,14 +1137,12 @@ def get_object_ids(self, measure: IMeasure) -> list[Any]: # POST-PROCESSING METHODS def add_exceedance_probability( - self, metrics_exposure: pd.DataFrame, column: str, threshold: float, period: int, + self, column: str, threshold: float, period: int, ) -> pd.DataFrame: """Calculate exceedance probabilities and append them to the results table. Parameters ---------- - metric_exposure : pd.DataFrame - A DataFrame with the mapped exposure columns for metrics calculation. column : str The name of the column to calculate exceedance probabilities for. threshold : float @@ -1153,10 +1157,12 @@ def add_exceedance_probability( """ self.logger.info("Calculating exceedance probabilities") fiat_results_df = ExceedanceProbabilityCalculator(column).append_probability( - metric_exposure, threshold, period, self.fiat_output_mapping["inundation_depth"] + self.outputs["table"], threshold, period ) - self.outputs["table"] = fiat_results_df - return self.outputs["table"] + fiat_results_df = fiat_results_df.rename(columns = self.fiat_output_mapping) + fiat_results_df = self.map_risk_metrics(fiat_results_df) + + return fiat_results_df def create_infometrics( self, metric_config_paths: list[os.PathLike], metrics_output_path: os.PathLike, metrics_exposure: pd.DataFrame From ffe24a55ab8e85df16eadb7203acb20612c317ec Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Mon, 3 Feb 2025 15:42:19 +0100 Subject: [PATCH 16/27] pre-commit --- flood_adapt/adapter/fiat_adapter.py | 150 +++++++++++++++--------- tests/test_adapter/test_fiat_adapter.py | 9 +- 2 files changed, 94 insertions(+), 65 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index c1fbbeb7..5ccc57d3 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -42,7 +42,7 @@ class FiatColumns: """Object with mapping of FIAT attributes to columns names.""" - + object_id = "object_id" object_name = "object_name" primary_object_type = "primary_object_type" @@ -58,14 +58,15 @@ class FiatColumns: ead_damage = "ead_damage" segment_length = "segment_length" damage_default = "damage" - fn_damage_structure= "fn_damage_structure" - fn_damage_content= "fn_damage_content" - fn_damage_other= "fn_damage_other" + fn_damage_structure = "fn_damage_structure" + fn_damage_content = "fn_damage_content" + fn_damage_other = "fn_damage_other" max_pot_damage_default = "max_damage_" - max_pot_damage_structure= "max_damage_structure" - max_pot_damage_content= "max_damage_content" - max_pot_damage_other= "max_damage_other" - aggr_label_default= "aggregation_label" + max_pot_damage_structure = "max_damage_structure" + max_pot_damage_content = "max_damage_content" + max_pot_damage_other = "max_damage_other" + aggr_label_default = "aggregation_label" + class FiatAdapter(IImpactAdapter): """ @@ -99,22 +100,25 @@ def __init__( self.config_base_path = config_base_path self.exe_path = exe_path self.delete_crashed_runs = delete_crashed_runs - self.fiat_columns_dict = {k: v for k, v in FiatColumns.__dict__.items() if not k.startswith("__")} - self.fiat_output_mapping = {FiatColumns.object_id: "Object ID", - FiatColumns.object_name : "Object Name", - FiatColumns.primary_object_type : "Primary Object Type", - FiatColumns.secondary_object_type : "Secondary Object Type", - FiatColumns.extraction_method : "Extraction Method", - FiatColumns.ground_floor_height : "Ground Floor Height", - FiatColumns.ground_elevation : "Ground Elevation", - FiatColumns.inundation_depth : "Inundation Depth", - FiatColumns.damage_default : "Damage: ", - FiatColumns.damage_structure : "Damage: Structure", - FiatColumns.damage_content : "Damage: Content", - FiatColumns.damage_other : "Damage: Other", - FiatColumns.total_damage : "Total Damage", - FiatColumns.ead_damage : "Risk (EAD)", - FiatColumns.segment_length : "Segment Length", + self.fiat_columns_dict = { + k: v for k, v in FiatColumns.__dict__.items() if not k.startswith("__") + } + self.fiat_output_mapping = { + FiatColumns.object_id: "Object ID", + FiatColumns.object_name: "Object Name", + FiatColumns.primary_object_type: "Primary Object Type", + FiatColumns.secondary_object_type: "Secondary Object Type", + FiatColumns.extraction_method: "Extraction Method", + FiatColumns.ground_floor_height: "Ground Floor Height", + FiatColumns.ground_elevation: "Ground Elevation", + FiatColumns.inundation_depth: "Inundation Depth", + FiatColumns.damage_default: "Damage: ", + FiatColumns.damage_structure: "Damage: Structure", + FiatColumns.damage_content: "Damage: Content", + FiatColumns.damage_other: "Damage: Other", + FiatColumns.total_damage: "Total Damage", + FiatColumns.ead_damage: "Risk (EAD)", + FiatColumns.segment_length: "Segment Length", FiatColumns.fn_damage_structure: "Damage Function: Structure", FiatColumns.fn_damage_content: "Damage Function: Content", FiatColumns.fn_damage_other: "Damage Function: Other", @@ -122,7 +126,8 @@ def __init__( FiatColumns.max_pot_damage_structure: "Max Potential Damage: Structure", FiatColumns.max_pot_damage_content: "Max Potential Damage: Content", FiatColumns.max_pot_damage_other: "Max Potential Damage: Other", - FiatColumns.aggr_label_default: "Aggregation level:"} + FiatColumns.aggr_label_default: "Aggregation level:", + } self._model = FiatModel(root=str(model_root.resolve()), mode="r") self._model.read() @@ -436,28 +441,32 @@ def postprocess(self, scenario): self.read_outputs() mode = scenario.event.attrs.mode - + # Map exposure columns for aggregation in self.config.aggregation: name = aggregation.name - self.fiat_output_mapping[f"{FiatColumns.aggr_label_default}:{name}"] = f"Aggregation Label: {name}" - metrics_exposure = self.outputs["table"].rename(columns = self.fiat_output_mapping) - + self.fiat_output_mapping[f"{FiatColumns.aggr_label_default}:{name}"] = ( + f"Aggregation Label: {name}" + ) + metrics_exposure = self.outputs["table"].rename( + columns=self.fiat_output_mapping + ) + # Define scenario output path scenario_output_path = scenario.impacts.results_path impacts_output_path = scenario.impacts.impacts_path - # Add exceedance probabilities if needed (only for risk) + # Add exceedance probabilities if needed (only for risk) if mode == Mode.risk: # Get config path # TODO check where this configs should be read from config_path = scenario.database.static_path.joinpath( "templates", "infometrics", "metrics_additional_risk_configs.toml" ) - + with open(config_path, mode="rb") as fp: config = tomli.load(fp)["flood_exceedance"] - + metrics_exposure = self.add_exceedance_probability( column=FiatColumns.inundation_depth, threshold=config["threshold"], @@ -494,7 +503,9 @@ def postprocess(self, scenario): metrics_outputs_path = scenario_output_path.joinpath( f"Infometrics_{scenario.attrs.name}.csv" ) - self.create_infometrics(metric_config_paths, metrics_outputs_path, metrics_exposure) + self.create_infometrics( + metric_config_paths, metrics_outputs_path, metrics_exposure + ) # Get paths of created aggregated infometrics aggr_metrics_paths = list( @@ -554,26 +565,36 @@ def postprocess(self, scenario): def map_risk_metrics(self, metrics_exposure: pd.DataFrame): for column in metrics_exposure: - if FiatColumns.inundation_depth in column: - inun_rp = column.split(FiatColumns.inundation_depth + "_")[-1] - rp = f"({inun_rp.split('.0y')[0]}Y)" - new_column = str(self.fiat_output_mapping[FiatColumns.inundation_depth] + " " + rp) - metrics_exposure = metrics_exposure.rename(columns = {column: new_column}) - elif FiatColumns.damage_default in column: - if FiatColumns.total_damage in column: - damage_rp = column.split(FiatColumns.total_damage + "_")[-1] - rp = f"({damage_rp.split('.0y')[0]}Y)" - new_column = str(self.fiat_output_mapping[FiatColumns.total_damage] + " " + rp) - else: - damage_rp = column.split(FiatColumns.damage_default + "_")[-1] - damage = damage_rp.split("_")[0].capitalize() - rp = f"({damage_rp.split('_')[1].split('.0y')[0]}Y)" - new_column = str(self.fiat_output_mapping[FiatColumns.damage_default] + damage + " " + rp) - metrics_exposure= metrics_exposure.rename(columns = {column: new_column}) + if FiatColumns.inundation_depth in column: + inun_rp = column.split(FiatColumns.inundation_depth + "_")[-1] + rp = f"({inun_rp.split('.0y')[0]}Y)" + new_column = str( + self.fiat_output_mapping[FiatColumns.inundation_depth] + " " + rp + ) + metrics_exposure = metrics_exposure.rename(columns={column: new_column}) + elif FiatColumns.damage_default in column: + if FiatColumns.total_damage in column: + damage_rp = column.split(FiatColumns.total_damage + "_")[-1] + rp = f"({damage_rp.split('.0y')[0]}Y)" + new_column = str( + self.fiat_output_mapping[FiatColumns.total_damage] + " " + rp + ) else: - metrics_exposure = metrics_exposure - + damage_rp = column.split(FiatColumns.damage_default + "_")[-1] + damage = damage_rp.split("_")[0].capitalize() + rp = f"({damage_rp.split('_')[1].split('.0y')[0]}Y)" + new_column = str( + self.fiat_output_mapping[FiatColumns.damage_default] + + damage + + " " + + rp + ) + metrics_exposure = metrics_exposure.rename(columns={column: new_column}) + else: + metrics_exposure = metrics_exposure + return metrics_exposure + def add_measure(self, measure: IMeasure): """ Add and apply a specific impact measure to the properties of the FIAT model. @@ -1137,7 +1158,10 @@ def get_object_ids(self, measure: IMeasure) -> list[Any]: # POST-PROCESSING METHODS def add_exceedance_probability( - self, column: str, threshold: float, period: int, + self, + column: str, + threshold: float, + period: int, ) -> pd.DataFrame: """Calculate exceedance probabilities and append them to the results table. @@ -1159,13 +1183,16 @@ def add_exceedance_probability( fiat_results_df = ExceedanceProbabilityCalculator(column).append_probability( self.outputs["table"], threshold, period ) - fiat_results_df = fiat_results_df.rename(columns = self.fiat_output_mapping) + fiat_results_df = fiat_results_df.rename(columns=self.fiat_output_mapping) fiat_results_df = self.map_risk_metrics(fiat_results_df) return fiat_results_df def create_infometrics( - self, metric_config_paths: list[os.PathLike], metrics_output_path: os.PathLike, metrics_exposure: pd.DataFrame + self, + metric_config_paths: list[os.PathLike], + metrics_output_path: os.PathLike, + metrics_exposure: pd.DataFrame, ) -> None: """ Create infometrics files based on the provided metric configuration paths. @@ -1189,7 +1216,9 @@ def create_infometrics( # Check if type of metric configuration is available for metric_file in metric_config_paths: if metric_file.exists(): - metrics_writer = MetricsFileWriter(metric_file, aggregation_label = "Aggregation Label: ") + metrics_writer = MetricsFileWriter( + metric_file, aggregation_label="Aggregation Label: " + ) metrics_writer.parse_metrics_to_file( df_results=metrics_exposure, @@ -1416,7 +1445,9 @@ def save_building_footprints(self, output_path: os.PathLike): ) # Add aggregation to fiat_columns for aggregation in self.config.aggregation: - self.fiat_columns_dict[f"aggr_label_default:{aggregation.name}"] = f"{self.fiat_columns_dict['aggr_label_default']}:{aggregation.name}" + self.fiat_columns_dict[f"aggr_label_default:{aggregation.name}"] = ( + f"{self.fiat_columns_dict['aggr_label_default']}:{aggregation.name}" + ) # Get damages footprints.aggregate(fiat_results_df, self.fiat_columns_dict) footprints.calc_normalized_damages(self.fiat_columns_dict) @@ -1442,7 +1473,10 @@ def save_roads(self, output_path: os.PathLike): aggr_cols = [ name for name in self.outputs["table"].columns - if any(f"aggregation_label:{aggregation.name}" in name for aggregation in self.config.aggregation) + if any( + f"aggregation_label:{aggregation.name}" in name + for aggregation in self.config.aggregation + ) ] inun_cols = [ name for name in roads.columns if FiatColumns.inundation_depth in name @@ -1455,4 +1489,4 @@ def save_roads(self, output_path: os.PathLike): on=FiatColumns.object_id, ) # Save as geopackage - roads.to_file(output_path, driver="GPKG") \ No newline at end of file + roads.to_file(output_path, driver="GPKG") diff --git a/tests/test_adapter/test_fiat_adapter.py b/tests/test_adapter/test_fiat_adapter.py index 0920e8f8..db702d49 100644 --- a/tests/test_adapter/test_fiat_adapter.py +++ b/tests/test_adapter/test_fiat_adapter.py @@ -89,9 +89,7 @@ def test_all_measures(self, run_scenario_all_measures): ) assert ( pytest.approx( - exposure_scenario.loc[ - inds_new_area, "max_damage_structure" - ].sum() + exposure_scenario.loc[inds_new_area, "max_damage_structure"].sum() ) == ( test_scenario.impacts.socio_economic_change.attrs.economic_growth / 100 @@ -147,10 +145,7 @@ def test_all_measures(self, run_scenario_all_measures): # Check if elevation took place correctly at each object for i, row in exposures.iterrows(): # If the initial elevation is smaller than the required one it should have been elevated to than one - if ( - row["ground_elevtn"] + row["ground_flht_1"] - < row["bfe"] + elevate_val - ): + if row["ground_elevtn"] + row["ground_flht_1"] < row["bfe"] + elevate_val: assert ( row["ground_elevtn"] + row["ground_flht_2"] == row["bfe"] + elevate_val From 1937f4fcaf11bc8bdf09f217e4c95072df36d19e Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 6 Feb 2025 13:13:42 +0100 Subject: [PATCH 17/27] add docstrings. Update add_exceedance column parameter --- flood_adapt/adapter/fiat_adapter.py | 43 ++++++++++++++++++----------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index 5ccc57d3..6a36c9f4 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -448,7 +448,7 @@ def postprocess(self, scenario): self.fiat_output_mapping[f"{FiatColumns.aggr_label_default}:{name}"] = ( f"Aggregation Label: {name}" ) - metrics_exposure = self.outputs["table"].rename( + mapped_exposure_df = self.outputs["table"].rename( columns=self.fiat_output_mapping ) @@ -466,9 +466,11 @@ def postprocess(self, scenario): with open(config_path, mode="rb") as fp: config = tomli.load(fp)["flood_exceedance"] - - metrics_exposure = self.add_exceedance_probability( - column=FiatColumns.inundation_depth, + column = column = [key for key, value in self.fiat_output_mapping.items() if value == config["column"]][0] + + self.fiat_output_mapping[column] + mapped_exposure_df = self.add_exceedance_probability( + column=column, threshold=config["threshold"], period=config["period"], ) @@ -478,7 +480,7 @@ def postprocess(self, scenario): f"Impacts_detailed_{scenario.attrs.name}.csv" ) - metrics_exposure.to_csv(fiat_results_path) + mapped_exposure_df.to_csv(fiat_results_path) # Create the infometrics files if mode == Mode.risk: @@ -504,7 +506,7 @@ def postprocess(self, scenario): f"Infometrics_{scenario.attrs.name}.csv" ) self.create_infometrics( - metric_config_paths, metrics_outputs_path, metrics_exposure + metric_config_paths, metrics_outputs_path, mapped_exposure_df ) # Get paths of created aggregated infometrics @@ -563,15 +565,22 @@ def postprocess(self, scenario): if not self.config.save_simulation: self.delete_model() - def map_risk_metrics(self, metrics_exposure: pd.DataFrame): - for column in metrics_exposure: + def map_risk_metrics(self, mapped_exposure_df: pd.DataFrame): + """Rename the Delft FIAT output columns per return period. + + Parameters + ---------- + mapped_exposure_df : pd.DataFrame + The exposure dataframe with the mapped headings for the exposure input (not yet the output). + """ + for column in mapped_exposure_df: if FiatColumns.inundation_depth in column: inun_rp = column.split(FiatColumns.inundation_depth + "_")[-1] rp = f"({inun_rp.split('.0y')[0]}Y)" new_column = str( self.fiat_output_mapping[FiatColumns.inundation_depth] + " " + rp ) - metrics_exposure = metrics_exposure.rename(columns={column: new_column}) + mapped_exposure_df = mapped_exposure_df.rename(columns={column: new_column}) elif FiatColumns.damage_default in column: if FiatColumns.total_damage in column: damage_rp = column.split(FiatColumns.total_damage + "_")[-1] @@ -589,11 +598,11 @@ def map_risk_metrics(self, metrics_exposure: pd.DataFrame): + " " + rp ) - metrics_exposure = metrics_exposure.rename(columns={column: new_column}) + mapped_exposure_df = mapped_exposure_df.rename(columns={column: new_column}) else: - metrics_exposure = metrics_exposure + mapped_exposure_df = mapped_exposure_df - return metrics_exposure + return mapped_exposure_df def add_measure(self, measure: IMeasure): """ @@ -1192,7 +1201,7 @@ def create_infometrics( self, metric_config_paths: list[os.PathLike], metrics_output_path: os.PathLike, - metrics_exposure: pd.DataFrame, + mapped_exposure_df: pd.DataFrame, ) -> None: """ Create infometrics files based on the provided metric configuration paths. @@ -1203,6 +1212,8 @@ def create_infometrics( A list of paths to the metric configuration files. metrics_output_path : os.PathLike The path where the metrics output file will be saved. + mapped_exposure_df: pd.DataFrame + The exposure dataframe with the mapped headings for the exposure input and output). Raises ------ @@ -1217,17 +1228,17 @@ def create_infometrics( for metric_file in metric_config_paths: if metric_file.exists(): metrics_writer = MetricsFileWriter( - metric_file, aggregation_label="Aggregation Label: " + metric_file, aggregation_prefix="Aggregation Label: " ) metrics_writer.parse_metrics_to_file( - df_results=metrics_exposure, + df_results=mapped_exposure_df, metrics_path=metrics_output_path, write_aggregate=None, ) metrics_writer.parse_metrics_to_file( - df_results=metrics_exposure, + df_results=mapped_exposure_df, metrics_path=metrics_output_path, write_aggregate="all", ) From 03abb396f27e61f088c4eb12ec34e965e0709f64 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 6 Feb 2025 13:25:34 +0100 Subject: [PATCH 18/27] update tests with FiatColumns class --- tests/test_adapter/test_fiat_adapter.py | 82 ++++++++++++------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/tests/test_adapter/test_fiat_adapter.py b/tests/test_adapter/test_fiat_adapter.py index db702d49..9c0dc709 100644 --- a/tests/test_adapter/test_fiat_adapter.py +++ b/tests/test_adapter/test_fiat_adapter.py @@ -7,7 +7,7 @@ db_path, ) from flood_adapt.object_model.scenario import Scenario - +from flood_adapt.adapter.fiat_adapter import FiatColumns class TestFiatAdapter: @pytest.fixture(scope="class") @@ -68,12 +68,12 @@ def test_all_measures(self, run_scenario_all_measures): assert len(exposure_scenario) > len(exposure_template) # check if growth has been applied correctly - inds1 = exposure_scenario["object_id"].isin(exposure_template["object_id"]) & ( - exposure_scenario["primary_object_type"] != "road" + inds1 = exposure_scenario[FiatColumns.object_id].isin(exposure_template[FiatColumns.object_id]) & ( + exposure_scenario[FiatColumns.primary_object_type] != "road" ) - exp1 = exposure_scenario.loc[inds1, "max_damage_structure"] - inds0 = exposure_template["primary_object_type"] != "road" - exp0 = exposure_template.loc[inds0, "max_damage_structure"] + exp1 = exposure_scenario.loc[inds1, FiatColumns.damage_structure] + inds0 = exposure_template[FiatColumns.primary_object_type] != "road" + exp0 = exposure_template.loc[inds0, FiatColumns.damage_structure] eg = test_scenario.impacts.socio_economic_change.attrs.economic_growth pg = ( test_scenario.impacts.socio_economic_change.attrs.population_growth_existing @@ -84,12 +84,12 @@ def test_all_measures(self, run_scenario_all_measures): ) # check if new area max damage is implemented correctly - inds_new_area = ~exposure_scenario["object_id"].isin( - exposure_template["object_id"] + inds_new_area = ~exposure_scenario[FiatColumns.object_id].isin( + exposure_template[FiatColumns.object_id] ) assert ( pytest.approx( - exposure_scenario.loc[inds_new_area, "max_damage_structure"].sum() + exposure_scenario.loc[inds_new_area, FiatColumns.damage_structure].sum() ) == ( test_scenario.impacts.socio_economic_change.attrs.economic_growth / 100 @@ -99,7 +99,7 @@ def test_all_measures(self, run_scenario_all_measures): test_scenario.impacts.socio_economic_change.attrs.population_growth_new / 100 ) - * exposure_template.loc[:, "max_damage_structure"].sum() + * exposure_template.loc[:, FiatColumns.damage_structure].sum() ) # check if buildings are elevated correctly @@ -120,39 +120,39 @@ def test_all_measures(self, run_scenario_all_measures): bfes = pd.read_csv(db_path(TopLevelDir.static) / "bfe" / "bfe.csv") # Create a dataframe to save the initial object attributes - exposures = exposure_template.merge(bfes, on="object_id")[ - ["object_id", "bfe", "ground_flht_"] - ].rename(columns={"ground_flht_": "ground_flht_1"}) + exposures = exposure_template.merge(bfes, on=FiatColumns.object_id)[ + [FiatColumns.object_id, "bfe", f"{FiatColumns.ground_floor_height}_"] + ].rename(columns={f"{FiatColumns.ground_floor_height}_": f"{FiatColumns.ground_floor_height}_1"}) # Merge with the adapted fiat model exposure - exposures = exposures.merge(exposure_scenario, on="object_id").rename( - columns={"ground_flht_": "ground_flht_2"} + exposures = exposures.merge(exposure_scenario, on=FiatColumns.object_id).rename( + columns={f"{FiatColumns.ground_floor_height}_": f"{FiatColumns.ground_floor_height}_2"} ) # Filter to only the objects affected by the measure exposures = exposures.loc[ - (exposure_scenario.loc[:, f"aggregation_label:_{aggr_label}"] == aggr_name) - & (exposure_scenario.loc[:, "primary_object_type"] == build_type), + (exposure_scenario.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] == aggr_name) + & (exposure_scenario.loc[:, FiatColumns.primary_object_type] == build_type), :, ] exposures = exposures[ [ - "object_id", - "ground_elevtn", + FiatColumns.object_id, + FiatColumns.ground_elevation, "bfe", - "ground_flht_1", - "ground_flht_2", + f"{FiatColumns.ground_floor_height}_1", + f"{FiatColumns.ground_floor_height}_2", ] ] # Check if elevation took place correctly at each object for i, row in exposures.iterrows(): # If the initial elevation is smaller than the required one it should have been elevated to than one - if row["ground_elevtn"] + row["ground_flht_1"] < row["bfe"] + elevate_val: + if row[FiatColumns.ground_elevation] + row[f"{FiatColumns.ground_floor_height}_1"] < row["bfe"] + elevate_val: assert ( - row["ground_elevtn"] + row["ground_flht_2"] + row[FiatColumns.ground_elevation] + row[f"{FiatColumns.ground_floor_height}_2"] == row["bfe"] + elevate_val ) # if not it should have stated the same else: - assert row["ground_flht_2"] == row["ground_flht_1"] + assert row[f"{FiatColumns.ground_floor_height}_2"] == row[f"{FiatColumns.ground_floor_height}_1"] # check if buildings are bought-out aggr_label = test_scenario.impacts.impact_strategy.measures[ @@ -165,10 +165,10 @@ def test_all_measures(self, run_scenario_all_measures): 1 ].attrs.property_type inds = ( - exposure_scenario.loc[:, f"aggregation_label:_{aggr_label}"] == aggr_name - ) & (exposure_scenario.loc[:, "primary_object_type"] == build_type) + exposure_scenario.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] == aggr_name + ) & (exposure_scenario.loc[:, FiatColumns.primary_object_type] == build_type) - assert all(exposure_scenario.loc[inds, "max_damage_structure"] == 0) + assert all(exposure_scenario.loc[inds, FiatColumns.damage_structure] == 0) # check if buildings are flood-proofed aggr_label = test_scenario.impacts.impact_strategy.measures[ @@ -181,15 +181,15 @@ def test_all_measures(self, run_scenario_all_measures): 2 ].attrs.property_type inds1 = ( - exposure_template.loc[:, f"aggregation_label:_{aggr_label}"] == aggr_name - ) & (exposure_template.loc[:, "primary_object_type"] == build_type) + exposure_template.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] == aggr_name + ) & (exposure_template.loc[:, FiatColumns.primary_object_type] == build_type) inds2 = ( - exposure_scenario.loc[:, f"aggregation_label:_{aggr_label}"] == aggr_name - ) & (exposure_scenario.loc[:, "primary_object_type"] == build_type) + exposure_scenario.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] == aggr_name + ) & (exposure_scenario.loc[:, FiatColumns.primary_object_type] == build_type) assert all( - exposure_scenario.loc[inds2, "fn_damage_structure"] - != exposure_template.loc[inds1, "fn_damage_structure"] + exposure_scenario.loc[inds2, FiatColumns.fn_damage_structure] + != exposure_template.loc[inds1, FiatColumns.fn_damage_structure] ) def test_raise_datum(self, run_scenario_raise_datum): @@ -214,17 +214,17 @@ def test_raise_datum(self, run_scenario_raise_datum): 0 ].attrs.property_type inds1 = ( - exposure_template.loc[:, f"aggregation_label:_{aggr_label}"] == aggr_name - ) & (exposure_template.loc[:, "primary_object_type"] == build_type) + exposure_template.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] == aggr_name + ) & (exposure_template.loc[:, FiatColumns.primary_object_type] == build_type) inds2 = ( - exposure_scenario.loc[:, f"aggregation_label:_{aggr_label}"] == aggr_name - ) & (exposure_scenario.loc[:, "primary_object_type"] == build_type) + exposure_scenario.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] == aggr_name + ) & (exposure_scenario.loc[:, FiatColumns.primary_object_type] == build_type) assert all( elev1 <= elev2 for elev1, elev2 in zip( - exposure_template.loc[inds1, "ground_flht_"], - exposure_scenario.loc[inds2, "ground_flht_"], + exposure_template.loc[inds1, f"{FiatColumns.ground_floor_height}_"], + exposure_scenario.loc[inds2, f"{FiatColumns.ground_floor_height}_"], ) ) @@ -232,8 +232,8 @@ def test_raise_datum(self, run_scenario_raise_datum): height + elev >= test_scenario.impacts.impact_strategy.measures[0].attrs.elevation.value for height, elev in zip( - exposure_scenario.loc[inds2, "ground_flht_"], - exposure_scenario.loc[inds2, "ground_elevtn"], + exposure_scenario.loc[inds2, f"{FiatColumns.ground_floor_height}_"], + exposure_scenario.loc[inds2, FiatColumns.ground_elevation], ) ) From 506c533ac4a0c2366a9e942e59fe7b6d844eaa06 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 6 Feb 2025 13:30:44 +0100 Subject: [PATCH 19/27] run pre-commit --- flood_adapt/adapter/fiat_adapter.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index 6a36c9f4..99dc0778 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -466,8 +466,12 @@ def postprocess(self, scenario): with open(config_path, mode="rb") as fp: config = tomli.load(fp)["flood_exceedance"] - column = column = [key for key, value in self.fiat_output_mapping.items() if value == config["column"]][0] - + column = column = [ + key + for key, value in self.fiat_output_mapping.items() + if value == config["column"] + ][0] + self.fiat_output_mapping[column] mapped_exposure_df = self.add_exceedance_probability( column=column, @@ -567,7 +571,7 @@ def postprocess(self, scenario): def map_risk_metrics(self, mapped_exposure_df: pd.DataFrame): """Rename the Delft FIAT output columns per return period. - + Parameters ---------- mapped_exposure_df : pd.DataFrame @@ -580,7 +584,9 @@ def map_risk_metrics(self, mapped_exposure_df: pd.DataFrame): new_column = str( self.fiat_output_mapping[FiatColumns.inundation_depth] + " " + rp ) - mapped_exposure_df = mapped_exposure_df.rename(columns={column: new_column}) + mapped_exposure_df = mapped_exposure_df.rename( + columns={column: new_column} + ) elif FiatColumns.damage_default in column: if FiatColumns.total_damage in column: damage_rp = column.split(FiatColumns.total_damage + "_")[-1] @@ -598,7 +604,9 @@ def map_risk_metrics(self, mapped_exposure_df: pd.DataFrame): + " " + rp ) - mapped_exposure_df = mapped_exposure_df.rename(columns={column: new_column}) + mapped_exposure_df = mapped_exposure_df.rename( + columns={column: new_column} + ) else: mapped_exposure_df = mapped_exposure_df From 8c7ee69c2fd9ebad39302d5428c20d98d2cf4e13 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 6 Feb 2025 13:30:53 +0100 Subject: [PATCH 20/27] run pre-commit --- tests/test_adapter/test_fiat_adapter.py | 55 ++++++++++++++++++------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/tests/test_adapter/test_fiat_adapter.py b/tests/test_adapter/test_fiat_adapter.py index 9c0dc709..7d764c4f 100644 --- a/tests/test_adapter/test_fiat_adapter.py +++ b/tests/test_adapter/test_fiat_adapter.py @@ -2,12 +2,13 @@ import pytest from pandas.testing import assert_frame_equal +from flood_adapt.adapter.fiat_adapter import FiatColumns from flood_adapt.object_model.interface.path_builder import ( TopLevelDir, db_path, ) from flood_adapt.object_model.scenario import Scenario -from flood_adapt.adapter.fiat_adapter import FiatColumns + class TestFiatAdapter: @pytest.fixture(scope="class") @@ -68,9 +69,9 @@ def test_all_measures(self, run_scenario_all_measures): assert len(exposure_scenario) > len(exposure_template) # check if growth has been applied correctly - inds1 = exposure_scenario[FiatColumns.object_id].isin(exposure_template[FiatColumns.object_id]) & ( - exposure_scenario[FiatColumns.primary_object_type] != "road" - ) + inds1 = exposure_scenario[FiatColumns.object_id].isin( + exposure_template[FiatColumns.object_id] + ) & (exposure_scenario[FiatColumns.primary_object_type] != "road") exp1 = exposure_scenario.loc[inds1, FiatColumns.damage_structure] inds0 = exposure_template[FiatColumns.primary_object_type] != "road" exp0 = exposure_template.loc[inds0, FiatColumns.damage_structure] @@ -122,14 +123,25 @@ def test_all_measures(self, run_scenario_all_measures): # Create a dataframe to save the initial object attributes exposures = exposure_template.merge(bfes, on=FiatColumns.object_id)[ [FiatColumns.object_id, "bfe", f"{FiatColumns.ground_floor_height}_"] - ].rename(columns={f"{FiatColumns.ground_floor_height}_": f"{FiatColumns.ground_floor_height}_1"}) + ].rename( + columns={ + f"{FiatColumns.ground_floor_height}_": f"{FiatColumns.ground_floor_height}_1" + } + ) # Merge with the adapted fiat model exposure exposures = exposures.merge(exposure_scenario, on=FiatColumns.object_id).rename( - columns={f"{FiatColumns.ground_floor_height}_": f"{FiatColumns.ground_floor_height}_2"} + columns={ + f"{FiatColumns.ground_floor_height}_": f"{FiatColumns.ground_floor_height}_2" + } ) # Filter to only the objects affected by the measure exposures = exposures.loc[ - (exposure_scenario.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] == aggr_name) + ( + exposure_scenario.loc[ + :, f"{FiatColumns.aggr_label_default}:{aggr_label}" + ] + == aggr_name + ) & (exposure_scenario.loc[:, FiatColumns.primary_object_type] == build_type), :, ] @@ -145,14 +157,22 @@ def test_all_measures(self, run_scenario_all_measures): # Check if elevation took place correctly at each object for i, row in exposures.iterrows(): # If the initial elevation is smaller than the required one it should have been elevated to than one - if row[FiatColumns.ground_elevation] + row[f"{FiatColumns.ground_floor_height}_1"] < row["bfe"] + elevate_val: + if ( + row[FiatColumns.ground_elevation] + + row[f"{FiatColumns.ground_floor_height}_1"] + < row["bfe"] + elevate_val + ): assert ( - row[FiatColumns.ground_elevation] + row[f"{FiatColumns.ground_floor_height}_2"] + row[FiatColumns.ground_elevation] + + row[f"{FiatColumns.ground_floor_height}_2"] == row["bfe"] + elevate_val ) # if not it should have stated the same else: - assert row[f"{FiatColumns.ground_floor_height}_2"] == row[f"{FiatColumns.ground_floor_height}_1"] + assert ( + row[f"{FiatColumns.ground_floor_height}_2"] + == row[f"{FiatColumns.ground_floor_height}_1"] + ) # check if buildings are bought-out aggr_label = test_scenario.impacts.impact_strategy.measures[ @@ -165,7 +185,8 @@ def test_all_measures(self, run_scenario_all_measures): 1 ].attrs.property_type inds = ( - exposure_scenario.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] == aggr_name + exposure_scenario.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] + == aggr_name ) & (exposure_scenario.loc[:, FiatColumns.primary_object_type] == build_type) assert all(exposure_scenario.loc[inds, FiatColumns.damage_structure] == 0) @@ -181,10 +202,12 @@ def test_all_measures(self, run_scenario_all_measures): 2 ].attrs.property_type inds1 = ( - exposure_template.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] == aggr_name + exposure_template.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] + == aggr_name ) & (exposure_template.loc[:, FiatColumns.primary_object_type] == build_type) inds2 = ( - exposure_scenario.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] == aggr_name + exposure_scenario.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] + == aggr_name ) & (exposure_scenario.loc[:, FiatColumns.primary_object_type] == build_type) assert all( @@ -214,10 +237,12 @@ def test_raise_datum(self, run_scenario_raise_datum): 0 ].attrs.property_type inds1 = ( - exposure_template.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] == aggr_name + exposure_template.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] + == aggr_name ) & (exposure_template.loc[:, FiatColumns.primary_object_type] == build_type) inds2 = ( - exposure_scenario.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] == aggr_name + exposure_scenario.loc[:, f"{FiatColumns.aggr_label_default}:{aggr_label}"] + == aggr_name ) & (exposure_scenario.loc[:, FiatColumns.primary_object_type] == build_type) assert all( From 1268de8dce4bf0085aec1671e81565f79599d2c1 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 6 Feb 2025 13:45:01 +0100 Subject: [PATCH 21/27] replace potential with pot --- flood_adapt/adapter/fiat_adapter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index 99dc0778..3c709c63 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -769,7 +769,7 @@ def apply_economic_growth( damage_cols = [ c for c in self._model.exposure.exposure_db.columns - if FiatColumns.max_potential_damage in c + if FiatColumns.max_pot_damage in c ] # Get objects that are buildings (using site info) @@ -816,7 +816,7 @@ def apply_population_growth_existing( damage_cols = [ c for c in self._model.exposure.exposure_db.columns - if FiatColumns.max_potential_damage in c + if FiatColumns.max_pot_damage in c ] # Get objects that are buildings (using site info) @@ -1009,7 +1009,7 @@ def buyout_properties(self, buyout: Buyout) -> None: damage_cols = [ c for c in self._model.exposure.exposure_db.columns - if FiatColumns.max_potential_damage in c + if FiatColumns.max_pot_damage in c ] # Get objects that are buildings (using site info) From b4cc6e9d55566c6949b9c5a240bfb13047a894a4 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 6 Feb 2025 13:52:50 +0100 Subject: [PATCH 22/27] fix max_pot_damage to max_pot_damage_default --- flood_adapt/adapter/fiat_adapter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index 3c709c63..0c99d6a8 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -769,7 +769,7 @@ def apply_economic_growth( damage_cols = [ c for c in self._model.exposure.exposure_db.columns - if FiatColumns.max_pot_damage in c + if FiatColumns.max_pot_damage_default in c ] # Get objects that are buildings (using site info) @@ -816,7 +816,7 @@ def apply_population_growth_existing( damage_cols = [ c for c in self._model.exposure.exposure_db.columns - if FiatColumns.max_pot_damage in c + if FiatColumns.max_pot_damage_default in c ] # Get objects that are buildings (using site info) @@ -1009,7 +1009,7 @@ def buyout_properties(self, buyout: Buyout) -> None: damage_cols = [ c for c in self._model.exposure.exposure_db.columns - if FiatColumns.max_pot_damage in c + if FiatColumns._default in c ] # Get objects that are buildings (using site info) From 1ce9e784180f40e74a8a680f5d7693a17a230876 Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 6 Feb 2025 14:09:54 +0100 Subject: [PATCH 23/27] update aggregation label to aggregation prefix --- flood_adapt/adapter/fiat_adapter.py | 12 ++++++------ flood_adapt/database_builder/create_database.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index 0c99d6a8..615e0939 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -65,7 +65,7 @@ class FiatColumns: max_pot_damage_structure = "max_damage_structure" max_pot_damage_content = "max_damage_content" max_pot_damage_other = "max_damage_other" - aggr_label_default = "aggregation_label" + aggregation_prefix = "aggregation_label" class FiatAdapter(IImpactAdapter): @@ -126,7 +126,7 @@ def __init__( FiatColumns.max_pot_damage_structure: "Max Potential Damage: Structure", FiatColumns.max_pot_damage_content: "Max Potential Damage: Content", FiatColumns.max_pot_damage_other: "Max Potential Damage: Other", - FiatColumns.aggr_label_default: "Aggregation level:", + FiatColumns.aggregation_prefix: "Aggregation level", } self._model = FiatModel(root=str(model_root.resolve()), mode="r") self._model.read() @@ -445,7 +445,7 @@ def postprocess(self, scenario): # Map exposure columns for aggregation in self.config.aggregation: name = aggregation.name - self.fiat_output_mapping[f"{FiatColumns.aggr_label_default}:{name}"] = ( + self.fiat_output_mapping[f"{FiatColumns.aggregation_prefix}{name}"] = ( f"Aggregation Label: {name}" ) mapped_exposure_df = self.outputs["table"].rename( @@ -896,7 +896,7 @@ def apply_population_growth_new( ] attribute_names = [aggr.field_name for aggr in self.config.aggregation] label_names = [ - f"{FiatColumns.aggregation_label}{aggr.name}" + f"{FiatColumns.aggregation_prefix}{aggr.name}" for aggr in self.config.aggregation ] # Use hydromt function @@ -1464,8 +1464,8 @@ def save_building_footprints(self, output_path: os.PathLike): ) # Add aggregation to fiat_columns for aggregation in self.config.aggregation: - self.fiat_columns_dict[f"aggr_label_default:{aggregation.name}"] = ( - f"{self.fiat_columns_dict['aggr_label_default']}:{aggregation.name}" + self.fiat_columns_dict[f"{FiatColumns.aggregation_prefix}{aggregation.name}"] = ( + f"{self.fiat_columns_dict[{FiatColumns.aggregation_prefix}]}{aggregation.name}" ) # Get damages footprints.aggregate(fiat_results_df, self.fiat_columns_dict) diff --git a/flood_adapt/database_builder/create_database.py b/flood_adapt/database_builder/create_database.py index 5c532cbf..e235b927 100644 --- a/flood_adapt/database_builder/create_database.py +++ b/flood_adapt/database_builder/create_database.py @@ -786,7 +786,7 @@ def read_fiat(self): self.buildings, region, "id", - rename=f"{FiatColumns.aggregation_label}region", + rename=f"{FiatColumns.aggregation_prefix}:region", ) exposure_csv = exposure_csv.merge( buildings_joined, on=FiatColumns.object_id, how="left" From e189e1423c3bfcf5c1fab7320659d57d4ed9d04d Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 6 Feb 2025 14:38:47 +0100 Subject: [PATCH 24/27] add : to aggregation prefix --- flood_adapt/adapter/fiat_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index 615e0939..73cc25cb 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -65,7 +65,7 @@ class FiatColumns: max_pot_damage_structure = "max_damage_structure" max_pot_damage_content = "max_damage_content" max_pot_damage_other = "max_damage_other" - aggregation_prefix = "aggregation_label" + aggregation_prefix = "aggregation_label:" class FiatAdapter(IImpactAdapter): From 57e2c887e5449cfec6170c18b62cb95126b61d3c Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 6 Feb 2025 15:21:38 +0100 Subject: [PATCH 25/27] bigfixes with aggregation naming --- flood_adapt/adapter/fiat_adapter.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index 73cc25cb..91f1719b 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -126,7 +126,7 @@ def __init__( FiatColumns.max_pot_damage_structure: "Max Potential Damage: Structure", FiatColumns.max_pot_damage_content: "Max Potential Damage: Content", FiatColumns.max_pot_damage_other: "Max Potential Damage: Other", - FiatColumns.aggregation_prefix: "Aggregation level", + FiatColumns.aggregation_prefix: "Aggregation Label: ", } self._model = FiatModel(root=str(model_root.resolve()), mode="r") self._model.read() @@ -444,9 +444,8 @@ def postprocess(self, scenario): # Map exposure columns for aggregation in self.config.aggregation: - name = aggregation.name - self.fiat_output_mapping[f"{FiatColumns.aggregation_prefix}{name}"] = ( - f"Aggregation Label: {name}" + self.fiat_output_mapping[f"{FiatColumns.aggregation_prefix}{aggregation.name}"] = ( + f"{self.fiat_output_mapping[FiatColumns.aggregation_prefix]}{aggregation.name}" ) mapped_exposure_df = self.outputs["table"].rename( columns=self.fiat_output_mapping @@ -1464,9 +1463,7 @@ def save_building_footprints(self, output_path: os.PathLike): ) # Add aggregation to fiat_columns for aggregation in self.config.aggregation: - self.fiat_columns_dict[f"{FiatColumns.aggregation_prefix}{aggregation.name}"] = ( - f"{self.fiat_columns_dict[{FiatColumns.aggregation_prefix}]}{aggregation.name}" - ) + self.fiat_columns_dict[f"{FiatColumns.aggregation_prefix}{aggregation.name}"] = f"{FiatColumns.aggregation_prefix}{aggregation.name}" # Get damages footprints.aggregate(fiat_results_df, self.fiat_columns_dict) footprints.calc_normalized_damages(self.fiat_columns_dict) From 3e6c27dbd494f1fb31cfbc7b6611715daba9e24d Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 6 Feb 2025 15:40:52 +0100 Subject: [PATCH 26/27] fix roads --- flood_adapt/adapter/fiat_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index 91f1719b..4ac32f2f 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -1490,7 +1490,7 @@ def save_roads(self, output_path: os.PathLike): name for name in self.outputs["table"].columns if any( - f"aggregation_label:{aggregation.name}" in name + f"{FiatColumns.aggregation_prefix}{aggregation.name}" in name for aggregation in self.config.aggregation ) ] From 91c51368f49fc06db98ef801e773789e7058b64a Mon Sep 17 00:00:00 2001 From: Santonia27 Date: Thu, 6 Feb 2025 16:17:44 +0100 Subject: [PATCH 27/27] bigfix max pote damage --- flood_adapt/adapter/fiat_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flood_adapt/adapter/fiat_adapter.py b/flood_adapt/adapter/fiat_adapter.py index 4ac32f2f..1a34e0f7 100644 --- a/flood_adapt/adapter/fiat_adapter.py +++ b/flood_adapt/adapter/fiat_adapter.py @@ -1008,7 +1008,7 @@ def buyout_properties(self, buyout: Buyout) -> None: damage_cols = [ c for c in self._model.exposure.exposure_db.columns - if FiatColumns._default in c + if FiatColumns.max_pot_damage_default in c ] # Get objects that are buildings (using site info)