diff --git a/hot_fair_utilities/preprocessing/multimasks_from_polygons.py b/hot_fair_utilities/preprocessing/multimasks_from_polygons.py new file mode 100644 index 00000000..2ddd76e8 --- /dev/null +++ b/hot_fair_utilities/preprocessing/multimasks_from_polygons.py @@ -0,0 +1,141 @@ +# Patched from ramp-code.scripts.multi_masks_from_polygons created for ramp project by carolyn.johnston@dev.global + +# Standard library imports +from pathlib import Path + +# Third party imports +import geopandas as gpd +import rasterio as rio +from ramp.data_mgmt.chip_label_pairs import ( + construct_mask_filepath, + get_tq_chip_label_pairs, +) +from ramp.utils.img_utils import to_channels_first +from ramp.utils.multimask_utils import df_to_px_mask, multimask_to_sparse_multimask +from solaris.utils.core import _check_rasterio_im_load +from solaris.utils.geo import get_crs +from solaris.vector.mask import crs_is_metric +from tqdm import tqdm + + +def get_rasterio_shape_and_transform(image_path): + # get the image shape and the affine transform to pass into df_to_px_mask. + with rio.open(image_path) as rio_dset: + shape = rio_dset.shape + transform = rio_dset.transform + return shape, transform + + +def multimasks_from_polygons( + in_poly_dir, + in_chip_dir, + out_mask_dir, + input_contact_spacing=8, + input_boundary_width=3, +): + """ + Create multichannel building footprint masks from a folder of geojson files. + This also requires the path to the matching image chips directory.Unit of input_contact_spacing and input_boundary_width is in pixel which is : + + ## Can not use meters for contact spacing and width because it won't maintain consistency in different zoom levels + + Real-world width (in meters)= Pixel width×Resolution (meters per pixel) + + Args: + in_poly_dir (str): Path to directory containing geojson files. + in_chip_dir (str): Path to directory containing image chip files with names matching geojson files. + out_mask_dir (str): Path to directory containing output SDT masks. + input_contact_spacing (int, optional): Pixels that are closer to two different polygons than contact_spacing will be labeled with the contact mask. + input_boundary_width (int, optional): Width in pixel of boundary inner buffer around building footprints + + Example: + multimasks_from_polygons( + "data/preprocessed/labels", + "data/preprocessed/chips", + "data/preprocessed/multimasks" + ) + """ + + # If output mask directory doesn't exist, try to create it. + Path(out_mask_dir).mkdir(parents=True, exist_ok=True) + + chip_label_pairs = get_tq_chip_label_pairs(in_chip_dir, in_poly_dir) + + chip_paths, label_paths = list(zip(*chip_label_pairs)) + + # construct the output mask file names from the chip file names. + # these will have the same base filenames as the chip files, + # with a mask.tif extension in place of the .tif extension. + mask_paths = [ + construct_mask_filepath(out_mask_dir, chip_path) for chip_path in chip_paths + ] + + # construct a list of full paths to the mask files + json_chip_mask_zips = zip(label_paths, chip_paths, mask_paths) + for json_path, chip_path, mask_path in tqdm( + json_chip_mask_zips, desc="Multimasks for input" + ): + + # We will run this on very large directories, and some label files might fail to process. + # We want to be able to resume mask creation from where we left off. + if Path(mask_path).is_file(): + continue + + # workaround for bug in solaris + mask_shape, mask_transform = get_rasterio_shape_and_transform(chip_path) + + gdf = gpd.read_file(json_path) + + # remove empty and null geometries + gdf = gdf[~gdf["geometry"].isna()] + gdf = gdf[~gdf.is_empty] + + reference_im = _check_rasterio_im_load(chip_path) + + if get_crs(gdf) != get_crs(reference_im): + # BUGFIX: if crs's don't match, reproject the geodataframe + gdf = gdf.to_crs(get_crs(reference_im)) + + if crs_is_metric(gdf): + meters = True + boundary_width = min(reference_im.res) * input_boundary_width + contact_spacing = min(reference_im.res) * input_contact_spacing + + else: + meters = False + boundary_width = input_boundary_width + contact_spacing = input_contact_spacing + + # NOTE: solaris does not support multipolygon geodataframes + # So first we call explode() to turn multipolygons into polygon dataframes + # ignore_index=True prevents polygons from the same multipolygon from being grouped into a series. -+ + gdf_poly = gdf.explode(ignore_index=True) + + # multi_mask is a one-hot, channels-last encoded mask + onehot_multi_mask = df_to_px_mask( + df=gdf_poly, + out_file=mask_path, + shape=mask_shape, + do_transform=True, + affine_obj=None, + channels=["footprint", "boundary", "contact"], + reference_im=reference_im, + boundary_width=boundary_width, + contact_spacing=contact_spacing, + out_type="uint8", + meters=meters, + ) + + # convert onehot_multi_mask to a sparse encoded mask + # of shape (1,H,W) for compatibility with rasterio writer + sparse_multi_mask = multimask_to_sparse_multimask(onehot_multi_mask) + sparse_multi_mask = to_channels_first(sparse_multi_mask) + + # write out sparse mask file with rasterio. + with rio.open(chip_path, "r") as src: + meta = src.meta.copy() + meta.update(count=sparse_multi_mask.shape[0]) + meta.update(dtype="uint8") + meta.update(nodata=None) + with rio.open(mask_path, "w", **meta) as dst: + dst.write(sparse_multi_mask) diff --git a/hot_fair_utilities/preprocessing/preprocess.py b/hot_fair_utilities/preprocessing/preprocess.py index 4536170f..6e7be250 100644 --- a/hot_fair_utilities/preprocessing/preprocess.py +++ b/hot_fair_utilities/preprocessing/preprocess.py @@ -4,6 +4,7 @@ from ..georeferencing import georeference from .clip_labels import clip_labels from .fix_labels import fix_labels +from .multimasks_from_polygons import multimasks_from_polygons from .reproject_labels import reproject_labels_to_epsg3857 @@ -13,6 +14,9 @@ def preprocess( rasterize=False, rasterize_options=None, georeference_images=False, + multimasks=False, + input_contact_spacing=8, # only required if multimasks is set to true + input_boundary_width=3, # only required if mulltimasks is set to true ) -> None: """Fully preprocess the input data. @@ -29,6 +33,7 @@ def preprocess( (if georeference_images=True), and the directories "binarymasks" and "grayscale_labels" if the corresponding rasterizing options are chosen. + "multimasks" - for the multimasks labels (if multimasks=True) rasterize: Whether to create the raster labels. rasterize_options: A list with options how to rasterize the label, if rasterize=True. Possible options: "grayscale" @@ -37,6 +42,13 @@ def preprocess( for the ramp model). If rasterize=False, rasterize_options will be ignored. georeference_images: Whether to georeference the OAM images. + multimasks: Whether to additionally output multimask labels. + input_contact_spacing (int, optional): Pixels that are closer to two different polygons than contact_spacing will be labeled with the contact mask. + input_boundary_width (int, optional): Width in pixel of boundary inner buffer around building footprints + + Unit of input_contact_spacing and input_boundary_width is in pixel, we couldn't use meters to maintain consistency based on different zoom level as pixel resolution will be different which is : + + Real-world width (in meters)= Pixel width×Resolution (meters per pixel) Example:: @@ -82,3 +94,15 @@ def preprocess( os.remove(f"{output_path}/corrected_labels.geojson") os.remove(f"{output_path}/labels_epsg3857.geojson") + + if multimasks: + assert os.path.isdir( + f"{output_path}/chips" + ), "Chips do not exist. Set georeference_images=True." + multimasks_from_polygons( + f"{output_path}/labels", + f"{output_path}/chips", + f"{output_path}/multimasks", + input_contact_spacing=input_contact_spacing, + input_boundary_width=input_boundary_width, + ) diff --git a/hot_fair_utilities/training/prepare_data.py b/hot_fair_utilities/training/prepare_data.py index c9f37b67..035c4b0f 100644 --- a/hot_fair_utilities/training/prepare_data.py +++ b/hot_fair_utilities/training/prepare_data.py @@ -14,7 +14,7 @@ def __init__(self, message): self.message = message -def split_training_2_validation(input_path, output_path): +def split_training_2_validation(input_path, output_path, multimasks=False): """Converts training 2 validation Currently supported for ramp , It converts training dataset provided by preprocessing script to validation datastes reuqired by ramp @@ -101,14 +101,21 @@ def split_training_2_validation(input_path, output_path): raise ex try: + if multimasks: + sd = f"{dst_path}/multimasks" + td = f"{dst_path}/val-multimasks" + else: + sd = f"{dst_path}/binarymasks" + td = f"{dst_path}/val-binarymasks" + subprocess.check_output( [ python_exec, f"{RAMP_HOME}/ramp-code/scripts/move_chips_from_csv.py", "-sd", - f"{dst_path}/binarymasks", + sd, "-td", - f"{dst_path}/val-binarymasks", + td, "-csv", f"{dst_path}/fair_split_val.csv", "-mv", diff --git a/hot_fair_utilities/training/ramp_config_base.json b/hot_fair_utilities/training/ramp_config_base.json index 5491a0c2..3a615077 100644 --- a/hot_fair_utilities/training/ramp_config_base.json +++ b/hot_fair_utilities/training/ramp_config_base.json @@ -1,13 +1,13 @@ { - "experiment_name":"HOT-OSM Efficient-Unet Finetune model_set1_batch20_epoch20_imgAug", - "discard_experiment":false, - "logging":{ - "log_experiment":true, - "experiment_log_path":"ramp-data/TRAIN/fAIr-experiments.csv", - "experiment_notes":"Binary Mask model, batchsize 20, 20 epochs on HOT-OSM dataset 1 Multizoom, finetuning from RAMP saved model", - "fields_to_log":[ - "experiment_name", - "experiment_notes", + "experiment_name": "HOT-OSM Efficient-Unet Finetune Model", + "discard_experiment": false, + "logging": { + "log_experiment": true, + "experiment_log_path": "ramp-data/TRAIN/fAIr-experiments.csv", + "experiment_notes": "Multi Mask model, batchsize 20, 8 epochs on HOT-OSM datasets Multizoom, finetuning from RAMP saved model", + "fields_to_log": [ + "experiment_name", + "experiment_notes", "timestamp", "num_epochs", "batch_size", @@ -30,108 +30,129 @@ "val_mask_dir" ] }, - "datasets":{ - "train_img_dir":"ramp-data/TRAIN/HOTOSM/1/chips", - "train_mask_dir":"ramp-data/TRAIN/HOTOSM/1/binarymasks", - "val_img_dir":"ramp-data/TRAIN/HOTOSM/1/val-chips", - "val_mask_dir":"ramp-data/TRAIN/HOTOSM/1/val-binarymasks" + "datasets": { + "train_img_dir": "ramp-data/TRAIN/HOTOSM/1/chips", + "train_mask_dir": "ramp-data/TRAIN/HOTOSM/1/multimasks", + "val_img_dir": "ramp-data/TRAIN/HOTOSM/1/val-chips", + "val_mask_dir": "ramp-data/TRAIN/HOTOSM/1/val-multimasks" }, - "num_classes":2, - "num_epochs":20, - "batch_size":20, - "input_img_shape":[256,256], - "output_img_shape":[256,256], - "loss":{ - "get_loss_fn_name":"get_sparse_categorical_crossentropy_fn", - "loss_fn_parms":{} + "num_classes": 4, + "num_epochs": 20, + "batch_size": 8, + "input_img_shape": [ + 256, + 256 + ], + "output_img_shape": [ + 256, + 256 + ], + "loss": { + "get_loss_fn_name": "get_sparse_categorical_crossentropy_fn", + "loss_fn_parms": {} }, - "metrics":{ - "use_metrics":true, - "get_metrics_fn_names":["get_sparse_categorical_accuracy_fn"], - "metrics_fn_parms":[{}] + "metrics": { + "use_metrics": true, + "get_metrics_fn_names": [ + "get_sparse_categorical_accuracy_fn" + ], + "metrics_fn_parms": [ + {} + ] }, - "optimizer":{ - "get_optimizer_fn_name":"get_adam_optimizer", - "optimizer_fn_parms":{ - "learning_rate":3E-04 + "optimizer": { + "get_optimizer_fn_name": "get_adam_optimizer", + "optimizer_fn_parms": { + "learning_rate": 3E-04 } }, - "model":{ - "get_model_fn_name":"get_effunet_model", - "model_fn_parms":{ - "backbone":"efficientnetb0", - "classes":["background","buildings"] + "model": { + "get_model_fn_name": "get_effunet_model", + "model_fn_parms": { + "backbone": "efficientnetb0", + "classes": [ + "background", + "buildings", + "boundary", + "close_contact" + ] } }, - "saved_model":{ - "use_saved_model":true, - "saved_model_path":"ramp-code/ramp/checkpoint.tf", - "save_optimizer_state":false + "saved_model": { + "use_saved_model": true, + "saved_model_path": "ramp-code/ramp/checkpoint.tf", + "save_optimizer_state": false }, - "augmentation":{ - "use_aug":true, - - "get_augmentation_fn_name":"get_augmentation_fn", - "aug_list":["Rotate","ColorJitter"], - "aug_parms":[ + "augmentation": { + "use_aug": true, + "get_augmentation_fn_name": "get_augmentation_fn", + "aug_list": [ + "Rotate", + "ColorJitter" + ], + "aug_parms": [ { - "border_mode":"BORDER_CONSTANT", - "interpolation":"INTER_NEAREST", - "value":[0.0,0.0,0.0], - "mask_value":0, - "p":0.7 + "border_mode": "BORDER_CONSTANT", + "interpolation": "INTER_NEAREST", + "value": [ + 0.0, + 0.0, + 0.0 + ], + "mask_value": 0, + "p": 0.7 }, { - "p":0.7 + "p": 0.7 } ] }, - "early_stopping":{ - "use_early_stopping":true, - "early_stopping_parms":{ - "monitor":"val_loss", - "min_delta":0.005, - "patience":50, - "verbose":0, - "mode":"auto", - "restore_best_weights":false + "early_stopping": { + "use_early_stopping": true, + "early_stopping_parms": { + "monitor": "val_loss", + "min_delta": 0.005, + "patience": 50, + "verbose": 0, + "mode": "auto", + "restore_best_weights": false } }, - "cyclic_learning_scheduler":{ - "use_clr":false, - "get_clr_callback_fn_name":"get_clr_callback_fn", - "clr_callback_parms":{ - "mode":"triangular2", - "stepsize":8, - "max_lr":1e-4, - "base_lr":3.25e-6 + "cyclic_learning_scheduler": { + "use_clr": false, + "get_clr_callback_fn_name": "get_clr_callback_fn", + "clr_callback_parms": { + "mode": "triangular2", + "stepsize": 8, + "max_lr": 1e-4, + "base_lr": 3.25e-6 }, - "clr_plot_dir":"ramp-data/TRAIN/HOTOSM/1/plots" + "clr_plot_dir": "ramp-data/TRAIN/HOTOSM/1/plots" }, - "tensorboard":{ - "use_tb":true, - "tb_logs_dir":"ramp-data/TRAIN/HOTOSM/1/logs", - "get_tb_callback_fn_name":"get_tb_callback_fn", - "tb_callback_parms":{ - "histogram_freq":1, - "update_freq":"batch" + "tensorboard": { + "use_tb": true, + "tb_logs_dir": "ramp-data/TRAIN/HOTOSM/1/logs", + "get_tb_callback_fn_name": "get_tb_callback_fn", + "tb_callback_parms": { + "histogram_freq": 1, + "update_freq": "batch" } - }, - "prediction_logging":{ - "use_prediction_logging":true, - "get_prediction_logging_fn_name":"get_pred_logging_callback_fn" }, - "model_checkpts":{ - "use_model_checkpts":true, + "prediction_logging": { + "use_prediction_logging": true, + "get_prediction_logging_fn_name": "get_pred_logging_callback_fn" + }, + "model_checkpts": { + "use_model_checkpts": true, "model_checkpts_dir": "ramp-data/TRAIN/HOTOSM/1/model-checkpts", - "get_model_checkpt_callback_fn_name":"get_model_checkpt_callback_fn", - "model_checkpt_callback_parms":{ - "mode":"max", - "save_best_only":false + "get_model_checkpt_callback_fn_name": "get_model_checkpt_callback_fn", + "model_checkpt_callback_parms": { + "mode": "max", + "save_best_only": false } }, - "feedback":{ - "freeze_layers" : false + "feedback": { + "freeze_layers": false }, - "random_seed":20220523 -} + "random_seed": 20220523 +} \ No newline at end of file diff --git a/hot_fair_utilities/training/run_training.py b/hot_fair_utilities/training/run_training.py index 2ebc413b..e43fbb93 100644 --- a/hot_fair_utilities/training/run_training.py +++ b/hot_fair_utilities/training/run_training.py @@ -61,13 +61,20 @@ def __init__(self, message): def apply_feedback( - pretrained_model_path, output_path, num_epochs, batch_size, freeze_layers + pretrained_model_path, + output_path, + num_epochs, + batch_size, + freeze_layers, + multimasks=False, ): if not os.path.exists(output_path): os.makedirs(output_path) # Update the fine-tuning configuration - fine_tuning_cfg = manage_fine_tuning_config(output_path, num_epochs, batch_size,freeze_layers) + fine_tuning_cfg = manage_fine_tuning_config( + output_path, num_epochs, batch_size, freeze_layers, multimasks + ) # Set the path of the pre-trained model in the configuration fine_tuning_cfg["saved_model"]["saved_model_path"] = pretrained_model_path @@ -76,7 +83,9 @@ def apply_feedback( run_main_train_code(fine_tuning_cfg) -def manage_fine_tuning_config(output_path, num_epochs, batch_size, freeze_layers): +def manage_fine_tuning_config( + output_path, num_epochs, batch_size, freeze_layers, multimasks=False +): # Define the paths to the source and destination JSON files working_dir = os.path.realpath(os.path.dirname(__file__)) config_base_path = os.path.join(working_dir, "ramp_config_base.json") @@ -88,9 +97,16 @@ def manage_fine_tuning_config(output_path, num_epochs, batch_size, freeze_layers # Modify the content of the data dictionary datasets data["datasets"]["train_img_dir"] = f"{output_path}/chips" - data["datasets"]["train_mask_dir"] = f"{output_path}/binarymasks" + if multimasks: + data["datasets"]["train_mask_dir"] = f"{output_path}/multimasks" + else: + data["datasets"]["train_mask_dir"] = f"{output_path}/binarymasks" + data["datasets"]["val_img_dir"] = f"{output_path}/val-chips" - data["datasets"]["val_mask_dir"] = f"{output_path}/val-binarymasks" + if multimasks: + data["datasets"]["val_mask_dir"] = f"{output_path}/val-multimasks" + else: + data["datasets"]["val_mask_dir"] = f"{output_path}/val-binarymasks" # epoch batchconfig data["num_epochs"] = num_epochs @@ -158,14 +174,13 @@ def run_main_train_code(cfg): ), f"the saved model was not constructed: {model_path}" if cfg["freeze_layers"]: - num_layers_to_freeze = 4 # freeze lower layers + num_layers_to_freeze = 4 # freeze lower layers for index, layer in enumerate(the_model.layers): if index < num_layers_to_freeze: layer.trainable = False else: layer.trainable = True - if not cfg["saved_model"]["save_optimizer_state"]: # If you don't want to save the original state of training, recompile the model. the_model.compile(optimizer=optimizer, loss=loss_fn, metrics=[the_metrics]) diff --git a/hot_fair_utilities/training/train.py b/hot_fair_utilities/training/train.py index d0641a80..37ab1e34 100644 --- a/hot_fair_utilities/training/train.py +++ b/hot_fair_utilities/training/train.py @@ -14,6 +14,7 @@ def train( model: str, model_home: str, freeze_layers: bool = False, + multimasks: bool = False, ): """Trains the input image with base model @@ -25,6 +26,13 @@ def train( batch_size: Batch size to be used for training model : Choose Model, Options supported are , ramp model_home : Model Home directory which contains necessary file in order to run model + freeze_layers : Either to freeze previous training knowleadge layers or not + multimasks : Either to use multimask labels in training or not , default : binary 0/1 , Multimaks classes include : "classes": [ + "background", + "buildings", + "boundary", + "close_contact" + ] Example:: final_accuracy, final_model_path = train( @@ -51,9 +59,9 @@ def train( os.environ["RAMP_HOME"] = model_home # Print the environment variables to verify that the new variable was added print("Starting to prepare data for training") - split_training_2_validation(input_path, output_path) + split_training_2_validation(input_path, output_path, multimasks) cfg = manage_fine_tuning_config( - output_path, epoch_size, batch_size, freeze_layers + output_path, epoch_size, batch_size, freeze_layers, multimasks ) print("Data is ready for training") run_main_train_code(cfg) @@ -70,17 +78,23 @@ def run_feedback( epoch_size: int, batch_size: int, freeze_layers: bool = True, + multimasks: bool = False, ): assert os.path.exists(input_path), "Input Feedback Path Doesn't Exist" assert os.path.exists(feedback_base_model), "Feedback base Model Doesn't Exist" os.environ.update(os.environ) os.environ["RAMP_HOME"] = model_home print("Starting to prepare data for training") - split_training_2_validation(input_path, output_path) + split_training_2_validation(input_path, output_path, multimasks) print("Data is ready for training") apply_feedback( - feedback_base_model, output_path, epoch_size, batch_size, freeze_layers + feedback_base_model, + output_path, + epoch_size, + batch_size, + freeze_layers, + multimasks, ) final_accuracy, final_model_path = extract_highest_accuracy_model(output_path) return (final_accuracy, final_model_path) diff --git a/test_app.py b/test_app.py index 71c52ea7..0b2a3385 100644 --- a/test_app.py +++ b/test_app.py @@ -38,6 +38,7 @@ rasterize=True, rasterize_options=["binary"], georeference_images=True, + multimasks=True, ) # Reader imports