Merge pull request Oslandia#88 from Oslandia/tanz

WIP: Tanzania dataset integration

Author: delhomer

.gitignore

@@ -8,6 +8,7 @@ deeposlandia/webapp/static/aerial
 deeposlandia/webapp/static/shapes
 deeposlandia/webapp/static/mapillary
 deeposlandia/webapp/static/mapillary_agg
+deeposlandia/webapp/static/tanzania
 deeposlandia/webapp/static/predicted
 deeposlandia/webapp/static/bower/
 deeposlandia.egg-info/

README.md

@@ -88,6 +88,23 @@ this image from this dataset is depicted below.
 
 ![Example of image, with labels and predictions](./images/aerial_prediction_example.png)
 
+## Open AI Tanzania
+
+This dataset comes from
+the
+[Tanzania challenge](https://blog.werobotics.org/2018/08/06/welcome-to-the-open-ai-tanzania-challenge/),
+that took place at the autumn 2018. The dataset contains 13 labelled images (2
+of them were assigned to validation in this project), and 9 additional images
+for testing purpose. The image resolution is very high (6~8 cm per pixel), that
+allowing a fine data preprocessing step.
+
+In such a dataset, one tries to automatically detect building footprints by
+distinguishing complete buildings, incomplete buildings and foudations.
+
+![Example of image, with labels and predictions](./images/tanzania_prediction_example.png)
+
+**(:warning: model training in processing... :-) )**
+
 ## Shapes
 
 To complete the project, and make the test easier, a randomly-generated shape
@@ -97,6 +114,10 @@ rectangle, one circle and/or one triangle per image, or neither of them. Their
 location into each image is randomly generated (they just can't be too close to
 image borders). The shape and background colors are randomly generated as well.
 
+## How to add a new dataset?
+
+If you want to contribute to the repo by adding a new dataset, please consult the [following instructions](./deeposlandia/add_a_dataset.md).
+
 # Flask application
 
 A Flask Web application may be launched locally through

config.ini.sample

@@ -7,6 +7,7 @@ shapes = /path/to/shape/dataset/
 mapillary = /path/to/mapillary/dataset/
 mapillary_agg = /path/to/agregated/mapillary/dataset/
 aerial = /path/to/aerial/dataset/
+tanzania = /path/to/tanzania/dataset/
 
 [folder]
 project_folder = /path/to/static/files/

deeposlandia/__init__.py

@@ -1,4 +1,15 @@
 """Deeposlandia package
 """
 
+import logging
+
+import daiquiri
+
 __version__ = '0.4'
+
+daiquiri.setup(level=logging.INFO,outputs=(
+    daiquiri.output.Stream(formatter=daiquiri.formatter.ColorFormatter(
+        fmt=("%(asctime)s :: %(levelname)s :: %(module)s :: "
+             "%(funcName)s : %(color)s%(message)s%(color_stop)s"))),
+))
+logger = daiquiri.getLogger("root")

deeposlandia/datagen.py

@@ -17,10 +17,19 @@
 import os
 import sys
 
+import daiquiri
 import pandas as pd
 
 from deeposlandia import utils
-from deeposlandia.dataset import AerialDataset, MapillaryDataset, ShapeDataset
+from deeposlandia.datasets import AVAILABLE_DATASETS
+from deeposlandia.datasets.mapillary import MapillaryDataset
+from deeposlandia.datasets.aerial import AerialDataset
+from deeposlandia.datasets.shapes import ShapeDataset
+from deeposlandia.datasets.tanzania import TanzaniaDataset
+
+
+logger = daiquiri.getLogger(__name__)
+
 
 def add_instance_arguments(parser):
     """Add instance-specific arguments from the command line
@@ -38,8 +47,9 @@ def add_instance_arguments(parser):
     parser.add_argument('-a', '--aggregate-label', action='store_true',
                         help="Aggregate labels with respect to their categories")
     parser.add_argument('-D', '--dataset',
-                        required=True,
-                        help="Dataset type (either mapillary, shapes or aerial)")
+                        required=True, choices=AVAILABLE_DATASETS,
+                        help=("Dataset type (to be chosen amongst available"
+                              "datasets)"))
     parser.add_argument('-p', '--datapath',
                         default="data",
                         help="Relative path towards data directory")
@@ -95,18 +105,22 @@ def add_instance_arguments(parser):
         train_dataset = AerialDataset(args.image_size)
         validation_dataset = AerialDataset(args.image_size)
         test_dataset = AerialDataset(args.image_size)
+    elif args.dataset == "tanzania":
+        train_dataset = TanzaniaDataset(args.image_size)
+        validation_dataset = TanzaniaDataset(args.image_size)
+        test_dataset = TanzaniaDataset(args.image_size)
     else:
-        utils.logger.error("Unsupported dataset type. Please choose "
-                           "'mapillary', 'shapes' or 'aerial'")
+        logger.error("Unsupported dataset type. Please choose amongst %s"
+                     % AVAILABLE_DATASETS)
         sys.exit(1)
 
     # Dataset populating/loading (depends on the existence of a specification file)
     if os.path.isfile(prepro_folder["training_config"]):
         train_dataset.load(prepro_folder["training_config"],
                            args.nb_training_image)
     else:
-        utils.logger.info(("No existing configuration file for this dataset. Create {}"
-                           "").format(prepro_folder["training_config"]))
+        logger.info(("No existing configuration file for this dataset. "
+                     "Create %s." % prepro_folder['training_config']))
         input_image_dir = os.path.join(input_folder, "training")
         train_dataset.populate(prepro_folder["training"], input_image_dir,
                                nb_images=args.nb_training_image,
@@ -117,8 +131,8 @@ def add_instance_arguments(parser):
         validation_dataset.load(prepro_folder["validation_config"],
                                 args.nb_validation_image)
     else:
-        utils.logger.info(("No existing configuration file for this dataset. Create {}"
-                           "").format(prepro_folder["validation_config"]))
+        logger.info(("No existing configuration file for this dataset. "
+                     "Create %s." % prepro_folder['validation_config']))
         input_image_dir = os.path.join(input_folder, "validation")
         validation_dataset.populate(prepro_folder["validation"],
                                     input_image_dir,
@@ -129,8 +143,8 @@ def add_instance_arguments(parser):
     if os.path.isfile(prepro_folder["testing_config"]):
         test_dataset.load(prepro_folder["testing_config"], args.nb_testing_image)
     else:
-        utils.logger.info(("No existing configuration file for this dataset. Create {}"
-                           "").format(prepro_folder["testing_config"]))
+        logger.info(("No existing configuration file for this dataset. "
+                     "Create %s." % prepro_folder['testing_config']))
         input_image_dir = os.path.join(input_folder, "testing")
         test_dataset.populate(prepro_folder["testing"],
                               input_image_dir,
@@ -141,5 +155,5 @@ def add_instance_arguments(parser):
 
     glossary = pd.DataFrame(train_dataset.labels)
     glossary["popularity"] = train_dataset.get_label_popularity()
-    utils.logger.info("Data glossary:\n{}".format(glossary))
+    logger.info("Data glossary:\n%s" % glossary)
     sys.exit(0)