Skip to content

Commit

Permalink
Public Code Commit
Browse files Browse the repository at this point in the history
  • Loading branch information
@mtrazzak
mtrazzak committed Jun 13, 2022
1 parent 5eb2010 commit 4331dbd
Show file tree
Hide file tree
Showing 60 changed files with 20,613 additions and 1 deletion.
74 changes: 73 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
@@ -1 +1,73 @@
# RobustMFSRforEO
# RobustMFSRforEO
Developing Robust MFSR for Earth Observation

## Environments

#### Pip

```
virtualenv --python=python3.8 ./venv
source ./venv/bin/activate
pip install -r requirements.txt
```


#### Conda
Install [miniconda](https://pytorch.org/get-started/locally/#anaconda), like so:

```
curl -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
sh Miniconda3-latest-Linux-x86_64.sh
```
Create the environment:
```
conda env create -f environments/solarisenv.yml
pip install solaris
```

## Containers

#### Singularity
The image is in `/containers/mfsr.simg` .

For an interactive shell:
```
singularity shell -B /data:/data /containers/mfsr.simg
```

and to run a script:
```
singularity exec /containers/mfsr.simg python3 main.py --epochs 8
```

#### Docker
(tbc)



## Tests

```
export PYTHONPATH=.
pytest --cov-report term-missing --cov=src
```

## Dataset folder structure

```
spacenet
├── csvs
└── train
└── L15-XXXE-XXXXN_XXXX_XXXX_XX
├── UDM_masks # mask of clouds of planet imagery
├── images # Raw Planet Images
├── images_masked # Planet images with cloud masked out
├── labels # GeoJSON labels of buildings and clouds (don’t worry about labels match)
├── masks # Labels in mask as geotiff form
├── sentinel # Sentinel imagery for that AOI
├── S2L2A # L2 processed Sentinel imagery (using this one)
├── sentinel_cloud # GeoTiff of Sentinel Cloud Masks
└── sentinel_processed # resampled sentinel imagery to 10m
└── sentinel_cloud_processed # resampled sentinel cloud imagery to 10m
```
167 changes: 167 additions & 0 deletions dataset_tools/(HOWTO) PrepareTiffLabels.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "signal-college",
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"sys.path.append('..')"
]
},
{
"cell_type": "markdown",
"id": "sunset-mandate",
"metadata": {},
"source": [
"# Burn Tiff Labels for SpaceNet Planet Imagery"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "complex-programmer",
"metadata": {},
"outputs": [],
"source": [
"#https://gist.github.com/avanetten/b295e89f6fa9654c9e9e480bdb2e4d60\n",
"from osgeo import gdal, ogr\n",
"\n",
"def create_building_mask(rasterSrc, vectorSrc, npDistFileName='', noDataValue=0, burn_values=1):\n",
" '''\n",
" Create building mask for rasterSrc,\n",
" Similar to labeltools/createNPPixArray() in spacenet utilities\n",
" Args:\n",
" - rasterSrc : str, base GeoTiff location\n",
" - vectorSrc : str, geoJSON to burn\n",
" - npDistFileName: str, output GeoTiff filename\n",
" '''\n",
" \n",
" ## open source vector file that truth data\n",
" source_ds = ogr.Open(vectorSrc)\n",
" source_layer = source_ds.GetLayer()\n",
"\n",
" ## extract data from src Raster File to be emulated\n",
" ## open raster file that is to be emulated\n",
" srcRas_ds = gdal.Open(rasterSrc)\n",
" cols = srcRas_ds.RasterXSize\n",
" rows = srcRas_ds.RasterYSize\n",
"\n",
" ## create First raster memory layer, units are pixels\n",
" # Change output to geotiff instead of memory \n",
" memdrv = gdal.GetDriverByName('GTiff') \n",
" dst_ds = memdrv.Create(npDistFileName, cols, rows, 1, gdal.GDT_Byte, options=['COMPRESS=LZW'])\n",
" dst_ds.SetGeoTransform(srcRas_ds.GetGeoTransform())\n",
" dst_ds.SetProjection(srcRas_ds.GetProjection())\n",
" band = dst_ds.GetRasterBand(1)\n",
" band.SetNoDataValue(noDataValue) \n",
" gdal.RasterizeLayer(dst_ds, [1], source_layer, burn_values=[burn_values])\n",
" dst_ds = 0"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "competitive-stylus",
"metadata": {},
"outputs": [],
"source": [
"from src.datasources import df_spacenet7\n",
"import os, tqdm\n",
"import pandas as pd\n",
"df_planet = df_spacenet7.loc['planet']"
]
},
{
"cell_type": "markdown",
"id": "appropriate-timeline",
"metadata": {},
"source": [
"## Burn Cloud Masks"
]
},
{
"cell_type": "code",
"execution_count": 48,
"id": "stretch-cradle",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 1423/1423 [00:37<00:00, 37.92it/s]\n"
]
}
],
"source": [
"df_planet_cloud = pd.DataFrame(list(df_planet['cloud_mask_path']))\n",
"for output_filename in tqdm.tqdm(list(df_planet_cloud[0])):\n",
" geojson_path = output_filename.replace(\"UDM_masks\",\"labels\").replace(\".tif\",\"_UDM.geojson\")\n",
" image_path = output_filename.replace(\"UDM_masks\",\"images\")\n",
" create_building_mask(rasterSrc=image_path, vectorSrc=geojson_path, npDistFileName=output_filename)"
]
},
{
"cell_type": "markdown",
"id": "durable-contrary",
"metadata": {},
"source": [
"## Burn Building Masks"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "super-documentary",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 1423/1423 [03:24<00:00, 6.97it/s]\n"
]
}
],
"source": [
"df_planet_building = pd.DataFrame(list(df_planet['label_mask_path']))\n",
"for output_filename in tqdm.tqdm(list(df_planet_building[0])):\n",
" geojson_path = output_filename.replace(\"masks\",\"labels\").replace(\".tif\",\".geojson\")\n",
" image_path = output_filename.replace(\"masks\",\"images\").replace(\"_Buildings\",\"\")\n",
" create_building_mask(rasterSrc=image_path, vectorSrc=geojson_path, npDistFileName=output_filename)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "distant-blackjack",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
136 changes: 136 additions & 0 deletions dataset_tools/(HOWTO) Resampling.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "heavy-scanner",
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"sys.path.append('..')"
]
},
{
"cell_type": "markdown",
"id": "durable-knight",
"metadata": {},
"source": [
"## Resample Sentinel Imagery"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "assisted-mambo",
"metadata": {},
"outputs": [],
"source": [
"from dataset_tools.resampleImagery import resample_dataset"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "impossible-moscow",
"metadata": {},
"outputs": [],
"source": [
"# Resample Imagery\n",
"resample_dataset(root_dir='/data/spacenet/train/',satellite='sentinel',GSD=10,False)"
]
},
{
"cell_type": "markdown",
"id": "national-microphone",
"metadata": {},
"source": [
"## Resample Sentinel Clouds"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "biblical-particle",
"metadata": {},
"outputs": [],
"source": [
"from src.datasources import df_spacenet7\n",
"from dataset_tools.resampleImagery import resample"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "opposed-inspection",
"metadata": {},
"outputs": [],
"source": [
"def resample_sentinel_cloud_dataset(root_dir,GSD,makeCOG):\n",
" \"\"\"\n",
" inputs - file: root directory [\"/data/spacenet/train/\"]\n",
" - output_file: satellite [\"planet\",\"sentinel\"]\n",
" - GSD: required ground sampling distance in m\n",
" \"\"\"\n",
" from src.datasources import spacenet7_index\n",
" satellite = 'sentinel'\n",
" df = spacenet7_index()\n",
" df_satellite = df.loc[satellite]\n",
" scenes = (df_satellite\n",
" .index\n",
" .get_level_values('scene') ## \"scene\" level of the multi-index\n",
" .unique()) ## Unique names\n",
" scenes = list(scenes)\n",
" for scene in tqdm.tqdm(scenes):\n",
" scene_df = df_satellite.query(f\"scene=='{scene}'\")\n",
" output_dir = os.path.join(root_dir,scene,satellite+'_cloud_processed')\n",
" if not os.path.exists(output_dir):\n",
" os.makedirs(output_dir)\n",
" for i in range(len(scene_df)):\n",
" file = scene_df.iloc[i]['cloud_mask_path']\n",
" filename = scene_df.iloc[i].basename.replace(\"TCI\",\"SCL\")\n",
" output_file = os.path.join(output_dir,filename)\n",
" resample(file, output_file, GSD, makeCOG=makeCOG)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "solar-ivory",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 46/46 [00:06<00:00, 7.07it/s]\n"
]
}
],
"source": [
"resample_sentinel_cloud_dataset('/data/spacenet/train/', 10,False)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading

0 comments on commit 4331dbd

Please sign in to comment.