Skip to content

Commit

Permalink
Merge pull request #70 from tsdataclinic/fema_national
Browse files Browse the repository at this point in the history 
Fema national
  • Loading branch information
@CanyonFoot
CanyonFoot authored Feb 16, 2024
2 parents f3b5e4f + 22c4f23 commit 7b0168b
Show file tree
Hide file tree
Showing 2 changed files with 280 additions and 0 deletions.
263 changes: 263 additions & 0 deletions analysis/src/process/fema_testing.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,263 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"import sys \n",
"import subprocess\n",
"import os"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"resolution = \"30\""
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"folder_path = f\"/home/data/national_fema_floodmap/rasters/{resolution}m\"\n",
"if not os.path.exists(folder_path):\n",
" os.mkdir(folder_path)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"raster100y = f\"{folder_path}/national_fema_raster_100y_{resolution}m.tiff\"\n",
"raster500y = f\"{folder_path}/national_fema_raster_500y_{resolution}m.tiff\"\n",
"raster_combined = f\"{folder_path}/national_fema_raster_combined_{resolution}m.tiff\""
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"ERROR 1: Error occurred in /home/conda/feedstock_root/build_artifacts/gdal-split_1704802395302/work/ogr/ogrsf_frmts/openfilegdb/filegdbindex.cpp at line 800\n",
"ERROR 1: Error occurred in /home/conda/feedstock_root/build_artifacts/gdal-split_1704802395302/work/ogr/ogrsf_frmts/openfilegdb/filegdbindex.cpp at line 800\n",
"Warning 1: organizePolygons() received a polygon with more than 100 parts. The processing may be really slow. You can skip the processing by setting METHOD=SKIP, or only make it analyze counter-clock wise parts by setting METHOD=ONLY_CCW if you can assume that the outline of holes is counter-clock wise defined\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"0...10...20...30...40...50...60...70...80...90...100 - done.\n"
]
},
{
"data": {
"text/plain": [
"CompletedProcess(args=['gdal_rasterize', '-tr', '30', '30', '-sql', \"SELECT * FROM USA_Flood_Hazard_Reduced_Set WHERE esri_symbology = '1% Annual Chance Flood Hazard'\", '-burn', '1', '-ot', 'Byte', '-co', 'COMPRESS=LZW', '/home/data/national_fema_floodmap/efa6409b-9dab-4b36-b0aa-8641f1b26b6e.gdb', '/home/data/national_fema_floodmap/rasters/30m/national_fema_raster_100y_30m.tiff'], returncode=0)"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Define the command as a list of arguments\n",
"command = [\n",
" \"gdal_rasterize\",\n",
" \"-tr\", resolution, resolution,\n",
" \"-sql\", \"SELECT * FROM USA_Flood_Hazard_Reduced_Set WHERE esri_symbology = '1% Annual Chance Flood Hazard'\",\n",
" \"-burn\", \"1\",\n",
" \"-ot\", \"Byte\",\n",
" \"-co\", \"COMPRESS=LZW\",\n",
" \"/home/data/national_fema_floodmap/efa6409b-9dab-4b36-b0aa-8641f1b26b6e.gdb\",\n",
" raster100y\n",
"]\n",
"\n",
"\n",
"# Execute the command\n",
"subprocess.run(command, check=True)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"ERROR 1: Error occurred in /home/conda/feedstock_root/build_artifacts/gdal-split_1704802395302/work/ogr/ogrsf_frmts/openfilegdb/filegdbindex.cpp at line 800\n",
"ERROR 1: Error occurred in /home/conda/feedstock_root/build_artifacts/gdal-split_1704802395302/work/ogr/ogrsf_frmts/openfilegdb/filegdbindex.cpp at line 800\n",
"Warning 1: organizePolygons() received a polygon with more than 100 parts. The processing may be really slow. You can skip the processing by setting METHOD=SKIP, or only make it analyze counter-clock wise parts by setting METHOD=ONLY_CCW if you can assume that the outline of holes is counter-clock wise defined\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"0...10...20...30...40...50...60...70...80...90...100 - done.\n"
]
},
{
"data": {
"text/plain": [
"CompletedProcess(args=['gdal_rasterize', '-tr', '30', '30', '-sql', \"SELECT * FROM USA_Flood_Hazard_Reduced_Set WHERE esri_symbology = '0.2% Annual Chance Flood Hazard'\", '-burn', '1', '-ot', 'Byte', '-co', 'COMPRESS=LZW', '/home/data/national_fema_floodmap/efa6409b-9dab-4b36-b0aa-8641f1b26b6e.gdb', '/home/data/national_fema_floodmap/rasters/30m/national_fema_raster_500y_30m.tiff'], returncode=0)"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import subprocess\n",
"\n",
"# Define the command as a list of arguments\n",
"command = [\n",
" \"gdal_rasterize\",\n",
" \"-tr\", resolution, resolution,\n",
" \"-sql\", \"SELECT * FROM USA_Flood_Hazard_Reduced_Set WHERE esri_symbology = '0.2% Annual Chance Flood Hazard'\",\n",
" \"-burn\", \"1\",\n",
" \"-ot\", \"Byte\",\n",
" \"-co\", \"COMPRESS=LZW\",\n",
" \"/home/data/national_fema_floodmap/efa6409b-9dab-4b36-b0aa-8641f1b26b6e.gdb\",\n",
" raster500y\n",
"]\n",
"\n",
"\n",
"# Execute the command\n",
"subprocess.run(command, check=True)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/canyon/miniconda3/envs/myenv/lib/python3.9/site-packages/osgeo/gdal.py:312: FutureWarning: Neither gdal.UseExceptions() nor gdal.DontUseExceptions() has been explicitly called. In GDAL 4.0, exceptions will be enabled by default.\n",
" warnings.warn(\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"0...10...20...30...40...50...60...70...80...90...100 - done.\n"
]
},
{
"data": {
"text/plain": [
"CompletedProcess(args=['/home/canyon/miniconda3/envs/myenv/bin/python', '-m', 'osgeo_utils.gdal_calc', '-A', '/home/data/national_fema_floodmap/rasters/30m/national_fema_raster_100y_30m.tiff', '-B', '/home/data/national_fema_floodmap/rasters/30m/national_fema_raster_500y_30m.tiff', '--outfile=/home/data/national_fema_floodmap/rasters/30m/national_fema_raster_combined_30m.tiff', '--calc=\"(A>0)*1 + (B>0)*(A<=0)*2\"', '--NoDataValue=0', '--config', 'CHECK_DISK_FREE_SPACE', 'FALSE', '--co', 'COMPRESS=LZW', '--overwrite'], returncode=0)"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Construct the command using sys.executable to ensure the correct Python interpreter is used\n",
"command = [\n",
" sys.executable, \"-m\", \"osgeo_utils.gdal_calc\",\n",
" \"-A\", raster100y,\n",
" \"-B\", raster500y,\n",
" \"--outfile=\" + raster_combined,\n",
" \"--calc=\\\"(A>0)*1 + (B>0)*(A<=0)*2\\\"\", # Conditional calculation\n",
" \"--NoDataValue=0\",\n",
" \"--config\", \"CHECK_DISK_FREE_SPACE\", \"FALSE\",\n",
" \"--co\", \"COMPRESS=LZW\",\n",
" \"--overwrite\"\n",
"]# Execute the command\n",
"subprocess.run(command, check=True)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"boundary_path = \"/home/data/national_fema_floodmap/nola_testing_file.geojson\" # Test paths\n",
"clipped_raster_path = f\"{folder_path}/national_fema_raster_combined_clipped.tiff\""
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'/home/data/national_fema_floodmap/rasters/30m/national_fema_raster_combined_30m.tiff'"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"raster_combined"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"command = [\n",
" \"gdalwarp\", \"-cutline\",\n",
" boundary_path, \"-crop_to_cutline\",\n",
" \"-dstalpha\", raster_combined,\n",
" clipped_raster_path\n",
"]\n",
"\n",
"# Execute the command\n",
"#subprocess.run(command, check=True)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "TOP-Sprint-lAvM2-mU",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.18"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
17 changes: 17 additions & 0 deletions analysis/src/process/process_FEMA_floodmaps.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,23 @@ def find_largest_file(directory):

return largest_file

def read_fema_file(config, city_key):
base_path = config["base_path"]
FEMA_file_prefix = config[city_key]["FEMA_file_name"]
city_path = f"{base_path}/cities/{city_key}"
zip_path = f"{base_path}/national/floodmaps/{FEMA_file_prefix}.zip"
extracted_folder_path = f"{city_path}/floodmaps/{FEMA_file_prefix}"
gdb_folder_path = f"{extracted_folder_path}/{FEMA_file_prefix}.gdb"

if not os.path.exists(extracted_folder_path):
extract_zip_file(zip_path, extracted_folder_path)

gdb_table_to_read = find_largest_file(gdb_folder_path)
FEMA_flood = geopd.read_file(gdb_table_to_read)

return FEMA_flood


def process_floodmap(config, city_key):
base_path = config["base_path"]
FEMA_file_prefix = config[city_key]["FEMA_file_name"]
Expand Down

0 comments on commit 7b0168b

Please sign in to comment.