[pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
worldbank · Jun 10, 2024 · 6d30f0a · 6d30f0a
1 parent 4e3d430
commit 6d30f0a
Show file tree

Hide file tree

Showing 11 changed files with 231 additions and 164 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -49,4 +49,4 @@ jobs:
           name: python-package-distributions
           path: dist/
       - name: Publish distribution 📦 to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
+        uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/notebooks/BUILT_compare_buildings_wsf.ipynb b/notebooks/BUILT_compare_buildings_wsf.ipynb
@@ -15,19 +15,18 @@
     }
    ],
    "source": [
-    "import sys, os\n",
+    "import sys\n",
+    "import os\n",
     "import rasterio\n",
     "\n",
     "import pandas as pd\n",
     "import geopandas as gpd\n",
-    "import numpy as np\n",
     "\n",
     "from shapely.wkt import loads\n",
     "\n",
     "sys.path.insert(0, \"../src\")\n",
     "\n",
     "import GOSTrocks.rasterMisc as rMisc\n",
-    "import GOSTrocks.dataMisc as dMisc\n",
     "\n",
     "%load_ext autoreload\n",
     "%autoreload 2"
@@ -49,19 +48,25 @@
    ],
    "source": [
     "# Local/input files\n",
-    "iso3 = 'KHM'\n",
+    "iso3 = \"KHM\"\n",
     "out_folder = \"c:/WBG/Work/KHM_Energy/data\"\n",
     "wsf_file = os.path.join(out_folder, \"WSF\", \"wsf.tif\")\n",
     "ghsl_file = os.path.join(out_folder, \"GHSL\", \"ghsl.tif\")\n",
-    "overture_buildings = os.path.join(out_folder, \"overture\", \"overture_download_2024_03_29.csv\")\n",
-    "overture_raster = os.path.join(out_folder, \"overture\", \"overture_download_2024_03_29.tif\")\n",
-    "overture_raster_points = os.path.join(out_folder, \"overture\", \"overture_download_2024_03_29_points.tif\")\n",
+    "overture_buildings = os.path.join(\n",
+    "    out_folder, \"overture\", \"overture_download_2024_03_29.csv\"\n",
+    ")\n",
+    "overture_raster = os.path.join(\n",
+    "    out_folder, \"overture\", \"overture_download_2024_03_29.tif\"\n",
+    ")\n",
+    "overture_raster_points = os.path.join(\n",
+    "    out_folder, \"overture\", \"overture_download_2024_03_29_points.tif\"\n",
+    ")\n",
     "for file in [wsf_file, ghsl_file]:\n",
     "    if not os.path.exists(os.path.dirname(file)):\n",
     "        os.makedirs(os.path.dirname(file))\n",
     "\n",
     "# get country extent from geopandas\n",
-    "world_filepath = gpd.datasets.get_path('naturalearth_lowres')\n",
+    "world_filepath = gpd.datasets.get_path(\"naturalearth_lowres\")\n",
     "world = gpd.read_file(world_filepath)\n",
     "country = world[world.iso_a3 == iso3]"
    ]
@@ -97,12 +102,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "#Clip GHSL using local files\n",
-    "local_version = r\"J:\\Data\\GLOBAL\\GHSL\\Built\\GHS_BUILT_S_E2020_GLOBE_R2023A_54009_100_V1_0.tif\"\n",
+    "# Clip GHSL using local files\n",
+    "local_version = (\n",
+    "    r\"J:\\Data\\GLOBAL\\GHSL\\Built\\GHS_BUILT_S_E2020_GLOBE_R2023A_54009_100_V1_0.tif\"\n",
+    ")\n",
     "if not os.path.exists(ghsl_file):\n",
     "    ghsl_raster = rasterio.open(local_version)\n",
     "    data, profile = rMisc.clipRaster(ghsl_raster, country)\n",
-    "    with rasterio.open(ghsl_file, 'w', **profile) as dst:\n",
+    "    with rasterio.open(ghsl_file, \"w\", **profile) as dst:\n",
     "        dst.write(data)\n",
     "ghsl_r = rasterio.open(ghsl_file)"
    ]
@@ -203,9 +210,9 @@
    "source": [
     "# read in and process Overture buildings\n",
     "ob = pd.read_csv(overture_buildings)\n",
-    "ob_geoms = ob['wkt'].apply(loads)\n",
+    "ob_geoms = ob[\"wkt\"].apply(loads)\n",
     "inB = gpd.GeoDataFrame(ob, geometry=ob_geoms, crs=4326)\n",
-    "inB.head()\n"
+    "inB.head()"
    ]
   },
   {
@@ -216,9 +223,11 @@
    "source": [
     "# attempt to rasterrize the buildings as polygons\n",
     "if not os.path.exists(overture_raster):\n",
-    "    rasterized_buildings = rMisc.rasterizeDataFrame(inB, templateRaster=ghsl_file, mergeAlg=\"ADD\", re_proj=True, nodata=0.)\n",
-    "    with rasterio.open(overture_raster, 'w', **rasterized_buildings['meta']) as dst:\n",
-    "        dst.write_band(1, rasterized_buildings['vals'])\n",
+    "    rasterized_buildings = rMisc.rasterizeDataFrame(\n",
+    "        inB, templateRaster=ghsl_file, mergeAlg=\"ADD\", re_proj=True, nodata=0.0\n",
+    "    )\n",
+    "    with rasterio.open(overture_raster, \"w\", **rasterized_buildings[\"meta\"]) as dst:\n",
+    "        dst.write_band(1, rasterized_buildings[\"vals\"])\n",
     "overture_r = rasterio.open(overture_raster)"
    ]
   },
@@ -231,11 +240,15 @@
     "# attempt to rasterrize the buildings as points\n",
     "if not os.path.exists(overture_raster_points):\n",
     "    inB_points = inB.copy()\n",
-    "    inB_points['geometry'] = inB_points['geometry'].centroid\n",
-    "    rasterized_buildings = rMisc.rasterizeDataFrame(inB_points, templateRaster=ghsl_file, mergeAlg=\"ADD\", re_proj=True, nodata=0.)\n",
-    "    with rasterio.open(overture_raster_points, 'w', **rasterized_buildings['meta']) as dst:\n",
-    "        dst.write_band(1, rasterized_buildings['vals'])\n",
-    "overture_r_points = rasterio.open(overture_raster_points) "
+    "    inB_points[\"geometry\"] = inB_points[\"geometry\"].centroid\n",
+    "    rasterized_buildings = rMisc.rasterizeDataFrame(\n",
+    "        inB_points, templateRaster=ghsl_file, mergeAlg=\"ADD\", re_proj=True, nodata=0.0\n",
+    "    )\n",
+    "    with rasterio.open(\n",
+    "        overture_raster_points, \"w\", **rasterized_buildings[\"meta\"]\n",
+    "    ) as dst:\n",
+    "        dst.write_band(1, rasterized_buildings[\"vals\"])\n",
+    "overture_r_points = rasterio.open(overture_raster_points)"
    ]
   },
   {
@@ -250,10 +263,10 @@
     "ghsl_thresh = 3000\n",
     "\n",
     "o_data = overture_r_points.read(1)\n",
-    "o_data = (o_data > o_thresh).astype('uint8')\n",
+    "o_data = (o_data > o_thresh).astype(\"uint8\")\n",
     "\n",
     "ghsl_data = ghsl_r.read(1)\n",
-    "ghsl_data = (ghsl_data > ghsl_thresh).astype('uint8') * 10\n",
+    "ghsl_data = (ghsl_data > ghsl_thresh).astype(\"uint8\") * 10\n",
     "\n",
     "combo_data = o_data + ghsl_data\n",
     "\n",
@@ -262,8 +275,8 @@
     "if not os.path.exists(out_file):\n",
     "    meta = overture_r_points.meta.copy()\n",
     "    meta.update(dtype=rasterio.uint8, nodata=0)\n",
-    "    with rasterio.open(out_file, 'w', **meta) as out_raster:\n",
-    "        out_raster.write_band(1, combo_data)\n"
+    "    with rasterio.open(out_file, \"w\", **meta) as out_raster:\n",
+    "        out_raster.write_band(1, combo_data)"
    ]
   },
   {

diff --git a/notebooks/FATHOM/CLIP_Flood_data_iso3.ipynb b/notebooks/FATHOM/CLIP_Flood_data_iso3.ipynb
@@ -19,7 +19,6 @@
    "source": [
     "import sys\n",
     "import os\n",
-    "import boto3\n",
     "import rasterio\n",
     "\n",
     "import geopandas as gpd\n",
@@ -63,11 +62,11 @@
     "out_folder = f\"/home/wb411133/temp/FATHOM/{iso3}\"\n",
     "if not os.path.exists(out_folder):\n",
     "    os.makedirs(out_folder)\n",
-    "    \n",
+    "\n",
     "# This demo uses the default national boundaries included with GeoPandas, but this can be changed here\n",
-    "world_filepath = gpd.datasets.get_path('naturalearth_lowres')\n",
+    "world_filepath = gpd.datasets.get_path(\"naturalearth_lowres\")\n",
     "world = gpd.read_file(world_filepath)\n",
-    "inB = world.loc[world['iso_a3'] == iso3].copy()"
+    "inB = world.loc[world[\"iso_a3\"] == iso3].copy()"
    ]
   },
   {
@@ -78,17 +77,21 @@
    "outputs": [],
    "source": [
     "# Select layer to downlaod\n",
-    "flood_type = [\"COASTAL\",\"FLUVIAL\",\"PLUVIAL\"]\n",
+    "flood_type = [\"COASTAL\", \"FLUVIAL\", \"PLUVIAL\"]\n",
     "defence = [\"DEFENDED\"]\n",
-    "return_period = ['1in5','1in10','1in50']\n",
+    "return_period = [\"1in5\", \"1in10\", \"1in50\"]\n",
     "climate_model = [\"PERCENTILE50\"]\n",
     "year = [\"2020\"]\n",
     "\n",
     "# all_vrts is a pandas dataframe with all the vrt paths to the global datasets, with columns defining\n",
     "# the various models' defining attributes\n",
     "all_vrts = dMisc.get_fathom_vrts(True)\n",
-    "sel_images = all_vrts.loc[(all_vrts['FLOOD_TYPE'].isin(flood_type)) & (all_vrts['DEFENCE'].isin(defence)) & \n",
-    "             (all_vrts['RETURN'].isin(return_period))  & (all_vrts['CLIMATE_MODEL'].isin(climate_model))]"
+    "sel_images = all_vrts.loc[\n",
+    "    (all_vrts[\"FLOOD_TYPE\"].isin(flood_type))\n",
+    "    & (all_vrts[\"DEFENCE\"].isin(defence))\n",
+    "    & (all_vrts[\"RETURN\"].isin(return_period))\n",
+    "    & (all_vrts[\"CLIMATE_MODEL\"].isin(climate_model))\n",
+    "]"
    ]
   },
   {
@@ -349,12 +352,12 @@
     "# For each image in the selected images dataframe, we clip out the area of interest\n",
     "#     which is defined by the ioso3 code, but could be any GeoDataFrame\n",
     "\n",
-    "for idx, row in sel_images.iterrows():    \n",
-    "    out_file = os.path.join(out_folder, os.path.basename(row['PATH']))\n",
+    "for idx, row in sel_images.iterrows():\n",
+    "    out_file = os.path.join(out_folder, os.path.basename(row[\"PATH\"]))\n",
     "    if not os.path.exists(out_file):\n",
-    "        cur_r = rasterio.open(row['PATH'])\n",
+    "        cur_r = rasterio.open(row[\"PATH\"])\n",
     "        rMisc.clipRaster(cur_r, inB, out_file)\n",
-    "    tPrint(os.path.basename(row['PATH']))"
+    "    tPrint(os.path.basename(row[\"PATH\"]))"
    ]
   }
  ],

diff --git a/notebooks/FATHOM/PROCESSING_NOTEBOOKS/Transfer_Data_AWS.ipynb b/notebooks/FATHOM/PROCESSING_NOTEBOOKS/Transfer_Data_AWS.ipynb
@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import sys, os\n",
+    "import os\n",
     "import boto3\n",
     "\n",
     "import pandas as pd"
@@ -39,14 +39,14 @@
     }
    ],
    "source": [
-    "in_file_list = '/home/wb411133/temp/World_Bank_Global_3_Complete.csv'\n",
+    "in_file_list = \"/home/wb411133/temp/World_Bank_Global_3_Complete.csv\"\n",
     "out_folder = os.path.join(os.path.dirname(in_file_list), \"FATHOM\")\n",
-    "s3_bucket = 'wbg-geography01'\n",
-    "s3_prefix = 'FATHOM/v2023/'\n",
-    "s3_out = os.path.join('s3://',s3_bucket, s3_prefix)\n",
+    "s3_bucket = \"wbg-geography01\"\n",
+    "s3_prefix = \"FATHOM/v2023/\"\n",
+    "s3_out = os.path.join(\"s3://\", s3_bucket, s3_prefix)\n",
     "\n",
     "in_files = pd.read_csv(in_file_list)\n",
-    "s3 = boto3.resource('s3')\n",
+    "s3 = boto3.resource(\"s3\")\n",
     "my_bucket = s3.Bucket(s3_bucket)"
    ]
   },
@@ -59,7 +59,7 @@
     "# Find all files already copied\n",
     "all_folders = []\n",
     "for obj in my_bucket.objects.filter(Prefix=s3_prefix):\n",
-    "    all_folders.append(obj.key.split(\"/\")[-2])\n"
+    "    all_folders.append(obj.key.split(\"/\")[-2])"
    ]
   },
   {
@@ -69,9 +69,9 @@
    "outputs": [],
    "source": [
     "processed_folders = list(set(all_folders))\n",
-    "delivered_folders = in_files['Layer'].values\n",
-    "sel_folders = [x for x in delivered_folders if not x in processed_folders]\n",
-    "sel_files = in_files.loc[in_files['Layer'].isin(sel_folders)].copy()\n",
+    "delivered_folders = in_files[\"Layer\"].values\n",
+    "sel_folders = [x for x in delivered_folders if x not in processed_folders]\n",
+    "sel_files = in_files.loc[in_files[\"Layer\"].isin(sel_folders)].copy()\n",
     "sel_files"
    ]
   },
@@ -83,31 +83,33 @@
    },
    "outputs": [],
    "source": [
-    "with open(os.path.join(out_folder, \"aaa_download_upload_2.sh\"), 'w') as out_file:\n",
-    "    out_file.write('#!/bin/bash\\n')\n",
+    "with open(os.path.join(out_folder, \"aaa_download_upload_2.sh\"), \"w\") as out_file:\n",
+    "    out_file.write(\"#!/bin/bash\\n\")\n",
     "    for idx, row in sel_files.iterrows():\n",
-    "        fathom_path = row['AWS_Path']\n",
-    "        local_folder = os.path.join(out_folder, row['Layer'])\n",
-    "        gost_folder = os.path.join(s3_out, row['Layer'])\n",
+    "        fathom_path = row[\"AWS_Path\"]\n",
+    "        local_folder = os.path.join(out_folder, row[\"Layer\"])\n",
+    "        gost_folder = os.path.join(s3_out, row[\"Layer\"])\n",
     "        if not os.path.exists(local_folder):\n",
     "            os.makedirs(local_folder)\n",
-    "        \n",
-    "        cur_out_folder = os.path.join(s3_prefix, row['Layer'])\n",
+    "\n",
+    "        cur_out_folder = os.path.join(s3_prefix, row[\"Layer\"])\n",
     "        obj_count = 0\n",
     "        for obj in my_bucket.objects.filter(Prefix=cur_out_folder):\n",
     "            obj_count += 1\n",
     "        print(f\"{row['Layer']}: {obj_count}\")\n",
     "        if obj_count == 0:\n",
-    "            download_command = f'aws s3 sync --profile fathom {fathom_path} {local_folder}'\n",
-    "            upload_command = f'aws s3 sync {local_folder} {gost_folder}'\n",
-    "            remove_command = f'rm -R {local_folder}'\n",
+    "            download_command = (\n",
+    "                f\"aws s3 sync --profile fathom {fathom_path} {local_folder}\"\n",
+    "            )\n",
+    "            upload_command = f\"aws s3 sync {local_folder} {gost_folder}\"\n",
+    "            remove_command = f\"rm -R {local_folder}\"\n",
     "\n",
     "            out_file.write(download_command)\n",
-    "            out_file.write('\\n')\n",
+    "            out_file.write(\"\\n\")\n",
     "            out_file.write(upload_command)\n",
-    "            out_file.write('\\n')\n",
+    "            out_file.write(\"\\n\")\n",
     "            out_file.write(remove_command)\n",
-    "            out_file.write('\\n')"
+    "            out_file.write(\"\\n\")"
    ]
   },
   {
@@ -118,7 +120,7 @@
    "source": [
     "all_vals = []\n",
     "for idx, row in in_files.iterrows():\n",
-    "    all_vals.append(row['Layer'].split('-'))"
+    "    all_vals.append(row[\"Layer\"].split(\"-\"))"
    ]
   },
   {
@@ -127,8 +129,23 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "xx = pd.DataFrame(all_vals, columns=['GLOBAL', \"Size\",'Offset','return','type','defense','depth','year','projection','v1','v2'])\n",
-    "xx.head()\n"
+    "xx = pd.DataFrame(\n",
+    "    all_vals,\n",
+    "    columns=[\n",
+    "        \"GLOBAL\",\n",
+    "        \"Size\",\n",
+    "        \"Offset\",\n",
+    "        \"return\",\n",
+    "        \"type\",\n",
+    "        \"defense\",\n",
+    "        \"depth\",\n",
+    "        \"year\",\n",
+    "        \"projection\",\n",
+    "        \"v1\",\n",
+    "        \"v2\",\n",
+    "    ],\n",
+    ")\n",
+    "xx.head()"
    ]
   },
   {
@@ -137,7 +154,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "xx['projection'].value_counts()"
+    "xx[\"projection\"].value_counts()"
    ]
   },
   {
@@ -146,7 +163,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "xx.loc[xx['projection'] == 'SSP1_2.6']['year'].value_counts()"
+    "xx.loc[xx[\"projection\"] == \"SSP1_2.6\"][\"year\"].value_counts()"
    ]
   },
   {
@@ -172,24 +189,24 @@
     }
    ],
    "source": [
-    "sel_scenario = 'GLOBAL-1ARCSEC-NW_OFFSET-1in10-COASTAL-DEFENDED-DEPTH-2030-SSP3_7.0-PERCENTILE50-v3.0'\n",
+    "sel_scenario = \"GLOBAL-1ARCSEC-NW_OFFSET-1in10-COASTAL-DEFENDED-DEPTH-2030-SSP3_7.0-PERCENTILE50-v3.0\"\n",
     "bucket = \"fathom-products-global\"\n",
     "prefix = f\"fathom-global/v3/{sel_scenario}\"\n",
     "\n",
     "local_folder = os.path.join(\"/home/wb411133/temp/FATHOM\", sel_scenario)\n",
-    "fathom_path = f's3://{bucket}/{prefix}'\n",
+    "fathom_path = f\"s3://{bucket}/{prefix}\"\n",
     "gost_folder = f\"s3://wbg-geography01/FATHOM/v2023/{sel_scenario}\"\n",
     "\n",
     "if not os.path.exists(local_folder):\n",
     "    os.makedirs(local_folder)\n",
     "\n",
-    "download_command = f'aws s3 sync --profile fathom {fathom_path} {local_folder}'\n",
-    "upload_command = f'aws s3 sync {local_folder} {gost_folder}'\n",
-    "remove_command = f'rm -R {local_folder}'\n",
+    "download_command = f\"aws s3 sync --profile fathom {fathom_path} {local_folder}\"\n",
+    "upload_command = f\"aws s3 sync {local_folder} {gost_folder}\"\n",
+    "remove_command = f\"rm -R {local_folder}\"\n",
     "\n",
     "print(download_command)\n",
     "print(upload_command)\n",
-    "print(remove_command)\n"
+    "print(remove_command)"
    ]
   },
   {