diff --git a/.ipynb_checkpoints/ECMWF_convert_to_ROMS-checkpoint.py b/.ipynb_checkpoints/ECMWF_convert_to_ROMS-checkpoint.py index 64f9580..d3fa31d 100644 --- a/.ipynb_checkpoints/ECMWF_convert_to_ROMS-checkpoint.py +++ b/.ipynb_checkpoints/ECMWF_convert_to_ROMS-checkpoint.py @@ -85,13 +85,13 @@ def change_reference_date(self, ds, config_ecmwf: ECMWF_query): dates = num2date(era5_time, units=era5_time_units, calendar=era5_time_cal) logging.debug( - "[ECMWF_convert_to_ROMS] Converted time: {} to {}".format(dates[0], dates[-1])) + f"[ECMWF_convert_to_ROMS] Converted time: {dates[0]} to {dates[-1]}") # Convert back to julian day and convert to days since 1948-01-01 as that is standard for ROMS # days_to_seconds = 86400.0 times = netCDF4.date2num(dates, units=config_ecmwf.time_units) # * days_to_seconds logging.debug( - "[ECMWF_convert_to_ROMS] Converted time: {} to {} units: {}".format(times[0], times[-1], config_ecmwf.time_units)) + f"[ECMWF_convert_to_ROMS] Converted time: {times[0]} to {times[-1]} units: {config_ecmwf.time_units}") return times, config_ecmwf.time_units, era5_time_cal @@ -117,8 +117,9 @@ def write_to_ROMS_netcdf_file(self, config_ecmwf: ECMWF_query, data_array, var_u latitude = ds.variables['latitude'][:] time, time_units, time_calendar = self.change_reference_date(ds, config_ecmwf) - netcdf_roms_filename = netcdf_file[0:-3] + '_roms.nc' - if os.path.exists(netcdf_roms_filename): os.remove(netcdf_roms_filename) + netcdf_roms_filename = f"{out_filename[0:-3]}_roms.nc" + if os.path.exists(netcdf_roms_filename): + os.remove(netcdf_roms_filename) logging.info( "[ECMWF_convert_to_ROMS] Writing final product to file {}".format(netcdf_roms_filename)) @@ -128,7 +129,7 @@ def write_to_ROMS_netcdf_file(self, config_ecmwf: ECMWF_query, data_array, var_u "Atmospheric data on original grid but converted to ROMS units and parameter names." \ "Files created using the ECMWF_tools toolbox:" \ "https://github.com/trondkr/ERA5-ROMS" - f1.history = "Created {}".format(datetime.now()) + f1.history = f"Created {datetime.now()}" f1.link = "https://github.com/trondkr/" f1.Conventions = "CF-1.0" fill_val = 1.0e35 @@ -163,12 +164,12 @@ def write_to_ROMS_netcdf_file(self, config_ecmwf: ECMWF_query, data_array, var_u vnc = f1.createVariable(metadata['roms_name'], 'd', (metadata['time_name'], 'lat', 'lon'), fill_value=fill_val) vnc.long_name = metadata["name"] vnc.standard_name = metadata["short_name"] - vnc.coordinates = "lon lat {}".format(metadata['time_name']) + vnc.coordinates = f"lon lat {metadata['time_name']}" vnc.units = var_units vnc.missing_value = fill_val vnc[:, :, :] = data_array[:,::-1,:] logging.info( - "[ECMWF_convert_to_ROMS] Finished writing to file {}".format(netcdf_roms_filename)) + f"[ECMWF_convert_to_ROMS] Finished writing to file {netcdf_roms_filename}") os.remove(netcdf_file) f1.close() diff --git a/.ipynb_checkpoints/ECMWF_query-checkpoint.py b/.ipynb_checkpoints/ECMWF_query-checkpoint.py index 7104918..f971fde 100644 --- a/.ipynb_checkpoints/ECMWF_query-checkpoint.py +++ b/.ipynb_checkpoints/ECMWF_query-checkpoint.py @@ -8,15 +8,15 @@ def __init__(self): self.setup_logging() self.use_era5 = True - self.start_year = 1991 - self.end_year = 2020 + self.start_year = 1981 + self.end_year = 1990 self.project = "A20" self.area = self.get_area_for_project(self.project) - + self.skip_existing_files=True self.resultsdir = "../oceanography/ERA5/{}/".format(self.project) self.debug = False - self.extract_data_every_6_hours = True + self.extract_data_every_N_hours = True self.time_units = "days since 1948-01-01 00:00:00" self.optionals = True # optional variables to extract depending on ROMS version (Rutgers or Kate) self.ROMS_version = "Kate" # "Rutgers" or "Kate" - the sea-ice component of Kates ROMS version uses downward @@ -44,7 +44,7 @@ def __init__(self): 'total_cloud_cover', 'total_precipitation', 'specific_humidity'] - + if self.ROMS_version == "Kate": self.parameters.append('mean_surface_downward_short_wave_radiation_flux') elif self.ROMS_version == "Rutgers": diff --git a/.ipynb_checkpoints/ECMWF_tools-checkpoint.py b/.ipynb_checkpoints/ECMWF_tools-checkpoint.py index 3f4aa37..fe67fc9 100644 --- a/.ipynb_checkpoints/ECMWF_tools-checkpoint.py +++ b/.ipynb_checkpoints/ECMWF_tools-checkpoint.py @@ -12,100 +12,175 @@ # # Login to your account here to get these : https://cds.climate.copernicus.eu -class ECMWF_tools: - def __init__(self): - # https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=form - # https://confluence.ecmwf.int/pages/viewpage.action?pageId=82870405#ERA5:datadocumentation-Table4 - # Check data availability: http://apps.ecmwf.int/datasets/ - - self.config_ecmwf = ECMWF_query.ECMWF_query() - self.server = cdsapi.Client(debug=self.config_ecmwf.debug) - - def create_requests(self): - years = [self.config_ecmwf.start_year + y for y in - range(self.config_ecmwf.end_year - self.config_ecmwf.start_year)] - - if not os.path.exists(self.config_ecmwf.resultsdir): - os.mkdir(self.config_ecmwf.resultsdir) - for year in years: - - print("=> Downloading for year {}".format(year)) - - for parameter in self.config_ecmwf.parameters: - print("=> getting data for : {} ".format(parameter)) - metadata = self.config_ecmwf.get_parameter_metadata(parameter) - - out_filename = "{}{}_{}_year_{}.nc".format(self.config_ecmwf.resultsdir, - self.config_ecmwf.dataset, - metadata["short_name"], - year) - if os.path.exists(out_filename): - os.remove(out_filename) - - self.submit_request(parameter, str(year), out_filename) - - def submit_request(self, parameter, year, out_filename): - - times = [ - '00:00', '01:00', '02:00', - '03:00', '04:00', '05:00', - '06:00', '07:00', '08:00', - '09:00', '10:00', '11:00', - '12:00', '13:00', '14:00', - '15:00', '16:00', '17:00', - '18:00', '19:00', '20:00', - '21:00', '22:00', '23:00', - ] - - if self.config_ecmwf.extract_data_every_6_hours is True: - times = ['00:00', '06:00', '12:00', '18:00'] - - options = { - 'product_type': 'reanalysis', - "year": year, - "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"], - 'day': [ - '01', '02', '03', - '04', '05', '06', - '07', '08', '09', - '10', '11', '12', - '13', '14', '15', - '16', '17', '18', - '19', '20', '21', - '22', '23', '24', - '25', '26', '27', - '28', '29', '30', - '31', - ], - 'time': times, - "variable": [parameter], - 'format': "netcdf", - "area": self.config_ecmwf.area, - "verbose": self.config_ecmwf.debug, - } - - # Add more specific options for variables on pressure surfaces - if parameter == "specific_humidity": - self.config_ecmwf.reanalysis = "reanalysis-era5-pressure-levels" - options["levtype"] = 'pl' - options["pressure_level"] = '1000' - else: - self.config_ecmwf.reanalysis = 'reanalysis-era5-single-levels' - - try: - # Do the request - self.server.retrieve(self.config_ecmwf.reanalysis, options, out_filename) - except Exception as e: - print(e) - print("[!] -------------------------- PROBLEMS WITH {0}".format(out_filename)) - - metadata = self.config_ecmwf.get_parameter_metadata(parameter) - - converter = ECMWF_convert_to_ROMS.ECMWF_convert_to_ROMS() - converter.convert_to_ROMS_units_standards(out_filename, metadata, parameter, self.config_ecmwf) - - -if __name__ == '__main__': - tool = ECMWF_tools() - tool.create_requests() +class ECMWF_tools: + def __init__(self): + # https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=form + # https://confluence.ecmwf.int/pages/viewpage.action?pageId=82870405#ERA5:datadocumentation-Table4 + # Check data availability: http://apps.ecmwf.int/datasets/ + + self.config_ecmwf = ECMWF_query.ECMWF_query() + self.server = cdsapi.Client(debug=self.config_ecmwf.debug) + + def create_requests(self): + years = [ + self.config_ecmwf.start_year + y + for y in range(self.config_ecmwf.end_year - self.config_ecmwf.start_year) + ] + + if not os.path.exists(self.config_ecmwf.resultsdir): + os.mkdir(self.config_ecmwf.resultsdir) + for year in years: + + print("=> Downloading for year {}".format(year)) + + for parameter in self.config_ecmwf.parameters: + print("=> getting data for : {} ".format(parameter)) + metadata = self.config_ecmwf.get_parameter_metadata(parameter) + + out_filename = "{}{}_{}_year_{}.nc".format( + self.config_ecmwf.resultsdir, + self.config_ecmwf.dataset, + metadata["short_name"], + year, + ) + roms_outfile = f"{out_filename[0:-3]}_roms.nc" + if os.path.exists(roms_outfile): + if not self.config_ecmwf.skip_existing_files: + os.remove(out_filename) + self.submit_request(parameter, str(year), out_filename) + else: + print(f"Skipping existing file: {out_filename}") + else: + self.submit_request(parameter, str(year), out_filename) + + def submit_request(self, parameter, year, out_filename): + + times = [ + "00:00", + "01:00", + "02:00", + "03:00", + "04:00", + "05:00", + "06:00", + "07:00", + "08:00", + "09:00", + "10:00", + "11:00", + "12:00", + "13:00", + "14:00", + "15:00", + "16:00", + "17:00", + "18:00", + "19:00", + "20:00", + "21:00", + "22:00", + "23:00", + ] + + if self.config_ecmwf.extract_data_every_N_hours is True: + times = times = [ + "00:00", + "02:00", + "04:00", + "06:00", + "08:00", + "10:00", + "12:00", + "14:00", + "16:00", + "18:00", + "20:00", + "22:00", + ] + + options = { + "product_type": "reanalysis", + "year": year, + "month": [ + "01", + "02", + "03", + "04", + "05", + "06", + "07", + "08", + "09", + "10", + "11", + "12", + ], + "day": [ + "01", + "02", + "03", + "04", + "05", + "06", + "07", + "08", + "09", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + ], + "time": times, + "variable": [parameter], + "format": "netcdf", + "area": self.config_ecmwf.area, + "verbose": self.config_ecmwf.debug, + } + + # Add more specific options for variables on pressure surfaces + if parameter == "specific_humidity": + self.config_ecmwf.reanalysis = "reanalysis-era5-pressure-levels" + options["levtype"] = "pl" + options["pressure_level"] = "1000" + else: + self.config_ecmwf.reanalysis = "reanalysis-era5-single-levels" + + try: + # Do the request + self.server.retrieve(self.config_ecmwf.reanalysis, options, out_filename) + except Exception as e: + print(e) + print( + "[!] -------------------------- PROBLEMS WITH {0}".format(out_filename) + ) + + metadata = self.config_ecmwf.get_parameter_metadata(parameter) + + converter = ECMWF_convert_to_ROMS.ECMWF_convert_to_ROMS() + converter.convert_to_ROMS_units_standards( + out_filename, metadata, parameter, self.config_ecmwf + ) + + +if __name__ == "__main__": + tool = ECMWF_tools() + tool.create_requests() diff --git a/ECMWF_convert_to_ROMS.py b/ECMWF_convert_to_ROMS.py index 64f9580..d3fa31d 100644 --- a/ECMWF_convert_to_ROMS.py +++ b/ECMWF_convert_to_ROMS.py @@ -85,13 +85,13 @@ def change_reference_date(self, ds, config_ecmwf: ECMWF_query): dates = num2date(era5_time, units=era5_time_units, calendar=era5_time_cal) logging.debug( - "[ECMWF_convert_to_ROMS] Converted time: {} to {}".format(dates[0], dates[-1])) + f"[ECMWF_convert_to_ROMS] Converted time: {dates[0]} to {dates[-1]}") # Convert back to julian day and convert to days since 1948-01-01 as that is standard for ROMS # days_to_seconds = 86400.0 times = netCDF4.date2num(dates, units=config_ecmwf.time_units) # * days_to_seconds logging.debug( - "[ECMWF_convert_to_ROMS] Converted time: {} to {} units: {}".format(times[0], times[-1], config_ecmwf.time_units)) + f"[ECMWF_convert_to_ROMS] Converted time: {times[0]} to {times[-1]} units: {config_ecmwf.time_units}") return times, config_ecmwf.time_units, era5_time_cal @@ -117,8 +117,9 @@ def write_to_ROMS_netcdf_file(self, config_ecmwf: ECMWF_query, data_array, var_u latitude = ds.variables['latitude'][:] time, time_units, time_calendar = self.change_reference_date(ds, config_ecmwf) - netcdf_roms_filename = netcdf_file[0:-3] + '_roms.nc' - if os.path.exists(netcdf_roms_filename): os.remove(netcdf_roms_filename) + netcdf_roms_filename = f"{out_filename[0:-3]}_roms.nc" + if os.path.exists(netcdf_roms_filename): + os.remove(netcdf_roms_filename) logging.info( "[ECMWF_convert_to_ROMS] Writing final product to file {}".format(netcdf_roms_filename)) @@ -128,7 +129,7 @@ def write_to_ROMS_netcdf_file(self, config_ecmwf: ECMWF_query, data_array, var_u "Atmospheric data on original grid but converted to ROMS units and parameter names." \ "Files created using the ECMWF_tools toolbox:" \ "https://github.com/trondkr/ERA5-ROMS" - f1.history = "Created {}".format(datetime.now()) + f1.history = f"Created {datetime.now()}" f1.link = "https://github.com/trondkr/" f1.Conventions = "CF-1.0" fill_val = 1.0e35 @@ -163,12 +164,12 @@ def write_to_ROMS_netcdf_file(self, config_ecmwf: ECMWF_query, data_array, var_u vnc = f1.createVariable(metadata['roms_name'], 'd', (metadata['time_name'], 'lat', 'lon'), fill_value=fill_val) vnc.long_name = metadata["name"] vnc.standard_name = metadata["short_name"] - vnc.coordinates = "lon lat {}".format(metadata['time_name']) + vnc.coordinates = f"lon lat {metadata['time_name']}" vnc.units = var_units vnc.missing_value = fill_val vnc[:, :, :] = data_array[:,::-1,:] logging.info( - "[ECMWF_convert_to_ROMS] Finished writing to file {}".format(netcdf_roms_filename)) + f"[ECMWF_convert_to_ROMS] Finished writing to file {netcdf_roms_filename}") os.remove(netcdf_file) f1.close() diff --git a/ECMWF_query.py b/ECMWF_query.py index 7104918..f971fde 100644 --- a/ECMWF_query.py +++ b/ECMWF_query.py @@ -8,15 +8,15 @@ def __init__(self): self.setup_logging() self.use_era5 = True - self.start_year = 1991 - self.end_year = 2020 + self.start_year = 1981 + self.end_year = 1990 self.project = "A20" self.area = self.get_area_for_project(self.project) - + self.skip_existing_files=True self.resultsdir = "../oceanography/ERA5/{}/".format(self.project) self.debug = False - self.extract_data_every_6_hours = True + self.extract_data_every_N_hours = True self.time_units = "days since 1948-01-01 00:00:00" self.optionals = True # optional variables to extract depending on ROMS version (Rutgers or Kate) self.ROMS_version = "Kate" # "Rutgers" or "Kate" - the sea-ice component of Kates ROMS version uses downward @@ -44,7 +44,7 @@ def __init__(self): 'total_cloud_cover', 'total_precipitation', 'specific_humidity'] - + if self.ROMS_version == "Kate": self.parameters.append('mean_surface_downward_short_wave_radiation_flux') elif self.ROMS_version == "Rutgers": diff --git a/ECMWF_tools.py b/ECMWF_tools.py index 3f4aa37..fe67fc9 100644 --- a/ECMWF_tools.py +++ b/ECMWF_tools.py @@ -12,100 +12,175 @@ # # Login to your account here to get these : https://cds.climate.copernicus.eu -class ECMWF_tools: - def __init__(self): - # https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=form - # https://confluence.ecmwf.int/pages/viewpage.action?pageId=82870405#ERA5:datadocumentation-Table4 - # Check data availability: http://apps.ecmwf.int/datasets/ - - self.config_ecmwf = ECMWF_query.ECMWF_query() - self.server = cdsapi.Client(debug=self.config_ecmwf.debug) - - def create_requests(self): - years = [self.config_ecmwf.start_year + y for y in - range(self.config_ecmwf.end_year - self.config_ecmwf.start_year)] - - if not os.path.exists(self.config_ecmwf.resultsdir): - os.mkdir(self.config_ecmwf.resultsdir) - for year in years: - - print("=> Downloading for year {}".format(year)) - - for parameter in self.config_ecmwf.parameters: - print("=> getting data for : {} ".format(parameter)) - metadata = self.config_ecmwf.get_parameter_metadata(parameter) - - out_filename = "{}{}_{}_year_{}.nc".format(self.config_ecmwf.resultsdir, - self.config_ecmwf.dataset, - metadata["short_name"], - year) - if os.path.exists(out_filename): - os.remove(out_filename) - - self.submit_request(parameter, str(year), out_filename) - - def submit_request(self, parameter, year, out_filename): - - times = [ - '00:00', '01:00', '02:00', - '03:00', '04:00', '05:00', - '06:00', '07:00', '08:00', - '09:00', '10:00', '11:00', - '12:00', '13:00', '14:00', - '15:00', '16:00', '17:00', - '18:00', '19:00', '20:00', - '21:00', '22:00', '23:00', - ] - - if self.config_ecmwf.extract_data_every_6_hours is True: - times = ['00:00', '06:00', '12:00', '18:00'] - - options = { - 'product_type': 'reanalysis', - "year": year, - "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"], - 'day': [ - '01', '02', '03', - '04', '05', '06', - '07', '08', '09', - '10', '11', '12', - '13', '14', '15', - '16', '17', '18', - '19', '20', '21', - '22', '23', '24', - '25', '26', '27', - '28', '29', '30', - '31', - ], - 'time': times, - "variable": [parameter], - 'format': "netcdf", - "area": self.config_ecmwf.area, - "verbose": self.config_ecmwf.debug, - } - - # Add more specific options for variables on pressure surfaces - if parameter == "specific_humidity": - self.config_ecmwf.reanalysis = "reanalysis-era5-pressure-levels" - options["levtype"] = 'pl' - options["pressure_level"] = '1000' - else: - self.config_ecmwf.reanalysis = 'reanalysis-era5-single-levels' - - try: - # Do the request - self.server.retrieve(self.config_ecmwf.reanalysis, options, out_filename) - except Exception as e: - print(e) - print("[!] -------------------------- PROBLEMS WITH {0}".format(out_filename)) - - metadata = self.config_ecmwf.get_parameter_metadata(parameter) - - converter = ECMWF_convert_to_ROMS.ECMWF_convert_to_ROMS() - converter.convert_to_ROMS_units_standards(out_filename, metadata, parameter, self.config_ecmwf) - - -if __name__ == '__main__': - tool = ECMWF_tools() - tool.create_requests() +class ECMWF_tools: + def __init__(self): + # https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=form + # https://confluence.ecmwf.int/pages/viewpage.action?pageId=82870405#ERA5:datadocumentation-Table4 + # Check data availability: http://apps.ecmwf.int/datasets/ + + self.config_ecmwf = ECMWF_query.ECMWF_query() + self.server = cdsapi.Client(debug=self.config_ecmwf.debug) + + def create_requests(self): + years = [ + self.config_ecmwf.start_year + y + for y in range(self.config_ecmwf.end_year - self.config_ecmwf.start_year) + ] + + if not os.path.exists(self.config_ecmwf.resultsdir): + os.mkdir(self.config_ecmwf.resultsdir) + for year in years: + + print("=> Downloading for year {}".format(year)) + + for parameter in self.config_ecmwf.parameters: + print("=> getting data for : {} ".format(parameter)) + metadata = self.config_ecmwf.get_parameter_metadata(parameter) + + out_filename = "{}{}_{}_year_{}.nc".format( + self.config_ecmwf.resultsdir, + self.config_ecmwf.dataset, + metadata["short_name"], + year, + ) + roms_outfile = f"{out_filename[0:-3]}_roms.nc" + if os.path.exists(roms_outfile): + if not self.config_ecmwf.skip_existing_files: + os.remove(out_filename) + self.submit_request(parameter, str(year), out_filename) + else: + print(f"Skipping existing file: {out_filename}") + else: + self.submit_request(parameter, str(year), out_filename) + + def submit_request(self, parameter, year, out_filename): + + times = [ + "00:00", + "01:00", + "02:00", + "03:00", + "04:00", + "05:00", + "06:00", + "07:00", + "08:00", + "09:00", + "10:00", + "11:00", + "12:00", + "13:00", + "14:00", + "15:00", + "16:00", + "17:00", + "18:00", + "19:00", + "20:00", + "21:00", + "22:00", + "23:00", + ] + + if self.config_ecmwf.extract_data_every_N_hours is True: + times = times = [ + "00:00", + "02:00", + "04:00", + "06:00", + "08:00", + "10:00", + "12:00", + "14:00", + "16:00", + "18:00", + "20:00", + "22:00", + ] + + options = { + "product_type": "reanalysis", + "year": year, + "month": [ + "01", + "02", + "03", + "04", + "05", + "06", + "07", + "08", + "09", + "10", + "11", + "12", + ], + "day": [ + "01", + "02", + "03", + "04", + "05", + "06", + "07", + "08", + "09", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + ], + "time": times, + "variable": [parameter], + "format": "netcdf", + "area": self.config_ecmwf.area, + "verbose": self.config_ecmwf.debug, + } + + # Add more specific options for variables on pressure surfaces + if parameter == "specific_humidity": + self.config_ecmwf.reanalysis = "reanalysis-era5-pressure-levels" + options["levtype"] = "pl" + options["pressure_level"] = "1000" + else: + self.config_ecmwf.reanalysis = "reanalysis-era5-single-levels" + + try: + # Do the request + self.server.retrieve(self.config_ecmwf.reanalysis, options, out_filename) + except Exception as e: + print(e) + print( + "[!] -------------------------- PROBLEMS WITH {0}".format(out_filename) + ) + + metadata = self.config_ecmwf.get_parameter_metadata(parameter) + + converter = ECMWF_convert_to_ROMS.ECMWF_convert_to_ROMS() + converter.convert_to_ROMS_units_standards( + out_filename, metadata, parameter, self.config_ecmwf + ) + + +if __name__ == "__main__": + tool = ECMWF_tools() + tool.create_requests()