Skip to content

Commit

Permalink
Added option to keep existing files when running extraction for many …
Browse files Browse the repository at this point in the history 
…variables and years. Useful if you need to restart the extraction and want to keep existing files
  • Loading branch information
@trondkr
trondkr committed Mar 3, 2022
1 parent 00c4ad2 commit 931ff1a
Show file tree
Hide file tree
Showing 6 changed files with 368 additions and 216 deletions.
15 changes: 8 additions & 7 deletions .ipynb_checkpoints/ECMWF_convert_to_ROMS-checkpoint.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,13 +85,13 @@ def change_reference_date(self, ds, config_ecmwf: ECMWF_query):

dates = num2date(era5_time, units=era5_time_units, calendar=era5_time_cal)
logging.debug(
"[ECMWF_convert_to_ROMS] Converted time: {} to {}".format(dates[0], dates[-1]))
f"[ECMWF_convert_to_ROMS] Converted time: {dates[0]} to {dates[-1]}")

# Convert back to julian day and convert to days since 1948-01-01 as that is standard for ROMS
# days_to_seconds = 86400.0
times = netCDF4.date2num(dates, units=config_ecmwf.time_units) # * days_to_seconds
logging.debug(
"[ECMWF_convert_to_ROMS] Converted time: {} to {} units: {}".format(times[0], times[-1], config_ecmwf.time_units))
f"[ECMWF_convert_to_ROMS] Converted time: {times[0]} to {times[-1]} units: {config_ecmwf.time_units}")

return times, config_ecmwf.time_units, era5_time_cal

Expand All @@ -117,8 +117,9 @@ def write_to_ROMS_netcdf_file(self, config_ecmwf: ECMWF_query, data_array, var_u
latitude = ds.variables['latitude'][:]
time, time_units, time_calendar = self.change_reference_date(ds, config_ecmwf)

netcdf_roms_filename = netcdf_file[0:-3] + '_roms.nc'
if os.path.exists(netcdf_roms_filename): os.remove(netcdf_roms_filename)
netcdf_roms_filename = f"{out_filename[0:-3]}_roms.nc"
if os.path.exists(netcdf_roms_filename):
os.remove(netcdf_roms_filename)
logging.info(
"[ECMWF_convert_to_ROMS] Writing final product to file {}".format(netcdf_roms_filename))

Expand All @@ -128,7 +129,7 @@ def write_to_ROMS_netcdf_file(self, config_ecmwf: ECMWF_query, data_array, var_u
"Atmospheric data on original grid but converted to ROMS units and parameter names." \
"Files created using the ECMWF_tools toolbox:" \
"https://github.com/trondkr/ERA5-ROMS"
f1.history = "Created {}".format(datetime.now())
f1.history = f"Created {datetime.now()}"
f1.link = "https://github.com/trondkr/"
f1.Conventions = "CF-1.0"
fill_val = 1.0e35
Expand Down Expand Up @@ -163,12 +164,12 @@ def write_to_ROMS_netcdf_file(self, config_ecmwf: ECMWF_query, data_array, var_u
vnc = f1.createVariable(metadata['roms_name'], 'd', (metadata['time_name'], 'lat', 'lon'), fill_value=fill_val)
vnc.long_name = metadata["name"]
vnc.standard_name = metadata["short_name"]
vnc.coordinates = "lon lat {}".format(metadata['time_name'])
vnc.coordinates = f"lon lat {metadata['time_name']}"
vnc.units = var_units
vnc.missing_value = fill_val

vnc[:, :, :] = data_array[:,::-1,:]
logging.info(
"[ECMWF_convert_to_ROMS] Finished writing to file {}".format(netcdf_roms_filename))
f"[ECMWF_convert_to_ROMS] Finished writing to file {netcdf_roms_filename}")
os.remove(netcdf_file)
f1.close()
10 changes: 5 additions & 5 deletions .ipynb_checkpoints/ECMWF_query-checkpoint.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ def __init__(self):
self.setup_logging()

self.use_era5 = True
self.start_year = 1991
self.end_year = 2020
self.start_year = 1981
self.end_year = 1990
self.project = "A20"
self.area = self.get_area_for_project(self.project)

self.skip_existing_files=True
self.resultsdir = "../oceanography/ERA5/{}/".format(self.project)
self.debug = False

self.extract_data_every_6_hours = True
self.extract_data_every_N_hours = True
self.time_units = "days since 1948-01-01 00:00:00"
self.optionals = True # optional variables to extract depending on ROMS version (Rutgers or Kate)
self.ROMS_version = "Kate" # "Rutgers" or "Kate" - the sea-ice component of Kates ROMS version uses downward
Expand Down Expand Up @@ -44,7 +44,7 @@ def __init__(self):
'total_cloud_cover',
'total_precipitation',
'specific_humidity']

if self.ROMS_version == "Kate":
self.parameters.append('mean_surface_downward_short_wave_radiation_flux')
elif self.ROMS_version == "Rutgers":
Expand Down
267 changes: 171 additions & 96 deletions .ipynb_checkpoints/ECMWF_tools-checkpoint.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,100 +12,175 @@
#
# Login to your account here to get these : https://cds.climate.copernicus.eu

class ECMWF_tools:

def __init__(self):
# https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=form
# https://confluence.ecmwf.int/pages/viewpage.action?pageId=82870405#ERA5:datadocumentation-Table4
# Check data availability: http://apps.ecmwf.int/datasets/

self.config_ecmwf = ECMWF_query.ECMWF_query()
self.server = cdsapi.Client(debug=self.config_ecmwf.debug)

def create_requests(self):
years = [self.config_ecmwf.start_year + y for y in
range(self.config_ecmwf.end_year - self.config_ecmwf.start_year)]

if not os.path.exists(self.config_ecmwf.resultsdir):
os.mkdir(self.config_ecmwf.resultsdir)
for year in years:

print("=> Downloading for year {}".format(year))

for parameter in self.config_ecmwf.parameters:
print("=> getting data for : {} ".format(parameter))
metadata = self.config_ecmwf.get_parameter_metadata(parameter)

out_filename = "{}{}_{}_year_{}.nc".format(self.config_ecmwf.resultsdir,
self.config_ecmwf.dataset,
metadata["short_name"],
year)
if os.path.exists(out_filename):
os.remove(out_filename)

self.submit_request(parameter, str(year), out_filename)

def submit_request(self, parameter, year, out_filename):

times = [
'00:00', '01:00', '02:00',
'03:00', '04:00', '05:00',
'06:00', '07:00', '08:00',
'09:00', '10:00', '11:00',
'12:00', '13:00', '14:00',
'15:00', '16:00', '17:00',
'18:00', '19:00', '20:00',
'21:00', '22:00', '23:00',
]

if self.config_ecmwf.extract_data_every_6_hours is True:
times = ['00:00', '06:00', '12:00', '18:00']

options = {
'product_type': 'reanalysis',
"year": year,
"month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"],
'day': [
'01', '02', '03',
'04', '05', '06',
'07', '08', '09',
'10', '11', '12',
'13', '14', '15',
'16', '17', '18',
'19', '20', '21',
'22', '23', '24',
'25', '26', '27',
'28', '29', '30',
'31',
],
'time': times,
"variable": [parameter],
'format': "netcdf",
"area": self.config_ecmwf.area,
"verbose": self.config_ecmwf.debug,
}

# Add more specific options for variables on pressure surfaces
if parameter == "specific_humidity":
self.config_ecmwf.reanalysis = "reanalysis-era5-pressure-levels"
options["levtype"] = 'pl'
options["pressure_level"] = '1000'
else:
self.config_ecmwf.reanalysis = 'reanalysis-era5-single-levels'

try:
# Do the request
self.server.retrieve(self.config_ecmwf.reanalysis, options, out_filename)
except Exception as e:
print(e)
print("[!] -------------------------- PROBLEMS WITH {0}".format(out_filename))

metadata = self.config_ecmwf.get_parameter_metadata(parameter)

converter = ECMWF_convert_to_ROMS.ECMWF_convert_to_ROMS()
converter.convert_to_ROMS_units_standards(out_filename, metadata, parameter, self.config_ecmwf)


if __name__ == '__main__':
tool = ECMWF_tools()
tool.create_requests()
class ECMWF_tools:
def __init__(self):
# https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=form
# https://confluence.ecmwf.int/pages/viewpage.action?pageId=82870405#ERA5:datadocumentation-Table4
# Check data availability: http://apps.ecmwf.int/datasets/

self.config_ecmwf = ECMWF_query.ECMWF_query()
self.server = cdsapi.Client(debug=self.config_ecmwf.debug)

def create_requests(self):
years = [
self.config_ecmwf.start_year + y
for y in range(self.config_ecmwf.end_year - self.config_ecmwf.start_year)
]

if not os.path.exists(self.config_ecmwf.resultsdir):
os.mkdir(self.config_ecmwf.resultsdir)
for year in years:

print("=> Downloading for year {}".format(year))

for parameter in self.config_ecmwf.parameters:
print("=> getting data for : {} ".format(parameter))
metadata = self.config_ecmwf.get_parameter_metadata(parameter)

out_filename = "{}{}_{}_year_{}.nc".format(
self.config_ecmwf.resultsdir,
self.config_ecmwf.dataset,
metadata["short_name"],
year,
)
roms_outfile = f"{out_filename[0:-3]}_roms.nc"
if os.path.exists(roms_outfile):
if not self.config_ecmwf.skip_existing_files:
os.remove(out_filename)
self.submit_request(parameter, str(year), out_filename)
else:
print(f"Skipping existing file: {out_filename}")
else:
self.submit_request(parameter, str(year), out_filename)

def submit_request(self, parameter, year, out_filename):

times = [
"00:00",
"01:00",
"02:00",
"03:00",
"04:00",
"05:00",
"06:00",
"07:00",
"08:00",
"09:00",
"10:00",
"11:00",
"12:00",
"13:00",
"14:00",
"15:00",
"16:00",
"17:00",
"18:00",
"19:00",
"20:00",
"21:00",
"22:00",
"23:00",
]

if self.config_ecmwf.extract_data_every_N_hours is True:
times = times = [
"00:00",
"02:00",
"04:00",
"06:00",
"08:00",
"10:00",
"12:00",
"14:00",
"16:00",
"18:00",
"20:00",
"22:00",
]

options = {
"product_type": "reanalysis",
"year": year,
"month": [
"01",
"02",
"03",
"04",
"05",
"06",
"07",
"08",
"09",
"10",
"11",
"12",
],
"day": [
"01",
"02",
"03",
"04",
"05",
"06",
"07",
"08",
"09",
"10",
"11",
"12",
"13",
"14",
"15",
"16",
"17",
"18",
"19",
"20",
"21",
"22",
"23",
"24",
"25",
"26",
"27",
"28",
"29",
"30",
"31",
],
"time": times,
"variable": [parameter],
"format": "netcdf",
"area": self.config_ecmwf.area,
"verbose": self.config_ecmwf.debug,
}

# Add more specific options for variables on pressure surfaces
if parameter == "specific_humidity":
self.config_ecmwf.reanalysis = "reanalysis-era5-pressure-levels"
options["levtype"] = "pl"
options["pressure_level"] = "1000"
else:
self.config_ecmwf.reanalysis = "reanalysis-era5-single-levels"

try:
# Do the request
self.server.retrieve(self.config_ecmwf.reanalysis, options, out_filename)
except Exception as e:
print(e)
print(
"[!] -------------------------- PROBLEMS WITH {0}".format(out_filename)
)

metadata = self.config_ecmwf.get_parameter_metadata(parameter)

converter = ECMWF_convert_to_ROMS.ECMWF_convert_to_ROMS()
converter.convert_to_ROMS_units_standards(
out_filename, metadata, parameter, self.config_ecmwf
)


if __name__ == "__main__":
tool = ECMWF_tools()
tool.create_requests()
Loading

0 comments on commit 931ff1a

Please sign in to comment.