Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Develop #2

Merged
merged 4 commits into from
Aug 7, 2014
Merged

Develop #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0b7560c
SceneDataset modified to extract pq tests run from the metadata.
matthew-hoyles May 30, 2014
a451656
Modified scene dataset to return alternate scene start and end dateti…
matthew-hoyles Jun 2, 2014
22e02d8
Replaced diagnostic prints in SceneDataset with logger calls.
matthew-hoyles Jun 2, 2014
dedcefc
Merge branch 'matt-hoyles/new-ingest-changes' into develop
matthew-hoyles Jul 18, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 66 additions & 27 deletions ULA3/dataset/_scene_dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from glob import glob
from datetime import datetime, date, time
import xml.dom.minidom
from pprint import pprint
from pprint import pprint, pformat
from osgeo import gdal, gdalconst, osr

import ULA3.geodesic as geocentric
Expand Down Expand Up @@ -384,12 +384,10 @@ def __find_root_dataset_pq(self):

try:
self._root_dataset = gdal.Open(self._root_dataset_pathname, self._eAccess)
except:
except (RuntimeError), e:
self._root_dataset = None
raise DSException(e.message)

self.pq_tests_run = int(suffix, base=2)

self._sub_datasets.append(self._root_dataset)

if self._data_dir: # Data directory has been determined
Expand Down Expand Up @@ -543,11 +541,12 @@ def __find_root_dataset_xml(self):
suffix = m.group(2).upper()
extension = m.group(3).upper()

print 'suffix ', suffix
print 'extension ', extension
print '_FILE_TYPE_INFO', self._FILE_TYPE_INFO
print 'extension in _FILE_TYPE_INFO', (extension in self._FILE_TYPE_INFO)
print 'satellite.root_band', self.satellite.root_band
logger.debug("__find_root_dataset_xml:")
logger.debug(" suffix = %s", suffix)
logger.debug(" extension = %s", extension)
logger.debug(" _FILE_TYPE_INFO = %s", self._FILE_TYPE_INFO)
logger.debug(" extension in _FILE_TYPE_INFO = %s", (extension in self._FILE_TYPE_INFO))
logger.debug(" satellite.root_band = %s", self.satellite.root_band)

# Determine the root suffix for the found file type
if extension in self._FILE_TYPE_INFO:
Expand All @@ -556,14 +555,14 @@ def __find_root_dataset_xml(self):
except (TypeError):
root_suffix = self._FILE_TYPE_INFO[extension]['ROOTSUFFIX']

print 'root_suffix', root_suffix, (suffix == root_suffix)
logger.debug(" root_suffix = %s", root_suffix)

if suffix == root_suffix:
self._root_dataset_pathname = os.path.abspath(os.path.join(datadir, f))
self._sub_dataset_type = extension
self._data_dir = datadir

print '_root_dataset_pathname', self._root_dataset_pathname
logger.debug(" _root_dataset_pathname = %s", self._root_dataset_pathname)

try: # ToDo: Need to deal with open failure here
self._root_dataset = gdal.Open(self._root_dataset_pathname, self._eAccess)
Expand All @@ -582,9 +581,11 @@ def __find_root_dataset_xml(self):
if self._data_dir and self._root_dataset_pathname: # Data directory has been determined
break # Stop searching

print
print '_data_dir ', self._data_dir
print '_root_dataset_pathname', self._root_dataset_pathname
logger.debug("")
logger.debug("__find_root_dataset_xml results:")
logger.debug(" self._data_dir = %s", self._data_dir)
logger.debug(" self._root_dataset_pathname = %s", self._root_dataset_pathname)
logger.debug("")

assert self._data_dir and self._root_dataset_pathname, 'Unable to find root dataset under ' + self._pathname

Expand Down Expand Up @@ -688,6 +689,15 @@ def __set_instance_values(self):
This is called after read_metadata(). Any derived vales should be set here.
"""

#
# Keep the original start and end datetimes (direct from the
# metadata) as alternate start and end datetimes. These can
# provide values where the scene_center_time is not available.
#

self.scene_alt_start_datetime = self.scene_start_datetime
self.scene_alt_end_datetime = self.scene_end_datetime

if self.scene_centre_date and self.scene_centre_time:
self.scene_centre_datetime = datetime(
self.scene_centre_date.year,
Expand Down Expand Up @@ -717,6 +727,12 @@ def __set_instance_values(self):
else:
self.completion_datetime = None

# Pixel Quality tests run - this information is extracted from the
# data quality statement in the metadata.

if self.processor_level == 'Pixel Quality':
self.pq_tests_run = self.__parse_pq_tests_run()

# SPATIAL REFERENCE GENERAL CASE
# Use the projection and geotransform that GDAL gives us.
# WARNING: in special case code below, always create a new spatial reference
Expand Down Expand Up @@ -829,15 +845,15 @@ def __set_instance_values(self):
# 'LR': (self.lr_lon, self.lr_lat)}

if self.IsGeographic():
print 'IsGeographic'
logger.debug("IsGeographic:")
self.lonlats = {
'CENTRE' : (extents[4][0], extents[4][1]),
'UL' : (extents[0][0], extents[0][1]),
'UR' : (extents[2][0], extents[2][1]),
'LL' : (extents[1][0], extents[1][1]),
'LR' : (extents[3][0], extents[3][1])
}
print self.lonlats
logger.debug("lonlats = %s", self.lonlats)
# If the scene is natively in geographics, we shouldn't need to
# project the co-ordinates to UTM.

Expand Down Expand Up @@ -867,7 +883,7 @@ def __set_instance_values(self):
for x,y in extents:
new_x, new_y, new_z = self.cxform_to_geo.TransformPoint(x,y)
re_prj_extents.append([new_x,new_y])
print new_x, new_y
logger.debug("new_x = %s, new_y = %s", new_x, new_y)

self.lonlats = {
'CENTRE' : (re_prj_extents[4][0], re_prj_extents[4][1]),
Expand Down Expand Up @@ -943,6 +959,27 @@ def __set_instance_values(self):
self.zone = abs(self.zone)
self.update_metadata('zone')

def __parse_pq_tests_run(self):
"""
Extract the pq_tests_run bit mask from the metadata.

This is obtained by parsing the data quatlity statement from the XML
metadata. The statement is availble the attribute dq_statement, set
by read_metadata as specified in _scene_dataset.xml.
"""

if self.dq_statement:
bit_string_list = re.findall(r'\(Bit (\d+)\): (Run|Not Run)',
self.dq_statement)
pq_tests_run = 0
for (bit_no_str, bit_run_str) in bit_string_list:
bit_no = int(bit_no_str)
bit_val = 1 if bit_run_str == 'Run' else 0
pq_tests_run |= bit_val << bit_no
else:
pq_tests_run = None

return pq_tests_run

def __get_mtl_bias_gain(self):
'''
Expand Down Expand Up @@ -1004,9 +1041,10 @@ def __get_mtl_bias_gain_landsat8_lookup(self):

rad_rescale_params = self._metadata.get_metadata('MTL,L1_METADATA_FILE,RADIOMETRIC_RESCALING')

print
print rad_rescale_params
pprint(rad_rescale_params)
logger.debug("")
logger.debug("__get_mtl_bias_gain_landsat8_lookup:")
logger.debug(str(rad_rescale_params))
logger.debug(pformat(rad_rescale_params))

# Need to check if we have a valid return from the metadata lookup
if rad_rescale_params:
Expand All @@ -1026,13 +1064,14 @@ def __get_mtl_bias_gain_landsat8_lookup(self):
else:
logger.debug('No RADIOMETRIC_RESCALING data found in MTL file. Unable to compute bias & gain.')

print
print 'bdict'
pprint(bdict)
print
print 'gdict'
pprint(gdict)
print
logger.debug("")
logger.debug("bdict")
logger.debug(pformat(bdict))

logger.debug("")
logger.debug("gdict")
logger.debug(pformat(gdict))
logger.debug("")

self.bias = bdict
self.gain = gdict
Expand Down
14 changes: 14 additions & 0 deletions ULA3/dataset/_scene_dataset.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -395,15 +395,23 @@
<VARIABLE NAME="scene_start_datetime" TYPE="datetime" REQUIRED="False">
<METADATA KEY="XML,EODS_DATASET,EXEXTENT,TEMPORALEXTENTFROM" REGEX="\d{4}\d{2}\d{2}T\d{2}:\d{2}:\d{2}"
FORMAT="%Y%m%dT%H:%M:%S"/>
<METADATA KEY="XML,EODS_DATASET,EXEXTENT,TEMPORALEXTENTFROM" REGEX="\d{4}\d{2}\d{2} \d{2}:\d{2}:\d{2}"
FORMAT="%Y%m%d %H:%M:%S"/>
<METADATA KEY="XML,METADATA,EXTENT,ACQUISITION_FROM_DT" REGEX="\d{4}\d{2}\d{2}T\d{2}:\d{2}:\d{2}"
FORMAT="%Y%m%dT%H:%M:%S"/>
<METADATA KEY="XML,METADATA,EXTENT,ACQUISITION_FROM_DT" REGEX="\d{4}\d{2}\d{2} \d{2}:\d{2}:\d{2}"
FORMAT="%Y%m%d %H:%M:%S"/>
</VARIABLE>

<VARIABLE NAME="scene_end_datetime" TYPE="datetime" REQUIRED="False">
<METADATA KEY="XML,EODS_DATASET,EXEXTENT,TEMPORALEXTENTTO" REGEX="\d{4}\d{2}\d{2}T\d{2}:\d{2}:\d{2}"
FORMAT="%Y%m%dT%H:%M:%S"/>
<METADATA KEY="XML,EODS_DATASET,EXEXTENT,TEMPORALEXTENTTO" REGEX="\d{4}\d{2}\d{2} \d{2}:\d{2}:\d{2}"
FORMAT="%Y%m%d %H:%M:%S"/>
<METADATA KEY="XML,METADATA,EXTENT,ACQUISITION_END_DT" REGEX="\d{4}\d{2}\d{2}T\d{2}:\d{2}:\d{2}"
FORMAT="%Y%m%dT%H:%M:%S"/>
<METADATA KEY="XML,METADATA,EXTENT,ACQUISITION_END_DT" REGEX="\d{4}\d{2}\d{2} \d{2}:\d{2}:\d{2}"
FORMAT="%Y%m%d %H:%M:%S"/>
</VARIABLE>

<VARIABLE NAME="scene_centre_x" TYPE="float" REQUIRED="False">
Expand Down Expand Up @@ -525,8 +533,14 @@
<METADATA KEY="XML,METADATA,DATAQUALITY,DQELEMENT,QUANTATIVEVALUEUNIT"/>
</VARIABLE>

<VARIABLE NAME="dq_statement" TYPE="string" REQUIRED="False">
<METADATA KEY="XML,EODS_DATASET,DATAQUALITY,LINEAGE,STATEMENT"/>
<METADATA KEY="XML,METADATA,DATAQUALITY,LINEAGE,STATEMENT"/>
</VARIABLE>

<VARIABLE NAME="cloud_cover_percentage" TYPE="float" REQUIRED="False">
<METADATA KEY="XML,EODS_DATASET,IMAGEDESCRIPTION,CLOUDCOVERPERCENTAGE"/>
</VARIABLE>

</CLASS>
</GA_DATASET>