diff --git a/.travis.yml b/.travis.yml index 919ba54c..d938cac4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,15 @@ install: - pip install coveralls script: - - for i in $(ls -d */); do cd $i; pytest --cov $i || exit 1; cd ..; done + - for i in $(ls -d */); + do + cd $i; + pytest --cov $i || break -1; + if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then + flake8 -v $i || brake -1; + fi; + cd ..; + done after_success: # If coveralls.io is set up for this package, uncomment the line diff --git a/caom2/caom2/__init__.py b/caom2/caom2/__init__.py index 0284e27e..60e8d452 100644 --- a/caom2/caom2/__init__.py +++ b/caom2/caom2/__init__.py @@ -72,19 +72,19 @@ """ This library implements the Common Archive Observation Model (CAOM), a general -purpose data model for use as the core data model of an astronomical data centre. -The details about the model and its components can be found at: +purpose data model for use as the core data model of an astronomical data +centre. The details about the model and its components can be found at: http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/caom2/ """ -from .caom_util import * -from .common import * -from .wcs import * -from .shape import * -from .chunk import * -from .part import * -from .artifact import * -from .plane import * -from .observation import * -from .obs_reader_writer import * -from .checksum import * +from .artifact import * # noqa +from .caom_util import * # noqa +from .checksum import * # noqa +from .chunk import * # noqa +from .common import * # noqa +from .obs_reader_writer import * # noqa +from .observation import * # noqa +from .part import * # noqa +from .plane import * # noqa +from .shape import * # noqa +from .wcs import * # noqa diff --git a/caom2/caom2/artifact.py b/caom2/caom2/artifact.py index 5fb20275..a9ee55d4 100644 --- a/caom2/caom2/artifact.py +++ b/caom2/caom2/artifact.py @@ -74,10 +74,9 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -from six.moves.urllib.parse import urlparse -from builtins import str, int - from aenum import Enum +from builtins import str, int +from six.moves.urllib.parse import urlparse from . import caom_util from .chunk import ProductType @@ -135,7 +134,7 @@ def __init__(self, self.content_length = content_length self.content_checksum = content_checksum if parts is None: - parts = caom_util.TypedOrderedDict(Part,) + parts = caom_util.TypedOrderedDict(Part, ) self.parts = parts def _key(self): @@ -225,16 +224,16 @@ def content_length(self, value): caom_util.type_check(value, int, "content_length") caom_util.value_check(value, 0, 1E10, "content_length") self._content_length = value - + @property def content_checksum(self): """the checksum value for the artifact data - + type: ChecksumURI - + """ return self._content_checksum - + @content_checksum.setter def content_checksum(self, value): if value is None: diff --git a/caom2/caom2/caom_util.py b/caom2/caom2/caom_util.py index 3d3f3091..fef9c251 100644 --- a/caom2/caom2/caom_util.py +++ b/caom2/caom2/caom_util.py @@ -78,16 +78,17 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -from builtins import bytes, int -import six + import collections import struct import sys import uuid from datetime import datetime -__all__ = ['TypedList', 'TypedSet', 'TypedOrderedDict', 'ClassProperty'] +import six +from builtins import bytes, int +__all__ = ['TypedList', 'TypedSet', 'TypedOrderedDict', 'ClassProperty'] # TODO both these are very bad, implement more sensibly IVOA_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%f" @@ -95,10 +96,11 @@ class int_32(int): """ - The checksum algorithm must distinguished between 32 bit integers and 64 bit - integers. This subtype of int is used to tell the algorithm to use only 4 bytes - in the checksum of an attribute of this type. + The checksum algorithm must distinguished between 32 bit integers and 64 + bit integers. This subtype of int is used to tell the algorithm to use + only 4 bytes in the checksum of an attribute of this type. """ + def __new__(cls, *args, **kwargs): return int.__new__(cls, *args, **kwargs) @@ -174,7 +176,7 @@ def long2uuid(l): if l.bit_length() > 63: raise ValueError("expected 64 bit long {}".format(l)) if l < 0: - l = (1<<64) + l + l = (1 << 64) + l return uuid.UUID(int=l) @@ -206,8 +208,9 @@ def value_check(value, min_value, max_value, variable, override=None): """Check if value is inside allowed range, or override""" sys.tracebacklimit = None - if value != override and not ((min_value is not None) and (min_value <= value) and - (max_value is not None) and (value <= max_value)): + if value != override and not ( + (min_value is not None) and (min_value <= value) and + (max_value is not None) and (value <= max_value)): if override is not False: raise ValueError( "Expected {} <= {} <= {} or {}, received {}".format( @@ -389,7 +392,8 @@ def __str__(self): def __repr__(self): return "TypeOrderedDict((%r))," % self._oktypes + ( - "(".join(["(%r,%r)" % (k, v) for k, v in six.iteritems(self)]) + ")") + "(".join( + ["(%r,%r)" % (k, v) for k, v in six.iteritems(self)]) + ")") def check(self, key, value): """ @@ -426,6 +430,6 @@ def __setitem__(self, key, value): class ClassProperty(property): """ """ + def __get__(self, cls, owner): return self.fget.__get__(None, owner)() - diff --git a/caom2/caom2/checksum.py b/caom2/caom2/checksum.py index 75b1b549..2c90f229 100644 --- a/caom2/caom2/checksum.py +++ b/caom2/caom2/checksum.py @@ -69,56 +69,60 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -from builtins import bytes, int, str +import hashlib +import logging +import struct import uuid from datetime import datetime -import struct -import hashlib + from aenum import Enum -import logging +from builtins import bytes, int, str -from caom2.common import CaomObject, AbstractCaomEntity, ObservationURI, ChecksumURI from caom2.caom_util import TypedSet, TypedList, TypedOrderedDict, int_32 +from caom2.common import CaomObject, AbstractCaomEntity, ObservationURI, \ + ChecksumURI __all__ = ['get_meta_checksum', 'get_acc_meta_checksum'] - """ -This modules contains the functionality for calculating the checksums corresponding -to entites in CAOM2. Two types of checksums are available: - get_meta_checksum - returns the checksum of the entity attributes except its - CAOM2 children +This modules contains the functionality for calculating the checksums +corresponding to entites in CAOM2. Two types of checksums are available: + get_meta_checksum - returns the checksum of the entity attributes except + its CAOM2 children get_acc_meta_checksum - returns the accumulated checksum of the entity attributes including those of the CAOM2 children The checksums can be used to get the state of the entity: the metachecksum -represents the state the entity itself while the accumulated metachecksum represents -the state of the entity and all the children below. These states can be saved -and later compared to quickly detect state changes. +represents the state the entity itself while the accumulated metachecksum +represents the state of the entity and all the children below. These states +can be saved and later compared to quickly detect state changes. -IMPORTANT NODE: The checksum algorithms use introspection to automatically find the -attributes that are part of the model. It is therefore very important that attributes -that are not part of the CAOM model (http://www.opencadc.org/caom2) be prefixed with an '_' -so that the checksum algorithms ignore them in their computations. Equally important -is to use the names of the attributes from the model (with the Python syntax) as -the algorithms parses them in alphabetical order. +IMPORTANT NODE: The checksum algorithms use introspection to automatically +find the attributes that are part of the model. It is therefore very important +that attributes that are not part of the CAOM model +(http://www.opencadc.org/caom2) be prefixed with an '_' so that the checksum +algorithms ignore them in their computations. Equally important +is to use the names of the attributes from the model (with the Python syntax) +as the algorithms parses them in alphabetical order. Gotchas to look for when calculating the checksum: -- There are two types of integers in the model: int (32 bit) and long (64 bit). In this -implementation the corresponding types are int_32 and int. -- Sets are ordered alphabetical. Therefore their members have to implement, at the minimum, -__eq__ and __lt__ that will result in proper sorting +- There are two types of integers in the model: int (32 bit) and long (64 bit). +In this implementation the corresponding types are int_32 and int. +- Sets are ordered alphabetical. Therefore their members have to implement, +at the minimum, __eq__ and __lt__ that will result in proper sorting """ logger = logging.getLogger('checksum') + def get_meta_checksum(entity): """ Uses md5sum algorithm to calculate the checksum of the caom2_object excluding its CAOM2 children. calculation order: 1. id for entities - 2.state fields in alphabetic order; depth - first recursion so foo.abc.x comes before foo.def + 2.state fields in alphabetic order; depth - first recursion so foo.abc.x + comes before foo.def value handling: - Date: truncate time to whole number of seconds and treat as a long @@ -128,11 +132,11 @@ def get_meta_checksum(entity): - byte: as- is (1 byte) - integer: 9(8 bytes, network byte order == big endian) - unrecognized classes: encode their string representation - + :param entity: CAOM2 entity :return: md5 checksum corresponding to the entity metadata """ - assert(isinstance(entity, AbstractCaomEntity)) + assert (isinstance(entity, AbstractCaomEntity)) md5 = hashlib.md5() update_caom_checksum(md5, entity) return ChecksumURI('md5:{}'.format(md5.hexdigest())) @@ -142,7 +146,7 @@ def get_acc_meta_checksum(entity): """ Similar to get_meta_checksum except that the accumulated checksum of the CAOM2 children are also included in alphabetical order of their ids - + :param entity: CAOM2 entity :return: md5 checksum corresponding to the entity metadata """ @@ -154,11 +158,11 @@ def get_acc_meta_checksum(entity): def update_acc_checksum(checksum, entity): """ - Updates the checksum alogrithm with the bytes corresponding to the entity. It first generates - the corresponding bytes for the meta checksum of the entity. To that, it then adds the - bytes corresponding to the accumulated meta checksum of each child listed in alphabetical - order of their id. - + Updates the checksum alogrithm with the bytes corresponding to the entity. + It first generates the corresponding bytes for the meta checksum of the + entity. To that, it then adds the bytes corresponding to the accumulated + meta checksum of each child listed in alphabeticals order of their id. + :param checksum: checksum algorithm that consumes the bytes (md5) :param entity: entity to generate the bytes for and consume them """ @@ -171,8 +175,10 @@ def update_acc_checksum(checksum, entity): for i in dir(entity): if not callable(getattr(entity, i)) and not i.startswith('_'): attrib = getattr(entity, i) - if (isinstance(attrib, TypedOrderedDict) or isinstance(attrib, TypedList) or - isinstance(attrib, TypedSet)) and (issubclass(attrib.key_type, AbstractCaomEntity)): + if (isinstance(attrib, TypedOrderedDict) or + isinstance(attrib, TypedList) or + isinstance(attrib, TypedSet)) and ( + issubclass(attrib.key_type, AbstractCaomEntity)): if (isinstance(attrib, TypedOrderedDict)): values = attrib.values() else: @@ -187,11 +193,13 @@ def update_acc_checksum(checksum, entity): def update_checksum(checksum, value, attribute=''): """ - Updates the checksum algorithm (md5) of (mostly) native types with corresponding bytes - (network byte order == big endian) + Updates the checksum algorithm (md5) of (mostly) native types with + corresponding bytes (network byte order == big endian) :param checksum: the checksum algorithm to update (md5) - :param value: value to translate into bytes in order to update the checksum algorithm - :param attribute: name of the attribute this value belongs to (used for debugging only) + :param value: value to translate into bytes in order to update the + checksum algorithm + :param attribute: name of the attribute this value belongs to + (used for debugging only) """ if type(value) is None: @@ -205,20 +213,23 @@ def update_checksum(checksum, value, attribute=''): logger.debug('Encoded attribute {}'.format(attribute)) update_caom_checksum(checksum, value, attribute) elif isinstance(value, bytes): - logger.debug('Encoded attribute bytes {} = {}'.format(attribute, value)) + logger.debug( + 'Encoded attribute bytes {} = {}'.format(attribute, value)) checksum.update(value) elif isinstance(value, bool): logger.debug('Encoded attribute bool {} = {}'.format(attribute, value)) checksum.update(struct.pack('!?', value)) - #elif isinstance(value, float_32): + # elif isinstance(value, float_32): # must be before float - #checksum.update(struct.pack('!f', value)) + # checksum.update(struct.pack('!f', value)) elif isinstance(value, float): - logger.debug('Encoded attribute float {} = {}'.format(attribute, value)) + logger.debug( + 'Encoded attribute float {} = {}'.format(attribute, value)) checksum.update(struct.pack('!d', value)) elif isinstance(value, int_32): # must be before int - logger.debug('Encoded attribute int_32 {} = {}'.format(attribute, value)) + logger.debug( + 'Encoded attribute int_32 {} = {}'.format(attribute, value)) checksum.update(struct.pack('!l', value)) elif isinstance(value, int): logger.debug('Encoded attribute int {} = {}'.format(attribute, value)) @@ -227,10 +238,13 @@ def update_checksum(checksum, value, attribute=''): logger.debug('Encoded attribute str {} = {}'.format(attribute, value)) checksum.update(value.encode('utf-8')) elif isinstance(value, datetime): - logger.debug('Encoded attribute datetime {} = {}'.format(attribute, value)) - checksum.update(struct.pack('!q', int((value - datetime(1970, 1, 1)).total_seconds()))) + logger.debug( + 'Encoded attribute datetime {} = {}'.format(attribute, value)) + checksum.update(struct.pack('!q', int( + (value - datetime(1970, 1, 1)).total_seconds()))) elif isinstance(value, set) or \ - (isinstance(value, TypedSet) and not isinstance(value.key_type, AbstractCaomEntity)): + (isinstance(value, TypedSet) and not + isinstance(value.key_type, AbstractCaomEntity)): for i in sorted(value): update_checksum(checksum, i, attribute) elif isinstance(value, list) or isinstance(value, TypedList): @@ -243,7 +257,8 @@ def update_checksum(checksum, value, attribute=''): logger.debug('Encoded attribute uuid {} = {}'.format(attribute, value)) checksum.update(value.bytes) elif isinstance(value, TypedOrderedDict): - # calculate the checksum of each component and add them in alphabetical order of their ids + # calculate the checksum of each component and add them in + # alphabetical order of their ids # Note: ignore dictionaries of AbstractCaomEntity types checksums = [] for i in value: @@ -252,12 +267,14 @@ def update_checksum(checksum, value, attribute=''): for i in sorted(checksums): update_checksum(checksum, checksum[i], attribute) else: - raise ValueError('Cannot transform in bytes: {}({})'.format(value, type(value))) + raise ValueError( + 'Cannot transform in bytes: {}({})'.format(value, type(value))) -def update_caom_checksum(checksum, entity, parent = None): +def update_caom_checksum(checksum, entity, parent=None): """ - Method to go through the attributes of a CAOM class and update the checksum of each of them. + Method to go through the attributes of a CAOM class and update the + checksum of each of them. Uses introspection and goes through the attributes in alphabetical order. :param checksum: checksum algorithm to update (md5) :param entity: entity to go through @@ -272,12 +289,14 @@ def update_caom_checksum(checksum, entity, parent = None): if not hasattr(update_caom_checksum, "checksum_excluded_fields"): # this is a way to do it statically checksum_excluded_fields = [i for i in dir(AbstractCaomEntity) - if not callable(getattr(AbstractCaomEntity, i)) + if not callable( + getattr(AbstractCaomEntity, i)) and not i.startswith('_')] - # get the fields in alphabetical order, depth first recursion but remove the + # get the fields in alphabetical order, depth first recursion + # but remove the for i in sorted(dir(entity)): - if not callable(getattr(entity, i)) and not i.startswith('_') and\ - i not in checksum_excluded_fields: + if not callable(getattr(entity, i)) and not i.startswith('_') and \ + i not in checksum_excluded_fields: if getattr(entity, i) is not None: atrib = '{}.{}'.format(parent, i) if parent is not None else i update_checksum(checksum, getattr(entity, i), atrib) diff --git a/caom2/caom2/chunk.py b/caom2/caom2/chunk.py index 65dbef17..baa16347 100644 --- a/caom2/caom2/chunk.py +++ b/caom2/caom2/chunk.py @@ -73,14 +73,15 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -from builtins import str, int from aenum import Enum -from .common import AbstractCaomEntity -from .common import CaomObject +from builtins import str + +from caom2.caom_util import int_32 from . import caom_util from . import wcs -from caom2.caom_util import int_32 +from .common import AbstractCaomEntity +from .common import CaomObject class ProductType(Enum): @@ -103,6 +104,7 @@ class ProductType(Enum): AUXILIARY = "auxiliary" THUMBNAIL = "thumbnail" + __all__ = ['ProductType', 'Chunk', 'ObservableAxis', 'SpatialWCS', 'SpectralWCS', 'TemporalWCS', 'PolarizationWCS'] @@ -142,7 +144,6 @@ def __init__(self, product_type=None, observable_axis=None, observable=None, ): - super(Chunk, self).__init__() self.product_type = product_type self.naxis = naxis @@ -210,7 +211,7 @@ def position_axis_1(self): @position_axis_1.setter def position_axis_1(self, value): caom_util.type_check(value, int_32, 'position_axis_1') -# util.valueCheck(value, 0, self.naxis, 'position_axis_1') + # util.valueCheck(value, 0, self.naxis, 'position_axis_1') self._position_axis_1 = int_32(value) if value is not None else None @property @@ -229,7 +230,7 @@ def position_axis_2(self): @position_axis_2.setter def position_axis_2(self, value): caom_util.type_check(value, int_32, 'position_axis_2') -# util.valueCheck(value, 0, self.naxis, 'position_axis_2') + # util.valueCheck(value, 0, self.naxis, 'position_axis_2') self._position_axis_2 = int_32(value) if value is not None else None @property @@ -249,7 +250,7 @@ def energy_axis(self): @energy_axis.setter def energy_axis(self, value): caom_util.type_check(value, int_32, 'energy_axis') -# util.valueCheck(value, 0, self.naxis, 'energy_axis') + # util.valueCheck(value, 0, self.naxis, 'energy_axis') self._energy_axis = int_32(value) if value is not None else None @property @@ -267,7 +268,7 @@ def time_axis(self): @time_axis.setter def time_axis(self, value): caom_util.type_check(value, int_32, 'polarization_axis') -# util.valueCheck(value, 0, self._naxis, 'polarization_axis') + # util.valueCheck(value, 0, self._naxis, 'polarization_axis') self._time_axis = int_32(value) if value is not None else None @property @@ -285,7 +286,7 @@ def polarization_axis(self): @polarization_axis.setter def polarization_axis(self, value): caom_util.type_check(value, int_32, 'polarization_axis') - caom_util.value_check(value, 0, 2**32, 'polariztion_axis') + caom_util.value_check(value, 0, 2 ** 32, 'polariztion_axis') self._polarization_axis = int_32(value) if value is not None else None @property @@ -300,7 +301,7 @@ def observable_axis(self): @observable_axis.setter def observable_axis(self, value): caom_util.type_check(value, int_32, 'obserable_axis') -# util.valueCheck(value, 0, 1E10, 'observable_axis') + # util.valueCheck(value, 0, 1E10, 'observable_axis') self._observable_axis = int_32(value) if value is not None else None @property @@ -425,8 +426,8 @@ class ObservableAxis(CaomObject): """ - def __init__(self, dependent, independent=None): + def __init__(self, dependent, independent=None): self.dependent = dependent self.independent = independent @@ -474,7 +475,6 @@ def __init__(self, coordsys=None, equinox=None, resolution=None): - self.axis = axis self.coordsys = coordsys self.equinox = equinox @@ -794,7 +794,6 @@ def __init__(self, exposure=None, resolution=None ): - self.axis = axis self.timesys = timesys self.trefpos = trefpos diff --git a/caom2/caom2/common.py b/caom2/caom2/common.py index 6ec0fdb4..2646346b 100644 --- a/caom2/caom2/common.py +++ b/caom2/caom2/common.py @@ -73,8 +73,9 @@ import inspect import uuid from datetime import datetime + +from builtins import int, str from six.moves.urllib.parse import SplitResult, urlsplit -from builtins import int, str, bytes from . import caom_util @@ -88,9 +89,10 @@ def get_current_ivoa_time(): IVOA date format to millisecond precision. """ now = datetime.now() - return datetime(now.year, now.month, now.day, now.hour, now.minute, \ + return datetime(now.year, now.month, now.day, now.hour, now.minute, now.second, int(str(now.microsecond)[:-3] + '000')) + class CaomObject(object): """ setup all objects with the same generic equality, str and repr methods @@ -104,7 +106,7 @@ def __str__(self): class_name = self.__class__.__name__ return "\n".join(["{}.{} : {}". format(class_name, arg, getattr(self, arg, None)) - for arg in args]) + for arg in args]) def __eq__(self, other): if type(other) == type(self): @@ -129,6 +131,7 @@ class AbstractCaomEntity(CaomObject): """Class that defines the persistence unique ID and last mod date """ def __init__(self, fulluuid=False): + super(CaomObject, self).__init__() self._id = AbstractCaomEntity._gen_id(fulluuid) self.last_modified = None self.max_last_modified = None @@ -151,15 +154,17 @@ def _gen_id(cls, fulluuid=False): return gen_id else: return uuid.UUID(fields=(0x00000000, 0x0000, 0x0000, - gen_id.clock_seq_hi_variant, gen_id.clock_seq_low, gen_id.node)) + gen_id.clock_seq_hi_variant, + gen_id.clock_seq_low, gen_id.node)) def compute_meta_checksum(self): - raise NotImplementedError("meta checksum calculation not yet implemented.") + raise NotImplementedError( + "meta checksum calculation not yet implemented.") @property def last_modified(self): return self._last_modified - + @last_modified.setter def last_modified(self, value): if value is None: @@ -167,11 +172,11 @@ def last_modified(self, value): else: caom_util.type_check(value, datetime, "last_modified", False) self._last_modified = value - + @property def max_last_modified(self): return self._max_last_modified - + @max_last_modified.setter def max_last_modified(self, value): if value is None: @@ -179,16 +184,16 @@ def max_last_modified(self, value): else: caom_util.type_check(value, datetime, "max_last_modified", False) self._max_last_modified = value - + @property def meta_checksum(self): """the meta checksum value - + type: ChecksumURI - + """ return self._meta_checksum - + @meta_checksum.setter def meta_checksum(self, value): if value is None: @@ -196,22 +201,23 @@ def meta_checksum(self, value): else: caom_util.type_check(value, ChecksumURI, "meta_checksum", False) self._meta_checksum = value - + @property def acc_meta_checksum(self): """the accumulated meta checksum value - + type: ChecksumURI - + """ return self._acc_meta_checksum - + @acc_meta_checksum.setter def acc_meta_checksum(self, value): if value is None: self._acc_meta_checksum = None else: - caom_util.type_check(value, ChecksumURI, "acc_meta_checksum", False) + caom_util.type_check(value, ChecksumURI, "acc_meta_checksum", + False) self._acc_meta_checksum = value @@ -227,12 +233,13 @@ def __init__(self, uri): Arguments: uri : URI corresponding to observation """ + super(CaomObject, self).__init__() tmp = urlsplit(uri) if tmp.scheme != ObservationURI._SCHEME: raise ValueError( "uri must be have scheme of {}. received: {}" - .format(ObservationURI._SCHEME, uri)) + .format(ObservationURI._SCHEME, uri)) if tmp.geturl() != uri: raise ValueError( "uri parsing failure. received: {}".format(uri)) @@ -242,13 +249,14 @@ def __init__(self, uri): if collection is None: raise ValueError( "uri did not contain a collection part. received: {}" - .format(uri)) + .format(uri)) caom_util.validate_path_component(self, "collection", collection) if observation_id is None: raise ValueError( "uri did not contain an observation_id part. received: {}" - .format(uri)) - caom_util.validate_path_component(self, "observation_id", observation_id) + .format(uri)) + caom_util.validate_path_component(self, "observation_id", + observation_id) (self._collection, self._observation_id) = (collection, observation_id) self._print_attributes = ['uri', 'collection', 'observation_id'] @@ -260,12 +268,14 @@ def __hash__(self): def __lt__(self, other): if not isinstance(other, ObservationURI): - raise ValueError('Canot compare ObservationURI with {}'.format(type(other))) + raise ValueError( + 'Canot compare ObservationURI with {}'.format(type(other))) return self.uri < other.uri def __eq__(self, other): if not isinstance(other, ObservationURI): - raise ValueError('Canot compare ObservationURI with {}'.format(type(other))) + raise ValueError( + 'Canot compare ObservationURI with {}'.format(type(other))) return self.uri == other.uri @classmethod @@ -279,12 +289,15 @@ def get_observation_uri(cls, collection, observation_id): """ caom_util.type_check(collection, str, "collection", override=False) - caom_util.type_check(observation_id, str, "observation_id", override=False) + caom_util.type_check(observation_id, str, "observation_id", + override=False) caom_util.validate_path_component(cls, "collection", collection) - caom_util.validate_path_component(cls, "observation_id", observation_id) + caom_util.validate_path_component(cls, "observation_id", + observation_id) - uri = SplitResult(ObservationURI._SCHEME, "", collection + "/" + observation_id, + uri = SplitResult(ObservationURI._SCHEME, "", + collection + "/" + observation_id, "", "").geturl() return cls(uri) @@ -312,7 +325,8 @@ def collection(self): def observation_id(self): """The observation_id of this Observations uri""" return self._observation_id - + + class ChecksumURI(CaomObject): """ Checksum URI """ @@ -323,23 +337,26 @@ def __init__(self, uri): Arguments: uri : Checksum URI in the format Algorithm:ChecksumValue """ + super(CaomObject, self).__init__() tmp = urlsplit(uri) - + algorithm = tmp.scheme checksum = tmp.path - + if algorithm is None: raise ValueError( - "A checksum scheme noting the algorithm is required.. received: {}" - .format(uri)) - + ("A checksum scheme noting the algorithm is " + "required.. received: {}") + .format(uri)) + if checksum is None: raise ValueError( "checksum uri did not contain an checksum part. received: {}" - .format(uri)) + .format(uri)) caom_util.validate_path_component(self, "checksum", checksum) - - (self._uri, self._algorithm, self._checksum) = (tmp.geturl(), algorithm, checksum) + + (self._uri, self._algorithm, self._checksum) = ( + tmp.geturl(), algorithm, checksum) self._print_attributes = ['uri', 'algorithm', 'checksum'] def _key(self): @@ -371,7 +388,3 @@ def algorithm(self): def checksum(self): """The checksum value""" return self._checksum - - - - diff --git a/caom2/caom2/obs_reader_writer.py b/caom2/caom2/obs_reader_writer.py index fc09e920..ad5d749e 100644 --- a/caom2/caom2/obs_reader_writer.py +++ b/caom2/caom2/obs_reader_writer.py @@ -79,7 +79,6 @@ import six from six.moves.urllib.parse import urlparse - from lxml import etree from . import artifact @@ -92,7 +91,6 @@ from . import wcs from . import common - DATA_PKG = 'data' CAOM20_SCHEMA_FILE = 'CAOM-2.0.xsd' @@ -121,7 +119,8 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) -__all__ = ['ObservationReader', 'ObservationWriter', 'ObservationParsingException'] +__all__ = ['ObservationReader', 'ObservationWriter', + 'ObservationParsingException'] class ObservationReader(object): @@ -157,7 +156,7 @@ def __init__(self, validate=False): namespace=CAOM22_NAMESPACE, schemaLocation=CAOM22_SCHEMA_FILE) xsd.getroot().insert(2, caom22_schema) - + caom23_schema = etree.Element( '{http://www.w3.org/2001/XMLSchema}import', namespace=CAOM23_NAMESPACE, @@ -185,13 +184,17 @@ def _set_entity_attributes(self, element, ns, caom2_entity): caom2_entity._id = uid if element_last_modified: - caom2_entity._last_modified = caom_util.str2ivoa(element_last_modified) + caom2_entity._last_modified = caom_util.str2ivoa( + element_last_modified) if element_max_last_modified: - caom2_entity._max_last_modified = caom_util.str2ivoa(element_max_last_modified) + caom2_entity._max_last_modified = caom_util.str2ivoa( + element_max_last_modified) if element_meta_checksum: - caom2_entity._meta_checksum = common.ChecksumURI(element_meta_checksum) + caom2_entity._meta_checksum = common.ChecksumURI( + element_meta_checksum) if element_acc_meta_checksum: - caom2_entity._acc_meta_checksum = common.ChecksumURI(element_acc_meta_checksum) + caom2_entity._acc_meta_checksum = common.ChecksumURI( + element_acc_meta_checksum) def _get_child_element(self, element_tag, parent, ns, required): for element in list(parent): @@ -210,41 +213,45 @@ def _get_child_element(self, element_tag, parent, ns, required): return None def _get_child_text(self, element_tag, parent, ns, required): - child_element = self._get_child_element(element_tag, parent, ns, required) + child_element = self._get_child_element(element_tag, parent, ns, + required) if child_element is None: return None else: return str(child_element.text) def _get_child_text_as_int(self, element_tag, parent, ns, required): - child_element = self._get_child_element(element_tag, parent, ns, required) + child_element = self._get_child_element(element_tag, parent, ns, + required) if child_element is None: return None else: return int(child_element.text) def _get_child_text_as_long(self, element_tag, parent, ns, required): - child_element = self._get_child_element(element_tag, parent, ns, required) + child_element = self._get_child_element(element_tag, parent, ns, + required) if child_element is None: return None else: return int(child_element.text) def _get_child_text_as_float(self, element_tag, parent, ns, required): - child_element = self._get_child_element(element_tag, parent, ns, required) + child_element = self._get_child_element(element_tag, parent, ns, + required) if child_element is None: return None else: return float(child_element.text) - + def _get_child_text_as_boolean(self, element_tag, parent, ns, required): - child_element = self._get_child_element(element_tag, parent, ns, required) + child_element = self._get_child_element(element_tag, parent, ns, + required) if child_element is None: return None else: return child_element.text.lower() == "true" - def _add_keywords(self, keywords_list, element, ns, required): """ Parses keywords sub-elements and adds them to a list @@ -260,12 +267,13 @@ def _add_keywords(self, keywords_list, element, ns, required): for keyword in keywords.split(): keywords_list.add(keyword) else: - keywords_element = self._get_child_element("keywords", element, ns, required) + keywords_element = self._get_child_element("keywords", element, ns, + required) if keywords_element is not None: - for keyword in keywords_element.iterchildren(tag=("{" + ns + "}keyword")): + for keyword in keywords_element.iterchildren( + tag=("{" + ns + "}keyword")): keywords_list.add(keyword.text) - def _get_algorithm(self, element_tag, parent, ns, required): """Build an Algorithm object from an XML representation @@ -282,7 +290,8 @@ def _get_algorithm(self, element_tag, parent, ns, required): if el is None: return None else: - return observation.Algorithm(self._get_child_text("name", el, ns, True)) + return observation.Algorithm( + self._get_child_text("name", el, ns, True)) def _get_meta_release(self, element_tag, parent, ns, required): """Build a MetaRelease object from an XML representation @@ -320,7 +329,8 @@ def _get_proposal(self, element_tag, parent, ns, required): if el is None: return None else: - proposal = observation.Proposal(self._get_child_text("id", el, ns, True)) + proposal = observation.Proposal( + self._get_child_text("id", el, ns, True)) proposal.pi_name = self._get_child_text("pi", el, ns, False) proposal.project = self._get_child_text("project", el, ns, False) proposal.title = self._get_child_text("title", el, ns, False) @@ -343,7 +353,8 @@ def _get_target(self, element_tag, parent, ns, required): if el is None: return None else: - target = observation.Target(self._get_child_text("name", el, ns, True)) + target = observation.Target( + self._get_child_text("name", el, ns, True)) target_type = self._get_child_text("type", el, ns, False) if target_type: target.target_type = observation.TargetType(target_type) @@ -372,7 +383,8 @@ def _get_target_position(self, element_tag, parent, ns, required): if el is None: return None else: - coords_element = self._get_child_element("coordinates", el, ns, required) + coords_element = self._get_child_element("coordinates", el, ns, + required) target_position = observation.TargetPosition( self._get_point(coords_element, ns, True), self._get_child_text("coordsys", el, ns, True)) @@ -416,7 +428,8 @@ def _get_telescope(self, element_tag, parent, ns, required): if el is None: return None else: - telescope = observation.Telescope(self._get_child_text("name", el, ns, True)) + telescope = observation.Telescope( + self._get_child_text("name", el, ns, True)) telescope.geo_location_x = ( self._get_child_text_as_float("geoLocationX", el, ns, False)) telescope.geo_location_y = ( @@ -442,7 +455,8 @@ def _get_instrument(self, element_tag, parent, ns, required): if el is None: return None else: - instrument = observation.Instrument(self._get_child_text("name", el, ns, True)) + instrument = observation.Instrument( + self._get_child_text("name", el, ns, True)) self._add_keywords(instrument.keywords, el, ns, False) return instrument @@ -476,7 +490,8 @@ def _get_environment(self, element_tag, parent, ns, required): environment.ambient_temp = ( self._get_child_text_as_float("ambientTemp", el, ns, False)) environment.photometric = ("true" == - self._get_child_text("photometric", el, ns, False)) + self._get_child_text("photometric", el, + ns, False)) return environment def _add_members(self, members, parent, ns): @@ -494,7 +509,8 @@ def _add_members(self, members, parent, ns): """ el = self._get_child_element("members", parent, ns, False) if el is not None: - for member_element in el.iterchildren("{" + ns + "}observationURI"): + for member_element in el.iterchildren( + "{" + ns + "}observationURI"): members.add(observation.ObservationURI(member_element.text)) if not members: @@ -570,13 +586,16 @@ def _get_metrics(self, element_tag, parent, ns, required): else: metrics = plane.Metrics() metrics.source_number_density = \ - self._get_child_text_as_float("sourceNumberDensity", el, ns, False) + self._get_child_text_as_float("sourceNumberDensity", el, ns, + False) metrics.background = \ self._get_child_text_as_float("background", el, ns, False) metrics.background_std_dev = \ - self._get_child_text_as_float("backgroundStddev", el, ns, False) + self._get_child_text_as_float("backgroundStddev", el, ns, + False) metrics.flux_density_limit = \ - self._get_child_text_as_float("fluxDensityLimit", el, ns, False) + self._get_child_text_as_float("fluxDensityLimit", el, ns, + False) metrics.mag_limit = \ self._get_child_text_as_float("magLimit", el, ns, False) return metrics @@ -612,8 +631,9 @@ def _get_point(self, point, ns, required): return : an Point object raise : ObservationParsingException """ - return shape.Point(self._get_child_text_as_float("cval1", point, ns, True), - self._get_child_text_as_float("cval2", point, ns, True)) + return shape.Point( + self._get_child_text_as_float("cval1", point, ns, True), + self._get_child_text_as_float("cval2", point, ns, True)) def _get_axis(self, element_tag, parent, ns, required): """Build an Axis object from an XML representation of an Axis element. @@ -831,9 +851,9 @@ def _get_coord_polygon2d(self, element_tag, parent, ns, required): for child_vertex_el in children_vertices: polygon.vertices.append(wcs.ValueCoord2D( self._get_child_text_as_float( - "coord1", child_vertex_el, ns, True), + "coord1", child_vertex_el, ns, True), self._get_child_text_as_float( - "coord2", child_vertex_el, ns, True))) + "coord2", child_vertex_el, ns, True))) return polygon def _get_coord_bounds2d(self, element_tag, parent, ns, required): @@ -861,8 +881,8 @@ def _get_coord_bounds2d(self, element_tag, parent, ns, required): if polygon is not None: return polygon else: - error = "Unsupported element not found in " + element_tag + \ - ": " + el.getText() + error = "Unsupported element not found in " + \ + element_tag + ": " + el.getText() raise ObservationParsingException(error) def _get_dimension2d(self, element_tag, parent, ns, required): @@ -934,7 +954,8 @@ def _get_coord_axis2d(self, element_tag, parent, ns, required): axis.error2 = self._get_coord_error("error2", el, ns, False) axis.range = self._get_coord_range2d("range", el, ns, False) axis.bounds = self._get_coord_bounds2d("bounds", el, ns, False) - axis.function = self._get_coord_function2d("function", el, ns, False) + axis.function = self._get_coord_function2d("function", el, ns, + False) return axis def _get_spatial_wcs(self, element_tag, parent, ns, required): @@ -954,7 +975,8 @@ def _get_spatial_wcs(self, element_tag, parent, ns, required): if el is None: return None else: - position = chunk.SpatialWCS(self._get_coord_axis2d("axis", el, ns, False)) + position = chunk.SpatialWCS( + self._get_coord_axis2d("axis", el, ns, False)) position.coordsys = self._get_child_text("coordsys", el, ns, False) position.equinox = \ self._get_child_text_as_float("equinox", el, ns, False) @@ -962,8 +984,8 @@ def _get_spatial_wcs(self, element_tag, parent, ns, required): self._get_child_text_as_float("resolution", el, ns, False) return position - def _add_children_to_coord_range1d_list(self, element_tag, ranges, parent, ns, - required): + def _add_children_to_coord_range1d_list(self, element_tag, ranges, parent, + ns, required): """Create CoordRange1D objects from an XML representation of the range elements and add them to the set of ranges. @@ -976,8 +998,8 @@ def _add_children_to_coord_range1d_list(self, element_tag, ranges, parent, ns, """ for range_element in parent.iterchildren("{" + ns + "}" + element_tag): ranges.append(wcs.CoordRange1D( - self._get_ref_coord("start", range_element, ns, True), - self._get_ref_coord("end", range_element, ns, True))) + self._get_ref_coord("start", range_element, ns, True), + self._get_ref_coord("end", range_element, ns, True))) def _get_coord_bounds1d(self, element_tag, parent, ns, required): """Build a CoordBounds1D object from an XML representation of a bounds @@ -1000,7 +1022,8 @@ def _get_coord_bounds1d(self, element_tag, parent, ns, required): samples_element = self._get_child_element("samples", el, ns, False) if samples_element is not None: self._add_children_to_coord_range1d_list( - "range", coord_bounds1d.samples, samples_element, ns, False) + "range", coord_bounds1d.samples, samples_element, ns, + False) return coord_bounds1d def _get_coord_range1d(self, element_tag, parent, ns, required): @@ -1067,7 +1090,8 @@ def _get_coord_axis1d(self, element_tag, parent, ns, required): axis.error = self._get_coord_error("error", el, ns, False) axis.range = self._get_coord_range1d("range", el, ns, False) axis.bounds = self._get_coord_bounds1d("bounds", el, ns, False) - axis.function = self._get_coord_function1d("function", el, ns, False) + axis.function = self._get_coord_function1d("function", el, ns, + False) return axis def _get_transition(self, element_tag, parent, ns, required): @@ -1183,11 +1207,11 @@ def _get_polarization_wcs(self, element_tag, parent, ns, required): else: return chunk.PolarizationWCS( self._get_coord_axis1d("axis", el, ns, False)) - + def _get_position(self, element_tag, parent, ns, required): """Build a Position object from an XML representation of a position element. - + Arguments: elTag : element tag which identifies the element parent : element containing the position element @@ -1203,15 +1227,18 @@ def _get_position(self, element_tag, parent, ns, required): pos = plane.Position() pos.bounds = self._get_shape("bounds", el, ns, False) pos.dimension = self._get_dimension2d("dimension", el, ns, False) - pos.resolution = self._get_child_text_as_float("resolution", el, ns, False) - pos.sample_size = self._get_child_text_as_float("sampleSize", el, ns, False) - pos.time_dependent = self._get_child_text_as_boolean("timeDependent", el, ns, False) + pos.resolution = self._get_child_text_as_float("resolution", el, ns, + False) + pos.sample_size = self._get_child_text_as_float("sampleSize", el, ns, + False) + pos.time_dependent = self._get_child_text_as_boolean("timeDependent", + el, ns, False) return pos - + def _get_energy(self, element_tag, parent, ns, required): """Build an Energy object from an XML representation of an energy element. - + Arguments: elTag : element tag which identifies the element parent : element containing the position element @@ -1226,26 +1253,33 @@ def _get_energy(self, element_tag, parent, ns, required): return None energy = plane.Energy() energy.bounds = self._get_interval("bounds", el, ns, False) - energy.dimension = self._get_child_text_as_int("dimension", el, ns, False) - energy.resolving_power = self._get_child_text_as_float("resolvingPower", el, ns, False) - energy.sample_size = self._get_child_text_as_float("sampleSize", el, ns, False) - energy.bandpass_name = self._get_child_text("bandpassName", el, ns, False) + energy.dimension = \ + self._get_child_text_as_int("dimension", el, ns, False) + energy.resolving_power = self._get_child_text_as_float( + "resolvingPower", el, ns, False) + energy.sample_size = \ + self._get_child_text_as_float("sampleSize", el, ns, False) + energy.bandpass_name = \ + self._get_child_text("bandpassName", el, ns, False) em_band = self._get_child_text("emBand", el, ns, False) if em_band: energy.em_band = plane.EnergyBand(em_band) - energy.restwav = self._get_child_text_as_float("restwav", el, ns, False) - _transition_el = self._get_child_element("transition", el, ns, required) + energy.restwav = \ + self._get_child_text_as_float("restwav", el, ns, False) + _transition_el = \ + self._get_child_element("transition", el, ns, required) if _transition_el is not None: species = self._get_child_text("species", _transition_el, ns, True) - transition = self._get_child_text("transition", _transition_el, ns, True) + transition = \ + self._get_child_text("transition", _transition_el, ns, True) energy.transition = wcs.EnergyTransition(species, transition) - + return energy def _get_time(self, element_tag, parent, ns, required): """Build a Time object from an XML representation of a time element. - + Arguments: elTag : element tag which identifies the element parent : element containing the position element @@ -1260,17 +1294,21 @@ def _get_time(self, element_tag, parent, ns, required): return None time = plane.Time() time.bounds = self._get_interval("bounds", el, ns, False) - time.dimension = self._get_child_text_as_int("dimension", el, ns, False) - time.resolution = self._get_child_text_as_float("resolution", el, ns, False) - time.sample_size = self._get_child_text_as_float("sampleSize", el, ns, False) - time.exposure = self._get_child_text_as_float("exposure", el, ns, False) - + time.dimension = \ + self._get_child_text_as_int("dimension", el, ns, False) + time.resolution = \ + self._get_child_text_as_float("resolution", el, ns, False) + time.sample_size = \ + self._get_child_text_as_float("sampleSize", el, ns, False) + time.exposure = \ + self._get_child_text_as_float("exposure", el, ns, False) + return time def _get_polarization(self, element_tag, parent, ns, required): """Build a Polarization object from an XML representation of a polarization element. - + Arguments: elTag : element tag which identifies the element parent : element containing the position element @@ -1292,55 +1330,73 @@ def _get_polarization(self, element_tag, parent, ns, required): _polarization_state = plane.PolarizationState(_pstate) _polarization_states.append(_polarization_state) polarization.polarization_states = _polarization_states - polarization.dimension = self._get_child_text_as_int("dimension", el, ns, False) - + polarization.dimension = self._get_child_text_as_int("dimension", el, + ns, False) + return polarization - + def _get_shape(self, element_tag, parent, ns, required): - shape_element = self._get_child_element(element_tag, parent, ns, required) + shape_element = self._get_child_element(element_tag, parent, ns, + required) if shape_element is None: return None shape_type = shape_element.get(XSI + "type") if "caom2:Polygon" == shape_type: if self.version < 23: - raise TypeError("Polygon element not supported for CAOM releases prior to 2.3") - points_element = self._get_child_element("points", shape_element, ns, True) + raise TypeError( + ("Polygon element not supported for " + "CAOM releases prior to 2.3")) + points_element = self._get_child_element("points", shape_element, + ns, True) points = list() - for point in points_element.iterchildren(tag=("{" + ns + "}point")): + for point in points_element.iterchildren( + tag=("{" + ns + "}point")): points.append(self._get_point(point, ns, True)) - samples_element = self._get_child_element("samples", shape_element, ns, True) + samples_element = self._get_child_element("samples", shape_element, + ns, True) vertices = list() self._add_vertices(vertices, samples_element, ns) - return shape.Polygon(points=points, samples=shape.MultiPolygon(vertices=vertices)) + return shape.Polygon(points=points, + samples=shape.MultiPolygon(vertices=vertices)) else: raise TypeError("Unsupported shape type " + shape_type) - + def _add_vertices(self, vertices, parent, ns): - _vertices_element = self._get_child_element("vertices", parent, ns, False) + _vertices_element = self._get_child_element("vertices", parent, ns, + False) if _vertices_element is None: return None else: - for _vertex_element in _vertices_element.iterchildren("{" + ns + "}vertex"): - cval1 = self._get_child_text_as_float("cval1", _vertex_element, ns, False) - cval2 = self._get_child_text_as_float("cval2", _vertex_element, ns, False) - seg_type_value = self._get_child_text_as_int("type", _vertex_element, ns, False) + for _vertex_element in _vertices_element.iterchildren( + "{" + ns + "}vertex"): + cval1 = self._get_child_text_as_float("cval1", _vertex_element, + ns, False) + cval2 = self._get_child_text_as_float("cval2", _vertex_element, + ns, False) + seg_type_value = self._get_child_text_as_int("type", + _vertex_element, + ns, False) seg_type = shape.SegmentType(seg_type_value) _vertex = shape.Vertex(cval1, cval2, seg_type) vertices.append(_vertex) - + def _get_interval(self, element_tag, parent, ns, required): - _interval_el = self._get_child_element(element_tag, parent, ns, required) + _interval_el = self._get_child_element(element_tag, parent, ns, + required) if _interval_el is None: return None _lower = self._get_child_text_as_float("lower", _interval_el, ns, True) _upper = self._get_child_text_as_float("upper", _interval_el, ns, True) - _samples_el = self._get_child_element("samples", _interval_el, ns, required) + _samples_el = self._get_child_element("samples", _interval_el, ns, + required) _interval = shape.Interval(_lower, _upper) if _samples_el is not None: _samples = list() for _sample_el in _samples_el.iterchildren("{" + ns + "}sample"): - _si_lower = self._get_child_text_as_float("lower", _sample_el, ns, required) - _si_upper = self._get_child_text_as_float("upper", _sample_el, ns, required) + _si_lower = self._get_child_text_as_float("lower", _sample_el, + ns, required) + _si_upper = self._get_child_text_as_float("upper", _sample_el, + ns, required) _sub_interval = shape.SubInterval(_si_lower, _si_upper) _samples.append(_sub_interval) _interval.samples = _samples @@ -1363,30 +1419,35 @@ def _add_chunks(self, chunks, parent, ns): for chunk_element in el.iterchildren("{" + ns + "}chunk"): _chunk = chunk.Chunk() product_type = \ - self._get_child_text("productType", chunk_element, ns, False) + self._get_child_text("productType", chunk_element, ns, + False) if product_type: _chunk.product_type = \ chunk.ProductType(product_type) _chunk.naxis = \ - self._get_child_text_as_int("naxis", chunk_element, ns, False) - _chunk.observable_axis = \ - self._get_child_text_as_int("observableAxis", chunk_element, ns, + self._get_child_text_as_int("naxis", chunk_element, ns, False) + _chunk.observable_axis = \ + self._get_child_text_as_int("observableAxis", + chunk_element, ns, False) _chunk.position_axis_1 = \ - self._get_child_text_as_int("positionAxis1", chunk_element, ns, - False) + self._get_child_text_as_int("positionAxis1", chunk_element, + ns, False) _chunk.position_axis_2 = \ - self._get_child_text_as_int("positionAxis2", chunk_element, ns, - False) + self._get_child_text_as_int("positionAxis2", chunk_element, + ns, False) _chunk.energy_axis = \ - self._get_child_text_as_int("energyAxis", chunk_element, ns, False) + self._get_child_text_as_int("energyAxis", chunk_element, + ns, False) _chunk.time_axis = \ - self._get_child_text_as_int("timeAxis", chunk_element, ns, False) - _chunk.polarization_axis = \ - self._get_child_text_as_int("polarizationAxis", chunk_element, ns, + self._get_child_text_as_int("timeAxis", chunk_element, ns, False) + _chunk.polarization_axis = \ + self._get_child_text_as_int("polarizationAxis", + chunk_element, ns, False) _chunk.observable = \ - self._get_observable_axis("observable", chunk_element, ns, False) + self._get_observable_axis("observable", chunk_element, ns, + False) _chunk.position = \ self._get_spatial_wcs("position", chunk_element, ns, False) _chunk.energy = \ @@ -1394,8 +1455,8 @@ def _add_chunks(self, chunks, parent, ns): _chunk.time = \ self._get_temporal_wcs("time", chunk_element, ns, False) _chunk.polarization = \ - self._get_polarization_wcs("polarization", chunk_element, ns, - False) + self._get_polarization_wcs("polarization", chunk_element, + ns, False) self._set_entity_attributes(chunk_element, ns, _chunk) chunks.append(_chunk) @@ -1415,9 +1476,11 @@ def _add_parts(self, parts, parent, ns): else: for part_element in el.iterchildren("{" + ns + "}part"): _part = \ - part.Part(self._get_child_text("name", part_element, ns, True)) + part.Part( + self._get_child_text("name", part_element, ns, True)) product_type = \ - self._get_child_text("productType", part_element, ns, False) + self._get_child_text("productType", part_element, ns, + False) if product_type: _part.product_type = \ chunk.ProductType(product_type) @@ -1442,27 +1505,41 @@ def _add_artifacts(self, artifacts, parent, ns): for artifact_element in el.iterchildren("{" + ns + "}artifact"): uri = self._get_child_text("uri", artifact_element, ns, True) - product_type = self._get_child_text("productType", artifact_element, ns, False) + product_type = self._get_child_text("productType", + artifact_element, ns, + False) if product_type is None: product_type = chunk.ProductType.SCIENCE - print("Using default Artifact.productType value {0}".format(str(chunk.ProductType.SCIENCE))) + print( + "Using default Artifact.productType value {0}".format( + str(chunk.ProductType.SCIENCE))) else: product_type = chunk.ProductType(product_type) - release_type = self._get_child_text("releaseType", artifact_element, ns, False) + release_type = self._get_child_text("releaseType", + artifact_element, ns, + False) if release_type is None: release_type = artifact.ReleaseType.DATA - print("Using default Artifact.releaseType value {0}".format(str(artifact.ReleaseType.DATA))) + print( + "Using default Artifact.releaseType value {0}".format( + str(artifact.ReleaseType.DATA))) else: release_type = artifact.ReleaseType(release_type) _artifact = artifact.Artifact(uri, product_type, release_type) - _artifact.content_type = self._get_child_text("contentType", artifact_element, ns, False) + _artifact.content_type = self._get_child_text("contentType", + artifact_element, + ns, False) _artifact.content_length = ( - self._get_child_text_as_long("contentLength", artifact_element, ns, False)) - content_checksum = self._get_child_text("contentChecksum", artifact_element, ns, False) + self._get_child_text_as_long("contentLength", + artifact_element, ns, False)) + content_checksum = self._get_child_text("contentChecksum", + artifact_element, ns, + False) if content_checksum: - _artifact.content_checksum = common.ChecksumURI(content_checksum) + _artifact.content_checksum = common.ChecksumURI( + content_checksum) self._add_parts(_artifact.parts, artifact_element, ns) self._set_entity_attributes(artifact_element, ns, _artifact) artifacts[_artifact.uri] = _artifact @@ -1485,39 +1562,51 @@ def _add_planes(self, planes, parent, ns): _plane = plane.Plane( self._get_child_text("productID", plane_element, ns, True)) _plane.meta_release = caom_util.str2ivoa( - self._get_child_text("metaRelease", plane_element, ns, False)) + self._get_child_text("metaRelease", plane_element, ns, + False)) _plane.data_release = caom_util.str2ivoa( - self._get_child_text("dataRelease", plane_element, ns, False)) + self._get_child_text("dataRelease", plane_element, ns, + False)) data_product_type = \ - self._get_child_text("dataProductType", plane_element, ns, False) + self._get_child_text("dataProductType", plane_element, ns, + False) if data_product_type: - if (data_product_type == 'catalog') and (self.version < 23): + if (data_product_type == 'catalog') and \ + (self.version < 23): # TODO backawards compatibility. To be removed when 2.2 # and older version no longer supported _plane.data_product_type = \ - plane.DataProductType('{}#{}'.format(plane._CAOM_VOCAB_NS, data_product_type)) + plane.DataProductType( + '{}#{}'.format(plane._CAOM_VOCAB_NS, + data_product_type)) else: _plane.data_product_type = \ plane.DataProductType(data_product_type) _plane.creator_id = \ self._get_child_text("creatorID", plane_element, ns, False) calibration_level = \ - self._get_child_text("calibrationLevel", plane_element, ns, False) + self._get_child_text("calibrationLevel", plane_element, ns, + False) if calibration_level: _plane.calibration_level = \ plane.CalibrationLevel(int(calibration_level)) _plane.provenance = \ - self._get_provenance("provenance", plane_element, ns, False) + self._get_provenance("provenance", plane_element, ns, + False) _plane.metrics = \ self._get_metrics("metrics", plane_element, ns, False) _plane.quality = \ self._get_quality("quality", plane_element, ns, False) - - _plane.position = self._get_position("position", plane_element, ns, False) - _plane.energy = self._get_energy("energy", plane_element, ns, False) + + _plane.position = self._get_position("position", plane_element, + ns, False) + _plane.energy = self._get_energy("energy", plane_element, ns, + False) _plane.time = self._get_time("time", plane_element, ns, False) - _plane.polarization = self._get_polarization("polarization", plane_element, ns, False) - + _plane.polarization = self._get_polarization("polarization", + plane_element, ns, + False) + self._add_artifacts(_plane.artifacts, plane_element, ns) self._set_entity_attributes(plane_element, ns, _plane) planes[_plane.product_id] = _plane @@ -1536,14 +1625,15 @@ def read(self, source): return : an Observation object raise : ObservationParsingException """ - - doc = etree.parse(source) + + doc = etree.parse(source) if self._validate and self._xmlschema: self._xmlschema.assertValid(doc) root = doc.getroot() ns = root.nsmap["caom2"] self.version = CAOM_VERSION[ns] - collection = str(self._get_child_element("collection", root, ns, True).text) + collection = str( + self._get_child_element("collection", root, ns, True).text) observation_id = \ str(self._get_child_element("observationID", root, ns, True).text) # Instantiate Algorithm @@ -1555,7 +1645,8 @@ def read(self, source): obs.algorithm = algorithm else: obs = \ - observation.CompositeObservation(collection, observation_id, algorithm) + observation.CompositeObservation(collection, observation_id, + algorithm) # Instantiate children of Observation obs.sequence_number = \ self._get_child_text_as_int("sequenceNumber", root, ns, False) @@ -1596,8 +1687,8 @@ def __init__(self, validate=False, write_empty_collections=False, """ Arguments: validate : If True enable schema validation, False otherwise - write_empty_collections : if True write empty elements for empty collections - namespace_prefix : a CAOM-2.x namespace prefix + write_empty_collections : if True write empty elements for empty + collections namespace_prefix : a CAOM-2.x namespace prefix namespace : a valid CAOM-2.x target namespace """ self._validate = validate @@ -1647,7 +1738,8 @@ def write(self, obs, out): assert isinstance(obs, observation.Observation), ( "observation is not an Observation") - obs_element = etree.Element(self._caom2_namespace + "Observation", nsmap=self._nsmap) + obs_element = etree.Element(self._caom2_namespace + "Observation", + nsmap=self._nsmap) if isinstance(obs, observation.SimpleObservation): obs_element.set(XSI + "type", "caom2:SimpleObservation") else: @@ -1657,13 +1749,14 @@ def write(self, obs, out): self._add_element("collection", obs.collection, obs_element) self._add_element("observationID", obs.observation_id, obs_element) - self._add_datetime_element("metaRelease", obs.meta_release, obs_element) + self._add_datetime_element("metaRelease", obs.meta_release, + obs_element) self._add_element("sequenceNumber", obs.sequence_number, obs_element) self._add_algorithm_element(obs.algorithm, obs_element) self._add_element("type", obs.type, obs_element) if obs.intent is not None: self._add_element( - "intent",obs.intent.value, obs_element) + "intent", obs.intent.value, obs_element) self._add_proposal_element(obs.proposal, obs_element) self._add_target_element(obs.target, obs_element) @@ -1680,7 +1773,8 @@ def write(self, obs, out): if self._validate and self._xmlschema: self._xmlschema.assertValid(obs_element) - doc = etree.tostring(obs_element, encoding='UTF-8', xml_declaration=True, pretty_print=True) + doc = etree.tostring(obs_element, encoding='UTF-8', + xml_declaration=True, pretty_print=True) out.write(doc) out.flush() @@ -1693,12 +1787,14 @@ def _add_entity_attributes(self, entity, element): if entity._last_modified is not None: self._add_attribute( - "lastModified", caom_util.date2ivoa(entity._last_modified), element) - + "lastModified", caom_util.date2ivoa(entity._last_modified), + element) + if self._output_version >= 23: if entity._max_last_modified is not None: self._add_attribute( - "maxLastModified", caom_util.date2ivoa(entity._max_last_modified), element) + "maxLastModified", + caom_util.date2ivoa(entity._max_last_modified), element) if entity._meta_checksum is not None: self._add_attribute( "metaChecksum", entity._meta_checksum.uri, element) @@ -1733,7 +1829,8 @@ def _add_target_element(self, target, parent): if target.target_type is not None: self._add_element("type", target.target_type.value, element) if target.standard is not None: - self._add_element("standard", str(target.standard).lower(), element) + self._add_element("standard", str(target.standard).lower(), + element) self._add_element("redshift", target.redshift, element) if target.moving is not None: self._add_element("moving", str(target.moving).lower(), element) @@ -1815,7 +1912,8 @@ def _add_planes_element(self, planes, parent): self._add_entity_attributes(_plane, plane_element) self._add_element("productID", _plane.product_id, plane_element) if self._output_version >= 23: - self._add_element("creatorID", _plane.creator_id, plane_element) + self._add_element("creatorID", _plane.creator_id, + plane_element) self._add_datetime_element("metaRelease", _plane.meta_release, plane_element) self._add_datetime_element("dataRelease", _plane.data_release, @@ -1841,30 +1939,32 @@ def _add_planes_element(self, planes, parent): self._add_provenance_element(_plane.provenance, plane_element) self._add_metrics_element(_plane.metrics, plane_element) self._add_quality_element(_plane.quality, plane_element) - + if self._output_version >= 22: self._add_position_element(_plane.position, plane_element) self._add_energy_element(_plane.energy, plane_element) self._add_time_element(_plane.time, plane_element) - self._add_polarization_element(_plane.polarization, plane_element) + self._add_polarization_element(_plane.polarization, + plane_element) self._add_artifacts_element(_plane.artifacts, plane_element) - + def _add_position_element(self, position, parent): if position is None: return - + element = self._get_caom_element("position", parent) self._add_shape_element("bounds", position.bounds, element) self._add_dimension2d_element("dimension", position.dimension, element) self._add_element("resolution", position.resolution, element) self._add_element("sampleSize", position.sample_size, element) - self._add_element("timeDependent", str(position.time_dependent).lower(), element) - + self._add_element("timeDependent", + str(position.time_dependent).lower(), element) + def _add_energy_element(self, energy, parent): if energy is None: return - + element = self._get_caom_element("energy", parent) self._add_interval_element("bounds", energy.bounds, element) self._add_element("dimension", energy.dimension, element) @@ -1876,36 +1976,38 @@ def _add_energy_element(self, energy, parent): if energy.transition: transition = self._get_caom_element("transition", element) self._add_element("species", energy.transition.species, transition) - self._add_element("transition", energy.transition.transition, transition) - + self._add_element("transition", energy.transition.transition, + transition) + def _add_time_element(self, time, parent): if time is None: return - + element = self._get_caom_element("time", parent) self._add_interval_element("bounds", time.bounds, element) self._add_element("dimension", time.dimension, element) self._add_element("resolution", time.resolution, element) self._add_element("sampleSize", time.sample_size, element) self._add_element("exposure", time.exposure, element) - + def _add_polarization_element(self, polarization, parent): if polarization is None: return - + element = self._get_caom_element("polarization", parent) if polarization.polarization_states: _pstates_el = self._get_caom_element("states", element) for _state in polarization.polarization_states: - self._add_element("state", _state.value, _pstates_el) - self._add_element("dimension", polarization.dimension, element) - + self._add_element("state", _state.value, _pstates_el) + self._add_element("dimension", polarization.dimension, element) + def _add_shape_element(self, name, the_shape, parent): if the_shape is None: return if self._output_version < 23: - raise TypeError('Polygon shape not supported in CAOM2 previous to v2.3') - + raise TypeError( + 'Polygon shape not supported in CAOM2 previous to v2.3') + if isinstance(the_shape, shape.Polygon): shape_element = self._get_caom_element(name, parent) shape_element.set(XSI + "type", "caom2:Polygon") @@ -1913,16 +2015,18 @@ def _add_shape_element(self, name, the_shape, parent): for point in the_shape.points: self._add_point_element("point", point, points_element) samples_element = self._get_caom_element("samples", shape_element) - vertices_element = self._get_caom_element("vertices", samples_element) + vertices_element = self._get_caom_element("vertices", + samples_element) for vertex in the_shape.samples.vertices: - vertex_element = self._get_caom_element("vertex", vertices_element) + vertex_element = self._get_caom_element("vertex", + vertices_element) self._add_element("cval1", vertex.cval1, vertex_element) self._add_element("cval2", vertex.cval2, vertex_element) self._add_element("type", vertex.type.value, vertex_element) else: raise TypeError("Unsupported shape type " - + the_shape.__class__.__name__) - + + the_shape.__class__.__name__) + def _add_interval_element(self, name, interval, parent): if interval is None: return @@ -1931,12 +2035,14 @@ def _add_interval_element(self, name, interval, parent): self._add_element("lower", interval.lower, _interval_element) self._add_element("upper", interval.upper, _interval_element) if interval.samples: - _samples_element = self._get_caom_element("samples", _interval_element) + _samples_element = self._get_caom_element("samples", + _interval_element) for _sample in interval.samples: - _sample_element = self._get_caom_element("sample", _samples_element) + _sample_element = self._get_caom_element("sample", + _samples_element) self._add_element("lower", _sample.lower, _sample_element) self._add_element("upper", _sample.upper, _sample_element) - + def _add_provenance_element(self, provenance, parent): if provenance is None: return @@ -1994,15 +2100,22 @@ def _add_artifacts_element(self, artifacts, parent): self._add_entity_attributes(_artifact, artifact_element) self._add_element("uri", _artifact.uri, artifact_element) if self._output_version > 21: - self._add_element("productType", _artifact.product_type.value, artifact_element) - self._add_element("releaseType", _artifact.release_type.value, artifact_element) - self._add_element("contentType", _artifact.content_type, artifact_element) - self._add_element("contentLength", _artifact.content_length, artifact_element) + self._add_element("productType", _artifact.product_type.value, + artifact_element) + self._add_element("releaseType", _artifact.release_type.value, + artifact_element) + self._add_element("contentType", _artifact.content_type, + artifact_element) + self._add_element("contentLength", _artifact.content_length, + artifact_element) if self._output_version > 22: if _artifact.content_checksum: - self._add_element("contentChecksum", _artifact.content_checksum.uri, artifact_element) + self._add_element("contentChecksum", + _artifact.content_checksum.uri, + artifact_element) if self._output_version < 22: - self._add_element("productType", _artifact.product_type.value, artifact_element) + self._add_element("productType", _artifact.product_type.value, + artifact_element) self._add_parts_element(_artifact.parts, artifact_element) def _add_parts_element(self, parts, parent): @@ -2015,7 +2128,8 @@ def _add_parts_element(self, parts, parent): self._add_entity_attributes(_part, part_element) self._add_element("name", _part.name, part_element) if _part.product_type is not None: - self._add_element("productType", _part.product_type.value, part_element) + self._add_element("productType", _part.product_type.value, + part_element) self._add_chunks_element(_part.chunks, part_element) def _add_chunks_element(self, chunks, parent): @@ -2046,7 +2160,8 @@ def _add_chunks_element(self, chunks, parent): self._add_spatial_wcs_element(_chunk.position, chunk_element) self._add_spectral_wcs_element(_chunk.energy, chunk_element) self._add_temporal_wcs_element(_chunk.time, chunk_element) - self._add_polarization_wcs_element(_chunk.polarization, chunk_element) + self._add_polarization_wcs_element(_chunk.polarization, + chunk_element) def _add_observable_axis_element(self, observable, parent): if observable is None: @@ -2191,7 +2306,8 @@ def _add_coord_bounds1d_element(self, name, bounds, parent): return element = self._get_caom_element(name, parent) - self._add_coord_range_1d_list_element("samples", bounds.samples, element) + self._add_coord_range_1d_list_element("samples", bounds.samples, + element) def _add_coord_bounds2d_element(self, name, bounds, parent): """Builds a representation of a CoordBounds2D and adds it to the @@ -2266,7 +2382,8 @@ def _add_coord_polygon2d_element(self, name, polygon, parent): if len(polygon.vertices) > 0: vertices_element = self._get_caom_element("vertices", element) for vertex in polygon.vertices: - self._add_value_coord2d_element("vertex", vertex, vertices_element) + self._add_value_coord2d_element("vertex", vertex, + vertices_element) def _add_coord_range1d_element(self, name, _range, parent): """ Builds a representation of a CoordRange1D and adds it to the diff --git a/caom2/caom2/observation.py b/caom2/caom2/observation.py index 22257506..3b3da045 100644 --- a/caom2/caom2/observation.py +++ b/caom2/caom2/observation.py @@ -73,17 +73,18 @@ unicode_literals) from datetime import datetime -from builtins import str, int -from aenum import Enum + import six +from aenum import Enum +from builtins import str from . import caom_util +from .caom_util import int_32 from .common import AbstractCaomEntity from .common import CaomObject from .common import ObservationURI from .plane import Plane from .shape import Point -from .caom_util import int_32 __all__ = ['ObservationIntentType', 'Status', 'TargetType', 'Observation', 'ObservationURI', 'Algorithm', 'SimpleObservation', @@ -160,7 +161,7 @@ class Observation(AbstractCaomEntity): -> SpectralWCS -> PolarizationWCS -> (Observable) - + """ def __init__(self, @@ -199,7 +200,7 @@ def __init__(self, self.algorithm = algorithm self._uri = ObservationURI.get_observation_uri(collection, - observation_id) + observation_id) self.sequence_number = sequence_number self.intent = intent @@ -213,7 +214,7 @@ def __init__(self, self.requirements = requirements self.meta_release = meta_release if planes is None: - self.planes = caom_util.TypedOrderedDict(Plane,) + self.planes = caom_util.TypedOrderedDict(Plane, ) else: self.planes = planes @@ -317,7 +318,6 @@ def sequence_number(self, value): caom_util.type_check(value, int_32, 'sequence_number') self._sequence_number = int_32(value) if value is not None else None - @property def type(self): """The OBSTYPE of the observation being recorded. @@ -715,7 +715,7 @@ def tau(self): @tau.setter def tau(self, value): caom_util.type_check(value, float, 'tau') - #Value must be >= 0, but has no upper limit + # Value must be >= 0, but has no upper limit self._tau = value @property diff --git a/caom2/caom2/part.py b/caom2/caom2/part.py index 265539e9..fc7f5462 100644 --- a/caom2/caom2/part.py +++ b/caom2/caom2/part.py @@ -72,6 +72,7 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) + from builtins import str from . import caom_util @@ -143,5 +144,6 @@ def chunks(self): @chunks.setter def chunks(self, value): - caom_util.type_check(value, caom_util.TypedList, 'chunks', override=False) + caom_util.type_check(value, caom_util.TypedList, 'chunks', + override=False) self._chunks = value diff --git a/caom2/caom2/plane.py b/caom2/caom2/plane.py index ed370833..57042ebc 100644 --- a/caom2/caom2/plane.py +++ b/caom2/caom2/plane.py @@ -73,10 +73,12 @@ unicode_literals) from datetime import datetime -from six.moves.urllib.parse import SplitResult, urlsplit -from builtins import str, int + from aenum import Enum, extend_enum +from builtins import str, int +from six.moves.urllib.parse import SplitResult, urlsplit +from caom2.caom_util import int_32 from . import caom_util from . import shape from . import wcs @@ -84,16 +86,16 @@ from .common import AbstractCaomEntity from .common import CaomObject from .common import ObservationURI -from caom2.caom_util import int_32 -__all__ = ['CalibrationLevel', 'DataProductType', 'EnergyBand', - 'VocabularyTerm', 'PolarizationState', 'Quality', 'Plane', - 'PlaneURI', 'DataQuality', 'Metrics', 'Provenance', 'Position', +__all__ = ['CalibrationLevel', 'DataProductType', 'EnergyBand', + 'VocabularyTerm', 'PolarizationState', 'Quality', 'Plane', + 'PlaneURI', 'DataQuality', 'Metrics', 'Provenance', 'Position', 'Energy', 'Polarization', 'Time'] _OBSCORE_VOCAB_NS = "http://www.ivoa.net/std/ObsCore" _CAOM_VOCAB_NS = "http://www.opencadc.org/caom2/DataProductType" + class CalibrationLevel(Enum): """ PLANNED: -1 @@ -116,9 +118,9 @@ class VocabularyTerm(object): def __init__(self, namespace, term, base=False): """ Construct a VocabularyTerm instance. This creates a term in the - specified vocabulary namespace. If the value of base is False, + specified vocabulary namespace. If the value of base is False, the string value (from getvalue()) will just be the namespace URI - plus the term added as a fragment. If the value of base is True, + plus the term added as a fragment. If the value of base is True, this is a term in a base vocabulary and the value will just be the term (without the namespace). @@ -182,11 +184,15 @@ class DataProductType(Enum): IMAGE = VocabularyTerm(_OBSCORE_VOCAB_NS, "image", True).get_value() CUBE = VocabularyTerm(_OBSCORE_VOCAB_NS, "cube", True).get_value() - EVENTLIST = VocabularyTerm(_OBSCORE_VOCAB_NS, "eventlist", True).get_value() + EVENTLIST = VocabularyTerm(_OBSCORE_VOCAB_NS, "eventlist", + True).get_value() SPECTRUM = VocabularyTerm(_OBSCORE_VOCAB_NS, "spectrum", True).get_value() - TIMESERIES = VocabularyTerm(_OBSCORE_VOCAB_NS, "timeseries", True).get_value() - VISIBILITY = VocabularyTerm(_OBSCORE_VOCAB_NS, "visibility", True).get_value() - MEASUREMENTS = VocabularyTerm(_OBSCORE_VOCAB_NS, "measurements", True).get_value() + TIMESERIES = VocabularyTerm(_OBSCORE_VOCAB_NS, "timeseries", + True).get_value() + VISIBILITY = VocabularyTerm(_OBSCORE_VOCAB_NS, "visibility", + True).get_value() + MEASUREMENTS = VocabularyTerm(_OBSCORE_VOCAB_NS, "measurements", + True).get_value() CATALOG = VocabularyTerm(_CAOM_VOCAB_NS, "catalog").get_value() @staticmethod @@ -196,7 +202,8 @@ def extend(namespace, name): :param namespace: Namespace for the new data product type :param name: Name of the new data product type """ - extend_enum(DataProductType, name.upper(), VocabularyTerm(namespace, name).get_value()) + extend_enum(DataProductType, name.upper(), + VocabularyTerm(namespace, name).get_value()) class EnergyBand(Enum): @@ -323,7 +330,7 @@ def creator_id(self): """A URI that identifies the creator of this plane. eg: ivo://cadc.nrc.ca/users?tester - type: URI + type: URI """ return self._creator_id @@ -348,7 +355,8 @@ def artifacts(self): @artifacts.setter def artifacts(self, value): - caom_util.type_check(value, caom_util.TypedOrderedDict, 'artifacts', override=False) + caom_util.type_check(value, caom_util.TypedOrderedDict, 'artifacts', + override=False) self._artifacts = value @property @@ -490,7 +498,7 @@ def position(self): of the python module at this time. """ return self._position - + @position.setter def position(self, value): caom_util.type_check(value, Position, "position") @@ -507,7 +515,7 @@ def energy(self): """ """ Energy """ return self._energy - + @energy.setter def energy(self, value): caom_util.type_check(value, Energy, "energy") @@ -524,12 +532,12 @@ def time(self): """ """ Time """ return self._time - + @time.setter def time(self, value): caom_util.type_check(value, Time, "time") self._time = value - + @property def polarization(self): """A caom2 Polarization object that is developed from @@ -540,7 +548,7 @@ def polarization(self): of the python module at this time. """ return self._polarization - + @polarization.setter def polarization(self, value): caom_util.type_check(value, Polarization, "polarization") @@ -593,12 +601,14 @@ def __hash__(self): def __lt__(self, other): if not isinstance(other, PlaneURI): - raise ValueError('Canot compare PlaneURI with {}'.format(type(other))) + raise ValueError( + 'Canot compare PlaneURI with {}'.format(type(other))) return self.uri < other.uri def __eq__(self, other): if not isinstance(other, PlaneURI): - raise ValueError('Canot compare PlaneURI with {}'.format(type(other))) + raise ValueError( + 'Canot compare PlaneURI with {}'.format(type(other))) return self.uri == other.uri @classmethod @@ -610,9 +620,11 @@ def get_plane_uri(cls, observation_uri, product_id): observation_uri : the uri of the observation product_id : ID of the product """ - caom_util.type_check(observation_uri, ObservationURI, "observation_uri", + caom_util.type_check(observation_uri, ObservationURI, + "observation_uri", + override=False) + caom_util.type_check(product_id, str, "observation_uri", override=False) - caom_util.type_check(product_id, str, "observation_uri", override=False) caom_util.validate_path_component(cls, "product_id", product_id) path = urlsplit(observation_uri.uri).path @@ -797,7 +809,9 @@ def __init__(self, name, """ assert name is not None, "No name provided" - assert isinstance(name, str), "name is not a unicode string: {0}".format(name) + assert isinstance(name, + str), "name is not a unicode string: {0}".format( + name) self._name = name self.version = version @@ -922,7 +936,8 @@ def bounds(self): @bounds.setter def bounds(self, value): if value is not None: - assert isinstance(value, (shape.Box, shape.Circle, shape.Polygon)), ( + assert isinstance(value, + (shape.Box, shape.Circle, shape.Polygon)), ( "bounds is not a Shape: {0}".format(value)) self._bounds = value @@ -1123,7 +1138,7 @@ def dimension(self, value): caom_util.type_check(value, int, 'dimension') caom_util.value_check(value, 0, 1E10, 'dimension') self._dimension = value - + @property def polarization_states(self): """ @@ -1134,7 +1149,8 @@ def polarization_states(self): @polarization_states.setter def polarization_states(self, value): if value is not None: - caom_util.type_check(value, list, 'polarization_states', override=False) + caom_util.type_check(value, list, 'polarization_states', + override=False) self._polarization_states = value diff --git a/caom2/caom2/shape.py b/caom2/caom2/shape.py index 9edea420..5efbe97a 100644 --- a/caom2/caom2/shape.py +++ b/caom2/caom2/shape.py @@ -70,13 +70,16 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +import math + from aenum import Enum -from . import common -from . import caom_util + from caom2.caom_util import int_32 -import math +from . import caom_util +from . import common -__all__ = ['SegmentType', 'Box', 'Circle', 'Interval', 'Point', 'Polygon', 'Vertex'] +__all__ = ['SegmentType', 'Box', 'Circle', 'Interval', 'Point', + 'Polygon', 'Vertex'] class SegmentType(Enum): @@ -91,7 +94,6 @@ class SegmentType(Enum): class Box(common.CaomObject): - def __init__(self, center, width, height): """ @@ -106,7 +108,8 @@ def get_area(self): return self._width * self._height def get_size(self): - return math.sqrt(self._width * self._width + self._height * self._height) + return math.sqrt(self._width * self._width + + self._height * self._height) # Properties @@ -150,7 +153,6 @@ def height(self, value): class Circle(common.CaomObject): - def __init__(self, center, radius): """ @@ -195,7 +197,6 @@ def radius(self, value): class SubInterval(common.CaomObject): - def __init__(self, lower, upper): self.lower = lower self.upper = upper @@ -218,7 +219,9 @@ def lower(self, value): except AttributeError: has_upper = False if has_upper: - assert not self._upper < value, "SubInterval: attempt to set upper < lower for " + str(self._upper) + "," + str(value) + assert not self._upper < value, \ + "SubInterval: attempt to set upper < lower for {}, {}".\ + format(str(value, str(self._lower))) self._lower = value @property @@ -237,12 +240,13 @@ def upper(self, value): except AttributeError: has_lower = False if has_lower: - assert not value < self._lower, "SubInterval: attempt to set upper < lower for " + str(value) + "," + str(self._lower) + assert not value < self._lower, \ + "SubInterval: attempt to set upper < lower for {}, {}".\ + format(value, self._lower) self._upper = value class Interval(common.CaomObject): - def __init__(self, lower, upper, samples=None): self.lower = lower @@ -252,7 +256,6 @@ def __init__(self, lower, upper, samples=None): def get_width(self): return self._upper - self._lower - @classmethod def intersection(cls, i1, i2): if i1.lower > i2.upper or i1.upper < i2.lower: @@ -280,7 +283,9 @@ def lower(self, value): except AttributeError: has_upper = False if has_upper: - assert not self._upper < value, "Interval: attempt to set upper < lower for " + str(self._upper) + "," + str(value) + assert not self._upper < value, \ + "Interval: attempt to set upper < lower for {], {}".\ + format(self._upper, value) self._lower = value @property @@ -299,7 +304,9 @@ def upper(self, value): except AttributeError: has_lower = False if has_lower: - assert not value < self._lower, "Interval: attempt to set upper < lower for " + str(value) + "," + str(self._lower) + assert not value < self._lower, \ + "Interval: attempt to set upper < lower for {}, {}".\ + format(value, self._lower) self._upper = value @property @@ -317,9 +324,7 @@ def samples(self, value): class Point(common.CaomObject): - def __init__(self, cval1, cval2): - self.cval1 = cval1 self.cval2 = cval2 @@ -349,7 +354,6 @@ def cval2(self, value): class Polygon(common.CaomObject): - def __init__(self, points=None, samples=None): if not points: self._points = [] @@ -366,7 +370,6 @@ def points(self): """ return self._points - @property def samples(self): """ @@ -377,12 +380,12 @@ def samples(self): @samples.setter def samples(self, value): if value is not None: - caom_util.type_check(value, MultiPolygon, 'multipolygon', override=False) + caom_util.type_check(value, MultiPolygon, 'multipolygon', + override=False) self._samples = value class MultiPolygon(common.CaomObject): - def __init__(self, vertices=None): if not vertices: self._vertices = [] @@ -390,7 +393,7 @@ def __init__(self, vertices=None): self._vertices = vertices # Properties - + @property def vertices(self): """ @@ -404,37 +407,41 @@ def validate(self): TODO: check the clockwise direction - An AssertionError is thrown when the object does not represent a multi polygon + An AssertionError is thrown when the object does not represent a + multi polygon """ if not self.vertices: raise AssertionError('invalid polygon: no vertices') lines = 0 if len(self._vertices) < 4: # triangle - raise AssertionError('invalid polygon: {} vertices (min 4)'.format(len(self._vertices))) + raise AssertionError('invalid polygon: {} vertices (min 4)'.format( + len(self._vertices))) open_loop = False for v in self._vertices: if v.type == SegmentType.MOVE: if open_loop: - raise AssertionError('invalid polygon: MOVE vertex when loop open') + raise AssertionError( + 'invalid polygon: MOVE vertex when loop open') lines = 0 open_loop = True elif v.type == SegmentType.CLOSE: if not open_loop: - raise AssertionError('invalid polygon: CLOSE vertex when loop close') + raise AssertionError( + 'invalid polygon: CLOSE vertex when loop close') if lines < 2: - raise AssertionError('invalid polygon: minimum 2 lines required') + raise AssertionError( + 'invalid polygon: minimum 2 lines required') open_loop = False else: if not open_loop: - raise AssertionError('invalid polygon: LINE vertex when loop close') + raise AssertionError( + 'invalid polygon: LINE vertex when loop close') lines += 1 - class Vertex(Point): - def __init__(self, cval1, cval2, type): super(Vertex, self).__init__(cval1, cval2) self.type = type @@ -452,4 +459,3 @@ def type(self): def type(self, value): caom_util.type_check(value, SegmentType, 'type', override=False) self._type = value - diff --git a/caom2/caom2/tests/caom_test_instances.py b/caom2/caom2/tests/caom_test_instances.py index 16ed6005..7e205dd1 100644 --- a/caom2/caom2/tests/caom_test_instances.py +++ b/caom2/caom2/tests/caom_test_instances.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -#*********************************************************************** -#****************** CANADIAN ASTRONOMY DATA CENTRE ******************* -#************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +# *********************************************************************** +# ****************** CANADIAN ASTRONOMY DATA CENTRE ******************* +# ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** # # (c) 2016. (c) 2016. # Government of Canada Gouvernement du Canada @@ -64,31 +64,32 @@ # # $Revision: 4 $ # -#*********************************************************************** +# *********************************************************************** # """ Defines Caom2TestInstances class """ from __future__ import (absolute_import, division, print_function, unicode_literals) -from builtins import int -import six + import collections from datetime import datetime +import six +from builtins import int + from caom2 import artifact from caom2 import caom_util from caom2 import chunk +from caom2 import common from caom2 import observation from caom2 import part from caom2 import plane from caom2 import shape from caom2 import wcs -from caom2 import common class Caom2TestInstances(object): - _collection = "collection" _observation_id = "observationID" _product_id = "productId" @@ -140,7 +141,8 @@ def get_simple_observation(self): if self.complete: simple_observation.sequence_number = int(5) simple_observation.obs_type = "flat" - simple_observation.intent = observation.ObservationIntentType.CALIBRATION + simple_observation.intent =\ + observation.ObservationIntentType.CALIBRATION simple_observation.meta_release = Caom2TestInstances._ivoa_date simple_observation.proposal = self.get_proposal() simple_observation.target = self.get_target() @@ -152,23 +154,29 @@ def get_simple_observation(self): simple_observation.environment = self.get_environment() simple_observation.last_modified = common.get_current_ivoa_time() if self.caom_version >= 23: - simple_observation.max_last_modified = common.get_current_ivoa_time() - simple_observation.meta_checksum = common.ChecksumURI("md5:9882dbbf9cadc221019b712fd402bcbd") - simple_observation.acc_meta_checksum = common.ChecksumURI("md5:844ce247db0844ad9f721430c80e7a21") + simple_observation.max_last_modified =\ + common.get_current_ivoa_time() + simple_observation.meta_checksum = common.ChecksumURI( + "md5:9882dbbf9cadc221019b712fd402bcbd") + simple_observation.acc_meta_checksum = common.ChecksumURI( + "md5:844ce247db0844ad9f721430c80e7a21") if self.depth > 1: simple_observation.planes.update(self.get_planes()) return simple_observation def get_composite_observation(self): composite_observation = \ - observation.CompositeObservation(Caom2TestInstances._collection, - Caom2TestInstances._observation_id, - self.get_algorithm()) - print("Creating test composite observation of version " + str(self.caom_version)) + observation.CompositeObservation( + Caom2TestInstances._collection, + Caom2TestInstances._observation_id, + self.get_algorithm()) + print("Creating test composite observation of version " + str( + self.caom_version)) if self.complete: composite_observation.sequence_number = int(10) composite_observation.obs_type = "filed" - composite_observation.intent = observation.ObservationIntentType.SCIENCE + composite_observation.intent =\ + observation.ObservationIntentType.SCIENCE composite_observation.meta_release = Caom2TestInstances._ivoa_date composite_observation.proposal = self.get_proposal() composite_observation.target = self.get_target() @@ -178,11 +186,15 @@ def get_composite_observation(self): composite_observation.telescope = self.get_telescope() composite_observation.instrument = self.get_instrument() composite_observation.environment = self.get_environment() - composite_observation.last_modified = common.get_current_ivoa_time() + composite_observation.last_modified =\ + common.get_current_ivoa_time() if self.caom_version >= 23: - composite_observation.max_last_modified = common.get_current_ivoa_time() - composite_observation.meta_checksum = common.ChecksumURI("md5:9882dbbf9cadc221019b712fd402bcbd") - composite_observation.acc_meta_checksum = common.ChecksumURI("md5:844ce247db0844ad9f721430c80e7a21") + composite_observation.max_last_modified = \ + common.get_current_ivoa_time() + composite_observation.meta_checksum = common.ChecksumURI( + "md5:9882dbbf9cadc221019b712fd402bcbd") + composite_observation.acc_meta_checksum = common.ChecksumURI( + "md5:844ce247db0844ad9f721430c80e7a21") if self.depth > 1: composite_observation.planes.update(self.get_planes()) composite_observation.members.update(self.get_members()) @@ -242,7 +254,8 @@ def get_environment(self): def get_members(self): members = caom_util.TypedSet( - observation.ObservationURI, observation.ObservationURI("caom:foo/bar")) + observation.ObservationURI, + observation.ObservationURI("caom:foo/bar")) return members def get_planes(self): @@ -259,8 +272,10 @@ def get_planes(self): if self.caom_version >= 23: _plane.creator_id = "ivo://cadc.nrc.ca?testuser" _plane.max_last_modified = common.get_current_ivoa_time() - _plane.meta_checksum = common.ChecksumURI("md5:9882dbbf9cadc221019b712fd402bcbd") - _plane.acc_meta_checksum = common.ChecksumURI("md5:844ce247db0844ad9f721430c80e7a21") + _plane.meta_checksum = common.ChecksumURI( + "md5:9882dbbf9cadc221019b712fd402bcbd") + _plane.acc_meta_checksum = common.ChecksumURI( + "md5:844ce247db0844ad9f721430c80e7a21") if self.caom_version >= 22: _plane.position = self.get_position() _plane.energy = self.get_energy() @@ -274,7 +289,7 @@ def get_planes(self): _plane.artifacts[k] = v planes["productID"] = _plane return planes - + def get_position(self): position = plane.Position() @@ -301,22 +316,24 @@ def get_position(self): position.resolution = 0.5 position.sample_size = 1.1 position.time_dependent = False - + return position - + def get_energy(self): energy = plane.Energy() - + lower = 1.0 upper = 2.0 lower1 = 1.1 upper1 = 2.1 lower2 = 1.2 upper2 = 2.2 - samples = [shape.SubInterval(lower,upper), shape.SubInterval(lower1,upper1), shape.SubInterval(lower2,upper2)] + samples = [shape.SubInterval(lower, upper), + shape.SubInterval(lower1, upper1), + shape.SubInterval(lower2, upper2)] interval = shape.Interval(lower, upper, samples) - + energy.bounds = interval energy.dimension = 100 energy.resolving_power = 2.0 @@ -324,38 +341,40 @@ def get_energy(self): energy.bandpass_name = "e" energy.em_band = plane.EnergyBand.GAMMARAY energy.transition = wcs.EnergyTransition("species", "transition") - + return energy - + def get_time(self): time = plane.Time() - + lower = 1.0 upper = 2.0 lower1 = 1.1 upper1 = 2.1 lower2 = 1.2 upper2 = 2.2 - samples = [shape.SubInterval(lower,upper), shape.SubInterval(lower1,upper1), shape.SubInterval(lower2,upper2)] + samples = [shape.SubInterval(lower, upper), + shape.SubInterval(lower1, upper1), + shape.SubInterval(lower2, upper2)] interval = shape.Interval(lower, upper, samples) - + time.bounds = interval time.dimension = 1 time.resolution = 2.1 time.sample_size = 3.0 time.exposure = 10.3 - + return time - + def get_polarization(self): polarization = plane.Polarization() - + p_states = [plane.PolarizationState.LL, plane.PolarizationState.XY] - + polarization.dimension = 2 polarization.polarization_states = p_states - + return polarization def get_provenance(self): @@ -398,8 +417,10 @@ def get_artifacts(self): _artifact.last_modified = common.get_current_ivoa_time() if self.caom_version >= 23: _artifact.max_last_modified = common.get_current_ivoa_time() - _artifact.meta_checksum = common.ChecksumURI("md5:9882dbbf9cadc221019b712fd402bcbd") - _artifact.acc_meta_checksum = common.ChecksumURI("md5:844ce247db0844ad9f721430c80e7a21") + _artifact.meta_checksum = common.ChecksumURI( + "md5:9882dbbf9cadc221019b712fd402bcbd") + _artifact.acc_meta_checksum = common.ChecksumURI( + "md5:844ce247db0844ad9f721430c80e7a21") if self.depth > 3: for k, v in six.iteritems(self.get_parts()): _artifact.parts[k] = v @@ -437,8 +458,10 @@ def get_chunks(self): _chunk.last_modified = common.get_current_ivoa_time() if self.caom_version >= 23: _chunk.max_last_modified = common.get_current_ivoa_time() - _chunk.meta_checksum = common.ChecksumURI("md5:9882dbbf9cadc221019b712fd402bcbd") - _chunk.acc_meta_checksum = common.ChecksumURI("md5:844ce247db0844ad9f721430c80e7a21") + _chunk.meta_checksum = common.ChecksumURI( + "md5:9882dbbf9cadc221019b712fd402bcbd") + _chunk.acc_meta_checksum = common.ChecksumURI( + "md5:844ce247db0844ad9f721430c80e7a21") chunks.append(_chunk) return chunks @@ -487,7 +510,7 @@ def get_temporal_wcs(self): def get_polarization_wcs(self): axis = wcs.Axis('STOKES') axis1d = wcs.CoordAxis1D(axis) - #IQUV + # IQUV axis1d.function = wcs.CoordFunction1D(int(4), 1.0, wcs.RefCoord(1.0, 1.0)) pol = chunk.PolarizationWCS(axis1d) @@ -501,7 +524,7 @@ def get_coord_axis1d(self): if self.complete: coord_axis1d.error = wcs.CoordError(1.0, 1.5) coord_axis1d.range = wcs.CoordRange1D(wcs.RefCoord(2.0, 2.5), - wcs.RefCoord(3.0, 3.5)) + wcs.RefCoord(3.0, 3.5)) coord_axis1d.function = ( wcs.CoordFunction1D(4, 4.5, wcs.RefCoord(5.0, 5.5))) bounds = wcs.CoordBounds1D() diff --git a/caom2/caom2/tests/setup_package.py b/caom2/caom2/tests/setup_package.py deleted file mode 100644 index 6e4be679..00000000 --- a/caom2/caom2/tests/setup_package.py +++ /dev/null @@ -1,7 +0,0 @@ -def get_package_data(): - return { - _ASTROPY_PACKAGE_NAME_ + '.tests': ['coveragerc']} - - -def requires_2to3(): - return False diff --git a/caom2/caom2/tests/test_artifact.py b/caom2/caom2/tests/test_artifact.py index 01be30ad..ae64ea24 100644 --- a/caom2/caom2/tests/test_artifact.py +++ b/caom2/caom2/tests/test_artifact.py @@ -73,22 +73,21 @@ unicode_literals) import unittest + from six.moves.urllib.parse import urlparse from .. import artifact -from .. import part from .. import common +from .. import part class TestEnums(unittest.TestCase): - def test_all(self): self.assertEqual(artifact.ReleaseType.DATA.value, "data") self.assertEqual(artifact.ReleaseType.META.value, "meta") class TestArtifact(unittest.TestCase): - def test_all(self): with self.assertRaises(TypeError): test_artifact = artifact.Artifact("caom:GEMINI/12345") @@ -130,18 +129,20 @@ def test_all(self): self.assertEquals(artifact.ProductType.PREVIEW, test_artifact.product_type, "Product type") - - self.assertIsNone(test_artifact.content_checksum, "Default content checksum") + + self.assertIsNone(test_artifact.content_checksum, + "Default content checksum") cs_uri = common.ChecksumURI("md5:e30580c1db513487f495fba09f64600e") test_artifact.content_checksum = cs_uri - self.assertEquals(test_artifact.content_checksum, cs_uri, "Content checksum") - + self.assertEquals(test_artifact.content_checksum, cs_uri, + "Content checksum") + self.assertEquals(0, len(test_artifact.parts), "Default parts") part1 = part.Part("1") test_artifact.parts["1"] = part1 self.assertEquals(1, len(test_artifact.parts), "Parts") self.assertTrue("1" in test_artifact.parts.keys()) - #add same part again + # add same part again part2 = part.Part("2") test_artifact.parts["2"] = part2 self.assertEquals(2, len(test_artifact.parts), "Parts") @@ -161,7 +162,7 @@ def test_all(self): self.assertTrue("1" in test_artifact.parts.keys()) self.assertTrue("2" in test_artifact.parts.keys()) - #incorrect URI + # incorrect URI exception = False try: test_artifact = artifact.Artifact( diff --git a/caom2/caom2/tests/test_caom_util.py b/caom2/caom2/tests/test_caom_util.py index 7a4e30cc..20eecbce 100644 --- a/caom2/caom2/tests/test_caom_util.py +++ b/caom2/caom2/tests/test_caom_util.py @@ -72,6 +72,7 @@ import unittest import uuid + from builtins import str, int from .. import artifact @@ -82,7 +83,6 @@ class TestCaomUtil(unittest.TestCase): - def test_typed_list(self): my_list1 = caom_util.TypedList(str, "Test1") self.assertEquals(1, len(my_list1), "list1 length") @@ -150,44 +150,49 @@ def test_typed_list(self): self.assertEqual("Test3", my_list1[3], "Non matching elements") self.assertEqual("Test4", my_list1[4], "Non matching elements") - my_list2 = caom_util.TypedList(plane.Energy,) + my_list2 = caom_util.TypedList(plane.Energy, ) self.assertEquals(0, len(my_list2), "list2 length") def test_validate_path_component(self): energy = plane.Energy() - caom_util.validate_path_component(energy, "something", "some:test\\path") + caom_util.validate_path_component(energy, "something", + "some:test\\path") exception = False try: - caom_util.validate_path_component(energy, "energyfield", "some:test path") + caom_util.validate_path_component(energy, "energyfield", + "some:test path") except AssertionError: exception = True self.assertTrue(exception, "Missing exception") exception = False try: - caom_util.validate_path_component(energy, "energyfield", "some:test/path") + caom_util.validate_path_component(energy, "energyfield", + "some:test/path") except AssertionError: exception = True self.assertTrue(exception, "Missing exception") exception = False try: - caom_util.validate_path_component(energy, "energyfield", "some:test||path") + caom_util.validate_path_component(energy, "energyfield", + "some:test||path") except AssertionError: exception = True self.assertTrue(exception, "Missing exception") exception = False try: - caom_util.validate_path_component(energy, "energyfield", "some:test %path") + caom_util.validate_path_component(energy, "energyfield", + "some:test %path") except AssertionError: exception = True self.assertTrue(exception, "Missing exception") def test_typed_set(self): - my_set = caom_util.TypedSet(str,) + my_set = caom_util.TypedSet(str, ) with self.assertRaises(AssertionError): my_set.add(float(1.0)) my_set.add(int(1)) @@ -208,7 +213,7 @@ def test_typed_set(self): my_set.add(bool(1)) my_set.add("Test1") - my_set = caom_util.TypedSet(str,) + my_set = caom_util.TypedSet(str, ) my_set.add("Test1") my_set.add("Test1") self.assertTrue(len(my_set) == 1) @@ -222,16 +227,16 @@ def test_typed_ordered_dict(self): artifact.ReleaseType.DATA) test_part10 = part.Part("10") test_plane_uri = plane.PlaneURI('caom:CFHT/55/66') - my_dict_plane = caom_util.TypedOrderedDict(plane.Plane,) + my_dict_plane = caom_util.TypedOrderedDict(plane.Plane, ) with self.assertRaises(ValueError): my_dict_plane['key11'] = test_plane10 - my_dict_artifact = caom_util.TypedOrderedDict(artifact.Artifact,) + my_dict_artifact = caom_util.TypedOrderedDict(artifact.Artifact, ) with self.assertRaises(ValueError): my_dict_artifact['caom:CFHT/55/6'] = test_artifact66 - my_dict_part = caom_util.TypedOrderedDict(part.Part,) + my_dict_part = caom_util.TypedOrderedDict(part.Part, ) with self.assertRaises(ValueError): my_dict_part['11'] = test_part10 - my_dict_wrong_type = caom_util.TypedOrderedDict(plane.PlaneURI,) + my_dict_wrong_type = caom_util.TypedOrderedDict(plane.PlaneURI, ) with self.assertRaises(ValueError): my_dict_wrong_type['caom:CFHT/55/67'] = test_plane_uri with self.assertRaises(TypeError): @@ -239,12 +244,13 @@ def test_typed_ordered_dict(self): with self.assertRaises(TypeError): my_dict_plane['key1'] = float(2.0) # test assignment - my_dict = caom_util.TypedOrderedDict(plane.Plane,) + my_dict = caom_util.TypedOrderedDict(plane.Plane, ) test_plane2 = plane.Plane('key2') test_plane1 = plane.Plane('key1') my_dict['key2'] = test_plane2 my_dict['key1'] = test_plane1 - # need to cast to list in order to make it work with both python 2 and 3 + # need to cast to list in order to make it work with both python + # 2 and 3 self.assertEqual(2, len(my_dict), 'mismatch in the number of entries in dictionary.') self.assertEqual('key2', list(my_dict.keys())[0], @@ -258,7 +264,8 @@ def test_typed_ordered_dict(self): # test constructor with non-empty dictionary test_plane1 = plane.Plane('key1') test_plane2 = plane.Plane('key2') - my_dict1 = caom_util.TypedOrderedDict(plane.Plane, ('key1', test_plane1), + my_dict1 = caom_util.TypedOrderedDict(plane.Plane, + ('key1', test_plane1), ('key2', test_plane2)) self.assertEqual(2, len(my_dict1), 'mismatch in the number of entries in dictionary.') diff --git a/caom2/caom2/tests/test_checksum.py b/caom2/caom2/tests/test_checksum.py index 66171454..5cec4280 100644 --- a/caom2/caom2/tests/test_checksum.py +++ b/caom2/caom2/tests/test_checksum.py @@ -72,20 +72,21 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -import os -from caom2 import obs_reader_writer, get_meta_checksum, get_acc_meta_checksum -from caom2.checksum import update_checksum, int_32 import hashlib -import struct +import os +from uuid import UUID + from builtins import int, str + +from caom2 import obs_reader_writer, get_meta_checksum, get_acc_meta_checksum from caom2.caom_util import str2ivoa -from uuid import UUID +from caom2.checksum import update_checksum, int_32 THIS_DIR = os.path.dirname(os.path.realpath(__file__)) TEST_DATA = 'data' -def test_primitive_checksum(): +def test_primitive_checksum(): md5 = hashlib.md5() # tests checksums of various primitives to match those in Java value = True @@ -107,10 +108,10 @@ def test_primitive_checksum(): value = int(12345678910) update_checksum(md5, value, False) assert ('f61cbb413a37d320af998a215530bc78' == md5.hexdigest()) - #md5 = hashlib.md5() - #value = common.float_32(1.1) - #common.get_primitive_to_bytes(md5, value, False) - #assert ('8ce670eb32869bc6b6109d970711f7c1' == md5.hexdigest()) + # md5 = hashlib.md5() + # value = common.float_32(1.1) + # common.get_primitive_to_bytes(md5, value, False) + # assert ('8ce670eb32869bc6b6109d970711f7c1' == md5.hexdigest()) md5 = hashlib.md5() value = 2.2 update_checksum(md5, value, False) @@ -129,14 +130,14 @@ def test_primitive_checksum(): assert ('5b71d023d4729575d550536dce8439e6' == md5.hexdigest()) - def test_compatibility(): - # tests loads a previously generated observation and checks the checksums against the previously - # calculated (in Java) checksums + # tests loads a previously generated observation and checks the checksums + # against the previously calculated (in Java) checksums - source_file_path = os.path.join(THIS_DIR, TEST_DATA, 'SampleComposite-CAOM-2.3.xml') + source_file_path = os.path.join(THIS_DIR, TEST_DATA, + 'SampleComposite-CAOM-2.3.xml') reader = obs_reader_writer.ObservationReader(True) - with open(source_file_path, 'r') as f: + with open(source_file_path, 'r'): obs = reader.read(source_file_path) for plane in obs.planes.values(): @@ -144,15 +145,16 @@ def test_compatibility(): for part in artifact.parts.values(): for chunk in part.chunks: assert chunk.meta_checksum == get_meta_checksum(chunk) - assert chunk.acc_meta_checksum == get_acc_meta_checksum(chunk) + assert chunk.acc_meta_checksum == get_acc_meta_checksum( + chunk) assert part.meta_checksum == get_meta_checksum(part) assert part.acc_meta_checksum == get_acc_meta_checksum(part) assert artifact.meta_checksum == get_meta_checksum(artifact) - assert artifact.acc_meta_checksum == get_acc_meta_checksum(artifact) + assert artifact.acc_meta_checksum == get_acc_meta_checksum( + artifact) assert plane.meta_checksum == get_meta_checksum(plane) assert plane.acc_meta_checksum == get_acc_meta_checksum(plane) - # check observation assert obs.meta_checksum == get_meta_checksum(obs) assert obs.acc_meta_checksum == get_acc_meta_checksum(obs) @@ -165,11 +167,13 @@ def test_compatibility(): for part in artifact.parts.values(): for chunk in part.chunks: assert chunk.meta_checksum == get_meta_checksum(chunk) - assert chunk.acc_meta_checksum == get_acc_meta_checksum(chunk) + assert chunk.acc_meta_checksum == get_acc_meta_checksum( + chunk) assert part.meta_checksum == get_meta_checksum(part) assert part.acc_meta_checksum == get_acc_meta_checksum(part) assert artifact.meta_checksum == get_meta_checksum(artifact) - assert artifact.acc_meta_checksum == get_acc_meta_checksum(artifact) + assert artifact.acc_meta_checksum == get_acc_meta_checksum( + artifact) assert plane.meta_checksum == get_meta_checksum(plane) assert plane.acc_meta_checksum == get_acc_meta_checksum(plane) assert obs.meta_checksum != get_meta_checksum(obs) @@ -185,11 +189,13 @@ def test_compatibility(): for part in artifact.parts.values(): for chunk in part.chunks: assert chunk.meta_checksum == get_meta_checksum(chunk) - assert chunk.acc_meta_checksum == get_acc_meta_checksum(chunk) + assert chunk.acc_meta_checksum == get_acc_meta_checksum( + chunk) assert part.meta_checksum == get_meta_checksum(part) assert part.acc_meta_checksum == get_acc_meta_checksum(part) assert artifact.meta_checksum == get_meta_checksum(artifact) - assert artifact.acc_meta_checksum == get_acc_meta_checksum(artifact) + assert artifact.acc_meta_checksum == get_acc_meta_checksum( + artifact) if plane._id == aplane._id: assert plane.meta_checksum != get_meta_checksum(plane) assert plane.acc_meta_checksum != get_acc_meta_checksum(plane) @@ -209,15 +215,18 @@ def test_compatibility(): for part in artifact.parts.values(): for chunk in part.chunks: assert chunk.meta_checksum == get_meta_checksum(chunk) - assert chunk.acc_meta_checksum == get_acc_meta_checksum(chunk) + assert chunk.acc_meta_checksum == get_acc_meta_checksum( + chunk) assert part.meta_checksum == get_meta_checksum(part) assert part.acc_meta_checksum == get_acc_meta_checksum(part) if artifact._id == anartifact._id: assert artifact.meta_checksum != get_meta_checksum(artifact) - assert artifact.acc_meta_checksum != get_acc_meta_checksum(artifact) + assert artifact.acc_meta_checksum != get_acc_meta_checksum( + artifact) else: assert artifact.meta_checksum == get_meta_checksum(artifact) - assert artifact.acc_meta_checksum == get_acc_meta_checksum(artifact) + assert artifact.acc_meta_checksum == get_acc_meta_checksum( + artifact) assert plane.meta_checksum == get_meta_checksum(plane) if plane._id == aplane._id: assert plane.acc_meta_checksum != get_acc_meta_checksum(plane) @@ -235,18 +244,23 @@ def test_compatibility(): for part in artifact.parts.values(): for chunk in part.chunks: assert chunk.meta_checksum == get_meta_checksum(chunk) - assert chunk.acc_meta_checksum == get_acc_meta_checksum(chunk) + assert chunk.acc_meta_checksum == get_acc_meta_checksum( + chunk) if part._id == apart._id: assert part.meta_checksum != get_meta_checksum(part) - assert part.acc_meta_checksum != get_acc_meta_checksum(part) + assert part.acc_meta_checksum != get_acc_meta_checksum( + part) else: assert part.meta_checksum == get_meta_checksum(part) - assert part.acc_meta_checksum == get_acc_meta_checksum(part) + assert part.acc_meta_checksum == get_acc_meta_checksum( + part) assert artifact.meta_checksum == get_meta_checksum(artifact) if artifact._id == anartifact._id: - assert artifact.acc_meta_checksum != get_acc_meta_checksum(artifact) + assert artifact.acc_meta_checksum != get_acc_meta_checksum( + artifact) else: - assert artifact.acc_meta_checksum == get_acc_meta_checksum(artifact) + assert artifact.acc_meta_checksum == get_acc_meta_checksum( + artifact) assert plane.meta_checksum == get_meta_checksum(plane) if plane._id == aplane._id: assert plane.acc_meta_checksum != get_acc_meta_checksum(plane) @@ -268,20 +282,26 @@ def test_compatibility(): for chunk in part.chunks: if chunk._id == achunk._id: assert chunk.meta_checksum != get_meta_checksum(chunk) - assert chunk.acc_meta_checksum != get_acc_meta_checksum(chunk) + assert chunk.acc_meta_checksum !=\ + get_acc_meta_checksum(chunk) else: assert chunk.meta_checksum == get_meta_checksum(chunk) - assert chunk.acc_meta_checksum == get_acc_meta_checksum(chunk) + assert chunk.acc_meta_checksum ==\ + get_acc_meta_checksum(chunk) assert part.meta_checksum == get_meta_checksum(part) if part._id == apart._id: - assert part.acc_meta_checksum != get_acc_meta_checksum(part) + assert part.acc_meta_checksum != get_acc_meta_checksum( + part) else: - assert part.acc_meta_checksum == get_acc_meta_checksum(part) + assert part.acc_meta_checksum == get_acc_meta_checksum( + part) assert artifact.meta_checksum == get_meta_checksum(artifact) if artifact._id == anartifact._id: - assert artifact.acc_meta_checksum != get_acc_meta_checksum(artifact) + assert artifact.acc_meta_checksum != get_acc_meta_checksum( + artifact) else: - assert artifact.acc_meta_checksum == get_acc_meta_checksum(artifact) + assert artifact.acc_meta_checksum == get_acc_meta_checksum( + artifact) assert plane.meta_checksum == get_meta_checksum(plane) if plane._id == aplane._id: assert plane.acc_meta_checksum != get_acc_meta_checksum(plane) diff --git a/caom2/caom2/tests/test_chunk.py b/caom2/caom2/tests/test_chunk.py index 92d90a58..29dce5aa 100644 --- a/caom2/caom2/tests/test_chunk.py +++ b/caom2/caom2/tests/test_chunk.py @@ -79,7 +79,6 @@ class TestEnums(unittest.TestCase): - def test_all(self): # test for invalid value with self.assertRaises(ValueError): @@ -92,10 +91,10 @@ def test_all(self): # test that we can get the object for each enum by name self.assertEqual(chunk.ProductType.SCIENCE.name, "SCIENCE") self.assertEqual(chunk.ProductType[ - chunk.ProductType.SCIENCE.name].name, "SCIENCE") + chunk.ProductType.SCIENCE.name].name, "SCIENCE") self.assertEqual(chunk.ProductType['SCIENCE'].value, "science") self.assertEqual(chunk.ProductType[ - chunk.ProductType.SCIENCE.name].value, "science") + chunk.ProductType.SCIENCE.name].value, "science") self.assertEqual(chunk.ProductType.SCIENCE.value, "science") self.assertEqual(chunk.ProductType.CALIBRATION.value, "calibration") @@ -108,9 +107,7 @@ def test_all(self): class TestChunk(unittest.TestCase): - def test_init(self): - test_chunk = chunk.Chunk() self.assertIsNone(test_chunk.product_type) self.assertIsNone(test_chunk.naxis) @@ -126,7 +123,6 @@ def test_init(self): self.assertIsNone(test_chunk.polarization) def test_attributes(self): - test_chunk = chunk.Chunk() with self.assertRaises(TypeError): test_chunk.product_type = float(1.0) @@ -143,7 +139,8 @@ def test_attributes(self): test_chunk.polarization = float(1.0) test_chunk.product_type = chunk.ProductType.SCIENCE - self.assertEqual(chunk.ProductType.SCIENCE.name, test_chunk.product_type.name) + self.assertEqual(chunk.ProductType.SCIENCE.name, + test_chunk.product_type.name) test_chunk.naxis = int(5) self.assertEqual(int(5), test_chunk.naxis) @@ -185,15 +182,14 @@ def test_attributes(self): test_chunk.time = time self.assertEqual(time, test_chunk.time) - polarization = chunk.PolarizationWCS(wcs.CoordAxis1D(wcs.Axis('STOKES'))) + polarization = chunk.PolarizationWCS( + wcs.CoordAxis1D(wcs.Axis('STOKES'))) test_chunk.polarization = polarization self.assertEqual(polarization, test_chunk.polarization) class TestObservableAxis(unittest.TestCase): - def test_init(self): - self.assertRaises(TypeError, chunk.ObservableAxis, None) self.assertRaises(TypeError, chunk.ObservableAxis, int(1)) @@ -208,9 +204,7 @@ def test_init(self): class TestSpatialWCS(unittest.TestCase): - def test_init(self): - self.assertRaises(TypeError, chunk.SpatialWCS, None) self.assertRaises(TypeError, chunk.SpatialWCS, int(1)) @@ -237,9 +231,7 @@ def test_init(self): class TestSpectralWCS(unittest.TestCase): - def test_init(self): - axis = wcs.Axis("ctype", "cunit") axis_1d = wcs.CoordAxis1D(axis) @@ -303,9 +295,7 @@ def test_init(self): class TestTemporalWCS(unittest.TestCase): - def test_init(self): - self.assertRaises(TypeError, chunk.TemporalWCS, None) self.assertRaises(TypeError, chunk.TemporalWCS, int(1)) @@ -337,9 +327,7 @@ def test_init(self): class TestPolarizationWCS(unittest.TestCase): - def test_init(self): - self.assertRaises(TypeError, chunk.PolarizationWCS, None) self.assertRaises(TypeError, chunk.PolarizationWCS, int(1)) diff --git a/caom2/caom2/tests/test_common.py b/caom2/caom2/tests/test_common.py index d4f3516b..2ddb30f4 100644 --- a/caom2/caom2/tests/test_common.py +++ b/caom2/caom2/tests/test_common.py @@ -72,20 +72,18 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -import unittest import binascii +import unittest -from .. import common +from .. import artifact from .. import chunk +from .. import common +from .. import observation from .. import part from .. import plane -from .. import artifact -from .. import observation -from datetime import datetime class TestCaom2IdGenerator(unittest.TestCase): - def test_all(self): # Not much for now. Just to make sure that all the clients work test_entity = common.AbstractCaomEntity() @@ -107,23 +105,31 @@ def test_all(self): test_plane = plane.Plane("prodid") print(test_plane._id, test_plane._last_modified) - + self.assertIsNone(test_plane.last_modified, "last_modified null") - self.assertIsNone(test_plane.max_last_modified, "max_last_modified null") + self.assertIsNone(test_plane.max_last_modified, + "max_last_modified null") self.assertIsNone(test_plane.meta_checksum, "meta_checksum null") - self.assertIsNone(test_plane.acc_meta_checksum, "acc_meta_checksum null") + self.assertIsNone(test_plane.acc_meta_checksum, + "acc_meta_checksum null") d1 = common.get_current_ivoa_time() d2 = common.get_current_ivoa_time() - cs_uri_meta = common.ChecksumURI("md5:e30580c1db513487f495fba09f64600e") - cs_uri_acc = common.ChecksumURI("sha1:7e2b74edf8ff7ddfda5ee3917dc65946b515b1f7") + cs_uri_meta = common.ChecksumURI( + "md5:e30580c1db513487f495fba09f64600e") + cs_uri_acc = common.ChecksumURI( + "sha1:7e2b74edf8ff7ddfda5ee3917dc65946b515b1f7") test_plane.last_modified = d1 test_plane.max_last_modified = d2 test_plane.meta_checksum = cs_uri_meta test_plane.acc_meta_checksum = cs_uri_acc self.assertEquals(test_plane.last_modified, d1, "last_modified") - self.assertEquals(test_plane.max_last_modified, d2, "max_last_modified") - self.assertEquals(test_plane.meta_checksum, cs_uri_meta, "meta_checksum") - self.assertEquals(test_plane.acc_meta_checksum, cs_uri_acc, "acc_meta_checksum") + self.assertEquals(test_plane.max_last_modified, d2, + "max_last_modified") + self.assertEquals(test_plane.meta_checksum, cs_uri_meta, + "meta_checksum") + self.assertEquals(test_plane.acc_meta_checksum, cs_uri_acc, + "acc_meta_checksum") + class TestMetadataChecksum(unittest.TestCase): def test_all(self): @@ -135,38 +141,45 @@ def test_all(self): with self.assertRaises(NotImplementedError): test_artifact.compute_meta_checksum() -class TestObservationURI(unittest.TestCase): +class TestObservationURI(unittest.TestCase): def test_all(self): obs_uri = observation.ObservationURI("caom:GEMINI/12345") self.assertEqual("caom:GEMINI/12345", obs_uri.uri, "Observation URI") self.assertEqual("GEMINI", obs_uri.collection, "Collection") self.assertEqual("12345", obs_uri.observation_id, "Observation ID") - obs_uri = observation.ObservationURI.get_observation_uri("CFHT", "654321") + obs_uri = observation.ObservationURI.get_observation_uri("CFHT", + "654321") self.assertEqual("caom:CFHT/654321", obs_uri.uri, "Observation URI") self.assertEqual("CFHT", obs_uri.collection, "Collection") self.assertEqual("654321", obs_uri.observation_id, "Observation ID") exception = False try: - obs_uri = observation.ObservationURI.get_observation_uri(None, "123") + obs_uri = observation.ObservationURI.get_observation_uri(None, + "123") except TypeError: exception = True self.assertTrue(exception, "Missing exception") exception = False try: - obs_uri = observation.ObservationURI.get_observation_uri("GEMINI", None) + obs_uri = observation.ObservationURI.get_observation_uri("GEMINI", + None) except TypeError: exception = True self.assertTrue(exception, "Missing exception") - -class TestChecksumURI(unittest.TestCase): + +class TestChecksumURI(unittest.TestCase): def test_all(self): cs_uri = common.ChecksumURI("md5:e30580c1db513487f495fba09f64600e") - self.assertEqual("md5:e30580c1db513487f495fba09f64600e", cs_uri.uri, "Checksum URI") + self.assertEqual("md5:e30580c1db513487f495fba09f64600e", cs_uri.uri, + "Checksum URI") self.assertEqual("md5", cs_uri.algorithm, "Algorithm") - self.assertEqual("e30580c1db513487f495fba09f64600e", cs_uri.checksum, "Checksum") - self.assertEqual(binascii.hexlify(bytearray.fromhex("e30580c1db513487f495fba09f64600e")), binascii.hexlify(cs_uri.get_bytes()), "Round trip") + self.assertEqual("e30580c1db513487f495fba09f64600e", cs_uri.checksum, + "Checksum") + self.assertEqual(binascii.hexlify( + bytearray.fromhex("e30580c1db513487f495fba09f64600e")), + binascii.hexlify(cs_uri.get_bytes()), "Round trip") diff --git a/caom2/caom2/tests/test_obs_reader_writer.py b/caom2/caom2/tests/test_obs_reader_writer.py index eb6c1f35..afdc8762 100644 --- a/caom2/caom2/tests/test_obs_reader_writer.py +++ b/caom2/caom2/tests/test_obs_reader_writer.py @@ -72,18 +72,18 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -from six import BytesIO import os import unittest from lxml import etree -from .xml_compare import xml_compare +from six import BytesIO from . import caom_test_instances +from .xml_compare import xml_compare from .. import obs_reader_writer from .. import observation -from .. import wcs from .. import shape +from .. import wcs THIS_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -125,7 +125,6 @@ def complete_composite(depth, bounds_is_circle, version): class TestObservationReaderWriter(unittest.TestCase): - def test_invalid_long_id(self): simple_observation = minimal_simple(1, False, 20) writer = obs_reader_writer.ObservationWriter( @@ -147,7 +146,8 @@ def test_invalid_long_id(self): def test_invalid_uuid(self): simple_observation = minimal_simple(1, False, 21) - writer = obs_reader_writer.ObservationWriter(False, False) # default writer is 2.1 + writer = obs_reader_writer.ObservationWriter(False, + False) # default is 2.1 output = BytesIO() writer.write(simple_observation, output) xml = output.getvalue() @@ -200,40 +200,51 @@ def test_complete_simple(self): def test_minimal_composite(self): for version in (20, 21, 22, 23): for i in range(1, 6): - print("Test Minimal Composite {} version {}".format(i, version)) + print( + "Test Minimal Composite {} version {}".format(i, version)) # CoordBounds2D as CoordCircle2D composite_observation = minimal_composite(i, True, version) # write empty elements - self.observation_test(composite_observation, True, True, version) + self.observation_test(composite_observation, True, True, + version) # do not write empty elements - self.observation_test(composite_observation, True, False, version) + self.observation_test(composite_observation, True, False, + version) # CoordBounds2D as CoordPolygon2D composite_observation = minimal_composite(i, False, version) # write empty elements - self.observation_test(composite_observation, True, True, version) + self.observation_test(composite_observation, True, True, + version) # do not write empty elements - self.observation_test(composite_observation, True, False, version) + self.observation_test(composite_observation, True, False, + version) def test_complete_composite(self): for version in (20, 21, 22, 23): for i in range(1, 6): - print("Test Complete Composite {} version {}".format(i, version)) + print( + "Test Complete Composite {} version {}".format(i, version)) # CoordBounds2D as CoordCircle2D composite_observation = complete_composite(i, True, version) # write empty elements - self.observation_test(composite_observation, True, True, version) + self.observation_test(composite_observation, True, True, + version) # do not write empty elements - self.observation_test(composite_observation, True, False, version) + self.observation_test(composite_observation, True, False, + version) # CoordBounds2D as CoordPolygon2D composite_observation = complete_composite(i, False, version) # write empty elements - self.observation_test(composite_observation, True, True, version) + self.observation_test(composite_observation, True, True, + version) # do not write empty elements - self.observation_test(composite_observation, True, False, version) + self.observation_test(composite_observation, True, False, + version) def test_versions(self): composite_observation = complete_composite(6, True, 20) - print("comp obs max lst mod: " + str(composite_observation.max_last_modified)) + print("comp obs max lst mod: " + str( + composite_observation.max_last_modified)) print("check 2.0 schema with 2.0 doc") self.observation_test(composite_observation, True, True, 20) print("check 2.1 schema with 2.0 doc") @@ -262,7 +273,7 @@ def test_versions(self): self.observation_test(composite_observation, True, True, 22) print("check 2.3 schema with 2.2 doc") self.observation_test(composite_observation, True, True, 23) - + composite_observation = complete_composite(6, True, 23) print("check 2.0 schema with 2.3 doc") self.observation_test(composite_observation, True, True, 20) @@ -278,7 +289,8 @@ def test_versions(self): composite_observation.planes['productID'].position.bounds = None self.observation_test(composite_observation, True, True, 22) - def observation_test(self, obs, validate, write_empty_collections, version): + def observation_test(self, obs, validate, write_empty_collections, + version): if version == 20: writer = obs_reader_writer.ObservationWriter( validate, write_empty_collections, "caom2", @@ -308,10 +320,10 @@ def observation_test(self, obs, validate, write_empty_collections, version): def compare_observations(self, expected, actual, version): assert ((isinstance(expected, observation.SimpleObservation) and - isinstance(actual, observation.SimpleObservation)) or + isinstance(actual, observation.SimpleObservation)) or (isinstance(expected, observation.CompositeObservation) and - isinstance(actual, observation.CompositeObservation))), ( - "Observation types do not match 0 vs 1". + isinstance(actual, observation.CompositeObservation))), \ + ("Observation types do not match 0 vs 1". format(expected.__class__.__name__, actual.__class__.__name__)) @@ -326,7 +338,7 @@ def compare_observations(self, expected, actual, version): self.assertIsNotNone(expected._id) self.assertIsNotNone(actual._id) self.assertEqual(expected._id, actual._id) - + self.compare_entity_attributes(expected, actual) self.assertIsNotNone(expected.algorithm) @@ -344,7 +356,8 @@ def compare_observations(self, expected, actual, version): self.compare_instrument(expected.instrument, actual.instrument) self.compare_environment(expected.environment, actual.environment) if version == 21: - self.compare_requirements(expected.requirements, actual.requirements) + self.compare_requirements(expected.requirements, + actual.requirements) self.compare_planes(expected.planes, actual.planes, version) @@ -458,15 +471,17 @@ def compare_planes(self, expected, actual, version): actual_plane = actual[key] self.assertIsNotNone(expected_plane) self.assertIsNotNone(actual_plane) - self.assertEqual(expected_plane.product_id, actual_plane.product_id) + self.assertEqual(expected_plane.product_id, + actual_plane.product_id) self.assertIsNotNone(expected_plane._id) self.assertIsNotNone(actual_plane._id) self.assertEqual(expected_plane._id, actual_plane._id) - + if version >= 23: self.assertEqual( - expected_plane.creator_id, actual_plane.creator_id, "creator_id") - + expected_plane.creator_id, actual_plane.creator_id, + "creator_id") + self.compare_entity_attributes(expected_plane, actual_plane) self.assertEqual(expected_plane.meta_release, @@ -480,32 +495,37 @@ def compare_planes(self, expected, actual, version): self.compare_provenance(expected_plane.provenance, actual_plane.provenance) self.compare_metrics(expected_plane.metrics, actual_plane.metrics) - + if version == 21: self.compare_quality(expected_plane.quality, actual_plane.quality) - + if version >= 22: - self.compare_position(expected_plane.position, actual_plane.position) + self.compare_position(expected_plane.position, + actual_plane.position) self.compare_energy(expected_plane.energy, actual_plane.energy) print('comparing time') self.compare_time(expected_plane.time, actual_plane.time) print('compared time') - self.compare_polarization(expected_plane.polarization, actual_plane.polarization) + self.compare_polarization(expected_plane.polarization, + actual_plane.polarization) self.compare_artifacts(expected_plane.artifacts, actual_plane.artifacts, version) - + def compare_position(self, expected, actual): if expected is None: self.assertIsNone(actual, "position") else: self.compare_shape(expected.bounds, actual.bounds) self.compare_dimension2d(expected.dimension, actual.dimension) - self.assertEqual(expected.resolution, actual.resolution, "resolution") - self.assertEqual(expected.sample_size, actual.sample_size, "sample_size") - self.assertEqual(expected.time_dependent, actual.time_dependent, "time_dependent") - + self.assertEqual(expected.resolution, actual.resolution, + "resolution") + self.assertEqual(expected.sample_size, actual.sample_size, + "sample_size") + self.assertEqual(expected.time_dependent, actual.time_dependent, + "time_dependent") + def compare_energy(self, expected, actual): print("comparing energy") if expected is None: @@ -513,62 +533,77 @@ def compare_energy(self, expected, actual): else: self.compare_interval(expected.bounds, actual.bounds) self.assertEqual(expected.dimension, actual.dimension, "dimension") - self.assertEqual(expected.resolving_power, actual.resolving_power, "resolving_power") - self.assertEqual(expected.sample_size, actual.sample_size, "sample_size") - self.assertEqual(expected.bandpass_name, actual.bandpass_name, "bandpass_name") + self.assertEqual(expected.resolving_power, actual.resolving_power, + "resolving_power") + self.assertEqual(expected.sample_size, actual.sample_size, + "sample_size") + self.assertEqual(expected.bandpass_name, actual.bandpass_name, + "bandpass_name") self.assertEqual(expected.em_band, actual.em_band, "em_band") - self.assertEqual(expected.bandpass_name, actual.bandpass_name, "bandpass_name") - self.compare_wcs_energy_transition(expected.transition, actual.transition) + self.assertEqual(expected.bandpass_name, actual.bandpass_name, + "bandpass_name") + self.compare_wcs_energy_transition(expected.transition, + actual.transition) - def compare_time(self, expected, actual): if expected is None: self.assertIsNone(actual, "time") else: self.compare_interval(expected.bounds, actual.bounds) self.assertEqual(expected.dimension, actual.dimension, "dimension") - self.assertEqual(expected.resolution, actual.resolution, "resolution") - self.assertEqual(expected.sample_size, actual.sample_size, "sample_size") + self.assertEqual(expected.resolution, actual.resolution, + "resolution") + self.assertEqual(expected.sample_size, actual.sample_size, + "sample_size") self.assertEqual(expected.exposure, actual.exposure, "exposure") - + def compare_polarization(self, expected, actual): if expected is None: self.assertIsNone(expected, "polarization") else: self.assertEqual(expected.dimension, actual.dimension, "dimension") if expected.polarization_states is None: - self.assertIsNone(actual.polarization_states, "polarization_states") + self.assertIsNone(actual.polarization_states, + "polarization_states") else: - self.assertEqual(len(expected.polarization_states), len(actual.polarization_states), "different number of polarization_states") + self.assertEqual(len(expected.polarization_states), + len(actual.polarization_states), + "different number of polarization_states") for index, state in enumerate(expected.polarization_states): - self.assertEqual(state, actual.polarization_states[index], "polarization_state") - + self.assertEqual(state, actual.polarization_states[index], + "polarization_state") + def compare_shape(self, expected, actual): if expected is None: self.assertIsNone(actual, "shape") else: if isinstance(expected, shape.Polygon): self.assertIsNotNone(actual, "shape is None") - self.assertTrue(isinstance(actual, shape.Polygon), "mismatched shapes" + actual.__class__.__name__) + self.assertTrue(isinstance(actual, shape.Polygon), + "mismatched shapes" + + actual.__class__.__name__) expected_points = expected.points actual_points = actual.points - self.assertEqual(len(expected_points), len(actual_points), "different number of points") + self.assertEqual(len(expected_points), len(actual_points), + "different number of points") for index, point in enumerate(expected_points): self.compare_point(point, actual_points[index]) actual_samples = actual.samples self.assertIsNotNone(actual_samples, "shape is None") self.assertTrue(isinstance(actual_samples, shape.MultiPolygon), - "mismatched shapes" + actual.__class__.__name__) + "mismatched shapes" + + actual.__class__.__name__) expected_samples = expected.samples expected_vertices = expected_samples.vertices actual_vertices = actual_samples.vertices - self.assertEqual(len(expected_vertices), len(actual_vertices), "different number of vertices") + self.assertEqual(len(expected_vertices), len(actual_vertices), + "different number of vertices") for index, vertex in enumerate(expected_vertices): self.compare_vertices(vertex, actual_vertices[index]) else: raise TypeError("Unsupported shape type " - + expected.__class__.__name__) - + + expected.__class__.__name__) + def compare_interval(self, expected, actual): if expected is None: self.assertIsNone(actual, "interval") @@ -578,29 +613,31 @@ def compare_interval(self, expected, actual): if expected.samples is None: self.assertIsNone(actual.samples, "samples") else: - self.assertEqual(len(actual.samples), len(expected.samples), "samples") + self.assertEqual(len(actual.samples), len(expected.samples), + "samples") for index, sample in enumerate(expected.samples): self.compare_sub_interval(sample, actual.samples[index]) - + def compare_sub_interval(self, expected, actual): if expected is None: self.assertIsNone(actual, "sub_interval") else: self.assertEqual(expected.lower, actual.lower, "lower") self.assertEqual(expected.upper, actual.upper, "upper") - + def compare_wcs_energy_transition(self, expected, actual): if expected is None: self.assertIsNone(actual, "wcs_energy_transition") else: self.assertEqual(expected.species, actual.species, "species") - self.assertEqual(expected.transition, actual.transition, "transition") - + self.assertEqual(expected.transition, actual.transition, + "transition") + def compare_vertices(self, expected, actual): self.assertEqual(expected.cval1, actual.cval1) self.assertEqual(expected.cval2, actual.cval2) self.assertEqual(expected.type, actual.type) - + def compare_provenance(self, expected, actual): if expected is None and actual is None: return @@ -622,8 +659,10 @@ def compare_metrics(self, expected, actual): self.assertEqual(expected.source_number_density, actual.source_number_density) self.assertEqual(expected.background, actual.background) - self.assertEqual(expected.background_std_dev, actual.background_std_dev) - self.assertEqual(expected.flux_density_limit, actual.flux_density_limit) + self.assertEqual(expected.background_std_dev, + actual.background_std_dev) + self.assertEqual(expected.flux_density_limit, + actual.flux_density_limit) self.assertEqual(expected.mag_limit, actual.mag_limit) def compare_quality(self, expected, actual): @@ -657,7 +696,7 @@ def compare_artifacts(self, expected, actual, version): self.assertIsNotNone(expected_artifact._id) self.assertIsNotNone(actual_artifact._id) self.assertEqual(expected_artifact._id, actual_artifact._id) - + self.compare_entity_attributes(expected_artifact, actual_artifact) self.assertEqual(expected_artifact.uri, actual_artifact.uri) @@ -688,11 +727,12 @@ def compare_parts(self, expected, actual, version): self.assertIsNotNone(expected_part._id) self.assertIsNotNone(actual_part._id) self.assertEqual(expected_part._id, actual_part._id) - + self.compare_entity_attributes(expected_part, actual_part) self.assertEqual(expected_part.name, actual_part.name) - self.assertEqual(expected_part.product_type, actual_part.product_type) + self.assertEqual(expected_part.product_type, + actual_part.product_type) self.compare_chunks(expected_part.chunks, actual_part.chunks) def compare_chunks(self, expected, actual): @@ -707,9 +747,9 @@ def compare_chunks(self, expected, actual): self.assertIsNotNone(expected_chunk._id) self.assertIsNotNone(actual_chunk._id) self.assertEqual(expected_chunk._id, actual_chunk._id) - + self.compare_entity_attributes(expected_chunk, actual_chunk) - + self.assertEqual(expected_chunk.product_type, actual_chunk.product_type) self.assertEqual(expected_chunk.naxis, actual_chunk.naxis) @@ -719,7 +759,8 @@ def compare_chunks(self, expected, actual): actual_chunk.position_axis_1) self.assertEqual(expected_chunk.position_axis_2, actual_chunk.position_axis_2) - self.assertEqual(expected_chunk.energy_axis, actual_chunk.energy_axis) + self.assertEqual(expected_chunk.energy_axis, + actual_chunk.energy_axis) self.assertEqual(expected_chunk.time_axis, actual_chunk.time_axis) self.assertEqual(expected_chunk.polarization_axis, actual_chunk.polarization_axis) @@ -727,7 +768,8 @@ def compare_chunks(self, expected, actual): actual_chunk.observable) self.compare_spatial_wcs(expected_chunk.position, actual_chunk.position) - self.compare_spectral_wcs(expected_chunk.energy, actual_chunk.energy) + self.compare_spectral_wcs(expected_chunk.energy, + actual_chunk.energy) self.compare_temporal_wcs(expected_chunk.time, actual_chunk.time) self.compare_polarization_wcs(expected_chunk.polarization, actual_chunk.polarization) @@ -845,7 +887,8 @@ def compare_coord_bounds1d(self, expected, actual): self.assertIsNotNone(expected.samples) self.assertIsNotNone(actual.samples) self.assertEqual(len(expected.samples), len(actual.samples)) - for expected_range, actual_range in zip(expected.samples, actual.samples): + for expected_range, actual_range in zip(expected.samples, + actual.samples): self.compare_coord_range1d(expected_range, actual_range) def compare_coord_bounds2d(self, expected, actual): @@ -977,22 +1020,30 @@ def compare_point(self, expected, actual): self.assertIsNotNone(actual.cval2) self.assertEqual(expected.cval1, actual.cval1) self.assertEqual(expected.cval2, actual.cval2) - + def compare_entity_attributes(self, expected, actual): - if expected.last_modified is not None and actual.last_modified is not None: - self.assertEqual(expected.last_modified, actual.last_modified, "last_modified") - - if expected.max_last_modified is not None and actual.max_last_modified is not None: - self.assertEqual(expected.max_last_modified, actual.max_last_modified, "max_last_modified") + if expected.last_modified is not None and\ + actual.last_modified is not None: + self.assertEqual(expected.last_modified, actual.last_modified, + "last_modified") - if expected.meta_checksum is not None and actual.meta_checksum is not None: - self.assertEqual(expected.meta_checksum, actual.meta_checksum, "meta_checksum") + if expected.max_last_modified is not None and\ + actual.max_last_modified is not None: + self.assertEqual(expected.max_last_modified, + actual.max_last_modified, "max_last_modified") - if expected.acc_meta_checksum is not None and actual.acc_meta_checksum is not None: - self.assertEqual(expected.acc_meta_checksum, actual.acc_meta_checksum, "acc_meta_checksum") + if expected.meta_checksum is not None and\ + actual.meta_checksum is not None: + self.assertEqual(expected.meta_checksum, actual.meta_checksum, + "meta_checksum") + + if expected.acc_meta_checksum is not None and\ + actual.acc_meta_checksum is not None: + self.assertEqual(expected.acc_meta_checksum, + actual.acc_meta_checksum, "acc_meta_checksum") -class TestRoundTrip(unittest.TestCase): +class TestRoundTrip(unittest.TestCase): TEST_DATA = 'data' def init(self): @@ -1003,8 +1054,8 @@ def get_file_list(self): if f.endswith('.xml')] def do_test(self, reader, writer, filename): - source_file_path = os.path.join(THIS_DIR + "/" + - TestRoundTrip.TEST_DATA + "/" + filename) + source_file_path = os.path.join( + THIS_DIR + "/" + TestRoundTrip.TEST_DATA + "/" + filename) source_xml_fp = open(source_file_path, 'r') obs = reader.read(source_file_path) source_xml_fp.close() @@ -1027,7 +1078,8 @@ def test_round_trip(self): try: self.init() files = self.get_file_list() - self.assertTrue(len(files) > 0, 'No XML files in test data directory') + self.assertTrue(len(files) > 0, + 'No XML files in test data directory') reader = obs_reader_writer.ObservationReader(True) writer20 = obs_reader_writer.ObservationWriter( diff --git a/caom2/caom2/tests/test_observation.py b/caom2/caom2/tests/test_observation.py index a7486b6d..cc97174b 100644 --- a/caom2/caom2/tests/test_observation.py +++ b/caom2/caom2/tests/test_observation.py @@ -82,7 +82,6 @@ class TestEnums(unittest.TestCase): - def test_all(self): # test for invalid value with self.assertRaises(KeyError): @@ -107,8 +106,10 @@ def test_all(self): observation.TargetType(1) # test that we can get the object for each enum by name - self.assertEqual(observation.ObservationIntentType.CALIBRATION.value, "calibration") - self.assertEqual(observation.ObservationIntentType.SCIENCE.value, "science") + self.assertEqual(observation.ObservationIntentType.CALIBRATION.value, + "calibration") + self.assertEqual(observation.ObservationIntentType.SCIENCE.value, + "science") self.assertEqual(observation.Status.FAIL.value, "fail") @@ -117,7 +118,6 @@ def test_all(self): class TestObservation(unittest.TestCase): - def test_all(self): algorithm = observation.Algorithm("myAlg") obs = observation.Observation("GSA", "A12345", algorithm) @@ -161,7 +161,8 @@ def test_all(self): self.assertEqual(target, obs.target, "Target") self.assertIsNone(obs.target_position, "Default target position") - target_position = observation.TargetPosition(shape.Point(1.0, 2.0), "coordsys") + target_position = observation.TargetPosition(shape.Point(1.0, 2.0), + "coordsys") obs.target_position = target_position self.assertEqual(target_position, obs.target_position, "TargetPosition") @@ -203,7 +204,7 @@ def test_all(self): self.assertTrue("myPlaneID" in obs.planes) self.assertTrue("myPlaneID2" in obs.planes.keys()) - obs2 = observation.Observation( + observation.Observation( obs.collection, obs.observation_id, obs.algorithm, @@ -221,7 +222,6 @@ def test_all(self): class TestSimpleObservation(unittest.TestCase): - def test_all(self): algorithm = observation.SimpleObservation._ALGORITHM obs = observation.SimpleObservation("GSA", "A12345") @@ -282,7 +282,8 @@ def test_all(self): obs.environment, "Environment") self.assertIsNone(obs.target_position, "Default target position") - target_position = observation.TargetPosition(shape.Point(1.0, 2.0), "coordsys") + target_position = observation.TargetPosition(shape.Point(1.0, 2.0), + "coordsys") obs.target_position = target_position self.assertEqual(target_position, obs.target_position, "TargetPosition") @@ -311,7 +312,7 @@ def test_complete_init(self): instrument = observation.Instrument("INST") target = observation.Target("LMC") meta_release = datetime.now() - planes = caom_util.TypedOrderedDict(plane.Plane,) + planes = caom_util.TypedOrderedDict(plane.Plane, ) environment = observation.Environment() obs = observation.SimpleObservation( @@ -367,7 +368,6 @@ def test_complete_init(self): class TestCompositeObservation(unittest.TestCase): - def test_all(self): algorithm = observation.Algorithm("mozaic") obs = observation.CompositeObservation("GSA", "A12345", algorithm) @@ -454,7 +454,8 @@ def test_all(self): obs.environment, "Environment") self.assertIsNone(obs.target_position, "Default target position") - target_position = observation.TargetPosition(shape.Point(1.0, 2.0), "coordsys") + target_position = observation.TargetPosition(shape.Point(1.0, 2.0), + "coordsys") obs.target_position = target_position self.assertEqual(target_position, obs.target_position, "TargetPosition") @@ -483,9 +484,10 @@ def test_complete_init(self): instrument = observation.Instrument("INST") target = observation.Target("LMC") meta_release = datetime.now() - planes = caom_util.TypedOrderedDict(plane.Plane,) + planes = caom_util.TypedOrderedDict(plane.Plane, ) environment = observation.Environment() - target_position = observation.TargetPosition(shape.Point(1.0, 2.0), "coordsys") + target_position = observation.TargetPosition(shape.Point(1.0, 2.0), + "coordsys") obs = observation.CompositeObservation( collection, @@ -552,14 +554,12 @@ def test_complete_init(self): class TestAlgorithm(unittest.TestCase): - def test_all(self): algorithm = observation.Algorithm("myAlgorithm") self.assertEqual("myAlgorithm", algorithm.name, "Algorithm name") class TestEnvironment(unittest.TestCase): - def test_all(self): environment = observation.Environment() @@ -589,11 +589,11 @@ def test_all(self): class TestIntrument(unittest.TestCase): - def test_all(self): instrument = observation.Instrument("myInstrument") self.assertEqual("myInstrument", instrument.name, "Instrument name") - self.assertEqual(0, len(instrument.keywords), "Default number of keywords") + self.assertEqual(0, len(instrument.keywords), + "Default number of keywords") instrument.keywords.add("optical") self.assertEqual(1, len(instrument.keywords), "Number of keywords") @@ -605,11 +605,11 @@ def test_all(self): class TestProposal(unittest.TestCase): - def test_all(self): proposal = observation.Proposal("myProposal") self.assertEqual("myProposal", proposal.id, "Proposal ID") - self.assertEqual(0, len(proposal.keywords), "Default number of keywords") + self.assertEqual(0, len(proposal.keywords), + "Default number of keywords") proposal.keywords.add("optical") self.assertEqual(1, len(proposal.keywords), "Number of keywords") self.assertTrue("optical" in proposal.keywords, "Keyword not found") @@ -625,7 +625,6 @@ def test_all(self): class TestRequirements(unittest.TestCase): - def test_all(self): self.assertRaises(TypeError, observation.Requirements, "string") requirements = observation.Requirements(observation.Status.FAIL) @@ -634,13 +633,13 @@ def test_all(self): class TestTarget(unittest.TestCase): - def test_all(self): target = observation.Target("myTarget") self.assertEqual("myTarget", target.name, "target name") target.target_type = observation.TargetType.FIELD - self.assertEqual(observation.TargetType.FIELD.name, target.target_type.name, "target type") + self.assertEqual(observation.TargetType.FIELD.name, + target.target_type.name, "target type") self.assertEqual(0, len(target.keywords), "Default number of keywords") target.keywords.add("optical") @@ -659,9 +658,12 @@ def test_all(self): target.moving = True self.assertTrue(target.moving, "Moving") - target = observation.Target("myOtherTarget", observation.TargetType.OBJECT, False, 1.2, {"radio"}, False) + target = observation.Target("myOtherTarget", + observation.TargetType.OBJECT, False, 1.2, + {"radio"}, False) self.assertEquals("myOtherTarget", target.name, "target name") - self.assertEquals(observation.TargetType.OBJECT, target.target_type, "target type") + self.assertEquals(observation.TargetType.OBJECT, target.target_type, + "target type") self.assertFalse(target.standard, "Standard") self.assertEquals(1.2, target.redshift, "Redshift") self.assertEquals(1, len(target.keywords), "Keywords") @@ -670,7 +672,6 @@ def test_all(self): class TestTargetPosition(unittest.TestCase): - def test_all(self): self.assertRaises(TypeError, observation.TargetPosition, "string") point = shape.Point(1.0, 2.0) @@ -695,11 +696,11 @@ def test_all(self): class TestTelescope(unittest.TestCase): - def test_all(self): telescope = observation.Telescope("myTelescope") self.assertEqual("myTelescope", telescope.name, "telescope name") - self.assertEqual(0, len(telescope.keywords), "Default number of keywords") + self.assertEqual(0, len(telescope.keywords), + "Default number of keywords") telescope.keywords.add("optical") self.assertEqual(1, len(telescope.keywords), "Number of keywords") diff --git a/caom2/caom2/tests/test_part.py b/caom2/caom2/tests/test_part.py index 62881a00..802ffbf7 100644 --- a/caom2/caom2/tests/test_part.py +++ b/caom2/caom2/tests/test_part.py @@ -79,9 +79,7 @@ class TestPart(unittest.TestCase): - def test_init(self): - test_part = part.Part(u"partName") self.assertEquals("partName", test_part.name, "Part name") self.assertIsNone(test_part.product_type) diff --git a/caom2/caom2/tests/test_plane.py b/caom2/caom2/tests/test_plane.py index 405cc70c..c27c8f7b 100644 --- a/caom2/caom2/tests/test_plane.py +++ b/caom2/caom2/tests/test_plane.py @@ -76,15 +76,14 @@ from datetime import datetime from .. import artifact +from .. import chunk from .. import observation from .. import plane -from .. import chunk -from .. import wcs from .. import shape +from .. import wcs class TestEnums(unittest.TestCase): - def test_all(self): # test for invalid value with self.assertRaises(ValueError): @@ -135,11 +134,15 @@ def test_all(self): self.assertEqual(plane.DataProductType.SPECTRUM.value, "spectrum") self.assertEqual(plane.DataProductType.TIMESERIES.value, "timeseries") self.assertEqual(plane.DataProductType.VISIBILITY.value, "visibility") - self.assertEqual(plane.DataProductType.MEASUREMENTS.value, "measurements") - self.assertEqual(plane.DataProductType.CATALOG.value, "http://www.opencadc.org/caom2/DataProductType#catalog") + self.assertEqual(plane.DataProductType.MEASUREMENTS.value, + "measurements") + self.assertEqual( + plane.DataProductType.CATALOG.value, + "http://www.opencadc.org/caom2/DataProductType#catalog") # extend data product type plane.DataProductType.extend('http://www.myorg/std', 'mytype') - self.assertEqual(plane.DataProductType.MYTYPE.value, 'http://www.myorg/std#mytype') + self.assertEqual(plane.DataProductType.MYTYPE.value, + 'http://www.myorg/std#mytype') self.assertEqual(plane.EnergyBand['RADIO'].value, "Radio") self.assertEqual(plane.EnergyBand['MILLIMETER'].value, "Millimeter") @@ -167,36 +170,42 @@ def test_all(self): class TestPlane(unittest.TestCase): - def test_all(self): test_plane = plane.Plane("ProdID") self.assertEqual("ProdID", test_plane.product_id, "Product ID") self.assertEqual(None, test_plane.creator_id, "Creator ID") test_plane.creator_id = "ivo://cadc.nrc.ca/users?tester" - self.assertEqual("ivo://cadc.nrc.ca/users?tester", test_plane.creator_id, "Creator ID") + self.assertEqual("ivo://cadc.nrc.ca/users?tester", + test_plane.creator_id, "Creator ID") self.assertEqual(0, len(test_plane.artifacts), "Default number of artifacts") self.assertIsNone(test_plane.meta_release, "Default meta release date") date_now = datetime.now() test_plane.meta_release = date_now - self.assertEqual(date_now, test_plane.meta_release, "Metadata release date") + self.assertEqual(date_now, test_plane.meta_release, + "Metadata release date") self.assertIsNone(test_plane.data_release, "Default data release date") date_now = datetime.now() test_plane.data_release = date_now - self.assertEqual(date_now, test_plane.data_release, "Data release date") - self.assertIsNone(test_plane.data_product_type, "Default data product type") + self.assertEqual(date_now, test_plane.data_release, + "Data release date") + self.assertIsNone(test_plane.data_product_type, + "Default data product type") test_plane.data_product_type = plane.DataProductType.IMAGE - self.assertEqual(plane.DataProductType.IMAGE, test_plane.data_product_type, + self.assertEqual(plane.DataProductType.IMAGE, + test_plane.data_product_type, "Data product type") plane.DataProductType.extend('http://www.myorg/std', 'mytype') test_plane.data_product_type = plane.DataProductType.MYTYPE - self.assertEqual(plane.DataProductType.MYTYPE, test_plane.data_product_type, + self.assertEqual(plane.DataProductType.MYTYPE, + test_plane.data_product_type, "Mytype product type") self.assertIsNone(test_plane.calibration_level, "Default calibration level") test_plane.calibration_level = plane.CalibrationLevel.CALIBRATED self.assertEqual(plane.CalibrationLevel.CALIBRATED, - test_plane.calibration_level, "plane.CalibrationLevel") + test_plane.calibration_level, + "plane.CalibrationLevel") self.assertIsNone(test_plane.quality, "Default quality") quality = plane.DataQuality(plane.Quality.JUNK) @@ -206,7 +215,8 @@ def test_all(self): self.assertIsNone(test_plane.provenance, "Default provenance") provenance = plane.Provenance("myProv") test_plane.provenance = provenance - self.assertEqual("myProv", test_plane.provenance.name, "Provenance - name") + self.assertEqual("myProv", test_plane.provenance.name, + "Provenance - name") self.assertIsNone(test_plane.metrics, "Default metrics") metrics = plane.Metrics() test_plane.metrics = metrics @@ -290,24 +300,26 @@ def test_all(self): class TestPlaneURI(unittest.TestCase): - def test_all(self): plane_uri = plane.PlaneURI("caom:GEMINI/12345/3333") self.assertEqual("caom:GEMINI/12345/3333", plane_uri.uri, "Plane URI") self.assertEqual("GEMINI", plane_uri.get_observation_uri().collection, "Collection") - self.assertEqual("12345", plane_uri.get_observation_uri().observation_id, + self.assertEqual("12345", + plane_uri.get_observation_uri().observation_id, "Observation ID") self.assertEqual("3333", plane_uri.get_product_id(), "Product ID") - plane_uri = plane.PlaneURI.get_plane_uri(observation.ObservationURI("caom:CFHT/654321"), - "555") + plane_uri = plane.PlaneURI.get_plane_uri( + observation.ObservationURI("caom:CFHT/654321"), + "555") self.assertEqual("caom:CFHT/654321/555", plane_uri.uri, "Observation URI") self.assertEqual("CFHT", plane_uri.get_observation_uri().collection, "Collection") - self.assertEqual("654321", plane_uri.get_observation_uri().observation_id, + self.assertEqual("654321", + plane_uri.get_observation_uri().observation_id, "Observation ID") self.assertEqual("555", plane_uri.get_product_id(), "Product ID") @@ -342,9 +354,7 @@ def test_all(self): class TestDataQuality(unittest.TestCase): - def test_all(self): - self.assertRaises(TypeError, plane.DataQuality, "string") quality = plane.DataQuality(plane.Quality.JUNK) self.assertEqual(plane.Quality.JUNK, quality.flag, @@ -352,7 +362,6 @@ def test_all(self): class TestMetrics(unittest.TestCase): - def test_all(self): metrics = plane.Metrics() @@ -380,7 +389,6 @@ def test_all(self): class TestProvenance(unittest.TestCase): - def test_all(self): provenance = plane.Provenance("MyProvenance") self.assertEqual("MyProvenance", provenance.name, "Name") @@ -459,7 +467,6 @@ def test_all(self): class TestPosition(unittest.TestCase): - def test_all(self): position = plane.Position() @@ -481,7 +488,6 @@ def test_all(self): class TestEnergy(unittest.TestCase): - def test_all(self): energy = plane.Energy() self.assertIsNone(energy.bounds, "Default energy bounds") @@ -504,15 +510,17 @@ def test_all(self): self.assertEqual("EBN", energy.bandpass_name, "Energy bandpass name") self.assertIsNone(energy.em_band, "Default energy em band") energy.em_band = plane.EnergyBand.OPTICAL - self.assertEqual(plane.EnergyBand.OPTICAL, energy.em_band, "Energy band") + self.assertEqual(plane.EnergyBand.OPTICAL, energy.em_band, + "Energy band") self.assertIsNone(energy.transition, "Default energy transition") energy.transition = wcs.EnergyTransition("aSpecies", "aTransition") - self.assertEqual("aSpecies", energy.transition.species, "Energy transition species") - self.assertEqual("aTransition", energy.transition.transition, "Energy transition transition") + self.assertEqual("aSpecies", energy.transition.species, + "Energy transition species") + self.assertEqual("aTransition", energy.transition.transition, + "Energy transition transition") class TestEnergyTransition(unittest.TestCase): - def test__init__(self): # test for invalid values self.assertRaises(TypeError, wcs.EnergyTransition, None, None) @@ -538,7 +546,6 @@ def test_setters(self): class TestPolarizaton(unittest.TestCase): - def test_all(self): polarization = plane.Polarization() @@ -549,7 +556,6 @@ def test_all(self): class TestTime(unittest.TestCase): - def test_all(self): time = plane.Time() self.assertIsNone(time.bounds, "Default bounds") diff --git a/caom2/caom2/tests/test_shape.py b/caom2/caom2/tests/test_shape.py index c268638b..c4b9751a 100644 --- a/caom2/caom2/tests/test_shape.py +++ b/caom2/caom2/tests/test_shape.py @@ -70,14 +70,13 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -import unittest import math +import unittest from .. import shape class TestEnums(unittest.TestCase): - def test_all(self): # test for invalid value with self.assertRaises(KeyError): @@ -99,9 +98,7 @@ def test_all(self): class TestBox(unittest.TestCase): - def test_all(self): - self.assertRaises(TypeError, shape.Box, None, None, None) self.assertRaises(TypeError, shape.Box, None, None, 1.0) self.assertRaises(TypeError, shape.Box, None, 1.0, None) @@ -118,16 +115,14 @@ def test_all(self): self.assertEqual(box.height, 4.0) self.assertEqual(box.center.cval1, 1.0) self.assertEqual(box.center.cval2, 2.0) - area = width * height + area = width * height self.assertEqual(box.get_area(), area) size = math.sqrt(width * width + height * height) self.assertEqual(box.get_size(), size) class TestCircle(unittest.TestCase): - def test_all(self): - self.assertRaises(TypeError, shape.Circle, None, None) self.assertRaises(TypeError, shape.Circle, None, 1.0) self.assertRaises(TypeError, shape.Circle, 1.0, None) @@ -141,13 +136,12 @@ def test_all(self): self.assertEqual(circle.radius, radius) self.assertEqual(circle.center.cval1, val1) self.assertEqual(circle.center.cval2, val2) - area = math.pi * radius * radius + area = math.pi * radius * radius self.assertEqual(circle.get_area(), area) self.assertEqual(circle.get_size(), 2.0 * radius) class TestInterval(unittest.TestCase): - def test_all(self): lower = 1.0 @@ -156,7 +150,9 @@ def test_all(self): upper1 = 2.1 lower2 = 1.2 upper2 = 2.2 - samples = [shape.SubInterval(lower,upper), shape.SubInterval(lower1,upper1), shape.SubInterval(lower2,upper2)] + samples = [shape.SubInterval(lower, upper), + shape.SubInterval(lower1, upper1), + shape.SubInterval(lower2, upper2)] self.assertRaises(TypeError, shape.Interval, None, None, None) self.assertRaises(TypeError, shape.Interval, None, None, 1.0) @@ -167,7 +163,8 @@ def test_all(self): self.assertRaises(TypeError, shape.Interval, int(1), None, samples) self.assertRaises(TypeError, shape.Interval, None, "string", samples) self.assertRaises(TypeError, shape.Interval, "string", None, samples) - self.assertRaises(TypeError, shape.Interval, "string1", "string2", int(1)) + self.assertRaises(TypeError, shape.Interval, "string1", "string2", + int(1)) self.assertRaises(AssertionError, shape.Interval, 2.0, 1.0, None) # test cannot set interval with upper < lower @@ -225,12 +222,10 @@ def test_all(self): ub = min(i1.upper, i3.upper) intersec3 = shape.Interval.intersection(i1, i3) self.assertEqual(intersec3, shape.Interval(lb, ub)) - -class TestPoint(unittest.TestCase): +class TestPoint(unittest.TestCase): def test_all(self): - self.assertRaises(TypeError, shape.Point, None, None) self.assertRaises(TypeError, shape.Point, None, 1.0) self.assertRaises(TypeError, shape.Point, 1.0, None) @@ -240,10 +235,9 @@ def test_all(self): point = shape.Point(1.0, 2.0) self.assertEqual(point.cval1, 1.0) self.assertEqual(point.cval2, 2.0) - -class TestSubInterval(unittest.TestCase): +class TestSubInterval(unittest.TestCase): def test_all(self): self.assertRaises(TypeError, shape.SubInterval, None, None) @@ -262,13 +256,11 @@ def test_all(self): self.assertEqual(has_assertionError, True) # test construction method - i1 = shape.SubInterval(10.0, 15.0) + shape.SubInterval(10.0, 15.0) class TestPolygon(unittest.TestCase): - def test_all(self): - p1 = shape.Point(1.0, 2.0) p2 = shape.Point(2.0, 3.0) p3 = shape.Point(3.0, 4.0) @@ -298,7 +290,6 @@ def test_all(self): class TestMultiPolygon(unittest.TestCase): - def test_all(self): v0 = shape.Vertex(1.0, 2.0, shape.SegmentType.MOVE) v1 = shape.Vertex(1.0, 2.0, shape.SegmentType.LINE) @@ -356,23 +347,28 @@ def test_all(self): with self.assertRaises(AssertionError): mp.validate() -class TestVertex(unittest.TestCase): +class TestVertex(unittest.TestCase): def test_all(self): - self.assertRaises(TypeError, shape.Vertex, None, None, None) self.assertRaises(TypeError, shape.Vertex, 1.0, 2.0, None) self.assertRaises(TypeError, shape.Vertex, 1.0, 2.0, 1.0) - self.assertRaises(TypeError, shape.Vertex, None, None, shape.SegmentType.LINE) - self.assertRaises(TypeError, shape.Vertex, None, 2.0, shape.SegmentType.LINE) - self.assertRaises(TypeError, shape.Vertex, 1.0, None, shape.SegmentType.LINE) - self.assertRaises(TypeError, shape.Vertex, None, "string", shape.SegmentType.LINE) - self.assertRaises(TypeError, shape.Vertex, "string", None, shape.SegmentType.LINE) - self.assertRaises(TypeError, shape.Vertex, None, int(1), shape.SegmentType.LINE) - self.assertRaises(TypeError, shape.Vertex, int(1), None, shape.SegmentType.LINE) + self.assertRaises(TypeError, shape.Vertex, None, None, + shape.SegmentType.LINE) + self.assertRaises(TypeError, shape.Vertex, None, 2.0, + shape.SegmentType.LINE) + self.assertRaises(TypeError, shape.Vertex, 1.0, None, + shape.SegmentType.LINE) + self.assertRaises(TypeError, shape.Vertex, None, "string", + shape.SegmentType.LINE) + self.assertRaises(TypeError, shape.Vertex, "string", None, + shape.SegmentType.LINE) + self.assertRaises(TypeError, shape.Vertex, None, int(1), + shape.SegmentType.LINE) + self.assertRaises(TypeError, shape.Vertex, int(1), None, + shape.SegmentType.LINE) vertex = shape.Vertex(1.0, 2.0, shape.SegmentType.LINE) self.assertEqual(vertex.cval1, 1.0) self.assertEqual(vertex.cval2, 2.0) self.assertEqual(vertex.type, shape.SegmentType.LINE) - diff --git a/caom2/caom2/tests/test_wcs.py b/caom2/caom2/tests/test_wcs.py index d8db39c8..7b166346 100644 --- a/caom2/caom2/tests/test_wcs.py +++ b/caom2/caom2/tests/test_wcs.py @@ -69,17 +69,16 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -from builtins import int import unittest +from builtins import int + from .. import wcs class TestAxis(unittest.TestCase): - def test_init(self): - self.assertRaises(TypeError, wcs.Axis, None, None) self.assertRaises(TypeError, wcs.Axis, None, "cunit") self.assertRaises(TypeError, wcs.Axis, "ctype", int(1)) @@ -91,9 +90,7 @@ def test_init(self): class TestCoord2D(unittest.TestCase): - def test_init(self): - coord1 = wcs.RefCoord(float(1.0), float(2.0)) coord2 = wcs.RefCoord(float(3.0), float(4.0)) @@ -109,9 +106,7 @@ def test_init(self): class TestCoordAxis1D(unittest.TestCase): - def test_init(self): - self.assertRaises(TypeError, wcs.CoordAxis1D, None) self.assertRaises(TypeError, wcs.CoordAxis1D, int(1)) @@ -147,9 +142,7 @@ def test_init(self): class TestCoordAxis2D(unittest.TestCase): - def test_init(self): - self.assertRaises(TypeError, wcs.CoordAxis2D, None, None) self.assertRaises(TypeError, wcs.CoordAxis2D, None, int(1)) self.assertRaises(TypeError, wcs.CoordAxis2D, int(1), None) @@ -206,9 +199,7 @@ def test_init(self): class TestCoordBounds1D(unittest.TestCase): - def test_init(self): - start = wcs.RefCoord(float(1.0), float(2.0)) end = wcs.RefCoord(float(3.0), float(4.0)) coord_range = wcs.CoordRange1D(start, end) @@ -223,9 +214,7 @@ def test_init(self): class TestCoordBounds2D(unittest.TestCase): - def test_init(self): - self.assertRaises(TypeError, wcs.CoordBounds2D, None) self.assertRaises(TypeError, wcs.CoordBounds2D, float(1.0)) @@ -244,9 +233,7 @@ def test_init(self): class TestCoordCircle2D(unittest.TestCase): - def test_init(self): - center = wcs.ValueCoord2D(float(1.0), float(2.0)) radius = float(1.5) @@ -262,7 +249,6 @@ def test_init(self): class TestCoordError(unittest.TestCase): - def test_init(self): self.assertRaises(TypeError, wcs.CoordError, None, None) self.assertRaises(TypeError, wcs.CoordError, None, float(1.0)) @@ -277,9 +263,7 @@ def test_init(self): class TestCoordFunction1D(unittest.TestCase): - def test_init(self): - naxis = int(1) delta = float(2.5) ref_coord = wcs.RefCoord(float(1.0), float(2.0)) @@ -306,9 +290,7 @@ def test_init(self): class TestCoordFunction2D(unittest.TestCase): - def test_init(self): - dimension = wcs.Dimension2D(int(1), int(2)) ref_coord = wcs.Coord2D(wcs.RefCoord(float(9.0), float(10.0)), wcs.RefCoord(float(11.0), float(12.0))) @@ -341,9 +323,7 @@ def test_init(self): class TestCoordPolygon2D(unittest.TestCase): - def test_init(self): - value_coord2d = wcs.ValueCoord2D(float(1.0), float(2.0)) polygon = wcs.CoordPolygon2D() @@ -356,9 +336,7 @@ def test_init(self): class TestCoordRange1D(unittest.TestCase): - def test_init(self): - start = wcs.RefCoord(float(1.0), float(2.0)) end = wcs.RefCoord(float(3.0), float(4.0)) @@ -374,9 +352,7 @@ def test_init(self): class TestCoordRange2D(unittest.TestCase): - def test_init(self): - start = wcs.Coord2D(wcs.RefCoord(float(1.0), float(2.0)), wcs.RefCoord(float(3.0), float(4.0))) end = wcs.Coord2D(wcs.RefCoord(float(5.0), float(6.0)), @@ -394,9 +370,7 @@ def test_init(self): class TestDimension2D(unittest.TestCase): - def test_init(self): - self.assertRaises(TypeError, wcs.Dimension2D, None, None) self.assertRaises(TypeError, wcs.Dimension2D, int(1), None) self.assertRaises(TypeError, wcs.Dimension2D, None, int(1)) @@ -409,9 +383,7 @@ def test_init(self): class TestRefCoord(unittest.TestCase): - def test_init(self): - self.assertRaises(TypeError, wcs.RefCoord, None, None) self.assertRaises(TypeError, wcs.RefCoord, None, float(1.0)) self.assertRaises(TypeError, wcs.RefCoord, float(1.0), None) @@ -425,9 +397,7 @@ def test_init(self): class TestSlice(unittest.TestCase): - def test_init(self): - axis = wcs.Axis("ctype", "cunit") my_bin = int(1) @@ -443,9 +413,7 @@ def test_init(self): class TestValueCoord2d(unittest.TestCase): - def test_init(self): - self.assertRaises(TypeError, wcs.ValueCoord2D, None, None) self.assertRaises(TypeError, wcs.ValueCoord2D, None, float(1.0)) self.assertRaises(TypeError, wcs.ValueCoord2D, float(1.0), None) diff --git a/caom2/caom2/tests/xml_compare.py b/caom2/caom2/tests/xml_compare.py index 1d95b597..24dcd207 100644 --- a/caom2/caom2/tests/xml_compare.py +++ b/caom2/caom2/tests/xml_compare.py @@ -1,11 +1,17 @@ """ -NOTE: This is the xml_compare v1.0.5 module available on pypi (https://pypi.python.org/pypi/xml_compare/1.0.5) -with a tiny patch on line 69 to make it work with Python 3. The module hasn't been updated in almost 10 years, +NOTE: This is the xml_compare v1.0.5 module available on pypi +(https://pypi.python.org/pypi/xml_compare/1.0.5) +with a tiny patch on line 69 to make it work with Python 3. +The module hasn't been updated in almost 10 years, hence the need to copy it over here and patch it. -Also patched the xml_compare function to deal with children nodes that are in different order. +Also patched the xml_compare function to deal with children nodes that are in +different order. """ + +import os + """ This module implements a XML comparer @@ -28,28 +34,31 @@ >>> xml5b = "4567.23" >>> doc5a = etree.fromstring(xml5a) >>> doc5b = etree.fromstring(xml5b) ->>> xml_compare(doc5a,doc5b,reporter=sys.stderr.write,strip_whitespaces=True,float_compare=True) +>>> xml_compare(doc5a,doc5b,reporter=sys.stderr.write,strip_whitespaces=True, + float_compare=True) True >>> xml6a = "" >>> xml6b = "" >>> doc6a = etree.fromstring(xml6a) >>> doc6b = etree.fromstring(xml6b) ->>> xml_compare(doc6a,doc6b,reporter=sys.stderr.write,strip_whitespaces=True,float_compare=True) +>>> xml_compare(doc6a,doc6b,reporter=sys.stderr.write,strip_whitespaces=True, + float_compare=True) True """ -import sys -import os + def getNodePath(node): return node.getroottree().getpath(node) + def doStripWhitespaces(text): - if text==None: + if text is None: return None else: - return text.strip().replace('\n','').replace('\r','') + return text.strip().replace('\n', '').replace('\r', '') -def text_compare(t1, t2,strip_whitespaces=False, float_compare=False): + +def text_compare(t1, t2, strip_whitespaces=False, float_compare=False): """ Text comparer for XML Text Nodes >>> text_compare("hi","hi") True @@ -57,9 +66,11 @@ def text_compare(t1, t2,strip_whitespaces=False, float_compare=False): False >>> text_compare("hi","hi ", strip_whitespaces=True) True - >>> text_compare("12.234","12.324", strip_whitespaces=True, float_compare=True) + >>> text_compare("12.234","12.324", strip_whitespaces=True, + float_compare=True) False - >>> text_compare("12.234","12.2340000", strip_whitespaces=True, float_compare=True) + >>> text_compare("12.234","12.2340000", strip_whitespaces=True, + float_compare=True) True >>> text_compare("Hola","Algo", strip_whitespaces=True, float_compare=True) False @@ -78,45 +89,59 @@ def text_compare(t1, t2,strip_whitespaces=False, float_compare=False): else: return (t1 or '') == (t2 or '') -def doReport(reporter,x1,x2,errorMsg): + +def doReport(reporter, x1, x2, errorMsg): if reporter: - reporter(getNodePath(x1)+" "+getNodePath(x2)+os.linesep+errorMsg+os.linesep) + reporter(getNodePath(x1) + " " + getNodePath( + x2) + os.linesep + errorMsg + os.linesep) + -def xml_compare(x1, x2, reporter=None, strip_whitespaces=False,ignore_order=False,float_compare=False): +def xml_compare(x1, x2, reporter=None, strip_whitespaces=False, + ignore_order=False, float_compare=False): if x1.tag != x2.tag: - doReport(reporter,x1,x2,'Tags do not match: %s and %s' % (x1.tag, x2.tag)) + doReport(reporter, x1, x2, + 'Tags do not match: %s and %s' % (x1.tag, x2.tag)) return False for name, value in x1.attrib.items(): - if not text_compare(value, x2.attrib.get(name), strip_whitespaces=strip_whitespaces, float_compare=float_compare): - doReport(reporter,x1,x2,'Attributes do not match: %s=%r, %s=%r' - % (name, value, name, x2.attrib.get(name))) + if not text_compare(value, x2.attrib.get(name), + strip_whitespaces=strip_whitespaces, + float_compare=float_compare): + doReport(reporter, x1, x2, 'Attributes do not match: %s=%r, %s=%r' + % (name, value, name, x2.attrib.get(name))) return False for name in x2.attrib.keys(): - if not x1.attrib.has_key(name): - doReport(reporter,x1,x2,'x2 has an attribute x1 is missing: %s' % name) + if name not in x1.attrib: + doReport(reporter, x1, x2, + 'x2 has an attribute x1 is missing: %s' % name) return False - if not text_compare(x1.text, x2.text, strip_whitespaces=strip_whitespaces, float_compare=float_compare): - doReport(reporter,x1,x2,'text: %r != %r' % (x1.text, x2.text)) + if not text_compare(x1.text, x2.text, strip_whitespaces=strip_whitespaces, + float_compare=float_compare): + doReport(reporter, x1, x2, 'text: %r != %r' % (x1.text, x2.text)) return False - if not text_compare(x1.tail, x2.tail, strip_whitespaces=strip_whitespaces, float_compare=float_compare): - doReport(reporter,x1,x2,'tail: %r != %r' % (x1.tail, x2.tail)) + if not text_compare(x1.tail, x2.tail, strip_whitespaces=strip_whitespaces, + float_compare=float_compare): + doReport(reporter, x1, x2, 'tail: %r != %r' % (x1.tail, x2.tail)) return False cl1 = x1.getchildren() cl2 = x2.getchildren() if len(cl1) != len(cl2): - doReport(reporter,x1,x2,'children length differs, %i != %i'% (len(cl1), len(cl2))) + doReport(reporter, x1, x2, + 'children length differs, %i != %i' % (len(cl1), len(cl2))) return False found = False for c1 in cl1: for c2 in cl2: - if xml_compare(c1, c2, reporter, strip_whitespaces=strip_whitespaces, float_compare=float_compare): + if xml_compare(c1, c2, reporter, + strip_whitespaces=strip_whitespaces, + float_compare=float_compare): found = True break if not found: - doReport(reporter,c1,c1,'child %s not found in destination'% (c1.tag)) + doReport(reporter, c1, c1, + 'child %s not found in destination' % (c1.tag)) return False return True @@ -125,6 +150,6 @@ def _test(): import doctest doctest.testmod() + if __name__ == "__main__": _test() - diff --git a/caom2/caom2/wcs.py b/caom2/caom2/wcs.py index dbaa781c..c1587e38 100644 --- a/caom2/caom2/wcs.py +++ b/caom2/caom2/wcs.py @@ -71,19 +71,21 @@ unicode_literals) from builtins import str, int -from . import common + from . import caom_util +from . import common -__all__ = ['Axis', 'Coord2D', 'CoordAxis1D', 'CoordAxis2D', 'CoordBounds1D', 'CoordBounds2D', 'CoordCircle2D', - 'CoordError', 'CoordFunction1D', 'CoordFunction2D', 'CoordPolygon2D', 'CoordRange1D', 'CoordRange2D', - 'Dimension2D', 'EnergyTransition', 'RefCoord', 'Slice', 'ValueCoord2D'] +__all__ = ['Axis', 'Coord2D', 'CoordAxis1D', 'CoordAxis2D', 'CoordBounds1D', + 'CoordBounds2D', 'CoordCircle2D', 'CoordError', 'CoordFunction1D', + 'CoordFunction2D', 'CoordPolygon2D', 'CoordRange1D', 'CoordRange2D', + 'Dimension2D', 'EnergyTransition', 'RefCoord', 'Slice', + 'ValueCoord2D'] class Axis(common.CaomObject): """the Axis class holds the definition of the axis type and units""" def __init__(self, ctype, cunit=None): - self.ctype = ctype self.cunit = cunit @@ -163,7 +165,6 @@ class CoordAxis1D(common.CaomObject): def __init__(self, axis, error=None, range=None, bounds=None, function=None): - self.axis = axis self.error = error self.range = range @@ -260,7 +261,6 @@ def __init__(self, axis1, axis2, error1=None, error2=None, range=None, bounds=None, function=None): - self.axis1 = axis1 self.axis2 = axis2 self.error1 = error1 @@ -378,9 +378,8 @@ class CoordBounds1D(common.CaomObject): """ def __init__(self, samples=None): - if samples is None: - samples = caom_util.TypedList(CoordRange1D,) + samples = caom_util.TypedList(CoordRange1D, ) self.samples = samples @property @@ -398,7 +397,8 @@ def samples(self): @samples.setter def samples(self, value): - caom_util.type_check(value, caom_util.TypedList, 'samples', override=False) + caom_util.type_check(value, caom_util.TypedList, 'samples', + override=False) self._samples = value @@ -414,7 +414,7 @@ def __init__(self, bounds): else: raise TypeError( "Expected CoordCircle2D or CoordPolygon2D, received {}" - .format(type(bounds))) + .format(type(bounds))) @property def bounds(self): @@ -490,7 +490,6 @@ class CoordError(common.CaomObject): """ def __init__(self, syser, rnder): - self.syser = syser self.rnder = rnder @@ -709,7 +708,7 @@ class CoordPolygon2D(common.CaomObject): def __init__(self, vertices=None): if vertices is None: - vertices = caom_util.TypedList(ValueCoord2D,) + vertices = caom_util.TypedList(ValueCoord2D, ) self.vertices = vertices @property @@ -730,7 +729,8 @@ def vertices(self): @vertices.setter def vertices(self, value): - caom_util.type_check(value, caom_util.TypedList, 'vertices', override=False) + caom_util.type_check(value, caom_util.TypedList, 'vertices', + override=False) self._vertices = value @@ -940,6 +940,7 @@ class Slice(common.CaomObject): values are stored in the bins """ + def __init__(self, axis, bin_): self.axis = axis self.bin = bin_ diff --git a/caom2/dev_requirements.txt b/caom2/dev_requirements.txt index f9feba1d..a3fd944f 100644 --- a/caom2/dev_requirements.txt +++ b/caom2/dev_requirements.txt @@ -1,6 +1,7 @@ -e . pytest>=3.0.5 pytest-cov>=2.5.1 +flake8>=3.4.1 funcsigs==1.0.2 mock==2.0.0 enum34==1.1.6 diff --git a/caom2/setup.py b/caom2/setup.py index f75e497e..6e0c72c7 100755 --- a/caom2/setup.py +++ b/caom2/setup.py @@ -41,7 +41,7 @@ def readme(): # generate the version file with open(os.path.join(PACKAGENAME, 'version.py'), 'w') as f: - f.write('version = \'{}\''.format(VERSION)) + f.write('version = \'{}\'\n'.format(VERSION)) # Treat everything in scripts except README.rst as a script to be installed scripts = [fname for fname in glob.glob(os.path.join('scripts', '*')) diff --git a/caom2repo/caom2repo/__init__.py b/caom2repo/caom2repo/__init__.py index 466309fe..651de670 100755 --- a/caom2repo/caom2repo/__init__.py +++ b/caom2repo/caom2repo/__init__.py @@ -1,4 +1,3 @@ - # Licensed under a 3-clause BSD style license - see LICENSE.rst """ @@ -10,9 +9,9 @@ 1. Instantiate CAOM2RepoClient and use it to access the caom2repo WS. The only mandatory argument that the CadcDataClient constructor takes is -a cadcutils.net.Subject that holds the user credentials. The client interacts with -the caom2repo WS through the get_observation, put_observation, post_observation, - delete_observation and visit functions of the client. +a cadcutils.net.Subject that holds the user credentials. The client interacts +with the caom2repo WS through the get_observation, put_observation, +post_observation, delete_observation and visit functions of the client. Example (appropriate credentials required): from caom2repo import CAOM2RepoClient @@ -37,11 +36,12 @@ import os os.system('caom2-repo-client read -n --collection CFHT 700000') -Method 1. is the recommended method as it does not required forking external processes -and also allows trapping the exceptions and reacting according to the type of the -error. Method 2 also works but the sys.exit needs to be trapped in order -to prevent the script from quiting. Method 3, while simple, must rely on inter -processes communication to determine the result of running the command. +Method 1. is the recommended method as it does not required forking external +processes and also allows trapping the exceptions and reacting according to +the type of the error. Method 2 also works but the sys.exit needs to be +trapped in order to prevent the script from quiting. Method 3, while simple, +must rely on inter processes communication to determine the result of +running the command. """ -from .core import * +from .core import * # noqa diff --git a/caom2repo/caom2repo/core.py b/caom2repo/caom2repo/core.py index ae974a3a..d06488dc 100755 --- a/caom2repo/caom2repo/core.py +++ b/caom2repo/caom2repo/core.py @@ -3,59 +3,59 @@ # *********************************************************************** # ****************** CANADIAN ASTRONOMY DATA CENTRE ******************* # ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** -# -# (c) 2016. (c) 2016. -# Government of Canada Gouvernement du Canada -# National Research Council Conseil national de recherches -# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 -# All rights reserved Tous droits réservés -# -# NRC disclaims any warranties, Le CNRC dénie toute garantie -# expressed, implied, or énoncée, implicite ou légale, -# statutory, of any kind with de quelque nature que ce -# respect to the software, soit, concernant le logiciel, -# including without limitation y compris sans restriction -# any warranty of merchantability toute garantie de valeur -# or fitness for a particular marchande ou de pertinence -# purpose. NRC shall not be pour un usage particulier. -# liable in any event for any Le CNRC ne pourra en aucun cas -# damages, whether direct or être tenu responsable de tout -# indirect, special or general, dommage, direct ou indirect, -# consequential or incidental, particulier ou général, -# arising from the use of the accessoire ou fortuit, résultant -# software. Neither the name de l'utilisation du logiciel. Ni -# of the National Research le nom du Conseil National de -# Council of Canada nor the Recherches du Canada ni les noms -# names of its contributors may de ses participants ne peuvent -# be used to endorse or promote être utilisés pour approuver ou -# products derived from this promouvoir les produits dérivés -# software without specific prior de ce logiciel sans autorisation -# written permission. préalable et particulière -# par écrit. -# -# This file is part of the Ce fichier fait partie du projet -# OpenCADC project. OpenCADC. -# -# OpenCADC is free software: OpenCADC est un logiciel libre ; -# you can redistribute it and/or vous pouvez le redistribuer ou le -# modify it under the terms of modifier suivant les termes de -# the GNU Affero General Public la “GNU Affero General Public -# License as published by the License” telle que publiée -# Free Software Foundation, par la Free Software Foundation -# either version 3 of the : soit la version 3 de cette -# License, or (at your option) licence, soit (à votre gré) -# any later version. toute version ultérieure. -# -# OpenCADC is distributed in the OpenCADC est distribué -# hope that it will be useful, dans l’espoir qu’il vous -# but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE -# without even the implied GARANTIE : sans même la garantie -# warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ -# or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF -# PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +# +# (c) 2016. (c) 2016. +# Government of Canada Gouvernement du Canada +# National Research Council Conseil national de recherches +# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +# All rights reserved Tous droits réservés +# +# NRC disclaims any warranties, Le CNRC dénie toute garantie +# expressed, implied, or énoncée, implicite ou légale, +# statutory, of any kind with de quelque nature que ce +# respect to the software, soit, concernant le logiciel, +# including without limitation y compris sans restriction +# any warranty of merchantability toute garantie de valeur +# or fitness for a particular marchande ou de pertinence +# purpose. NRC shall not be pour un usage particulier. +# liable in any event for any Le CNRC ne pourra en aucun cas +# damages, whether direct or être tenu responsable de tout +# indirect, special or general, dommage, direct ou indirect, +# consequential or incidental, particulier ou général, +# arising from the use of the accessoire ou fortuit, résultant +# software. Neither the name de l'utilisation du logiciel. Ni +# of the National Research le nom du Conseil National de +# Council of Canada nor the Recherches du Canada ni les noms +# names of its contributors may de ses participants ne peuvent +# be used to endorse or promote être utilisés pour approuver ou +# products derived from this promouvoir les produits dérivés +# software without specific prior de ce logiciel sans autorisation +# written permission. préalable et particulière +# par écrit. +# +# This file is part of the Ce fichier fait partie du projet +# OpenCADC project. OpenCADC. +# +# OpenCADC is free software: OpenCADC est un logiciel libre ; +# you can redistribute it and/or vous pouvez le redistribuer ou le +# modify it under the terms of modifier suivant les termes de +# the GNU Affero General Public la “GNU Affero General Public +# License as published by the License” telle que publiée +# Free Software Foundation, par la Free Software Foundation +# either version 3 of the : soit la version 3 de cette +# License, or (at your option) licence, soit (à votre gré) +# any later version. toute version ultérieure. +# +# OpenCADC is distributed in the OpenCADC est distribué +# hope that it will be useful, dans l’espoir qu’il vous +# but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +# without even the implied GARANTIE : sans même la garantie +# warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +# or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +# PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence # General Public License for Générale Publique GNU Affero # more details. pour plus de détails. -# +# # You should have received Vous devriez avoir reçu une # a copy of the GNU Affero copie de la Licence Générale # General Public License along Publique GNU Affero avec @@ -76,17 +76,13 @@ import os import os.path import sys -from six import BytesIO from datetime import datetime from cadcutils import net from cadcutils import util - -from caom2repo import version - from caom2.obs_reader_writer import ObservationReader, ObservationWriter from caom2.version import version as caom2_version -from six.moves.urllib.parse import urlparse +from six import BytesIO # from . import version as caom2repo_version from caom2repo import version @@ -95,7 +91,8 @@ BATCH_SIZE = int(10000) -CAOM2REPO_OBS_CAPABILITY_ID = 'vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.1' +CAOM2REPO_OBS_CAPABILITY_ID =\ + 'vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.1' # resource ID for info DEFAULT_RESOURCE_ID = 'ivo://cadc.nrc.ca/caom2repo' @@ -103,7 +100,6 @@ class CAOM2RepoClient(object): - """Class to do CRUD + visitor actions on a CAOM2 collection repo.""" logger = logging.getLogger('CAOM2RepoClient') @@ -117,9 +113,11 @@ def __init__(self, subject, resource_id=DEFAULT_RESOURCE_ID, host=None): """ agent = '{}/{}'.format(APP_NAME, version.version) self.host = host - self._repo_client = net.BaseWsClient(resource_id, subject, agent, retry=True, host=host) + self._repo_client = net.BaseWsClient(resource_id, subject, agent, + retry=True, host=host) - agent = "caom2-repo-client/{} caom2/{}".format(version.version, caom2_version) + agent = "caom2-repo-client/{} caom2/{}".format(version.version, + caom2_version) self._repo_client = net.BaseWsClient(resource_id, subject, agent, retry=True, host=self.host) @@ -127,7 +125,7 @@ def __init__(self, subject, resource_id=DEFAULT_RESOURCE_ID, host=None): # shortcuts for the CRUD operations def create(self, observation): """ - Creates an observation in the repo. + Creates an observation in the repo. :param observation: Observation to create :return: Created observation """ @@ -158,21 +156,24 @@ def delete(self, collection, observation_id): """ self.delete_observation(collection, observation_id) - - def visit(self, plugin, collection, start=None, end=None, halt_on_error=False): + def visit(self, plugin, collection, start=None, end=None, + halt_on_error=False): """ Main processing function that iterates through the observations of the collection and updates them according to the algorithm of the plugin function - :param plugin: path to python file that contains the algorithm to be applied to visited - observations + :param plugin: path to python file that contains the algorithm to be + applied to visited observations :param collection: name of the CAOM2 collection - :param start: optional earliest date-time of the targeted observation set + :param start: optional earliest date-time of the targeted observation + set :param end: optional latest date-time of the targeted observation set - :param halt_on_error if True halts the execution on the first exception raised by - the plugin update function otherwise logs the error and continues - :return: tuple (list of visited observations, list of updated observation, - list of skipped observations, list of failured observation) + :param halt_on_error if True halts the execution on the first exception + raised by the plugin update function otherwise logs the error + and continues + :return: tuple (list of visited observations, list of updated + observation, list of skipped observations, list of failure + observations) """ if not os.path.isfile(plugin): raise Exception('Cannot find plugin file ' + plugin) @@ -183,7 +184,8 @@ def visit(self, plugin, collection, start=None, end=None, halt_on_error=False): assert type(end) is datetime self._load_plugin_class(plugin) - # this is updated by _get_observations with the timestamp of last observation in the batch + # this is updated by _get_observations with the timestamp of last + # observation in the batch self._start = start visited = [] failed = [] @@ -196,21 +198,24 @@ def visit(self, plugin, collection, start=None, end=None, halt_on_error=False): observation = self.get_observation(collection, observationID) try: if self.plugin.update(observation) is False: - self.logger.info('SKIP {}'.format(observation.observation_id)) + self.logger.info( + 'SKIP {}'.format(observation.observation_id)) skipped.append(observation.observation_id) else: self.post_observation(observation) - self.logger.debug('UPDATED {}'.format(observation.observation_id)) + self.logger.debug( + 'UPDATED {}'.format(observation.observation_id)) updated.append(observation.observation_id) except Exception as e: failed.append(observation.observation_id) self.logger.error('FAILED {} - Reason: {}'. - format(observation.observation_id, e)) + format(observation.observation_id, e)) if halt_on_error: raise e visited.append(observation.observation_id) if len(observations) == BATCH_SIZE: - observations = self._get_observations(collection, self._start, end) + observations = self._get_observations(collection, self._start, + end) else: # the last batch was smaller so it must have been the last break @@ -232,8 +237,9 @@ def _get_observations(self, collection, start=None, end=None): if end is not None: params['END'] = util.utils.date2ivoa(end) - response = self._repo_client.get((CAOM2REPO_OBS_CAPABILITY_ID, collection), - params=params) + response = self._repo_client.get( + (CAOM2REPO_OBS_CAPABILITY_ID, collection), + params=params) last_datetime = None for line in response.text.splitlines(): columns = line.split('\t') @@ -253,22 +259,24 @@ def _load_plugin_class(self, filepath): :param filepath: path to the file containing the python function """ expected_class = 'ObservationUpdater' - + mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1]) if file_ext.lower() == '.pyc': py_mod = imp.load_compiled(mod_name, filepath) else: py_mod = imp.load_source(mod_name, filepath) - + if hasattr(py_mod, expected_class): self.plugin = getattr(py_mod, expected_class)() else: raise Exception( - 'Cannot find ObservationUpdater class in pluging file ' + filepath) - + 'Cannot find ObservationUpdater class in pluging file ' + + filepath) + if not hasattr(self.plugin, 'update'): - raise Exception('Cannot find update method in plugin class ' + filepath) + raise Exception( + 'Cannot find update method in plugin class ' + filepath) def get_observation(self, collection, observation_id): """ @@ -299,14 +307,15 @@ def post_observation(self, observation): """ assert observation.collection is not None assert observation.observation_id is not None - path = '/{}/{}'.format(observation.collection, observation.observation_id) + path = '/{}/{}'.format(observation.collection, + observation.observation_id) logging.debug('POST {}'.format(path)) ibuffer = BytesIO() ObservationWriter().write(observation, ibuffer) obs_xml = ibuffer.getvalue() headers = {'Content-Type': 'application/xml'} - response = self._repo_client.post( + self._repo_client.post( (CAOM2REPO_OBS_CAPABILITY_ID, path), headers=headers, data=obs_xml) logging.debug('Successfully updated Observation\n') @@ -318,14 +327,15 @@ def put_observation(self, observation): """ assert observation.collection is not None assert observation.observation_id is not None - path = '/{}/{}'.format(observation.collection, observation.observation_id) + path = '/{}/{}'.format(observation.collection, + observation.observation_id) logging.debug('PUT {}'.format(path)) ibuffer = BytesIO() ObservationWriter().write(observation, ibuffer) obs_xml = ibuffer.getvalue() headers = {'Content-Type': 'application/xml'} - response = self._repo_client.put( + self._repo_client.put( (CAOM2REPO_OBS_CAPABILITY_ID, path), headers=headers, data=obs_xml) logging.debug('Successfully put Observation\n') @@ -338,7 +348,8 @@ def delete_observation(self, collection, observation_id): assert observation_id is not None path = '/{}/{}'.format(collection, observation_id) logging.debug('DELETE {}'.format(path)) - response = self._repo_client.delete((CAOM2REPO_OBS_CAPABILITY_ID, path)) + self._repo_client.delete( + (CAOM2REPO_OBS_CAPABILITY_ID, path)) logging.info('Successfully deleted Observation {}\n') @@ -352,74 +363,85 @@ def str2date(s): return None return datetime.strptime(s, date_format) -def main_app(): +def main_app(): + parser = util.get_base_parser(version=version.version, + default_resource_id=DEFAULT_RESOURCE_ID) - parser = util.get_base_parser(version=version.version, default_resource_id=DEFAULT_RESOURCE_ID) + parser.description = ( + 'Client for a CAOM2 repo. In addition to CRUD (Create, Read, Update ' + 'and Delete) operations it also implements a visitor operation that ' + 'allows for updating multiple observations in a collection') - parser.description = ('Client for a CAOM2 repo. In addition to CRUD (Create, Read, Update and Delete) ' - 'operations it also implements a visitor operation that allows for updating ' - 'multiple observations in a collection') - parser.add_argument("-s", "--server", help='URL of the CAOM2 repo server') - + parser.formatter_class = argparse.RawTextHelpFormatter subparsers = parser.add_subparsers(dest='cmd') - create_parser = subparsers.add_parser('create', description='Create a new observation', - help='Create a new observation') - create_parser.add_argument('observation', help='XML file containing the observation', + create_parser = subparsers.add_parser( + 'create', description='Create a new observation', + help='Create a new observation') + create_parser.add_argument('observation', + help='XML file containing the observation', type=argparse.FileType('r')) - read_parser = subparsers.add_parser('read', - description='Read an existing observation', - help='Read an existing observation') - read_parser.add_argument('--output', '-o', help='destination file', required=False) - read_parser.add_argument('collection', help='collection name in CAOM2 repo') + read_parser = subparsers.add_parser( + 'read', description='Read an existing observation', + help='Read an existing observation') + read_parser.add_argument('--output', '-o', help='destination file', + required=False) + read_parser.add_argument('collection', + help='collection name in CAOM2 repo') read_parser.add_argument('observationID', help='observation identifier') - update_parser = subparsers.add_parser('update', - description='Update an existing observation', - help='Update an existing observation') - update_parser.add_argument('observation', help='XML file containing the observation', + update_parser = subparsers.add_parser( + 'update', description='Update an existing observation', + help='Update an existing observation') + update_parser.add_argument('observation', + help='XML file containing the observation', type=argparse.FileType('r')) - delete_parser = subparsers.add_parser('delete', - description='Delete an existing observation', - help='Delete an existing observation') - delete_parser.add_argument('collection', help='collection name in CAOM2 repo') + delete_parser = subparsers.add_parser( + 'delete', description='Delete an existing observation', + help='Delete an existing observation') + delete_parser.add_argument('collection', + help='collection name in CAOM2 repo') delete_parser.add_argument('observationID', help='observation identifier') # Note: RawTextHelpFormatter allows for the use of newline in epilog - visit_parser = subparsers.add_parser('visit', - formatter_class=argparse.RawTextHelpFormatter, - description='Visit observations in a collection', - help='Visit observations in a collection') - visit_parser.add_argument('--plugin', required=True, type=argparse.FileType('r'), + visit_parser = subparsers.add_parser( + 'visit', formatter_class=argparse.RawTextHelpFormatter, + description='Visit observations in a collection', + help='Visit observations in a collection') + visit_parser.add_argument('--plugin', required=True, + type=argparse.FileType('r'), help='plugin class to update each observation') - visit_parser.add_argument('--start', type=str2date, - - help='earliest observation to visit (UTC IVOA format: YYYY-mm-ddTH:M:S)') - visit_parser.add_argument('--end', type=str2date, - help='latest observation to visit (UTC IVOA format: YYYY-mm-ddTH:M:S)') - visit_parser.add_argument('--halt-on-error', action='store_true', - help='stop visitor on first update exception raised by plugin') - visit_parser.add_argument('collection', help='data collection in CAOM2 repo') - - visit_parser.epilog =\ -""" -Minimum plugin file format: ----- - from caom2 import Observation - - class ObservationUpdater: + visit_parser.add_argument( + '--start', type=str2date, help=('earliest observation to visit ' + '(UTC IVOA format: YYYY-mm-ddTH:M:S)')) + visit_parser.add_argument( + '--end', type=str2date, + help='latest observation to visit (UTC IVOA format: YYYY-mm-ddTH:M:S)') + visit_parser.add_argument( + '--halt-on-error', action='store_true', + help='stop visitor on first update exception raised by plugin') + visit_parser.add_argument('collection', + help='data collection in CAOM2 repo') + + visit_parser.epilog = \ + """ + Minimum plugin file format: + ---- + from caom2 import Observation - def update(self, observation): - assert isinstance(observation, Observation), ( - 'observation {} is not an Observation'.format(observation)) - # custom code to update the observation ----- -""" + class ObservationUpdater: + + def update(self, observation): + assert isinstance(observation, Observation), ( + 'observation {} is not an Observation'.format(observation)) + # custom code to update the observation + ---- + """ args = parser.parse_args() if len(sys.argv) < 2: parser.print_usage(file=sys.stderr) @@ -436,17 +458,21 @@ def update(self, observation): server = None if args.server: server = args.server - + client = CAOM2RepoClient(subject, args.resource_id, host=server) if args.cmd == 'visit': - print ("Visit") - logging.debug("Call visitor with plugin={}, start={}, end={}, collection={}". - format(args.plugin.name, args.start, args.end, args.collection)) + print("Visit") + logging.debug( + "Call visitor with plugin={}, start={}, end={}, collection={}". + format(args.plugin.name, args.start, args.end, + args.collection)) (visited, updated, skipped, failed) = \ - client.visit(args.plugin.name, args.collection, start=args.start, end=args.end, + client.visit(args.plugin.name, args.collection, start=args.start, + end=args.end, halt_on_error=args.halt_on_error) - logging.info('Visitor stats: visited/updated/skipped/errors: {}/{}/{}/{}'.format( - len(visited), len(updated), len(skipped), len(failed))) + logging.info( + 'Visitor stats: visited/updated/skipped/errors: {}/{}/{}/{}'. + format(len(visited), len(updated), len(skipped), len(failed))) elif args.cmd == 'create': logging.info("Create") @@ -454,7 +480,8 @@ def update(self, observation): client.put_observation(obs_reader.read(args.observation)) elif args.cmd == 'read': logging.info("Read") - observation = client.get_observation(args.collection, args.observationID) + observation = client.get_observation(args.collection, + args.observationID) observation_writer = ObservationWriter() if args.output: with open(args.output, 'wb') as obsfile: @@ -468,9 +495,11 @@ def update(self, observation): client.post_observation(obs_reader.read(args.observation)) else: logging.info("Delete") - client.delete_observation(collection=args.collection, observation_id=args.observationID) + client.delete_observation(collection=args.collection, + observation_id=args.observationID) logging.info("DONE") + if __name__ == '__main__': main_app() diff --git a/caom2repo/caom2repo/tests/__init__.py b/caom2repo/caom2repo/tests/__init__.py index 2a4a293e..e1f8eaaa 100755 --- a/caom2repo/caom2repo/tests/__init__.py +++ b/caom2repo/caom2repo/tests/__init__.py @@ -1,8 +1,8 @@ # # -*- coding: utf-8 -*- -#*********************************************************************** -#****************** CANADIAN ASTRONOMY DATA CENTRE ******************* -#************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +# *********************************************************************** +# ****************** CANADIAN ASTRONOMY DATA CENTRE ******************* +# ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** # # (c) 2010. (c) 2010. # Government of Canada Gouvernement du Canada @@ -65,8 +65,7 @@ # # $Revision: 4 $ # -#*********************************************************************** +# *********************************************************************** # """ Deines __init__ """ - diff --git a/caom2repo/caom2repo/tests/addplaneplugin.py b/caom2repo/caom2repo/tests/addplaneplugin.py index c442baed..4d65d123 100755 --- a/caom2repo/caom2repo/tests/addplaneplugin.py +++ b/caom2repo/caom2repo/tests/addplaneplugin.py @@ -1,60 +1,61 @@ +#!/usr/bin/env python2.7 # # -*- coding: utf-8 -*- # *********************************************************************** # ****************** CANADIAN ASTRONOMY DATA CENTRE ******************* # ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** -# -# (c) 2016. (c) 2016. -# Government of Canada Gouvernement du Canada -# National Research Council Conseil national de recherches -# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 -# All rights reserved Tous droits réservés -# -# NRC disclaims any warranties, Le CNRC dénie toute garantie -# expressed, implied, or énoncée, implicite ou légale, -# statutory, of any kind with de quelque nature que ce -# respect to the software, soit, concernant le logiciel, -# including without limitation y compris sans restriction -# any warranty of merchantability toute garantie de valeur -# or fitness for a particular marchande ou de pertinence -# purpose. NRC shall not be pour un usage particulier. -# liable in any event for any Le CNRC ne pourra en aucun cas -# damages, whether direct or être tenu responsable de tout -# indirect, special or general, dommage, direct ou indirect, -# consequential or incidental, particulier ou général, -# arising from the use of the accessoire ou fortuit, résultant -# software. Neither the name de l'utilisation du logiciel. Ni -# of the National Research le nom du Conseil National de -# Council of Canada nor the Recherches du Canada ni les noms -# names of its contributors may de ses participants ne peuvent -# be used to endorse or promote être utilisés pour approuver ou -# products derived from this promouvoir les produits dérivés -# software without specific prior de ce logiciel sans autorisation -# written permission. préalable et particulière -# par écrit. -# -# This file is part of the Ce fichier fait partie du projet -# OpenCADC project. OpenCADC. -# -# OpenCADC is free software: OpenCADC est un logiciel libre ; -# you can redistribute it and/or vous pouvez le redistribuer ou le -# modify it under the terms of modifier suivant les termes de -# the GNU Affero General Public la “GNU Affero General Public -# License as published by the License” telle que publiée -# Free Software Foundation, par la Free Software Foundation -# either version 3 of the : soit la version 3 de cette -# License, or (at your option) licence, soit (à votre gré) -# any later version. toute version ultérieure. -# -# OpenCADC is distributed in the OpenCADC est distribué -# hope that it will be useful, dans l’espoir qu’il vous -# but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE -# without even the implied GARANTIE : sans même la garantie -# warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ -# or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF -# PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +# +# (c) 2016. (c) 2016. +# Government of Canada Gouvernement du Canada +# National Research Council Conseil national de recherches +# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +# All rights reserved Tous droits réservés +# +# NRC disclaims any warranties, Le CNRC dénie toute garantie +# expressed, implied, or énoncée, implicite ou légale, +# statutory, of any kind with de quelque nature que ce +# respect to the software, soit, concernant le logiciel, +# including without limitation y compris sans restriction +# any warranty of merchantability toute garantie de valeur +# or fitness for a particular marchande ou de pertinence +# purpose. NRC shall not be pour un usage particulier. +# liable in any event for any Le CNRC ne pourra en aucun cas +# damages, whether direct or être tenu responsable de tout +# indirect, special or general, dommage, direct ou indirect, +# consequential or incidental, particulier ou général, +# arising from the use of the accessoire ou fortuit, résultant +# software. Neither the name de l'utilisation du logiciel. Ni +# of the National Research le nom du Conseil National de +# Council of Canada nor the Recherches du Canada ni les noms +# names of its contributors may de ses participants ne peuvent +# be used to endorse or promote être utilisés pour approuver ou +# products derived from this promouvoir les produits dérivés +# software without specific prior de ce logiciel sans autorisation +# written permission. préalable et particulière +# par écrit. +# +# This file is part of the Ce fichier fait partie du projet +# OpenCADC project. OpenCADC. +# +# OpenCADC is free software: OpenCADC est un logiciel libre ; +# you can redistribute it and/or vous pouvez le redistribuer ou le +# modify it under the terms of modifier suivant les termes de +# the GNU Affero General Public la “GNU Affero General Public +# License as published by the License” telle que publiée +# Free Software Foundation, par la Free Software Foundation +# either version 3 of the : soit la version 3 de cette +# License, or (at your option) licence, soit (à votre gré) +# any later version. toute version ultérieure. +# +# OpenCADC is distributed in the OpenCADC est distribué +# hope that it will be useful, dans l’espoir qu’il vous +# but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +# without even the implied GARANTIE : sans même la garantie +# warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +# or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +# PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence # General Public License for Générale Publique GNU Affero # more details. pour plus de détails. -# +# # You should have received Vous devriez avoir reçu une # a copy of the GNU Affero copie de la Licence Générale # General Public License along Publique GNU Affero avec @@ -68,14 +69,14 @@ # from __future__ import (absolute_import, division, print_function, unicode_literals) + from caom2.observation import Observation from caom2.plane import Plane class ObservationUpdater(object): - """ObservationUpdater that adds a plane to the observation.""" - + def update(self, observation): """ Processes an observation and updates it diff --git a/caom2repo/caom2repo/tests/data/create_help.txt b/caom2repo/caom2repo/tests/data/create_help.txt new file mode 100644 index 00000000..60196b38 --- /dev/null +++ b/caom2repo/caom2repo/tests/data/create_help.txt @@ -0,0 +1,28 @@ +usage: caom2-repo create [-h] + [--cert CERT | -n | --netrc-file NETRC_FILE | -u USER] + [--host HOST] [--resource-id RESOURCE_ID] + [-d | -q | -v] + observation + +Create a new observation + +positional arguments: + observation XML file containing the observation + +optional arguments: + --cert CERT location of your X509 certificate to use for + authentication (unencrypted, in PEM format) + -d, --debug debug messages + -h, --help show this help message and exit + --host HOST base hostname for services - used mainly for testing + (default: www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca) + -n use .netrc in $HOME for authentication + --netrc-file NETRC_FILE + netrc file to use for authentication + -q, --quiet run quietly + --resource-id RESOURCE_ID + resource identifier (default + ivo://cadc.nrc.ca/caom2repo) + -u, --user USER name of user to authenticate. Note: application + prompts for the corresponding password! + -v, --verbose verbose messages diff --git a/caom2repo/caom2repo/tests/data/delete_help.txt b/caom2repo/caom2repo/tests/data/delete_help.txt new file mode 100644 index 00000000..2e907d6b --- /dev/null +++ b/caom2repo/caom2repo/tests/data/delete_help.txt @@ -0,0 +1,29 @@ +usage: caom2-repo delete [-h] + [--cert CERT | -n | --netrc-file NETRC_FILE | -u USER] + [--host HOST] [--resource-id RESOURCE_ID] + [-d | -q | -v] + collection observationID + +Delete an existing observation + +positional arguments: + collection collection name in CAOM2 repo + observationID observation identifier + +optional arguments: + --cert CERT location of your X509 certificate to use for + authentication (unencrypted, in PEM format) + -d, --debug debug messages + -h, --help show this help message and exit + --host HOST base hostname for services - used mainly for testing + (default: www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca) + -n use .netrc in $HOME for authentication + --netrc-file NETRC_FILE + netrc file to use for authentication + -q, --quiet run quietly + --resource-id RESOURCE_ID + resource identifier (default + ivo://cadc.nrc.ca/caom2repo) + -u, --user USER name of user to authenticate. Note: application + prompts for the corresponding password! + -v, --verbose verbose messages diff --git a/caom2repo/caom2repo/tests/data/help.txt b/caom2repo/caom2repo/tests/data/help.txt new file mode 100644 index 00000000..458291c2 --- /dev/null +++ b/caom2repo/caom2repo/tests/data/help.txt @@ -0,0 +1,17 @@ +usage: caom2-repo [-h] [-V] [-s SERVER] {create,read,update,delete,visit} ... + +Client for a CAOM2 repo. In addition to CRUD (Create, Read, Update and Delete) operations it also implements a visitor operation that allows for updating multiple observations in a collection + +positional arguments: + {create,read,update,delete,visit} + create Create a new observation + read Read an existing observation + update Update an existing observation + delete Delete an existing observation + visit Visit observations in a collection + +optional arguments: + -h, --help show this help message and exit + -V, --version show program's version number and exit + -s SERVER, --server SERVER + URL of the CAOM2 repo server diff --git a/caom2repo/caom2repo/tests/data/read_help.txt b/caom2repo/caom2repo/tests/data/read_help.txt new file mode 100644 index 00000000..71d7fdb9 --- /dev/null +++ b/caom2repo/caom2repo/tests/data/read_help.txt @@ -0,0 +1,30 @@ +usage: caom2-repo read [-h] + [--cert CERT | -n | --netrc-file NETRC_FILE | -u USER] + [--host HOST] [--resource-id RESOURCE_ID] + [-d | -q | -v] [--output OUTPUT] + collection observationID + +Read an existing observation + +positional arguments: + collection collection name in CAOM2 repo + observationID observation identifier + +optional arguments: + --cert CERT location of your X509 certificate to use for + authentication (unencrypted, in PEM format) + -d, --debug debug messages + -h, --help show this help message and exit + --host HOST base hostname for services - used mainly for testing + (default: www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca) + -n use .netrc in $HOME for authentication + --netrc-file NETRC_FILE + netrc file to use for authentication + --output, -o OUTPUT destination file + -q, --quiet run quietly + --resource-id RESOURCE_ID + resource identifier (default + ivo://cadc.nrc.ca/caom2repo) + -u, --user USER name of user to authenticate. Note: application + prompts for the corresponding password! + -v, --verbose verbose messages diff --git a/caom2repo/caom2repo/tests/data/update_help.txt b/caom2repo/caom2repo/tests/data/update_help.txt new file mode 100644 index 00000000..82f71653 --- /dev/null +++ b/caom2repo/caom2repo/tests/data/update_help.txt @@ -0,0 +1,28 @@ +usage: caom2-repo update [-h] + [--cert CERT | -n | --netrc-file NETRC_FILE | -u USER] + [--host HOST] [--resource-id RESOURCE_ID] + [-d | -q | -v] + observation + +Update an existing observation + +positional arguments: + observation XML file containing the observation + +optional arguments: + --cert CERT location of your X509 certificate to use for + authentication (unencrypted, in PEM format) + -d, --debug debug messages + -h, --help show this help message and exit + --host HOST base hostname for services - used mainly for testing + (default: www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca) + -n use .netrc in $HOME for authentication + --netrc-file NETRC_FILE + netrc file to use for authentication + -q, --quiet run quietly + --resource-id RESOURCE_ID + resource identifier (default + ivo://cadc.nrc.ca/caom2repo) + -u, --user USER name of user to authenticate. Note: application + prompts for the corresponding password! + -v, --verbose verbose messages diff --git a/caom2repo/caom2repo/tests/data/visit_help.txt b/caom2repo/caom2repo/tests/data/visit_help.txt new file mode 100644 index 00000000..7552ef8b --- /dev/null +++ b/caom2repo/caom2repo/tests/data/visit_help.txt @@ -0,0 +1,49 @@ +usage: caom2-repo visit [-h] + [--cert CERT | -n | --netrc-file NETRC_FILE | -u USER] + [--host HOST] [--resource-id RESOURCE_ID] + [-d | -q | -v] --plugin PLUGIN [--start START] + [--end END] [--halt-on-error] + collection + +Visit observations in a collection + +positional arguments: + collection data collection in CAOM2 repo + +optional arguments: + --cert CERT location of your X509 certificate to use for + authentication (unencrypted, in PEM format) + -d, --debug debug messages + --end END latest observation to visit (UTC IVOA format: YYYY-mm- + ddTH:M:S) + --halt-on-error stop visitor on first update exception raised by + plugin + -h, --help show this help message and exit + --host HOST base hostname for services - used mainly for testing + (default: www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca) + -n use .netrc in $HOME for authentication + --netrc-file NETRC_FILE + netrc file to use for authentication + --plugin PLUGIN plugin class to update each observation + -q, --quiet run quietly + --resource-id RESOURCE_ID + resource identifier (default + ivo://cadc.nrc.ca/caom2repo) + --start START earliest observation to visit (UTC IVOA format: YYYY- + mm-ddTH:M:S) + -u, --user USER name of user to authenticate. Note: application + prompts for the corresponding password! + -v, --verbose verbose messages + + Minimum plugin file format: + ---- + from caom2 import Observation + + class ObservationUpdater: + + def update(self, observation): + assert isinstance(observation, Observation), ( + 'observation {} is not an Observation'.format(observation)) + # custom code to update the observation + ---- + diff --git a/caom2repo/caom2repo/tests/errorplugin.py b/caom2repo/caom2repo/tests/errorplugin.py index 0501c1f0..a0f68f08 100755 --- a/caom2repo/caom2repo/tests/errorplugin.py +++ b/caom2repo/caom2repo/tests/errorplugin.py @@ -1,60 +1,61 @@ +#!/usr/bin/env python2.7 # # -*- coding: utf-8 -*- # *********************************************************************** # ****************** CANADIAN ASTRONOMY DATA CENTRE ******************* # ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** -# -# (c) 2017. (c) 2017. -# Government of Canada Gouvernement du Canada -# National Research Council Conseil national de recherches -# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 -# All rights reserved Tous droits réservés -# -# NRC disclaims any warranties, Le CNRC dénie toute garantie -# expressed, implied, or énoncée, implicite ou légale, -# statutory, of any kind with de quelque nature que ce -# respect to the software, soit, concernant le logiciel, -# including without limitation y compris sans restriction -# any warranty of merchantability toute garantie de valeur -# or fitness for a particular marchande ou de pertinence -# purpose. NRC shall not be pour un usage particulier. -# liable in any event for any Le CNRC ne pourra en aucun cas -# damages, whether direct or être tenu responsable de tout -# indirect, special or general, dommage, direct ou indirect, -# consequential or incidental, particulier ou général, -# arising from the use of the accessoire ou fortuit, résultant -# software. Neither the name de l'utilisation du logiciel. Ni -# of the National Research le nom du Conseil National de -# Council of Canada nor the Recherches du Canada ni les noms -# names of its contributors may de ses participants ne peuvent -# be used to endorse or promote être utilisés pour approuver ou -# products derived from this promouvoir les produits dérivés -# software without specific prior de ce logiciel sans autorisation -# written permission. préalable et particulière -# par écrit. -# -# This file is part of the Ce fichier fait partie du projet -# OpenCADC project. OpenCADC. -# -# OpenCADC is free software: OpenCADC est un logiciel libre ; -# you can redistribute it and/or vous pouvez le redistribuer ou le -# modify it under the terms of modifier suivant les termes de -# the GNU Affero General Public la “GNU Affero General Public -# License as published by the License” telle que publiée -# Free Software Foundation, par la Free Software Foundation -# either version 3 of the : soit la version 3 de cette -# License, or (at your option) licence, soit (à votre gré) -# any later version. toute version ultérieure. -# -# OpenCADC is distributed in the OpenCADC est distribué -# hope that it will be useful, dans l’espoir qu’il vous -# but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE -# without even the implied GARANTIE : sans même la garantie -# warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ -# or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF -# PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +# +# (c) 2016. (c) 2016. +# Government of Canada Gouvernement du Canada +# National Research Council Conseil national de recherches +# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +# All rights reserved Tous droits réservés +# +# NRC disclaims any warranties, Le CNRC dénie toute garantie +# expressed, implied, or énoncée, implicite ou légale, +# statutory, of any kind with de quelque nature que ce +# respect to the software, soit, concernant le logiciel, +# including without limitation y compris sans restriction +# any warranty of merchantability toute garantie de valeur +# or fitness for a particular marchande ou de pertinence +# purpose. NRC shall not be pour un usage particulier. +# liable in any event for any Le CNRC ne pourra en aucun cas +# damages, whether direct or être tenu responsable de tout +# indirect, special or general, dommage, direct ou indirect, +# consequential or incidental, particulier ou général, +# arising from the use of the accessoire ou fortuit, résultant +# software. Neither the name de l'utilisation du logiciel. Ni +# of the National Research le nom du Conseil National de +# Council of Canada nor the Recherches du Canada ni les noms +# names of its contributors may de ses participants ne peuvent +# be used to endorse or promote être utilisés pour approuver ou +# products derived from this promouvoir les produits dérivés +# software without specific prior de ce logiciel sans autorisation +# written permission. préalable et particulière +# par écrit. +# +# This file is part of the Ce fichier fait partie du projet +# OpenCADC project. OpenCADC. +# +# OpenCADC is free software: OpenCADC est un logiciel libre ; +# you can redistribute it and/or vous pouvez le redistribuer ou le +# modify it under the terms of modifier suivant les termes de +# the GNU Affero General Public la “GNU Affero General Public +# License as published by the License” telle que publiée +# Free Software Foundation, par la Free Software Foundation +# either version 3 of the : soit la version 3 de cette +# License, or (at your option) licence, soit (à votre gré) +# any later version. toute version ultérieure. +# +# OpenCADC is distributed in the OpenCADC est distribué +# hope that it will be useful, dans l’espoir qu’il vous +# but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +# without even the implied GARANTIE : sans même la garantie +# warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +# or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +# PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence # General Public License for Générale Publique GNU Affero # more details. pour plus de détails. -# +# # You should have received Vous devriez avoir reçu une # a copy of the GNU Affero copie de la Licence Générale # General Public License along Publique GNU Affero avec @@ -71,9 +72,8 @@ class ObservationUpdater(object): - """Plugin that does not update the observation""" - + def update(self, observation): """ Processes an observation and updates it diff --git a/caom2repo/caom2repo/tests/passplugin.py b/caom2repo/caom2repo/tests/passplugin.py index ac62915b..b639a4c0 100755 --- a/caom2repo/caom2repo/tests/passplugin.py +++ b/caom2repo/caom2repo/tests/passplugin.py @@ -1,60 +1,61 @@ +#!/usr/bin/env python2.7 # # -*- coding: utf-8 -*- # *********************************************************************** # ****************** CANADIAN ASTRONOMY DATA CENTRE ******************* # ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** -# -# (c) 2016. (c) 2016. -# Government of Canada Gouvernement du Canada -# National Research Council Conseil national de recherches -# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 -# All rights reserved Tous droits réservés -# -# NRC disclaims any warranties, Le CNRC dénie toute garantie -# expressed, implied, or énoncée, implicite ou légale, -# statutory, of any kind with de quelque nature que ce -# respect to the software, soit, concernant le logiciel, -# including without limitation y compris sans restriction -# any warranty of merchantability toute garantie de valeur -# or fitness for a particular marchande ou de pertinence -# purpose. NRC shall not be pour un usage particulier. -# liable in any event for any Le CNRC ne pourra en aucun cas -# damages, whether direct or être tenu responsable de tout -# indirect, special or general, dommage, direct ou indirect, -# consequential or incidental, particulier ou général, -# arising from the use of the accessoire ou fortuit, résultant -# software. Neither the name de l'utilisation du logiciel. Ni -# of the National Research le nom du Conseil National de -# Council of Canada nor the Recherches du Canada ni les noms -# names of its contributors may de ses participants ne peuvent -# be used to endorse or promote être utilisés pour approuver ou -# products derived from this promouvoir les produits dérivés -# software without specific prior de ce logiciel sans autorisation -# written permission. préalable et particulière -# par écrit. -# -# This file is part of the Ce fichier fait partie du projet -# OpenCADC project. OpenCADC. -# -# OpenCADC is free software: OpenCADC est un logiciel libre ; -# you can redistribute it and/or vous pouvez le redistribuer ou le -# modify it under the terms of modifier suivant les termes de -# the GNU Affero General Public la “GNU Affero General Public -# License as published by the License” telle que publiée -# Free Software Foundation, par la Free Software Foundation -# either version 3 of the : soit la version 3 de cette -# License, or (at your option) licence, soit (à votre gré) -# any later version. toute version ultérieure. -# -# OpenCADC is distributed in the OpenCADC est distribué -# hope that it will be useful, dans l’espoir qu’il vous -# but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE -# without even the implied GARANTIE : sans même la garantie -# warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ -# or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF -# PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +# +# (c) 2016. (c) 2016. +# Government of Canada Gouvernement du Canada +# National Research Council Conseil national de recherches +# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +# All rights reserved Tous droits réservés +# +# NRC disclaims any warranties, Le CNRC dénie toute garantie +# expressed, implied, or énoncée, implicite ou légale, +# statutory, of any kind with de quelque nature que ce +# respect to the software, soit, concernant le logiciel, +# including without limitation y compris sans restriction +# any warranty of merchantability toute garantie de valeur +# or fitness for a particular marchande ou de pertinence +# purpose. NRC shall not be pour un usage particulier. +# liable in any event for any Le CNRC ne pourra en aucun cas +# damages, whether direct or être tenu responsable de tout +# indirect, special or general, dommage, direct ou indirect, +# consequential or incidental, particulier ou général, +# arising from the use of the accessoire ou fortuit, résultant +# software. Neither the name de l'utilisation du logiciel. Ni +# of the National Research le nom du Conseil National de +# Council of Canada nor the Recherches du Canada ni les noms +# names of its contributors may de ses participants ne peuvent +# be used to endorse or promote être utilisés pour approuver ou +# products derived from this promouvoir les produits dérivés +# software without specific prior de ce logiciel sans autorisation +# written permission. préalable et particulière +# par écrit. +# +# This file is part of the Ce fichier fait partie du projet +# OpenCADC project. OpenCADC. +# +# OpenCADC is free software: OpenCADC est un logiciel libre ; +# you can redistribute it and/or vous pouvez le redistribuer ou le +# modify it under the terms of modifier suivant les termes de +# the GNU Affero General Public la “GNU Affero General Public +# License as published by the License” telle que publiée +# Free Software Foundation, par la Free Software Foundation +# either version 3 of the : soit la version 3 de cette +# License, or (at your option) licence, soit (à votre gré) +# any later version. toute version ultérieure. +# +# OpenCADC is distributed in the OpenCADC est distribué +# hope that it will be useful, dans l’espoir qu’il vous +# but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +# without even the implied GARANTIE : sans même la garantie +# warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +# or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +# PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence # General Public License for Générale Publique GNU Affero # more details. pour plus de détails. -# +# # You should have received Vous devriez avoir reçu une # a copy of the GNU Affero copie de la Licence Générale # General Public License along Publique GNU Affero avec @@ -71,9 +72,8 @@ class ObservationUpdater(object): - """Plugin that does not update the observation""" - + def update(self, observation): """ Processes an observation and updates it diff --git a/caom2repo/caom2repo/tests/setup_package.py b/caom2repo/caom2repo/tests/setup_package.py deleted file mode 100644 index 6e4be679..00000000 --- a/caom2repo/caom2repo/tests/setup_package.py +++ /dev/null @@ -1,7 +0,0 @@ -def get_package_data(): - return { - _ASTROPY_PACKAGE_NAME_ + '.tests': ['coveragerc']} - - -def requires_2to3(): - return False diff --git a/caom2repo/caom2repo/tests/test_core.py b/caom2repo/caom2repo/tests/test_core.py index 26b97e6c..c8701fee 100644 --- a/caom2repo/caom2repo/tests/test_core.py +++ b/caom2repo/caom2repo/tests/test_core.py @@ -1,60 +1,61 @@ +#!/usr/bin/env python2.7 # # -*- coding: utf-8 -*- # *********************************************************************** # ****************** CANADIAN ASTRONOMY DATA CENTRE ******************* # ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** -# -# (c) 2016. (c) 2016. -# Government of Canada Gouvernement du Canada -# National Research Council Conseil national de recherches -# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 -# All rights reserved Tous droits réservés -# -# NRC disclaims any warranties, Le CNRC dénie toute garantie -# expressed, implied, or énoncée, implicite ou légale, -# statutory, of any kind with de quelque nature que ce -# respect to the software, soit, concernant le logiciel, -# including without limitation y compris sans restriction -# any warranty of merchantability toute garantie de valeur -# or fitness for a particular marchande ou de pertinence -# purpose. NRC shall not be pour un usage particulier. -# liable in any event for any Le CNRC ne pourra en aucun cas -# damages, whether direct or être tenu responsable de tout -# indirect, special or general, dommage, direct ou indirect, -# consequential or incidental, particulier ou général, -# arising from the use of the accessoire ou fortuit, résultant -# software. Neither the name de l'utilisation du logiciel. Ni -# of the National Research le nom du Conseil National de -# Council of Canada nor the Recherches du Canada ni les noms -# names of its contributors may de ses participants ne peuvent -# be used to endorse or promote être utilisés pour approuver ou -# products derived from this promouvoir les produits dérivés -# software without specific prior de ce logiciel sans autorisation -# written permission. préalable et particulière -# par écrit. -# -# This file is part of the Ce fichier fait partie du projet -# OpenCADC project. OpenCADC. -# -# OpenCADC is free software: OpenCADC est un logiciel libre ; -# you can redistribute it and/or vous pouvez le redistribuer ou le -# modify it under the terms of modifier suivant les termes de -# the GNU Affero General Public la “GNU Affero General Public -# License as published by the License” telle que publiée -# Free Software Foundation, par la Free Software Foundation -# either version 3 of the : soit la version 3 de cette -# License, or (at your option) licence, soit (à votre gré) -# any later version. toute version ultérieure. -# -# OpenCADC is distributed in the OpenCADC est distribué -# hope that it will be useful, dans l’espoir qu’il vous -# but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE -# without even the implied GARANTIE : sans même la garantie -# warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ -# or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF -# PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +# +# (c) 2016. (c) 2016. +# Government of Canada Gouvernement du Canada +# National Research Council Conseil national de recherches +# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +# All rights reserved Tous droits réservés +# +# NRC disclaims any warranties, Le CNRC dénie toute garantie +# expressed, implied, or énoncée, implicite ou légale, +# statutory, of any kind with de quelque nature que ce +# respect to the software, soit, concernant le logiciel, +# including without limitation y compris sans restriction +# any warranty of merchantability toute garantie de valeur +# or fitness for a particular marchande ou de pertinence +# purpose. NRC shall not be pour un usage particulier. +# liable in any event for any Le CNRC ne pourra en aucun cas +# damages, whether direct or être tenu responsable de tout +# indirect, special or general, dommage, direct ou indirect, +# consequential or incidental, particulier ou général, +# arising from the use of the accessoire ou fortuit, résultant +# software. Neither the name de l'utilisation du logiciel. Ni +# of the National Research le nom du Conseil National de +# Council of Canada nor the Recherches du Canada ni les noms +# names of its contributors may de ses participants ne peuvent +# be used to endorse or promote être utilisés pour approuver ou +# products derived from this promouvoir les produits dérivés +# software without specific prior de ce logiciel sans autorisation +# written permission. préalable et particulière +# par écrit. +# +# This file is part of the Ce fichier fait partie du projet +# OpenCADC project. OpenCADC. +# +# OpenCADC is free software: OpenCADC est un logiciel libre ; +# you can redistribute it and/or vous pouvez le redistribuer ou le +# modify it under the terms of modifier suivant les termes de +# the GNU Affero General Public la “GNU Affero General Public +# License as published by the License” telle que publiée +# Free Software Foundation, par la Free Software Foundation +# either version 3 of the : soit la version 3 de cette +# License, or (at your option) licence, soit (à votre gré) +# any later version. toute version ultérieure. +# +# OpenCADC is distributed in the OpenCADC est distribué +# hope that it will be useful, dans l’espoir qu’il vous +# but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +# without even the implied GARANTIE : sans même la garantie +# warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +# or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +# PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence # General Public License for Générale Publique GNU Affero # more details. pour plus de détails. -# +# # You should have received Vous devriez avoir reçu une # a copy of the GNU Affero copie de la Licence Générale # General Public License along Publique GNU Affero avec @@ -66,6 +67,7 @@ # # *********************************************************************** # + from __future__ import (absolute_import, division, print_function, unicode_literals) @@ -73,8 +75,6 @@ import os import sys import unittest -# TODO to be changed to io.BytesIO when caom2 is prepared for python3 -from six import BytesIO, StringIO from datetime import datetime import requests @@ -83,14 +83,19 @@ from caom2.obs_reader_writer import ObservationWriter from caom2.observation import SimpleObservation from mock import Mock, patch, MagicMock, ANY, call - -# The following is a temporary workaround for Python issue 25532 (https://bugs.python.org/issue25532) -call.__wrapped__ = None +# TODO to be changed to io.BytesIO when caom2 is prepared for python3 +from six import BytesIO, StringIO from caom2repo import core from caom2repo.core import CAOM2RepoClient +# The following is a temporary workaround for Python issue 25532 +# (https://bugs.python.org/issue25532) +call.__wrapped__ = None + + THIS_DIR = os.path.dirname(os.path.realpath(__file__)) +TESTDATA_DIR = os.path.join(THIS_DIR, 'data') class MyExitError(Exception): @@ -98,7 +103,6 @@ class MyExitError(Exception): class TestCAOM2Repo(unittest.TestCase): - """Test the Caom2Visitor class""" @patch('caom2repo.core.net.BaseWsClient', Mock()) @@ -112,7 +116,7 @@ def test_plugin_class(self): visitor._load_plugin_class(os.path.join(THIS_DIR, 'passplugin.py')) visitor.plugin.update(obs) self.assertEquals(expect_obs, obs) - + # plugin class adds a plane to the observation visitor = CAOM2RepoClient(auth.Subject()) obs = SimpleObservation('cfht', '7000000o') @@ -121,18 +125,20 @@ def test_plugin_class(self): visitor.plugin.update(obs) self.assertNotEquals(expect_obs, obs) self.assertEquals(len(expect_obs.planes) + 1, len(obs.planes)) - + # non-existent the plugin file with self.assertRaises(Exception): visitor._load_plugin_class(os.path.join(THIS_DIR, 'blah.py')) - + # non-existent ObservationUpdater class in the plugin file with self.assertRaises(Exception): - visitor._load_plugin_class(os.path.join(THIS_DIR, 'test_visitor.py')) - + visitor._load_plugin_class( + os.path.join(THIS_DIR, 'test_visitor.py')) + # non-existent update method in ObservationUpdater class - with self.assertRaises(Exception): - visitor._load_plugin_class(os.path.join(THIS_DIR, 'noupdateplugin.py')) + with self.assertRaises(Exception): + visitor._load_plugin_class( + os.path.join(THIS_DIR, 'noupdateplugin.py')) # patch sleep to stop the test from sleeping and slowing down execution @patch('cadcutils.net.ws.WsCapabilities') @@ -141,7 +147,8 @@ def test_plugin_class(self): @patch('cadcutils.net.ws.Session.send') def test_get_observation(self, mock_get, caps_mock): caps_mock.get_service_host.return_value = 'some.host.com' - caps_mock.return_value.get_access_url.return_value = 'http://serviceurl/caom2repo/pub' + caps_mock.return_value.get_access_url.return_value =\ + 'http://serviceurl/caom2repo/pub' collection = 'cfht' observation_id = '7000000o' service_url = 'www.cadc.nrc.ca/caom2repo' @@ -155,8 +162,9 @@ def test_get_observation(self, mock_get, caps_mock): mock_get.return_value = response ibuffer.seek(0) # reposition the buffer for reading visitor = CAOM2RepoClient(auth.Subject(), host=service_url) - self.assertEquals(obs, visitor.get_observation(collection, observation_id)) - + self.assertEquals(obs, + visitor.get_observation(collection, observation_id)) + # signal problems http_error = requests.HTTPError() response.status_code = 500 @@ -164,7 +172,7 @@ def test_get_observation(self, mock_get, caps_mock): response.raise_for_status.side_effect = [http_error] with self.assertRaises(exceptions.InternalServerException): visitor.get_observation(collection, observation_id) - + # temporary transient errors http_error = requests.HTTPError() response.status_code = 503 @@ -178,6 +186,7 @@ def test_get_observation(self, mock_get, caps_mock): http_error.response = response def raise_error(): raise http_error + response.raise_for_status.side_effect = raise_error with self.assertRaises(exceptions.HttpException): visitor.get_observation(collection, observation_id) @@ -192,38 +201,49 @@ def test_get_observations(self, mock_get, caps_mock): # observations matching a collection and start/end criteria # Also, patch the CAOM2RepoClient now. caps_mock.get_service_host.return_value = 'some.host.com' - caps_mock.return_value.get_access_url.return_value = 'http://serviceurl/caom2repo/pub' + caps_mock.return_value.get_access_url.return_value =\ + 'http://serviceurl/caom2repo/pub' response = MagicMock() response.status_code = 200 last_datetime = '2000-10-10T12:30:00.333' - response.text = 'CFHT\t700000o\t2000-10-10T12:20:11.123\t3e00ca6129dc8358315015204ab9fe15\nCFHT\t700001o\t' +\ - last_datetime + '\t3e00ca6129dc8358315015204ab9fe15' + response.text = \ + ('CFHT\t700000o\t2000-10-10T12:20:11.123\t' + '3e00ca6129dc8358315015204ab9fe15\nCFHT\t700001o\t' + + last_datetime + '\t3e00ca6129dc8358315015204ab9fe15') mock_get.return_value = response - + visitor = CAOM2RepoClient(auth.Subject()) end_date = util.utils.str2ivoa(last_datetime) - + expect_observations = ['700000o', '700001o'] - self.assertEquals(expect_observations, visitor._get_observations('cfht')) + self.assertEquals(expect_observations, + visitor._get_observations('cfht')) self.assertEquals(end_date, visitor._start) mock_get.assert_called_once_with(( - 'vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.1', 'cfht'), + 'vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.1', + 'cfht'), params={'MAXREC': core.BATCH_SIZE}) mock_get.reset_mock() - visitor._get_observations('cfht', end=datetime.strptime('2000-11-11', '%Y-%m-%d')) + visitor._get_observations('cfht', end=datetime.strptime('2000-11-11', + '%Y-%m-%d')) mock_get.assert_called_once_with(( - 'vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.1', 'cfht'), - params={'END': '2000-11-11T00:00:00.000', 'MAXREC': core.BATCH_SIZE}) + 'vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.1', + 'cfht'), + params={'END': '2000-11-11T00:00:00.000', + 'MAXREC': core.BATCH_SIZE}) mock_get.reset_mock() visitor._get_observations('cfht', - start=datetime.strptime('2000-11-11', '%Y-%m-%d'), - end=datetime.strptime('2000-11-12', '%Y-%m-%d')) + start=datetime.strptime('2000-11-11', + '%Y-%m-%d'), + end=datetime.strptime('2000-11-12', + '%Y-%m-%d')) mock_get.assert_called_once_with(( - 'vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.1', 'cfht') - , params={'START': '2000-11-11T00:00:00.000', - 'END': '2000-11-12T00:00:00.000', 'MAXREC': core.BATCH_SIZE}) + 'vos://cadc.nrc.ca~vospace/CADC/std/CAOM2Repository#obs-1.1', + 'cfht'), params={'START': '2000-11-11T00:00:00.000', + 'END': '2000-11-12T00:00:00.000', + 'MAXREC': core.BATCH_SIZE}) # patch sleep to stop the test from sleeping and slowing down execution @patch('cadcutils.net.ws.WsCapabilities') @@ -232,14 +252,16 @@ def test_get_observations(self, mock_get, caps_mock): @patch('cadcutils.net.ws.Session.send') def test_post_observation(self, mock_conn, caps_mock): caps_mock.get_service_host.return_value = 'some.host.com' - caps_mock.return_value.get_access_url.return_value = 'http://serviceurl/caom2repo/auth' + caps_mock.return_value.get_access_url.return_value =\ + 'http://serviceurl/caom2repo/auth' collection = 'cfht' observation_id = '7000000o' service = 'caom2repo' service_url = 'www.cadc.nrc.ca' obs = SimpleObservation(collection, observation_id) - visitor = CAOM2RepoClient(auth.Subject(netrc='somenetrc'), host=service_url) + visitor = CAOM2RepoClient(auth.Subject(netrc='somenetrc'), + host=service_url) response = MagicMock() response.status = 200 mock_conn.return_value = response @@ -250,9 +272,11 @@ def test_post_observation(self, mock_conn, caps_mock): visitor.post_observation(obs) self.assertEqual('POST', mock_conn.call_args[0][0].method) - self.assertEqual('/{}/auth/{}/{}'.format(service, collection, observation_id), - mock_conn.call_args[0][0].path_url) - self.assertEqual('application/xml', mock_conn.call_args[0][0].headers['Content-Type']) + self.assertEqual( + '/{}/auth/{}/{}'.format(service, collection, observation_id), + mock_conn.call_args[0][0].path_url) + self.assertEqual('application/xml', + mock_conn.call_args[0][0].headers['Content-Type']) self.assertEqual(obsxml, mock_conn.call_args[0][0].body) # signal problems @@ -276,6 +300,7 @@ def test_post_observation(self, mock_conn, caps_mock): http_error.response = response def raise_error(): raise http_error + response.raise_for_status.side_effect = raise_error with self.assertRaises(exceptions.HttpException): visitor.post_observation(obs) @@ -287,7 +312,8 @@ def raise_error(): raise http_error @patch('cadcutils.net.ws.Session.send') def test_put_observation(self, mock_conn, caps_mock): caps_mock.get_service_host.return_value = 'some.host.com' - caps_mock.return_value.get_access_url.return_value = 'http://serviceurl/caom2repo/pub' + caps_mock.return_value.get_access_url.return_value =\ + 'http://serviceurl/caom2repo/pub' collection = 'cfht' observation_id = '7000000o' service = 'caom2repo' @@ -306,9 +332,11 @@ def test_put_observation(self, mock_conn, caps_mock): visitor.put_observation(obs) self.assertEqual('PUT', mock_conn.call_args[0][0].method) - self.assertEqual('/{}/pub/{}/{}'.format(service, collection, observation_id), - mock_conn.call_args[0][0].path_url) - self.assertEqual('application/xml', mock_conn.call_args[0][0].headers['Content-Type']) + self.assertEqual( + '/{}/pub/{}/{}'.format(service, collection, observation_id), + mock_conn.call_args[0][0].path_url) + self.assertEqual('application/xml', + mock_conn.call_args[0][0].headers['Content-Type']) self.assertEqual(obsxml, mock_conn.call_args[0][0].body) # signal problems @@ -343,12 +371,12 @@ def raise_error(): raise http_error @patch('cadcutils.net.ws.Session.send') def test_delete_observation(self, mock_conn, caps_mock): caps_mock.get_service_host.return_value = 'some.host.com' - caps_mock.return_value.get_access_url.return_value = 'http://serviceurl/caom2repo/pub' + caps_mock.return_value.get_access_url.return_value =\ + 'http://serviceurl/caom2repo/pub' collection = 'cfht' observation_id = '7000000o' service_url = 'www.cadc.nrc.ca' - obs = SimpleObservation(collection, observation_id) visitor = CAOM2RepoClient(auth.Subject(), host=service_url) response = MagicMock() response.status = 200 @@ -383,17 +411,18 @@ def raise_error(): raise http_error with self.assertRaises(exceptions.HttpException): visitor.delete_observation(collection, observation_id) - @patch('caom2repo.core.net.BaseWsClient', Mock()) def test_process(self): core.BATCH_SIZE = 3 # size of the batch is 3 obs = [['a', 'b', 'c'], ['d'], []] visitor = CAOM2RepoClient(auth.Subject()) - visitor.get_observation = MagicMock(return_value=MagicMock(spec=SimpleObservation)) + visitor.get_observation = MagicMock( + return_value=MagicMock(spec=SimpleObservation)) visitor.post_observation = MagicMock() visitor._get_observations = MagicMock(side_effect=obs) - (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht') + (visited, updated, skipped, failed) = visitor.visit( + os.path.join(THIS_DIR, 'passplugin.py'), 'cfht') self.assertEqual(4, len(visited)) self.assertEqual(4, len(updated)) self.assertEqual(0, len(skipped)) @@ -401,7 +430,8 @@ def test_process(self): obs = [['a', 'b', 'c'], ['d', 'e', 'f'], []] visitor._get_observations = MagicMock(side_effect=obs) - (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht') + (visited, updated, skipped, failed) = visitor.visit( + os.path.join(THIS_DIR, 'passplugin.py'), 'cfht') self.assertEqual(6, len(visited)) self.assertEqual(6, len(updated)) self.assertEqual(0, len(skipped)) @@ -412,11 +442,12 @@ def test_process(self): # raises exception for 'ERROR' obs_ids = [['UPDATE', 'SKIP', 'ERROR'], []] obs = [SimpleObservation(collection='TEST', observation_id='UPDATE'), - SimpleObservation(collection='TEST', observation_id='SKIP'), - SimpleObservation(collection='TEST', observation_id='ERROR')] + SimpleObservation(collection='TEST', observation_id='SKIP'), + SimpleObservation(collection='TEST', observation_id='ERROR')] visitor._get_observations = MagicMock(side_effect=obs_ids) visitor.get_observation = MagicMock(side_effect=obs) - (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'), 'cfht') + (visited, updated, skipped, failed) = visitor.visit( + os.path.join(THIS_DIR, 'errorplugin.py'), 'cfht') self.assertEqual(3, len(visited)) self.assertEqual(1, len(updated)) self.assertEqual(1, len(skipped)) @@ -431,7 +462,8 @@ def test_process(self): SimpleObservation(collection='TEST', observation_id='SKIP')] visitor._get_observations = MagicMock(side_effect=obs_ids) visitor.get_observation = MagicMock(side_effect=obs) - (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'errorplugin.py'), 'cfht') + (visited, updated, skipped, failed) = visitor.visit( + os.path.join(THIS_DIR, 'errorplugin.py'), 'cfht') self.assertEqual(5, len(visited)) self.assertEqual(2, len(updated)) self.assertEqual(2, len(skipped)) @@ -459,24 +491,27 @@ def test_process(self): response2 = MagicMock() response2.text = """ARCHIVE\td\t2011-02-02T11:00:00.000""" visitor = CAOM2RepoClient(auth.Subject()) - visitor.get_observation = MagicMock(return_value=MagicMock(spec=SimpleObservation)) + visitor.get_observation = MagicMock( + return_value=MagicMock(spec=SimpleObservation)) visitor.post_observation = MagicMock() visitor._repo_client.get = MagicMock(side_effect=[response, response2]) start = '2010-10-10T12:00:00.000' end = '2012-12-12T11:11:11.000' - (visited, updated, skipped, failed) = visitor.visit(os.path.join(THIS_DIR, 'passplugin.py'), 'cfht', - start=util.str2ivoa(start), - end=util.str2ivoa(end)) + (visited, updated, skipped, failed) = visitor.visit( + os.path.join(THIS_DIR, 'passplugin.py'), 'cfht', + start=util.str2ivoa(start), + end=util.str2ivoa(end)) self.assertEqual(4, len(visited)) self.assertEqual(4, len(updated)) self.assertEqual(0, len(skipped)) self.assertEqual(0, len(failed)) calls = [call((core.CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'), - params={'START': start, 'END': end, 'MAXREC' :3}), + params={'START': start, 'END': end, 'MAXREC': 3}), call((core.CAOM2REPO_OBS_CAPABILITY_ID, 'cfht'), - params={'START': '2011-01-01T12:00:00.000', # datetime of the last record in the batch + params={'START': '2011-01-01T12:00:00.000', + # datetime of the last record in the batch 'END': end, 'MAXREC': 3})] visitor._repo_client.get.assert_has_calls(calls) @@ -501,12 +536,10 @@ def test_shortcuts(self): target.delete('CFHT', 'abc') target.delete_observation.assert_called_with('CFHT', 'abc') - @patch('caom2repo.core.CAOM2RepoClient') def test_main_app(self, client_mock): collection = 'cfht' observation_id = '7000000o' - service = 'caom2repo' ifile = '/tmp/inputobs' obs = SimpleObservation(collection, observation_id) @@ -514,40 +547,50 @@ def test_main_app(self, client_mock): # test create with open(ifile, 'wb') as infile: ObservationWriter().write(obs, infile) - sys.argv = ["caom2tools", "create", '--resource-id', 'ivo://ca.nrc.ca/resource', ifile] + sys.argv = ["caom2tools", "create", '--resource-id', + 'ivo://ca.nrc.ca/resource', ifile] core.main_app() client_mock.return_value.put_observation.assert_called_with(obs) # test update - sys.argv = ["caom2tools", "update", '--resource-id', 'ivo://ca.nrc.ca/resource', ifile] + sys.argv = ["caom2tools", "update", '--resource-id', + 'ivo://ca.nrc.ca/resource', ifile] core.main_app() client_mock.return_value.post_observation.assert_called_with(obs) # test read - sys.argv = ["caom2tools", "read", '--resource-id', 'ivo://ca.nrc.ca/resource', + sys.argv = ["caom2tools", "read", '--resource-id', + 'ivo://ca.nrc.ca/resource', collection, observation_id] client_mock.return_value.get_observation.return_value = obs core.main_app() - client_mock.return_value.get_observation.assert_called_with(collection, observation_id) + client_mock.return_value.get_observation.\ + assert_called_with(collection, observation_id) # repeat with output argument - sys.argv = ["caom2tools", "read", '--resource-id', 'ivo://ca.nrc.ca/resource', + sys.argv = ["caom2tools", "read", '--resource-id', + 'ivo://ca.nrc.ca/resource', "--output", ifile, collection, observation_id] client_mock.return_value.get_observation.return_value = obs core.main_app() - client_mock.return_value.get_observation.assert_called_with(collection, observation_id) + client_mock.return_value.get_observation.\ + assert_called_with(collection, observation_id) os.remove(ifile) # test delete - sys.argv = ["caom2tools", "delete", '--resource-id', 'ivo://ca.nrc.ca/resource', + sys.argv = ["caom2tools", "delete", '--resource-id', + 'ivo://ca.nrc.ca/resource', collection, observation_id] core.main_app() - client_mock.return_value.delete_observation.assert_called_with(collection=collection, - observation_id=observation_id) + client_mock.return_value.delete_observation.assert_called_with( + collection=collection, + observation_id=observation_id) # test visit - # get the absolute path to be able to run the tests with the astropy frameworks + # get the absolute path to be able to run the tests with the + # astropy frameworks plugin_file = THIS_DIR + "/passplugin.py" - sys.argv = ["caom2tools", "visit", '--resource-id', 'ivo://ca.nrc.ca/resource', + sys.argv = ["caom2tools", "visit", '--resource-id', + 'ivo://ca.nrc.ca/resource', "--plugin", plugin_file, "--start", "2012-01-01T11:22:33", "--end", "2013-01-01T11:33:22", collection] client_mock.return_value.visit.return_value = ['1'], ['1'], [], [] @@ -559,8 +602,9 @@ def test_main_app(self, client_mock): end=core.str2date("2013-01-01T11:33:22")) # repeat visit test with halt-on-error - sys.argv = ["caom2tools", "visit", '--resource-id', 'ivo://ca.nrc.ca/resource', - "--plugin", plugin_file,'--halt-on-error', + sys.argv = ["caom2tools", "visit", '--resource-id', + 'ivo://ca.nrc.ca/resource', + "--plugin", plugin_file, '--halt-on-error', "--start", "2012-01-01T11:22:33", "--end", "2013-01-01T11:33:22", collection] client_mock.return_value.visit.return_value = ['1'], ['1'], [], [] @@ -572,254 +616,72 @@ def test_main_app(self, client_mock): end=core.str2date("2013-01-01T11:33:22")) @patch('sys.exit', Mock(side_effect=[MyExitError, MyExitError, MyExitError, - MyExitError, MyExitError, MyExitError])) + MyExitError, MyExitError, + MyExitError])) def test_help(self): """ Tests the helper displays for commands and subcommands in main""" # expected helper messages - usage =\ -"""usage: caom2-client [-h] [-V] [-s SERVER] - {create,read,update,delete,visit} ... - -Client for a CAOM2 repo. In addition to CRUD (Create, Read, Update and Delete) operations it also implements a visitor operation that allows for updating multiple observations in a collection - -positional arguments: - {create,read,update,delete,visit} - create Create a new observation - read Read an existing observation - update Update an existing observation - delete Delete an existing observation - visit Visit observations in a collection - -optional arguments: - -h, --help show this help message and exit - -V, --version show program's version number and exit - -s SERVER, --server SERVER - URL of the CAOM2 repo server -""" - - create_usage =\ -"""usage: caom2-client create [-h] - [--cert CERT | -n | --netrc-file NETRC_FILE | -u USER] - [--host HOST] [--resource-id RESOURCE_ID] - [-d | -q | -v] - observation - -Create a new observation - -positional arguments: - observation XML file containing the observation - -optional arguments: - --cert CERT location of your X509 certificate to use for - authentication (unencrypted, in PEM format) - -d, --debug debug messages - -h, --help show this help message and exit - --host HOST base hostname for services - used mainly for testing - (default: www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca) - -n use .netrc in $HOME for authentication - --netrc-file NETRC_FILE - netrc file to use for authentication - -q, --quiet run quietly - --resource-id RESOURCE_ID - resource identifier (default - ivo://cadc.nrc.ca/caom2repo) - -u, --user USER name of user to authenticate. Note: application - prompts for the corresponding password! - -v, --verbose verbose messages -""" - - read_usage =\ -"""usage: caom2-client read [-h] - [--cert CERT | -n | --netrc-file NETRC_FILE | -u USER] - [--host HOST] [--resource-id RESOURCE_ID] - [-d | -q | -v] [--output OUTPUT] - collection observationID - -Read an existing observation - -positional arguments: - collection collection name in CAOM2 repo - observationID observation identifier - -optional arguments: - --cert CERT location of your X509 certificate to use for - authentication (unencrypted, in PEM format) - -d, --debug debug messages - -h, --help show this help message and exit - --host HOST base hostname for services - used mainly for testing - (default: www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca) - -n use .netrc in $HOME for authentication - --netrc-file NETRC_FILE - netrc file to use for authentication - --output, -o OUTPUT destination file - -q, --quiet run quietly - --resource-id RESOURCE_ID - resource identifier (default - ivo://cadc.nrc.ca/caom2repo) - -u, --user USER name of user to authenticate. Note: application - prompts for the corresponding password! - -v, --verbose verbose messages -""" - - update_usage =\ -"""usage: caom2-client update [-h] - [--cert CERT | -n | --netrc-file NETRC_FILE | -u USER] - [--host HOST] [--resource-id RESOURCE_ID] - [-d | -q | -v] - observation - -Update an existing observation - -positional arguments: - observation XML file containing the observation - -optional arguments: - --cert CERT location of your X509 certificate to use for - authentication (unencrypted, in PEM format) - -d, --debug debug messages - -h, --help show this help message and exit - --host HOST base hostname for services - used mainly for testing - (default: www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca) - -n use .netrc in $HOME for authentication - --netrc-file NETRC_FILE - netrc file to use for authentication - -q, --quiet run quietly - --resource-id RESOURCE_ID - resource identifier (default - ivo://cadc.nrc.ca/caom2repo) - -u, --user USER name of user to authenticate. Note: application - prompts for the corresponding password! - -v, --verbose verbose messages -""" - - delete_usage =\ -"""usage: caom2-client delete [-h] - [--cert CERT | -n | --netrc-file NETRC_FILE | -u USER] - [--host HOST] [--resource-id RESOURCE_ID] - [-d | -q | -v] - collection observationID - -Delete an existing observation - -positional arguments: - collection collection name in CAOM2 repo - observationID observation identifier - -optional arguments: - --cert CERT location of your X509 certificate to use for - authentication (unencrypted, in PEM format) - -d, --debug debug messages - -h, --help show this help message and exit - --host HOST base hostname for services - used mainly for testing - (default: www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca) - -n use .netrc in $HOME for authentication - --netrc-file NETRC_FILE - netrc file to use for authentication - -q, --quiet run quietly - --resource-id RESOURCE_ID - resource identifier (default - ivo://cadc.nrc.ca/caom2repo) - -u, --user USER name of user to authenticate. Note: application - prompts for the corresponding password! - -v, --verbose verbose messages -""" - - visit_usage =\ -"""usage: caom2-client visit [-h] - [--cert CERT | -n | --netrc-file NETRC_FILE | -u USER] - [--host HOST] [--resource-id RESOURCE_ID] - [-d | -q | -v] --plugin PLUGIN [--start START] - [--end END] [--halt-on-error] - collection - -Visit observations in a collection - -positional arguments: - collection data collection in CAOM2 repo - -optional arguments: - --cert CERT location of your X509 certificate to use for - authentication (unencrypted, in PEM format) - -d, --debug debug messages - --end END latest observation to visit (UTC IVOA format: YYYY-mm- - ddTH:M:S) - --halt-on-error stop visitor on first update exception raised by - plugin - -h, --help show this help message and exit - --host HOST base hostname for services - used mainly for testing - (default: www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca) - -n use .netrc in $HOME for authentication - --netrc-file NETRC_FILE - netrc file to use for authentication - --plugin PLUGIN plugin class to update each observation - -q, --quiet run quietly - --resource-id RESOURCE_ID - resource identifier (default - ivo://cadc.nrc.ca/caom2repo) - --start START earliest observation to visit (UTC IVOA format: YYYY- - mm-ddTH:M:S) - -u, --user USER name of user to authenticate. Note: application - prompts for the corresponding password! - -v, --verbose verbose messages - -Minimum plugin file format: ----- - from caom2 import Observation - - class ObservationUpdater: - - def update(self, observation): - assert isinstance(observation, Observation), ( - 'observation {} is not an Observation'.format(observation)) - # custom code to update the observation ----- -""" + with open(os.path.join(TESTDATA_DIR, 'help.txt'), 'r') as myfile: + usage = myfile.read() + with open( + os.path.join(TESTDATA_DIR, 'create_help.txt'), 'r') as myfile: + create_usage = myfile.read() + with open(os.path.join(TESTDATA_DIR, 'read_help.txt'), 'r') as myfile: + read_usage = myfile.read() + with open( + os.path.join(TESTDATA_DIR, 'update_help.txt'), 'r') as myfile: + update_usage = myfile.read() + with open( + os.path.join(TESTDATA_DIR, 'delete_help.txt'), 'r') as myfile: + delete_usage = myfile.read() + with open(os.path.join(TESTDATA_DIR, 'visit_help.txt'), 'r') as myfile: + visit_usage = myfile.read() self.maxDiff = None # Display the entire difference # --help with patch('sys.stdout', new_callable=StringIO) as stdout_mock: - sys.argv = ["caom2-client", "--help"] + sys.argv = ["caom2-repo", "--help"] with self.assertRaises(MyExitError): core.main_app() self.assertEqual(usage, stdout_mock.getvalue()) # create --help with patch('sys.stdout', new_callable=StringIO) as stdout_mock: - sys.argv = ["caom2-client", "create", "--help"] + sys.argv = ["caom2-repo", "create", "--help"] with self.assertRaises(MyExitError): core.main_app() self.assertEqual(create_usage, stdout_mock.getvalue()) - #print(stdout_mock.getvalue()) + # print(stdout_mock.getvalue()) # read --help with patch('sys.stdout', new_callable=StringIO) as stdout_mock: - sys.argv = ["caom2-client", "read", "--help"] + sys.argv = ["caom2-repo", "read", "--help"] with self.assertRaises(MyExitError): core.main_app() self.assertEqual(read_usage, stdout_mock.getvalue()) - #print(stdout_mock.getvalue()) + # print(stdout_mock.getvalue()) # update --help with patch('sys.stdout', new_callable=StringIO) as stdout_mock: - sys.argv = ["caom2-client", "update", "--help"] + sys.argv = ["caom2-repo", "update", "--help"] with self.assertRaises(MyExitError): core.main_app() self.assertEqual(update_usage, stdout_mock.getvalue()) - #print(stdout_mock.getvalue()) + # print(stdout_mock.getvalue()) # delete --help with patch('sys.stdout', new_callable=StringIO) as stdout_mock: - sys.argv = ["caom2-client", "delete", "--help"] + sys.argv = ["caom2-repo", "delete", "--help"] with self.assertRaises(MyExitError): core.main_app() self.assertEqual(delete_usage, stdout_mock.getvalue()) - #print(stdout_mock.getvalue()) + # print(stdout_mock.getvalue()) # visit --help with patch('sys.stdout', new_callable=StringIO) as stdout_mock: - sys.argv = ["caom2-client", "visit", "--help"] + sys.argv = ["caom2-repo", "visit", "--help"] with self.assertRaises(MyExitError): core.main_app() self.assertEqual(visit_usage, stdout_mock.getvalue()) - #print(stdout_mock.getvalue()) + # print(stdout_mock.getvalue()) diff --git a/caom2repo/dev_requirements.txt b/caom2repo/dev_requirements.txt index 1d926ae9..ec660112 100644 --- a/caom2repo/dev_requirements.txt +++ b/caom2repo/dev_requirements.txt @@ -2,6 +2,7 @@ -e . pytest>=3.0.5 pytest-cov>=2.5.1 +flake8>=3.4.1 funcsigs==1.0.2 mock==2.0.0 xml-compare==1.0.5 diff --git a/caom2repo/setup.py b/caom2repo/setup.py index e497a3af..9490bdd0 100755 --- a/caom2repo/setup.py +++ b/caom2repo/setup.py @@ -42,7 +42,7 @@ def readme(): # generate the version file with open(os.path.join(PACKAGENAME, 'version.py'), 'w') as f: - f.write('version = \'{}\''.format(VERSION)) + f.write('version = \'{}\'\n'.format(VERSION)) # Treat everything in scripts except README.rst as a script to be installed scripts = [fname for fname in glob.glob(os.path.join('scripts', '*'))