s2766 - make it possible to preserve Observation- and Plane-level met…

…adata during update. (#136) Add an update flag to the BluePrint
opencadc · Sep 14, 2020 · 70fd87f · 70fd87f
1 parent 8886a86
commit 70fd87f
Show file tree

Hide file tree

Showing 17 changed files with 331 additions and 315 deletions.
diff --git a/caom2utils/caom2utils/fits2caom2.py b/caom2utils/caom2utils/fits2caom2.py
diff --git a/caom2utils/caom2utils/tests/data/cfht/1709071og/cfhtwircam.override b/caom2utils/caom2utils/tests/data/cfht/1709071og/cfhtwircam.override
@@ -7,6 +7,7 @@ proposal.project =
 proposal.pi = Jean-Gabriel Cuby
 proposal.title =  
 plane.calibrationLevel = 1
+plane.dataProductType = image
 plane.dataRelease = 2015-02-01T00:00:00
 observation.metaRelease = 2014-05-12T10:18:55
 plane.metaRelease = 2014-05-12T10:18:55

diff --git a/caom2utils/caom2utils/tests/data/cfht/1709071p/1709071p.override b/caom2utils/caom2utils/tests/data/cfht/1709071p/1709071p.override
@@ -7,6 +7,7 @@ proposal.project =
 proposal.pi = Jean-Gabriel Cuby
 proposal.title =  
 plane.calibrationLevel = 2
+plane.dataProductType = image
 plane.dataRelease = 2015-02-01T00:00:00
 observation.metaRelease = 2014-05-12T10:18:55
 plane.metaRelease = 2014-05-12T10:18:55

diff --git a/caom2utils/caom2utils/tests/data/cfht/1916216i/1916216i.override b/caom2utils/caom2utils/tests/data/cfht/1916216i/1916216i.override
@@ -7,6 +7,7 @@ target.standard = false
 proposal.project = BINAMICS
 proposal.title = Binamics: Binarity and Magnetic Interactions in various classes of Stars
 plane.calibrationLevel = 2
+plane.dataProductType = spectrum
 plane.dataRelease = 2017-08-31T00:00:00
 provenance.inputs = caom:CFHT/1916216/1916216o
 provenance.lastExecuted = 2016-03-29T02:18:53

diff --git a/caom2utils/caom2utils/tests/data/cfht/1916216o/1916216o.override b/caom2utils/caom2utils/tests/data/cfht/1916216o/1916216o.override
@@ -7,6 +7,7 @@ target.standard = false
 proposal.project = BINAMICS
 proposal.title = Binamics: Binarity and Magnetic Interactions in various classes of Stars
 plane.calibrationLevel = 1
+plane.dataProductType = spectrum
 plane.dataRelease = 2017-08-31T00:00:00
 artifact.productType = science
 artifact.contentChecksum = md5:147df938dceac8e987b99b241bf1d794

diff --git a/caom2utils/caom2utils/tests/data/cfht/1916216p/1916216p.override b/caom2utils/caom2utils/tests/data/cfht/1916216p/1916216p.override
@@ -7,6 +7,7 @@ target.standard = false
 proposal.project = BINAMICS
 proposal.title = Binamics: Binarity and Magnetic Interactions in various classes of Stars
 plane.calibrationLevel = 2
+plane.dataProductType = spectrum
 plane.dataRelease = 2017-08-31T00:00:00
 provenance.inputs = caom:CFHT/1916216/1916216o caom:CFHT/1916217/1916217o caom:CFHT/1916218/1916218o caom:CFHT/1916219/1916219o
 provenance.lastExecuted = 2016-03-29T02:20:54

diff --git a/caom2utils/caom2utils/tests/data/cfht/2087482o/2087482o.override b/caom2utils/caom2utils/tests/data/cfht/2087482o/2087482o.override
@@ -6,6 +6,7 @@ proposal.project =
 proposal.pi = Mary Laychak
 proposal.title =  
 plane.calibrationLevel = 1
+plane.dataProductType = image
 plane.dataRelease = 2018-08-31T00:00:00
 artifact.productType = science
 artifact.contentChecksum = md5:214f02f46f936d26ebc7c79af82a1f63

diff --git a/caom2utils/caom2utils/tests/data/cfht/2087482p/2087482p.override b/caom2utils/caom2utils/tests/data/cfht/2087482p/2087482p.override
@@ -6,6 +6,7 @@ proposal.project =
 proposal.pi = Mary Laychak
 proposal.title =  
 plane.calibrationLevel = 2
+plane.dataProductType = image
 plane.dataRelease = 2018-08-31T00:00:00
 provenance.inputs = caom:CFHT/2087482/2087482o
 provenance.keywords = 

diff --git a/caom2utils/caom2utils/tests/data/cfht/2216850f/2216850f.override b/caom2utils/caom2utils/tests/data/cfht/2216850f/2216850f.override
@@ -7,6 +7,7 @@ target.standard = false
 proposal.project =  
 proposal.title =  
 plane.calibrationLevel = 1
+plane.dataProductType = spectrum
 plane.dataRelease = 2017-11-05T00:00:00
 artifact.productType = calibration
 artifact.contentChecksum = md5:6b166e52dc8618ddb7a9ad2b175641b2

diff --git a/caom2utils/caom2utils/tests/data/cfht/2216860b/2216860b.override b/caom2utils/caom2utils/tests/data/cfht/2216860b/2216860b.override
@@ -7,6 +7,7 @@ target.standard = false
 proposal.project =  
 proposal.title =  
 plane.calibrationLevel = 1
+plane.dataProductType = spectrum
 plane.dataRelease = 2017-11-05T00:00:00
 artifact.productType = calibration
 artifact.contentChecksum = md5:a448e8df9e6261f7161e76a5db87c072

diff --git a/caom2utils/caom2utils/tests/data/cgps/myOBS/cgps.override b/caom2utils/caom2utils/tests/data/cgps/myOBS/cgps.override
@@ -0,0 +1,2 @@
+plane.calibrationLevel = 2
+plane.dataProductType = cube
diff --git a/caom2utils/caom2utils/tests/data/help.txt b/caom2utils/caom2utils/tests/data/help.txt
@@ -1,7 +1,7 @@
 usage: fits2caom2 [-h] [--cert CERT | -n | --netrc-file NETRC_FILE | -u USER]
                   [--host HOST] [--resource-id RESOURCE_ID] [-d | -q | -v]
                   [-V] [--dumpconfig] [--not_connected] [--no_validate]
-                  [-o OUT_OBS_XML] [--caom_namespace CAOM_NAMESPACE]
+                  [-o OUT_OBS_XML]
                   (-i IN_OBS_XML | --observation collection observationID)
                   [--local LOCAL [LOCAL ...]] [--productID PRODUCTID]
                   [--config CONFIG] [--default DEFAULT] [--override OVERRIDE]
@@ -13,10 +13,6 @@ positional arguments:
   fileURI               URI of a fits file
 
 optional arguments:
-  --caom_namespace CAOM_NAMESPACE
-                        if this parameter is specified, over-ride the default
-                        CAOM2 version when writing XML. The default is the
-                        latest version of CAOM2.3.
   --cert CERT           location of your X509 certificate to use for
                         authentication (unencrypted, in PEM format)
   --config CONFIG       optional CAOM2 utype to keyword config file to merge

diff --git a/caom2utils/caom2utils/tests/data/omm/Cdemo_ext2_SCIRED/omm.blueprint b/caom2utils/caom2utils/tests/data/omm/Cdemo_ext2_SCIRED/omm.blueprint
@@ -1,4 +1,5 @@
 Observation.type = OBJECT
+Observation.algorithm.name = composite
 DerivedObservation.members = ('BINTABLE', ['FICS'], 1)
 Plane.dataProductType = image
 Plane.calibrationLevel = 2

diff --git a/caom2utils/caom2utils/tests/test_fits2caom2.py b/caom2utils/caom2utils/tests/test_fits2caom2.py
@@ -963,7 +963,7 @@ def _get_obs(from_xml_string):
           <caom2:productType>thumbnail</caom2:productType>
           <caom2:releaseType>data</caom2:releaseType>
           <caom2:contentType>text/plain</caom2:contentType>
-          <caom2:contentLength>2981</caom2:contentLength>
+          <caom2:contentLength>2709</caom2:contentLength>
           <caom2:contentChecksum>md5:e6c08f3b8309f05a5a3330e27e3b44eb</caom2:contentChecksum>
           <caom2:uri>file://""" + text_file + """</caom2:uri>
         </caom2:artifact>

diff --git a/caom2utils/caom2utils/tests/test_obs_blueprint.py b/caom2utils/caom2utils/tests/test_obs_blueprint.py
@@ -86,13 +86,13 @@ def test_obs_blueprint():
     assert elems != ObsBlueprint.CAOM2_ELEMENTS
 
     # default config (one entry per row...)
-    assert str(ObsBlueprint()).count('\n') == 23
+    assert str(ObsBlueprint()).count('\n') == 24
     print(ObsBlueprint())
 
     # default config with WCS info
     assert str(ObsBlueprint(position_axes=(1, 2), energy_axis=3,
                polarization_axis=4, time_axis=5,
-                            obs_axis=6)).count('\n') == 83
+                            obs_axis=6)).count('\n') == 84
 
     ob = ObsBlueprint()
     ob.configure_position_axes(axes=(1, 2))

diff --git a/caom2utils/setup.cfg b/caom2utils/setup.cfg
@@ -34,7 +34,7 @@ edit_on_github = False
 github_project = opencadc/caom2tools
 install_requires = cadcdata>=1.2.3 caom2>=2.4 astropy>=2.0 spherical-geometry==1.2.11;python_version=="2.7" spherical-geometry>=1.2.17;python_version>="3.4" vos>=3.1.1
 # version should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386)
-version = 1.4.6
+version = 1.5
 
 
 

diff --git a/doc/user/script_description.md b/doc/user/script_description.md
@@ -148,7 +148,7 @@ The following script is an end-to-end example of describing and loading a CAOM2
                        verbose=False,
                        debug=True,
                        quiet=False,
-                       caom_namespace=obs_reader_writer.CAOM23_NAMESPACE,
+                       caom_namespace=obs_reader_writer.CAOM24_NAMESPACE,
                        **kwargs)
 
     # load the observation into memory
@@ -161,17 +161,33 @@ The following script is an end-to-end example of describing and loading a CAOM2
     # The service generates a parallel set of database keys that must be honoured.
     # The id values must be consistent when doing 'create' and 'update' calls, or
     # a "This observation already exists" error will occur.
-    #
-    # existing_obs = repo_client.read('COLLECTION', 'TEST_OBS_ID')
-    # writer = obs_reader_writer.ObservationWriter()
-    # writer.write(existing_obs, '/fully/qualified/EXISTING.XML')
-    # fits2caom2.augment(...
-    #                    in_obs_xml='/fully/qualified/EXISTING.XML',
-    #                    ...)
-    # load the observation into memory
-    # then use:
-    # repo_client.update(observation)
-    repo_client.create(observation)
+    try:
+        existing_obs = repo_client.read(collection, observation_id)
+        existing_xml = os.path.join(this_dir, 'EXISTING.XML')
+        writer = obs_reader_writer.ObservationWriter()
+        writer.write(existing_obs, existing_xml)
+        fits2caom2.augment(blueprints=blueprints,
+                           no_validate=False,
+                           dump_config=False,
+                           plugin=None,
+                           out_obs_xml='./TEST_OBS.XML',
+                           in_obs_xml=existing_xml,
+                           collection='COLLECTION',
+                           observation='TEST_OBS',
+                           product_id='TEST_PRODUCT_ID',
+                           uri=uri,
+                           netrc=netrc_fqn,
+                           file_name='file:///test_files/TEST_FILE.FITS',
+                           verbose=False,
+                           debug=True,
+                           quiet=False,
+                           caom_namespace=obs_reader_writer.CAOM24_NAMESPACE,
+                           **kwargs)
+        reader = obs_reader_writer.ObservationReader(False)
+        observation = reader.read(xml_file)
+        repo_client.update(observation)
+    except:
+        repo_client.create(observation)
 
 ## More Information
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		plane.calibrationLevel = 2
		plane.dataProductType = cube