Minor updates to reporting

Author: geojunky

seismic/extract_event_traces.py

@@ -288,14 +288,14 @@ def extract_data(recording_timespan_getter, waveform_getter,
 
             stream_count = 0
             sta_stream = Stream()
-            status = DataFrame()
+            status = defaultdict(int)
+            log.info('Data extraction stats:\n')
             for s in safe_iter_event_data(curr_cat, curr_inv, waveform_getter,
                                           use_rfstats=rfstats_map[wave],
                                           phase=phase_map[wave],
                                           tt_model=tt_model, pbar=None,
                                           request_window=request_window,
-                                          pad=pad,
-                                          status=status):
+                                          pad=pad, status=status, log=log):
                 # Write traces to output file in append mode so that arbitrarily large file
                 # can be processed. If the file already exists, then existing streams will
                 # be overwritten rather than duplicated.
@@ -350,27 +350,24 @@ def extract_data(recording_timespan_getter, waveform_getter,
                     if(len(sta_stream)):
                         write_h5_event_stream(event_trace_datafile, sta_stream, index=h5_index, mode='a')
                     else:
-                        t = Trace(data=np.array([]),
+                        t = Trace(data=np.array([0]),
                                   header={'network': net, 'station': sta,
                                           'location': loc, 'channel': 'XXX',
                                           'wave_type': wave,
                                           'station_longitude': sta_lon,
                                           'station_latitude': sta_lat,
-                                          'event_time': UTCDateTime.now()})
+                                          'event_time': UTCDateTime(0)})
                         write_h5_event_stream(event_trace_datafile, Stream([t]), index=h5_index, mode='a')
                     # end if
                 # end if
                 comm.Barrier()
             # end for
 
-            if(len(status)):
-                status.index += 1
-                log.info('Data extraction stats:\n{}\n'.format(status.to_string()))
-            # end if
-            log.info('Summary: good data found for {}/{} events.'.format\
-                         (np.sum(np.array(status['status']=='Good data')) if(len(status)) else 0,
-                          len(status)))
-            if(len(curr_cat) != len(status)): log.warning('All events may not have been processed..')
+            log.info('\nSummary:\n', extra={'simple': True})
+            for k, v in status.items():
+                log.info('{}: good data found for {}/{} events.'.format \
+                         (k, v, len(curr_cat)), extra={'simple': True})
+            # end for
 
             warn_str = \
             " No {} traces found for {}! Added a null trace.".format(DESCS[wave], nsl)

seismic/misc.py

@@ -23,12 +23,22 @@ def setup_logger(name, log_file=None, level=logging.INFO, propagate=False):
     """
     Function to setup a logger; adapted from stackoverflow
     """
+    class ConditionalFormatter(logging.Formatter):
+        def format(self, record):
+            if hasattr(record, 'simple') and record.simple:
+                return record.getMessage()
+            else:
+                return logging.Formatter.format(self, record)
+            # end if
+        # end func
+    # end class
+
     handler = None
     if(log_file):
         handler = logging.FileHandler(log_file, mode='w')
     # end if
 
-    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
+    formatter = ConditionalFormatter('%(asctime)s %(levelname)s %(message)s')
 
     logger = logging.getLogger(name+log_file if log_file else '')
     logger.setLevel(level)

seismic/rf_station_orientations.py

@@ -120,8 +120,8 @@ def analyze_station_orientations(ned, curation_opts=DEFAULT_CURATION_OPTS,
     logger = logging.getLogger(__name__ + ':' + full_code)
     logger.setLevel(logging.INFO)
 
-    # check if ned has valid data
-    if not np.any(np.array([len(stream[0].data) for _, _, stream in ned])):
+    # check if ned has valid data. Note that null traces are length 1
+    if not np.any(np.array([(len(stream[0].data) > 1) for _, _, stream in ned])):
         return results
     # end if
 

seismic/stream_io.py

@@ -14,11 +14,10 @@
 import obspyh5
 from obspyh5 import dataset2trace, is_obspyh5, trace2group
 from os.path import splitext
-
+from typing import DefaultDict
 from seismic.units_utils import KM_PER_DEG
 from rf.rfstream import rfstats, obj2stats
 from collections import defaultdict
-from pandas import DataFrame
 # pylint: disable=invalid-name
 
 
@@ -34,7 +33,7 @@
 
 def safe_iter_event_data(events, inventory, get_waveforms, use_rfstats=True, phase='P',
                          request_window=None, pad=10, pbar=None,
-                         status:DataFrame=None, **kwargs):
+                         status: DefaultDict[str, int] = None, log=None, **kwargs):
     """
     Return iterator yielding three component streams per station and event.
 
@@ -48,7 +47,8 @@ def safe_iter_event_data(events, inventory, get_waveforms, use_rfstats=True, pha
     :param request_window: requested time window around the onset of the phase
     :param float pad: padding in seconds around request window
     :param pbar: tqdm_ instance for displaying a progressbar
-    :param status: an empty pandas DataFrame for retrieving statistics
+    :param status: a defaultdict for retrieving statistics
+    :param log: a python logging instance
     :param kwargs: all other kwargs are passed to `~rf.rfstream.rfstats()`
 
     :return: three component streams with raw data
@@ -82,19 +82,18 @@ def _get_stations(inventory):
         pbar.total = len(events) * len(stations)
     # end if
 
+    fmt = "{:<15} {:<30} {:>9} {:>9} {:>7} {:>5} {:>15}"
+    if(log is not None):
+        log.info(fmt.format("seed_id", "origin_time", "lon", "lat", "depth", "mag", "status")+'\n',
+                 extra={'simple': True})
+    # end if
     for i, (event, seedid) in enumerate(itertools.product(events, stations)):
         if pbar is not None: pbar.update(1)
         origin = (event.preferred_origin() or event.origins[0])
         magnitude = (event.preferred_magnitude() or event.magnitudes[0])
         origin_time, elon, elat, edepth, eMw = origin['time'], origin['longitude'], \
             origin['latitude'], origin['depth'], magnitude.mag
-        row_items = [seedid, origin_time, elon, elat, edepth/1e3, eMw]
-
-        # initialize status data-frame
-        if (i == 0 and status is not None):
-            cols = ['seed_id', 'origin_time', 'lon', 'lat', 'depth', 'magnitude', 'status']
-            for col in cols: status[col] = None
-        # end if
+        row_items = [seedid, origin_time.strftime('%Y-%m-%dT%H:%M:%S.%f'), elon, elat, edepth/1e3, eMw]
 
         try:
             # exclude datetime from call to get_coordinates to ensure incorrect
@@ -103,7 +102,7 @@ def _get_stations(inventory):
             args = (seedid[:-1] + stations[seedid], None)
             coords = inventory.get_coordinates(*args)
         except Exception:  # station not available at that time
-            if(status is not None): status.loc[i] = [*row_items, 'Invalid inventory']
+            if(log is not None): log.info(fmt.format(*row_items, 'Invalid inventory'), extra={'simple': True})
             continue
         # end try
 
@@ -115,11 +114,11 @@ def _get_stations(inventory):
                 from warnings import warn
                 warn('Error "%s" in rfstats call for event %s, station %s.'
                      % (exception, event.resource_id, seedid))
-                if(status is not None): status.loc[i] = [*row_items, 'Invalid rfstats']
+                if (log is not None): log.info(fmt.format(*row_items, 'Invalid rfstats'), extra={'simple': True})
                 continue
             # end try
             if not stats:
-                if(status is not None): status.loc[i] = [*row_items, 'Invalid rfstats']
+                if (log is not None): log.info(fmt.format(*row_items, 'Invalid rfstats'), extra={'simple': True})
                 continue
             # end if
         # end if
@@ -146,11 +145,11 @@ def _get_stations(inventory):
             stream.merge()
 
             if(len(stream) == 0):
-                if(status is not None): status.loc[i] = [*row_items, 'No data']
+                if (log is not None): log.info(fmt.format(*row_items, 'No data'), extra={'simple': True})
                 continue
             # end if
         except Exception:  # no data available
-            if(status is not None): status.loc[i] = [*row_items, 'Bad data']
+            if (log is not None): log.info(fmt.format(*row_items, 'No data'), extra={'simple': True})
             continue
         # end try
 
@@ -178,7 +177,7 @@ def _get_stations(inventory):
             warn('Need 3 component seismograms. %d components '
                  'detected for event %s, station %s.'
                  % (len(stream), event.resource_id, seedid))
-            if(status is not None): status.loc[i] = [*row_items, 'Missing components']
+            if (log is not None): log.info(fmt.format(*row_items, 'Missing components'), extra={'simple': True})
             continue
         # end if
 
@@ -200,7 +199,7 @@ def has_masked_values(data_stream):
                 from warnings import warn
                 warn('Gaps or overlaps detected for event %s, station %s.'
                      % (event.resource_id, seedid))
-                if(status is not None): status.loc[i] = [*row_items, 'Patchy data']
+                if (log is not None): log.info(fmt.format(*row_items, 'Patchy data'), extra={'simple': True})
                 continue
             else:
                 for tr in stream: tr.data = np.array(tr.data)
@@ -211,7 +210,8 @@ def has_masked_values(data_stream):
             tr.stats.update(stats)
         # end for
 
-        if(status is not None): status.loc[i] = [*row_items, 'Good data']
+        if (log is not None): log.info(fmt.format(*row_items, 'Good data'), extra={'simple': True})
+        if(status is not None): status[seedid] += 1
         yield RFStream(stream)
     # end for
 # end func

seismic/swp_station_orientations.py

@@ -723,8 +723,8 @@ def dump_swp_data(r1phi, r1cc, r2phi, r2cc, e_array):
     logger = logging.getLogger(__name__ + ':' + full_code)
     logger.setLevel(logging.INFO)
 
-    # check if ned has valid data
-    if not np.any(np.array([len(stream[0].data) for _, _, stream in ned])):
+    # check if ned has valid data. Note that null traces are length 1
+    if not np.any(np.array([(len(stream[0].data) > 1) for _, _, stream in ned])):
         return results
     # end if