[update] first working version of gap check, testing needed

[minor] soft-coded data channels

parameters.yaml

@@ -44,8 +44,9 @@ THRESHOLDS:
   unclassified: 5              # min voltage samples not classified for warning
   max_vm_warn: 1.5             # threshold for mass offset (warn), fail)
   max_vm_fail: 2.5             # threshold for mass offset (warn), fail)
-  clockquality_warn: 90        # clock quality ranges from 0 % to 100 % with 100 % being the best level
+  clockquality_warn: 90        # warn level - clock quality ranges from 0 % to 100 % with 100 % being the best level
-  clockquality_fail: 70
+  clockquality_fail: 70        # fail level
+  min_gap: 0.1                 # minimum for gap declaration, should be > 0 [s]
 
 
 # ---------------------------------- Specification of input channels ---------------------------------------------------
@@ -108,6 +109,9 @@ CHANNELS:
     warn: "clockquality_warn"
     fail: "clockquality_fail"
 
+# specify data channels (can be additional to the above). From these channels only headers will be read
+data_channels: ["HHZ", "HHN", "HHE"]
+
 
 # ---------------------------------------- OPTIONAL PARAMETERS ---------------------------------------------------------
 

survBot.py

@@ -68,7 +68,8 @@ def fancy_timestr(dt, thresh=600, modif='+'):
 
 class SurveillanceBot(object):
     def __init__(self, parameter_path, outpath_html=None):
-        self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'mass', 'clock', 'temp', 'other']
+        self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'mass', 'clock', 'gaps', 'temp',
+                     'other']
         self.parameter_path = parameter_path
         self.update_parameters()
         self.starttime = UTCDateTime()
@@ -76,6 +77,7 @@ class SurveillanceBot(object):
         self.current_day = self.starttime.julday
         self.outpath_html = outpath_html
         self.filenames = []
+        self.filenames_wf_data = []
         self.filenames_read = []
         self.station_list = []
         self.analysis_print_list = []
@@ -83,6 +85,7 @@ class SurveillanceBot(object):
         self.status_track = {}
         self.dataStream = Stream()
         self.data = {}
+        self.gaps = []
         self.print_count = 0
         self.status_message = ''
         self.html_fig_dir = 'figures'
@@ -92,8 +95,12 @@ class SurveillanceBot(object):
 
     def update_parameters(self):
         self.parameters = read_yaml(self.parameter_path)
-        # add channels to list in parameters dicitonary
+        # add channels to list in parameters dictionary, also add data channels
-        self.parameters['channels'] = list(self.parameters.get('CHANNELS').keys())
+        channels = list(self.parameters.get('CHANNELS').keys())
+        for channel in self.parameters.get('data_channels'):
+            if not channel in channels:
+                channels.append(channel)
+        self.parameters['channels'] = channels
         self.reread_parameters = self.parameters.get('reread_parameters')
         self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')]
         self.verbosity = self.parameters.get('verbosity')
@@ -129,18 +136,25 @@ class SurveillanceBot(object):
 
     def get_filenames(self):
         self.filenames = []
+        self.filenames_wf_data = []
         time_now = UTCDateTime()
         t1 = time_now - self.parameters.get('timespan') * 24 * 3600
         networks = self.parameters.get('networks')
         stations = self.parameters.get('stations')
         locations = self.parameters.get('locations')
         channels = self.parameters.get('channels')
+        channels_wf_data = self.parameters.get('data_channels')
         for network in networks:
             for station in stations:
                 for location in locations:
                     for channel in channels:
-                        self.filenames += list(self.cl._get_filenames(network, station, location, channel,
+                        fnames = list(self.cl._get_filenames(network, station, location, channel,
                                                              starttime=t1, endtime=time_now))
+                        self.filenames += fnames
+
+                        # keep track of filenames with wf data (only read headers later)
+                        if channel in channels_wf_data:
+                            self.filenames_wf_data += fnames
 
     def read_data(self, re_read_at_hour=1, daily_overlap=2):
         '''
@@ -166,6 +180,10 @@ class SurveillanceBot(object):
             if filename in self.filenames_read:
                 continue
             try:
+                # read only header of wf_data
+                if filename in self.filenames_wf_data:
+                    st_new = read(filename, headonly=True)
+                else:
                     st_new = read(filename, dtype=float)
                 # add file to read filenames to prevent re-reading in case it is not the current day (or end of
                 # previous day)
@@ -176,7 +194,8 @@ class SurveillanceBot(object):
                 print(f'Could not read file {filename}:', e)
                 continue
             self.dataStream += st_new
-        self.dataStream.merge(fill_value=np.nan)
+        self.gaps = self.dataStream.get_gaps(min_gap=self.parameters['THRESHOLDS'].get('min_gap'))
+        self.dataStream.merge()
 
         # organise data in dictionary with key for each station
         for trace in self.dataStream:
@@ -251,7 +270,7 @@ class SurveillanceBot(object):
     def get_station_delay(self, nwst_id):
         """ try to get station delay from SDS archive using client"""
         locations = ['', '0', '00']
-        channels = ['HHZ', 'HHE', 'HHN'] + self.parameters.get('channels')
+        channels = self.parameters.get('channels') + self.parameters.get('data_channels')
         network, station = nwst_id.split('.')[:2]
 
         times = []
@@ -706,6 +725,7 @@ class StationQC(object):
         self.pb_rout_charge_analysis()
         self.mass_analysis()
         self.clock_quality_analysis()
+        self.gaps_analysis()
 
     def return_print_analysis(self):
         items = [self.nwst_id]
@@ -981,6 +1001,26 @@ class StationQC(object):
                     if volt_lvl == last_val or (volt_lvl == -1 and last_val < 1):
                         self.error(key, detailed_message=f'Last PowBox voltage state {last_val}V: {message}')
 
+    def gaps_analysis(self, key='gaps'):
+        """ return gaps of a given nwst_id """
+
+        gaps = []
+        for gap_list in self.parent.gaps:
+            nw_gap, st_gap = gap_list[:2]
+            if nw_gap == self.network and st_gap == self.station:
+                gaps.append(gap_list)
+
+        if not gaps:
+            self.status_ok(key=key)
+            return
+
+        detailed_message = ''
+        for gap_list in gaps:
+            text = '{}.{}.{}.{}: last sample - {}, next sample - {}, delta {}, samples {}\n'.format(*gap_list)
+            detailed_message += text
+
+        self.warn(key=key, detailed_message=detailed_message, count=len(gaps))
+
     def calc_occurrences(self, ind_array):
         # try calculate number of voltage peaks/plateaus from gaps between indices
         if len(ind_array) == 0: