[update] re-worked channel definition in parameters.yaml, each channel now has its own dictionary with optional plotting flags

[WIP] clock quality work in progress, currently disabled

parameters.yaml

@@ -3,8 +3,6 @@ datapath: "/data/SDS/"                    # SC3 Datapath
 networks: ["1Y", "HA"]                    # select networks, list or str
 stations: "*"                             # select stations, list or str
 locations: "*"                            # select locations, list or str
-channels: ["EX1", "EX2", "EX3", "VEI",
-           "VM1", "VM2", "VM3", "LCQ"]    # Specify SOH channels, currently supported EX[1-3], VEI and VM[1-3]
 stations_blacklist: ["TEST", "EREA"]      # exclude these stations
 networks_blacklist: []                    # exclude these networks
 interval: 60               # Perform checks every x seconds
@@ -44,10 +42,73 @@ THRESHOLDS:
   low_volt: 12                 # min voltage for low voltage warning
   high_volt: 14.8              # max voltage for over voltage warning
   unclassified: 5              # min voltage samples not classified for warning
-  max_vm: [1.5, 2.5]           # thresholds for mass offset (warn, fail)
+  max_vm_warn: 1.5             # threshold for mass offset (warn), fail)
+  max_vm_fail: 2.5             # threshold for mass offset (warn), fail)
   clockquality_warn: 90        # clock quality ranges from 0 % to 100 % with 100 % being the best level
   clockquality_fail: 70
 
+
+# ---------------------------------- Specification of input channels ---------------------------------------------------
+# Currently supported: EX[1-3], VEI, VM[1-3], LCQ
+#
+# For each channel a factor 'unit' for unit conversion (e.g. to SI) can be provided, as well as a 'name'
+# and 'ticks' [ymin, ymax, ystep] for plotting.
+# 'warn' and 'fail' plot horizontal lines in corresponding colors (can be str in TRESHOLDS, int/float or iterable)
+#
+# 'transform' can be provided for plotting to perform arithmetic operations in given order, e.g.:
+#  transform: - ["*", 20]
+#             - ["-", 20]
+#  --> PBox EX1 V to deg C: 20 * x -20
+CHANNELS:
+  EX1:
+    unit: 1e-6
+    name: "Temperature (°C)"
+    ticks: [-10, 50, 10]
+    transform:
+      - ["*", 20]
+      - ["-", 20]
+    warn: "max_temp"
+  EX2:
+    unit: 1e-6
+    name: "230V/12V (V)"
+    ticks: [1, 5, 1]
+    warn: [2, 3, 4, 4.5, 5]
+  EX3:
+    unit: 1e-6
+    name: "Rout/Charge (V)"
+    ticks: [1, 5, 1]
+    warn: [2, 2.5, 3, 4, 5]
+  VEI:
+    unit: 1e-3
+    name: "Logger (V)"
+    ticks: [9, 15, 1]
+    warn: ["low_volt", "high_volt"]
+    fail: 10.5
+  VM1:
+    unit: 1e-6
+    name: "Mass 1 (V)"
+    ticks: [-2.5, 2.5, 1]
+    warn: [-1.5, 1.5]
+    fail: [-2.5, 2.5]
+  VM2:
+    unit: 1e-6
+    name: "Mass 2 (V)"
+    ticks: [-2.5, 2.5, 1]
+    warn: [-1.5, 1.5]
+    fail: [-2.5, 2.5]
+  VM3:
+    unit: 1e-6
+    name: "Mass 3 (V)"
+    ticks: [-2.5, 2.5, 1]
+    warn: [-1.5, 1.5]
+    fail: [-2.5, 2.5]
+  LCQ:
+    name: "Clock Q (%)"
+    ticks: [0, 100, 20]
+    warn: "clockquality_warn"
+    fail: "clockquality_fail"
+
+
 # ---------------------------------------- OPTIONAL PARAMETERS ---------------------------------------------------------
 
 # add links to html table with specified key as column and value as relative link, interpretable string parameters:
@@ -64,40 +125,3 @@ EMAIL:
   sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender
   stations_blacklist: ['GR33']                     # do not send emails for specific stations
   networks_blacklist: []                           # do not send emails for specific network
-
-# names for plotting of the above defined parameter "channels" in the same order
-channel_names: ["Temperature (°C)",
-                "230V/12V (V)",
-                "Rout/Charge (V)",
-                "Logger (V)",
-                "Mass 1 (V)",
-                "Mass 2 (V)",
-                "Mass 3 (V)",
-                "Clock Q (%)"]
-
-# specify y-ticks (and ylims) giving, (ymin, ymax, step) for each of the above channels (0: default)
-CHANNEL_TICKS:
-  - [-10, 50, 10]
-  - [1, 5, 1]
-  - [1, 5, 1]
-  - [9, 15, 1]
-  - [-2, 2, 1]
-  - [-2, 2, 1]
-  - [-2, 2, 1]
-  - [0, 100, 20]
-
-# Factor for channel to SI-units (for plotting)
-CHANNEL_UNITS:
-  EX1: 1e-6
-  EX2: 1e-6
-  EX3: 1e-6
-  VEI: 1e-3
-  VM1: 1e-6
-  VM2: 1e-6
-  VM3: 1e-6
-
-# Transform channel for plotting, perform arithmetic operations in given order, e.g.: PBox EX1 V to deg C: 20 * x -20
-CHANNEL_TRANSFORM:
-  EX1:
-    - ["*", 20]
-    - ["-", 20]

survBot.py

@@ -19,7 +19,7 @@ from obspy.clients.filesystem.sds import Client
 
 from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str, \
     init_html_table, finish_html_table
-from utils import get_bg_color, modify_stream_for_plot, trace_ylabels, trace_yticks
+from utils import get_bg_color, modify_stream_for_plot, trace_yticks, trace_thresholds
 
 try:
     import smtplib
@@ -68,7 +68,7 @@ def fancy_timestr(dt, thresh=600, modif='+'):
 
 class SurveillanceBot(object):
     def __init__(self, parameter_path, outpath_html=None):
-        self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'mass', 'clock', 'temp', 'other']
+        self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'mass', 'temp', 'other']
         self.parameter_path = parameter_path
         self.update_parameters()
         self.starttime = UTCDateTime()
@@ -92,6 +92,8 @@ class SurveillanceBot(object):
 
     def update_parameters(self):
         self.parameters = read_yaml(self.parameter_path)
+        # add channels to list in parameters dicitonary
+        self.parameters['channels'] = list(self.parameters.get('CHANNELS').keys())
         self.reread_parameters = self.parameters.get('reread_parameters')
         self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')]
         self.verbosity = self.parameters.get('verbosity')
@@ -348,8 +350,9 @@ class SurveillanceBot(object):
             try:
                 st = modify_stream_for_plot(st, parameters=self.parameters)
                 st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full')
-                trace_ylabels(fig, self.parameters, self.verbosity)
+                # trace_ylabels(fig, self.parameters, self.verbosity)
                 trace_yticks(fig, self.parameters, self.verbosity)
+                trace_thresholds(fig, self.parameters, self.verbosity)
             except Exception as e:
                 print(f'Could not generate plot for {nwst_id}:')
                 print(traceback.format_exc())
@@ -441,6 +444,9 @@ class SurveillanceBot(object):
             print(f'Could not write HTML table to {fnout}:')
             print(traceback.format_exc())
 
+        if self.verbosity:
+            print(f'Wrote html table to {fnout}')
+
     def update_status_message(self):
         timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
         self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \
@@ -693,7 +699,7 @@ class StationQC(object):
         self.pb_power_analysis()
         self.pb_rout_charge_analysis()
         self.mass_analysis()
-        self.clock_quality_analysis()
+        #self.clock_quality_analysis()
 
     def return_print_analysis(self):
         items = [self.nwst_id]
@@ -860,17 +866,17 @@ class StationQC(object):
         common_highest_val = np.nanmax(abs(last_val_mean))
         common_highest_val = round(common_highest_val, 1)
 
-        # get thresholds for WARN (max_vm1) and FAIL (max_vm2)
+        # get thresholds for WARN (max_vm_warn) and FAIL (max_vm_fail)
         thresholds = self.parameters.get('THRESHOLDS')
-        max_vm = thresholds.get('max_vm')
+        max_vm_warn = thresholds.get('max_vm_warn')
-        if not max_vm:
+        max_vm_fail = thresholds.get('max_vm_fail')
+        if not max_vm_warn or not max_vm_fail:
             return
-        max_vm1, max_vm2 = max_vm
 
         # change status depending on common_highest_val
-        if common_highest_val < max_vm1:
+        if common_highest_val < max_vm_warn:
             self.status_ok(key, detailed_message=f'{common_highest_val}V')
-        elif max_vm1 <= common_highest_val < max_vm2:
+        elif max_vm_warn <= common_highest_val < max_vm_fail:
             self.warn(key=key,
                       detailed_message=f'Warning raised for mass centering. Highest val {common_highest_val}V', )
         else:

survBotGUI.py

@@ -34,7 +34,7 @@ from obspy import UTCDateTime
 
 from survBot import SurveillanceBot
 from write_utils import *
-from utils import get_bg_color, modify_stream_for_plot, trace_ylabels, trace_yticks
+from utils import get_bg_color, modify_stream_for_plot, trace_yticks, trace_thresholds
 
 try:
     from rest_api.utils import get_station_iccid
@@ -316,8 +316,9 @@ class MainWindow(QtWidgets.QMainWindow):
             self.plot_widget.setWindowTitle(nwst_id)
             st = modify_stream_for_plot(st, parameters=self.parameters)
             st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig)
-            trace_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
+            # trace_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
             trace_yticks(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
+            trace_thresholds(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
             self.plot_widget.show()
 
     def notification(self, text):

utils.py

@@ -2,6 +2,9 @@
 # -*- coding: utf-8 -*-
 
 import matplotlib
+import numpy as np
+
+from obspy import Stream
 
 
 def get_bg_color(check_key, status, dt_thresh=None, hex=False):
@@ -37,6 +40,11 @@ def get_color(key):
     return colors_dict.get(key)
 
 
+def get_color_mpl(key):
+    color_tup = get_color(key)
+    return np.array([color/255. for color in color_tup])
+
+
 def get_time_delay_color(dt, dt_thresh):
     """ Set color of time delay after thresholds specified in self.dt_thresh """
     if dt < dt_thresh[0]:
@@ -68,33 +76,43 @@ def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
     return rgba
 
 
-def modify_stream_for_plot(st, parameters):
+def modify_stream_for_plot(input_stream, parameters):
     """ copy (if necessary) and modify stream for plotting """
-    ch_units = parameters.get('CHANNEL_UNITS')
-    ch_transf = parameters.get('CHANNEL_TRANSFORM')
 
     # make a copy
-    st = st.copy()
+    st = Stream()
 
-    # modify trace for plotting by multiplying unit factor (e.g. 1e-3 mV to V)
+    channels_dict = parameters.get('CHANNELS')
-    if ch_units:
+
-        for tr in st:
+    # iterate over all channels and put them to new stream in order
-            channel = tr.stats.channel
+    for index, ch_tup in enumerate(channels_dict.items()):
-            unit_factor = ch_units.get(channel)
+        # unpack tuple from items
+        channel, channel_dict = ch_tup
+
+        # get correct channel from stream
+        st_sel = input_stream.select(channel=channel)
+        # in case there are != 1 there is ambiguity
+        if not len(st_sel) == 1:
+            continue
+
+        # make a copy to not modify original stream!
+        tr = st_sel[0].copy()
+
+        # multiply with conversion factor for unit
+        unit_factor = channel_dict.get('unit')
         if unit_factor:
             tr.data = tr.data * float(unit_factor)
 
-    # modify trace for plotting by other arithmetic expressions
+        # apply transformations if provided
-    if ch_transf:
+        transform = channel_dict.get('transform')
-        for tr in st:
+        if transform:
-            channel = tr.stats.channel
+            tr.data = transform_trace(tr.data, transform)
-            transf = ch_transf.get(channel)
-            if transf:
-                tr.data = transform_trace(tr.data, transf)
 
-    # change channel IDs to prevent re-sorting in obspy routine
+        # modify trace id to maintain plotting order
-    for index, trace in enumerate(st):
+        name = channel_dict.get('name')
-        trace.id = f'trace {index + 1}: {trace.id}'
+        tr.id = f'trace {index + 1}: {name} - {tr.id}'
+
+        st.append(tr)
 
     return st
 
@@ -124,10 +142,8 @@ def transform_trace(data, transf):
 def trace_ylabels(fig, parameters, verbosity=0):
     """
     Adds channel names to y-axis if defined in parameters.
-    Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is
-    difficult to assess the correct order from Obspy plotting routing.
     """
-    names = parameters.get('channel_names')
+    names = [channel.get('name') for channel in parameters.get('CHANNELS').values()]
     if not names: # or not len(st.traces):
         return
     if not len(names) == len(fig.axes):
@@ -142,10 +158,8 @@ def trace_ylabels(fig, parameters, verbosity=0):
 def trace_yticks(fig, parameters, verbosity=0):
     """
     Adds channel names to y-axis if defined in parameters.
-    Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is
-    difficult to assess the correct order from Obspy plotting routing.
     """
-    ticks = parameters.get('CHANNEL_TICKS')
+    ticks = [channel.get('ticks') for channel in parameters.get('CHANNELS').values()]
     if not ticks:
         return
     if not len(ticks) == len(fig.axes):
@@ -157,6 +171,38 @@ def trace_yticks(fig, parameters, verbosity=0):
             continue
         ymin, ymax, step = ytick_tripple
 
-        yticks = list(range(ymin, ymax + step, step))
+        yticks = list(np.arange(ymin, ymax + step, step))
         ax.set_yticks(yticks)
         ax.set_ylim(ymin - 0.33 * step, ymax + 0.33 * step)
+
+
+def trace_thresholds(fig, parameters, verbosity=0):
+    """
+    Adds channel thresholds (warn, fail) to y-axis if defined in parameters.
+    """
+    if verbosity > 0:
+        print('Plotting trace thresholds')
+
+    keys_colors = {'warn': dict(color=0.8 * get_color_mpl('WARN'), linestyle=(0, (5, 10)), alpha=0.5, linewidth=0.7),
+                   'fail': dict(color=0.8 * get_color_mpl('FAIL'), linestyle='solid', alpha=0.5, linewidth=0.7)}
+
+    for key, kwargs in keys_colors.items():
+        channel_threshold_list = [channel.get(key) for channel in parameters.get('CHANNELS').values()]
+        if not channel_threshold_list:
+            continue
+        plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs)
+
+
+def plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs):
+    for channel_thresholds, ax in zip(channel_threshold_list, fig.axes):
+        if not channel_thresholds:
+            continue
+
+        if not isinstance(channel_thresholds, (list, tuple)):
+            channel_thresholds = [channel_thresholds]
+
+        for warn_thresh in channel_thresholds:
+            if isinstance(warn_thresh, str):
+                warn_thresh = parameters.get('THRESHOLDS').get(warn_thresh)
+            if type(warn_thresh in (float, int)):
+                ax.axhline(warn_thresh, **kwargs)