[update] added possibility to modify voltage level in plots from settings in parameters.yaml

parameters.yaml

@@ -55,3 +55,15 @@ EMAIL:
   addresses: ['marcel.paffrath@rub.de', 'kasper.fischer@rub.de']  # list of mail addresses for info mails
   sender: 'webmaster@geophysik.ruhr-uni-bochum.de' # mail sender
 
+# Factor for channel to SI-units (for plotting)
+CHANNEL_UNITS:
+  EX1: 1e-6
+  EX2: 1e-6
+  EX3: 1e-6
+  VEI: 1e-3
+
+# Transform channel for plotting, perform arithmetic operations in given order, e.g.: PBox EX1 V to deg C: 20 * x -20
+CHANNEL_TRANSFORM:
+  EX1:
+    - ['*', 20]
+    - ['-', 20]

survBot.py

@@ -17,13 +17,14 @@ import matplotlib.pyplot as plt
 from obspy import read, UTCDateTime, Stream
 from obspy.clients.filesystem.sds import Client
 
-from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str,\
+from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str, \
     init_html_table, finish_html_table
-from utils import get_bg_color
+from utils import get_bg_color, modify_stream_for_plot
 
 try:
     import smtplib
     from email.mime.text import MIMEText
+
     mail_functionality = True
 except ImportError:
     print('Could not import smtplib or mail. Disabled sending mails.')
@@ -188,7 +189,7 @@ class SurveillanceBot(object):
             stream = self.data.get(nwst_id)
             if stream:
                 nsl = nsl_from_id(nwst_id)
-                station_qc = StationQC(stream, nsl, self.parameters, self.keys, qc_starttime,
+                station_qc = StationQC(self, stream, nsl, self.parameters, self.keys, qc_starttime,
                                        self.verbosity, print_func=self.print,
                                        status_track=self.status_track.get(nwst_id))
                 analysis_print_result = station_qc.return_print_analysis()
@@ -277,7 +278,7 @@ class SurveillanceBot(object):
             if self.outpath_html:
                 self.write_html_table()
                 if self.parameters.get('html_figures'):
-                    self.write_html_figures(check_plot_time=not(first_exec))
+                    self.write_html_figures(check_plot_time=not (first_exec))
             else:
                 self.print_analysis()
             time.sleep(self.refresh_period)
@@ -321,21 +322,28 @@ class SurveillanceBot(object):
             os.mkdir(self.outpath_html)
 
     def write_html_figures(self, check_plot_time=True):
-        """ Write figures for html, right now hardcoded hourly """
+        """ Write figures for html (e.g. hourly) """
         if check_plot_time and not self.check_plot_hour():
             return
-        self.check_fig_dir()
 
         for nwst_id in self.station_list:
-            fig = plt.figure(figsize=(16, 9))
+            self.write_html_figure(nwst_id)
-            fnout = self.get_fig_path_abs(nwst_id)
+
-            st = self.data.get(nwst_id)
+    def write_html_figure(self, nwst_id):
-            if st:
+        """ Write figure for html for specified station """
-                st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full')
+        self.check_fig_dir()
-                ax = fig.axes[0]
+
-                ax.set_title(f'Hourly refreshed plot at (UTC) {UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S")}')
+        fig = plt.figure(figsize=(16, 9))
-                fig.savefig(fnout, dpi=150., bbox_inches='tight')
+        fnout = self.get_fig_path_abs(nwst_id)
-            plt.close(fig)
+        st = self.data.get(nwst_id)
+        if st:
+            st = modify_stream_for_plot(st, parameters=self.parameters)
+            st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full')
+            ax = fig.axes[0]
+            ax.set_title(f'Plot refreshed at (UTC) {UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S")}. '
+                         f'Refreshed hourly or on FAIL status.')
+            fig.savefig(fnout, dpi=150., bbox_inches='tight')
+        plt.close(fig)
 
     def write_html_table(self, default_color='#e6e6e6'):
         self.check_html_dir()
@@ -345,7 +353,7 @@ class SurveillanceBot(object):
         try:
             with open(fnout, 'w') as outfile:
                 write_html_header(outfile, self.refresh_period)
-                #write_html_table_title(outfile, self.parameters)
+                # write_html_table_title(outfile, self.parameters)
                 init_html_table(outfile)
 
                 # First write header items
@@ -405,7 +413,7 @@ class SurveillanceBot(object):
         timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
         self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \
                               f'Current time (UTC) {UTCDateTime().strftime("%Y-%m-%d %H:%M:%S")} | ' \
-                              f'Refresh period: {self.refresh_period}s | '\
+                              f'Refresh period: {self.refresh_period}s | ' \
                               f'Showing data of last {timespan}'
 
     def print(self, string, **kwargs):
@@ -422,12 +430,13 @@ class SurveillanceBot(object):
 
 
 class StationQC(object):
-    def __init__(self, stream, nsl, parameters, keys, starttime, verbosity, print_func, status_track={}):
+    def __init__(self, parent, stream, nsl, parameters, keys, starttime, verbosity, print_func, status_track={}):
         """
         Station Quality Check class.
         :param nsl: dictionary containing network, station and location (key: str)
         :param parameters: parameters dictionary from parameters.yaml file
         """
+        self.parent = parent
         self.stream = stream
         self.nsl = nsl
         self.network = nsl.get('network')
@@ -448,6 +457,10 @@ class StationQC(object):
 
         self.start()
 
+    @property
+    def nwst_id(self):
+        return f'{self.network}.{self.station}'
+
     def status_ok(self, key, detailed_message="Everything OK", status_message='OK', overwrite=False):
         current_status = self.status_dict.get(key)
         # do not overwrite existing warnings or errors
@@ -497,6 +510,9 @@ class StationQC(object):
             current_status.count += count
         else:
             current_status = new_error
+            # refresh plot (using parent class) if error is new and not on program-startup
+            if self.search_previous_errors(key, n_errors=1):
+                self.parent.write_html_figure(self.nwst_id)
 
         self._update_status(key, current_status, detailed_message, last_occurrence)
 
@@ -507,7 +523,7 @@ class StationQC(object):
         if self.search_previous_errors(key):
             self.send_mail(key, detailed_message)
 
-    def search_previous_errors(self, key):
+    def search_previous_errors(self, key, n_errors=None):
         """
         Check n_track + 1 previous statuses for errors.
         If first item in list is no error but all others are return True (first time n_track errors appeared --
@@ -515,9 +531,12 @@ class StationQC(object):
         In all other cases return True.
         This also prevents sending status (e.g. mail) in case of program startup
         """
+        if n_errors is not None:
+            n_errors = self.parameters.get('n_track') + 1
+
         previous_errors = self.status_track.get(key)
         # only if error list is filled n_track times
-        if previous_errors and len(previous_errors) == self.parameters.get('n_track') + 1:
+        if previous_errors and len(previous_errors) == n_errors:
             # if first entry was no error but all others are, return True (-> new Fail n_track times)
             if not previous_errors[0] and all(previous_errors[1:]):
                 return True
@@ -547,7 +566,7 @@ class StationQC(object):
         dt = timedelta(seconds=n_track * interval)
         text = f'{key} FAIL status longer than {dt}: ' + message
         msg = MIMEText(text)
-        msg['Subject'] = f'new FAIL status on station {self.network}.{self.station}'
+        msg['Subject'] = f'new FAIL status on station {self.nwst_id}'
         msg['From'] = sender
         msg['To'] = ', '.join(addresses)
 
@@ -556,7 +575,6 @@ class StationQC(object):
         s.sendmail(sender, addresses, msg.as_string())
         s.quit()
 
-        
     def status_other(self, detailed_message, status_message, last_occurrence=None, count=1):
         key = 'other'
         new_status = StatusOther(count=count, messages=[status_message])
@@ -611,7 +629,7 @@ class StationQC(object):
         self.pb_rout_charge_analysis()
 
     def return_print_analysis(self):
-        items = [f'{self.network}.{self.station}']
+        items = [self.nwst_id]
         for key in self.keys:
             status = self.status_dict[key]
             message = status.message
@@ -670,11 +688,10 @@ class StationQC(object):
             self.warn(key, detailed_message=detailed_message, count=n_overvolt,
                       last_occurrence=self.get_last_occurrence(trace, overvolt))
 
-
         if len(undervolt) > 0:
             # try calculate number of voltage peaks from gaps between indices
             n_undervolt = len(np.where(np.diff(undervolt) > 1)[0]) + 1
-            detailed_message = warn_message + f' {n_undervolt}x Voltage under {low_volt}V '\
+            detailed_message = warn_message + f' {n_undervolt}x Voltage under {low_volt}V ' \
                                + self.get_last_occurrence_timestring(trace, undervolt)
             self.warn(key, detailed_message=detailed_message, count=n_undervolt,
                       last_occurrence=self.get_last_occurrence(trace, undervolt))
@@ -693,7 +710,7 @@ class StationQC(object):
         nsamp_av = int(trace.stats.sampling_rate) * timespan
         av_temp_str = str(round(np.mean(temp[-nsamp_av:]), 1)) + deg_str
         # dt of average
-        dt_t_str = str(timedelta(seconds=int(timespan)))
+        dt_t_str = str(timedelta(seconds=int(timespan))).replace(', 0:00:00', '')
         # current temp
         cur_temp = round(temp[-1], 1)
         if self.verbosity > 1:
@@ -778,7 +795,7 @@ class StationQC(object):
                         n_occurrences = len(np.where(np.diff(ind_array) > 1)[0]) + 1
                         self.warn(key=key,
                                   detailed_message=f'Trace {trace.get_id()}: '
-                                          f'Found {n_occurrences} occurrence(s) of {volt_lvl}V: {key}: {message}'
+                                                   f'Found {n_occurrences} occurrence(s) of {volt_lvl}V: {key}: {message}'
                                                    + self.get_last_occurrence_timestring(trace, ind_array),
                                   count=n_occurrences,
                                   last_occurrence=self.get_last_occurrence(trace, ind_array))
@@ -839,9 +856,9 @@ class StationQC(object):
             n_occurrences = len(np.where(np.diff(under) > 1)[0]) + 1
             voltage_dict[-1] = under
             self.status_other(detailed_message=f'Trace {trace.get_id()}: '
-                              f'Voltage below {pb_ok}V in {len(under)} samples, {n_occurrences} time(s). '
+                                               f'Voltage below {pb_ok}V in {len(under)} samples, {n_occurrences} time(s). '
-                              f'Mean voltage: {np.mean(voltage):.2}'
+                                               f'Mean voltage: {np.mean(voltage):.2}'
-                                       + self.get_last_occurrence_timestring(trace, under),
+                                               + self.get_last_occurrence_timestring(trace, under),
                               status_message='under 1V ({})'.format(n_occurrences))
 
         # classify last voltage values
@@ -860,8 +877,8 @@ class StationQC(object):
             max_uncl = self.parameters.get('THRESHOLDS').get('unclassified')
             if max_uncl and n_unclassified > max_uncl:
                 self.status_other(detailed_message=f'Trace {trace.get_id()}: '
-                                               f'{n_unclassified}/{len(all_indices)} '
+                                                   f'{n_unclassified}/{len(all_indices)} '
-                                               f'unclassified voltage values in channel {trace.get_id()}',
+                                                   f'unclassified voltage values in channel {trace.get_id()}',
                                   status_message=f'{channel}: {n_unclassified} uncl.')
 
         return False, voltage_dict, last_val

survBotGUI.py

@@ -22,7 +22,6 @@ except ImportError:
         except ImportError:
             raise ImportError('Could import neither of PySide2, PySide6 or PyQt5')
 
-import matplotlib
 from matplotlib.figure import Figure
 
 if QtGui.__package__ in ['PySide2', 'PyQt5', 'PySide6']:
@@ -35,7 +34,7 @@ from obspy import UTCDateTime
 
 from survBot import SurveillanceBot
 from write_utils import *
-from utils import get_bg_color
+from utils import get_bg_color, modify_stream_for_plot
 
 try:
     from rest_api.utils import get_station_iccid
@@ -315,6 +314,7 @@ class MainWindow(QtWidgets.QMainWindow):
         if st:
             self.plot_widget = PlotWidget(self)
             self.plot_widget.setWindowTitle(nwst_id)
+            st = modify_stream_for_plot(st, parameters=self.parameters)
             st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig)
             self.plot_widget.show()
 

utils.py

@@ -3,6 +3,7 @@
 
 import matplotlib
 
+
 def get_bg_color(check_key, status, dt_thresh=None, hex=False):
     message = status.message
     if check_key == 'last active':
@@ -23,6 +24,7 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False):
         bg_color = '#{:02x}{:02x}{:02x}'.format(*bg_color[:3])
     return bg_color
 
+
 def get_color(key):
     # some GUI default colors
     colors_dict = {'FAIL': (255, 50, 0, 255),
@@ -33,6 +35,7 @@ def get_color(key):
                    'undefined': (230, 230, 230, 255)}
     return colors_dict.get(key)
 
+
 def get_time_delay_color(dt, dt_thresh):
     """ Set color of time delay after thresholds specified in self.dt_thresh """
     if dt < dt_thresh[0]:
@@ -41,6 +44,7 @@ def get_time_delay_color(dt, dt_thresh):
         return get_color('WARN')
     return get_color('FAIL')
 
+
 def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
     """ Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """
     if type(temp) in [str]:
@@ -50,3 +54,51 @@ def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
     rgba = [int(255 * c) for c in cmap(val)]
     return rgba
 
+
+def modify_stream_for_plot(st, parameters):
+    """ copy (if necessary) and modify stream for plotting """
+    ch_units = parameters.get('CHANNEL_UNITS')
+    ch_transf = parameters.get('CHANNEL_TRANSFORM')
+
+    # if either of both are defined make copy
+    if ch_units or ch_transf:
+        st = st.copy()
+
+    # modify trace for plotting by multiplying unit factor (e.g. 1e-3 mV to V)
+    if ch_units:
+        for tr in st:
+            channel = tr.stats.channel
+            unit_factor = ch_units.get(channel)
+            if unit_factor:
+                tr.data = tr.data * float(unit_factor)
+    # modify trace for plotting by other arithmetic expressions
+    if ch_transf:
+        for tr in st:
+            channel = tr.stats.channel
+            transf = ch_transf.get(channel)
+            if transf:
+                tr.data = transform_trace(tr.data, transf)
+
+    return st
+
+
+def transform_trace(data, transf):
+    """
+    Transform trace with arithmetic operations in order, specified in transf
+    @param data: numpy array
+    @param transf: list of lists with arithmetic operations (e.g. [['*', '20'], ] -> multiply data by 20
+    """
+    # This looks a little bit hardcoded, however it is safer than using e.g. "eval"
+    for operator_str, val in transf:
+        if operator_str == '+':
+            data = data + val
+        elif operator_str == '-':
+            data = data - val
+        elif operator_str == '*':
+            data = data * val
+        elif operator_str == '/':
+            data = data / val
+        else:
+            raise IOError(f'Unknown arithmethic operator string: {operator_str}')
+
+    return data

write_utils.py

@@ -53,6 +53,6 @@ def write_html_row(fobj, items, html_key='td'):
 
 def get_print_title_str(parameters):
     timespan = parameters.get('timespan') * 24 * 3600
-    tdelta_str = str(timedelta(seconds=int(timespan)))
+    tdelta_str = str(timedelta(seconds=int(timespan))).replace(', 0:00:00', '')
     return f'Analysis table of router quality within the last {tdelta_str}'