[bugfix] compared timedelta with int

parameters.yaml

@@ -1,14 +1,15 @@
 # Parameters file for Surveillance Bot
-datapath: '/data/SDS/'      # SC3 Datapath
+datapath: "/data/SDS/"      # SC3 Datapath
-networks: ['1Y', 'HA']
+networks: ["1Y", "HA"]
-stations: '*'
+stations: "*"
-locations: '*'
+locations: "*"
-channels: ['EX1', 'EX2', 'EX3', 'VEI']    # Specify SOH channels, currently supported EX[1-3] and VEI
+channels: ["EX1", "EX2", "EX3", "VEI"]    # Specify SOH channels, currently supported EX[1-3] and VEI
-stations_blacklist: ['TEST', 'EREA']
+channel_names: ["Temperature (°)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"] # names for plotting (optional)
+stations_blacklist: ["TEST", "EREA"]
 networks_blacklist: []
 interval: 60               # Perform checks every x seconds
-n_track: 120               # wait number of intervals after FAIL before performing an action (i.e. send mail)
+n_track: 120               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
-timespan: 3                # Check data of the recent x days
+timespan: 7                # Check data of the recent x days
 verbosity: 0
 track_changes: True        # tracks all changes since GUI startup by text highlighting (GUI only)
 warn_count: False          # show number of warnings and errors in table
@@ -16,17 +17,11 @@ dt_thresh: [300, 1800]     # threshold (s) for timing delay colourisation (yello
 html_figures: True         # Create html figure directory and links
 reread_parameters: True    # reread parameters file (change parameters on runtime, not for itself/GUI refresh/datapath)
 
-# add links to html table with specified key as column and value as relative link, interpretable string parameters:
-# nw (e.g. 1Y), st (e.g. GR01A), nwst_id (e.g. 1Y.GR01A)
-# can also be empty!
-add_links:
-  slmon: {"URL": "{nw}_{st}.html", "text": "show"}   # for example: slmon: {"URL": "{nw}_{st}.html", "text": "link"}
-
 POWBOX:
   pb_ok: 1  # Voltage for PowBox OK
   pb_SOH2: # PowBox channel 2 voltage translations
     -1: {"230V": "PBox under 1V", "12V": "PBox under 1V"}
-    1: {"230V": 'OK', "12V": "OK"}
+    1: {"230V": "OK", "12V": "OK"}
     2: {"230V": "OFF", "12V": "OK"}
     3: {"230V": "OK", "12V": "overvoltage"}
     4: {"230V": "OK", "12V": "undervoltage"}
@@ -49,13 +44,19 @@ THRESHOLDS:
   high_volt: 14.8               # max voltage for over voltage warning
   unclassified: 5              # min voltage samples not classified for warning
 
-# E-mail notifications
+# add links to html table with specified key as column and value as relative link, interpretable string parameters:
-EMAIL:
+# nw (e.g. 1Y), st (e.g. GR01A), nwst_id (e.g. 1Y.GR01A)
-  mailserver: 'localhost'
+# optional!
-  addresses: ['marcel.paffrath@rub.de', 'kasper.fischer@rub.de']  # list of mail addresses for info mails
+add_links:
-  sender: 'webmaster@geophysik.ruhr-uni-bochum.de' # mail sender
+  slmon: {"URL": "{nw}_{st}.html", "text": "show"}  # for example: slmon: {"URL": "path/{nw}_{st}.html", "text": "link"}
 
-# Factor for channel to SI-units (for plotting)
+# E-mail notifications (optional)
+EMAIL:
+  mailserver: "localhost"
+  addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"]  # list of mail addresses for info mails
+  sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender
+
+# Factor for channel to SI-units (for plotting, optional)
 CHANNEL_UNITS:
   EX1: 1e-6
   EX2: 1e-6
@@ -63,7 +64,8 @@ CHANNEL_UNITS:
   VEI: 1e-3
 
 # Transform channel for plotting, perform arithmetic operations in given order, e.g.: PBox EX1 V to deg C: 20 * x -20
+# optional!
 CHANNEL_TRANSFORM:
   EX1:
-    - ['*', 20]
+    - ["*", 20]
-    - ['-', 20]
+    - ["-", 20]

survBot.py

@@ -19,7 +19,7 @@ from obspy.clients.filesystem.sds import Client
 
 from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str, \
     init_html_table, finish_html_table
-from utils import get_bg_color, modify_stream_for_plot
+from utils import get_bg_color, modify_stream_for_plot, annotate_trace_axes
 
 try:
     import smtplib
@@ -337,15 +337,22 @@ class SurveillanceBot(object):
         fnout = self.get_fig_path_abs(nwst_id)
         st = self.data.get(nwst_id)
         if st:
-            st = modify_stream_for_plot(st, parameters=self.parameters)
+            # TODO: this section might fail, adding try-except block for analysis and to prevent program from crashing
-            st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full')
+            try:
-            ax = fig.axes[0]
+                st = modify_stream_for_plot(st, parameters=self.parameters)
-            ax.set_title(f'Plot refreshed at (UTC) {UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S")}. '
+                st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full')
-                         f'Refreshed hourly or on FAIL status.')
+                annotate_trace_axes(fig, self.parameters, self.verbosity)
-            fig.savefig(fnout, dpi=150., bbox_inches='tight')
+            except Exception as e:
+                print(f'Could not generate plot for {nwst_id}:')
+                print(traceback.format_exc())
+            if len(fig.axes) > 0:
+                ax = fig.axes[0]
+                ax.set_title(f'Plot refreshed at (UTC) {UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S")}. '
+                             f'Refreshed hourly or on FAIL status.')
+                fig.savefig(fnout, dpi=150., bbox_inches='tight')
         plt.close(fig)
 
-    def write_html_table(self, default_color='#e6e6e6'):
+    def write_html_table(self, default_color='#e6e6e6', default_header_color='#999'):
         self.check_html_dir()
         fnout = pjoin(self.outpath_html, 'survBot_out.html')
         if not fnout:
@@ -361,16 +368,18 @@ class SurveillanceBot(object):
                 # add columns for additional links
                 for key in self.add_links:
                     header.insert(-1, key)
-                header_items = [dict(text='Station', color=default_color)]
+                header_items = [dict(text='Station', color=default_header_color)]
                 for check_key in header:
-                    item = dict(text=check_key, color=default_color)
+                    item = dict(text=check_key, color=default_header_color)
                     header_items.append(item)
                 write_html_row(outfile, header_items, html_key='th')
 
                 # Write all cells
                 for nwst_id in self.station_list:
                     fig_name = self.get_fig_path_rel(nwst_id)
-                    col_items = [dict(text=nwst_id.rstrip('.'), color=default_color, hyperlink=fig_name)]
+                    nwst_id_str = nwst_id.rstrip('.')
+                    col_items = [dict(text=nwst_id_str, color=default_color, hyperlink=fig_name,
+                                      bold=True, tooltip=f'Show plot of {nwst_id_str}')]
                     for check_key in header:
                         if check_key in self.keys:
                             status_dict = self.analysis_results.get(nwst_id)
@@ -388,7 +397,8 @@ class SurveillanceBot(object):
                                 if not type(message) in [str]:
                                     message = str(message) + deg_str
 
-                            item = dict(text=str(message), tooltip=str(detailed_message), color=bg_color)
+                            item = dict(text=str(message), tooltip=str(detailed_message), color=bg_color,
+                                        blink=status.is_active)
                         elif check_key in self.add_links:
                             value = self.add_links.get(check_key).get('URL')
                             link_text = self.add_links.get(check_key).get('text')
@@ -510,25 +520,30 @@ class StationQC(object):
             current_status.count += count
         else:
             current_status = new_error
-            # refresh plot (using parent class) if error is new and not on program-startup
+            # if error is new and not on program-startup set active and refresh plot (using parent class)
-            if self.search_previous_errors(key, n_errors=1):
+            if self.search_previous_errors(key, n_errors=1) is True:
                 self.parent.write_html_figure(self.nwst_id)
 
-        self._update_status(key, current_status, detailed_message, last_occurrence)
-
         if self.verbosity:
             self.print(f'{UTCDateTime()}: {detailed_message}', flush=False)
 
         # do not send error mail if this is the first run (e.g. program startup) or state was already error (unchanged)
-        if self.search_previous_errors(key):
+        if self.search_previous_errors(key) is True:
-            self.send_mail(key, detailed_message)
+            self.send_mail(key, status_type='FAIL', additional_message=detailed_message)
+            # set status to "inactive" after sending info mail
+            current_status.is_active = False
+        elif self.search_previous_errors(key) == 'active':
+            current_status.is_active = True
+
+        self._update_status(key, current_status, detailed_message, last_occurrence)
 
     def search_previous_errors(self, key, n_errors=None):
         """
         Check n_track + 1 previous statuses for errors.
-        If first item in list is no error but all others are return True (first time n_track errors appeared --
+        If first item in list is no error but all others are return True
-        if ALL n_track + 1 are error: error is old)
+        (first time n_track errors appeared if ALL n_track + 1 are error: error is old)
-        In all other cases return True.
+        If last item is error but not all items are error yet return keyword 'active' -> error active, no message sent
+        In all other cases return False.
         This also prevents sending status (e.g. mail) in case of program startup
         """
         if n_errors is not None:
@@ -540,10 +555,12 @@ class StationQC(object):
             # if first entry was no error but all others are, return True (-> new Fail n_track times)
             if not previous_errors[0] and all(previous_errors[1:]):
                 return True
-        else:
+        # in case previous_errors exists, last item is error but not all items are error, error still active
-            return False
+        elif previous_errors and previous_errors[-1] and not all(previous_errors):
+            return 'active'
+        return False
 
-    def send_mail(self, key, message):
+    def send_mail(self, key, status_type, additional_message=''):
         """ Send info mail using parameters specified in parameters file """
         if not mail_functionality:
             if self.verbosity:
@@ -561,12 +578,10 @@ class StationQC(object):
             if self.verbosity:
                 print('Mail sender or addresses not correctly defined. Return')
             return
-        n_track = self.parameters.get('n_track')
+        dt = self.get_dt_for_action()
-        interval = self.parameters.get('interval')
+        text = f'{key}: Status {status_type} longer than {dt}: ' + additional_message
-        dt = timedelta(seconds=n_track * interval)
-        text = f'{key} FAIL status longer than {dt}: ' + message
         msg = MIMEText(text)
-        msg['Subject'] = f'new FAIL status on station {self.nwst_id}'
+        msg['Subject'] = f'new message on station {self.nwst_id}'
         msg['From'] = sender
         msg['To'] = ', '.join(addresses)
 
@@ -575,6 +590,12 @@ class StationQC(object):
         s.sendmail(sender, addresses, msg.as_string())
         s.quit()
 
+    def get_dt_for_action(self):
+        n_track = self.parameters.get('n_track')
+        interval = self.parameters.get('interval')
+        dt = timedelta(seconds=n_track * interval)
+        return dt
+
     def status_other(self, detailed_message, status_message, last_occurrence=None, count=1):
         key = 'other'
         new_status = StatusOther(count=count, messages=[status_message])
@@ -593,13 +614,15 @@ class StationQC(object):
 
         self.status_dict[key] = current_status
 
-    def activity_check(self):
+    def activity_check(self, key='last_active'):
         self.last_active = self.last_activity()
         if not self.last_active:
             status = StatusError()
         else:
-            message = timedelta(seconds=int(self.program_starttime - self.last_active))
+            dt_active = timedelta(seconds=int(self.program_starttime - self.last_active))
-            status = Status(message=message)
+            status = Status(message=dt_active)
+            self.check_for_inactive_message(key, dt_active)
+
         self.status_dict['last active'] = status
 
     def last_activity(self):
@@ -611,6 +634,12 @@ class StationQC(object):
         if len(endtimes) > 0:
             return max(endtimes)
 
+    def check_for_inactive_message(self, key, dt_active):
+        dt_action = self.get_dt_for_action()
+        interval = self.parameters.get('interval')
+        if dt_action <= dt_active < dt_action + timedelta(seconds=interval):
+            self.send_mail(key, status_type='Inactive')
+
     def start(self):
         self.analyse_channels()
 
@@ -889,7 +918,9 @@ class StationQC(object):
 
 
 class Status(object):
-    def __init__(self, message='-', detailed_messages=None, count: int = 0, last_occurrence=None, show_count=True):
+    def __init__(self, message=None, detailed_messages=None, count: int = 0, last_occurrence=None, show_count=True):
+        if message is None:
+            message = '-'
         if detailed_messages is None:
             detailed_messages = []
         self.show_count = show_count
@@ -901,6 +932,7 @@ class Status(object):
         self.is_warn = None
         self.is_error = None
         self.is_other = False
+        self.is_active = False
 
     def set_warn(self):
         self.is_warn = True

survBotGUI.py

@@ -34,7 +34,7 @@ from obspy import UTCDateTime
 
 from survBot import SurveillanceBot
 from write_utils import *
-from utils import get_bg_color, modify_stream_for_plot
+from utils import get_bg_color, modify_stream_for_plot, annotate_trace_axes
 
 try:
     from rest_api.utils import get_station_iccid
@@ -316,6 +316,7 @@ class MainWindow(QtWidgets.QMainWindow):
             self.plot_widget.setWindowTitle(nwst_id)
             st = modify_stream_for_plot(st, parameters=self.parameters)
             st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig)
+            annotate_trace_axes(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
             self.plot_widget.show()
 
     def notification(self, text):

utils.py

@@ -102,3 +102,23 @@ def transform_trace(data, transf):
             raise IOError(f'Unknown arithmethic operator string: {operator_str}')
 
     return data
+
+
+def annotate_trace_axes(fig, parameters, verbosity=0):
+    """
+    Adds channel names to y-axis if defined in parameters.
+    Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is
+    difficult to assess the correct order from Obspy plotting routing.
+    """
+    names = parameters.get('channel_names')
+    if not names: # or not len(st.traces):
+        return
+    if not len(names) == len(fig.axes):
+        if verbosity:
+            print('Mismatch in axis and label lengths. Not adding plot labels')
+        return
+    for channel_name, ax in zip(names, fig.axes):
+        if channel_name:
+            ax.set_ylabel(channel_name)
+
+

write_utils.py

@@ -1,12 +1,15 @@
 from datetime import timedelta
 
+
 def write_html_table_title(fobj, parameters):
     title = get_print_title_str(parameters)
     fobj.write(f'<h3>{title}</h3>\n')
 
+
 def write_html_text(fobj, text):
     fobj.write(f'<p>{text}</p>\n')
 
+
 def write_html_header(fobj, refresh_rate=10):
     header = ['<!DOCTYPE html>',
               '<html>',
@@ -24,35 +27,44 @@ def write_html_header(fobj, refresh_rate=10):
     for item in header:
         fobj.write(item + '\n')
 
+
 def init_html_table(fobj):
     fobj.write('<table style="width:100%">\n')
 
+
 def finish_html_table(fobj):
     fobj.write('</table>\n')
 
+
 def write_html_footer(fobj):
     footer = ['</body>',
               '</html>']
     for item in footer:
         fobj.write(item + '\n')
 
+
 def write_html_row(fobj, items, html_key='td'):
     default_space = '  '
     fobj.write(default_space + '<tr>\n')
     for item in items:
         text = item.get('text')
+        if item.get('bold'):
+            text = '<b>' + text + '</b>'
+        if item.get('italic'):
+            text = '<i>' + text + '</i>'
         tooltip = item.get('tooltip')
         color = item.get('color')
         # check for black background of headers (shouldnt happen anymore)
         color = '#e6e6e6' if color == '#000000' else color
         hyperlink = item.get('hyperlink')
         image_str = f'<a href="{hyperlink}">' if hyperlink else ''
-        fobj.write(2 * default_space + f'<{html_key} bgcolor="{color}" title="{tooltip}"> {image_str}'
+        blink_str = f' class="blink-bg"' if item.get('blink') else ''
+        fobj.write(2 * default_space + f'<{html_key}{blink_str} bgcolor="{color}" title="{tooltip}"> {image_str}'
                    + text + f'</{html_key}>\n')
     fobj.write(default_space + '</tr>\n')
 
+
 def get_print_title_str(parameters):
     timespan = parameters.get('timespan') * 24 * 3600
     tdelta_str = str(timedelta(seconds=int(timespan))).replace(', 0:00:00', '')
     return f'Analysis table of router quality within the last {tdelta_str}'
-