Merge pull request 'release/1.0' (#3) from release/1.0 into main

Reviewed-on: #3

README.md

@@ -32,6 +32,8 @@ The main program with html output is executed by entering
 python survBot.py -html path_for_html_output
 ```
 
+There are example stylesheets in the folder *stylesheets* that can be copied into the path_for_html_output if desired.
+
 The GUI can be loaded via
 
 ```shell script

parameters.yaml

@@ -1,18 +1,18 @@
 # Parameters file for Surveillance Bot
-datapath: "/data/SDS/"      # SC3 Datapath
+datapath: "/data/SDS/"                    # SC3 Datapath
-networks: ["1Y", "HA"]
+networks: ["1Y", "HA"]                    # select networks, list or str
-stations: "*"
+stations: "*"                             # select stations, list or str
-locations: "*"
+locations: "*"                            # select locations, list or str
 channels: ["EX1", "EX2", "EX3", "VEI"]    # Specify SOH channels, currently supported EX[1-3] and VEI
-channel_names: ["Temperature (°)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"] # names for plotting (optional)
+stations_blacklist: ["TEST", "EREA"]      # exclude these stations
-stations_blacklist: ["TEST", "EREA"]
+networks_blacklist: []                    # exclude these networks
-networks_blacklist: []
 interval: 60               # Perform checks every x seconds
-n_track: 120               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
+n_track: 300               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
 timespan: 7                # Check data of the recent x days
-verbosity: 0
+verbosity: 0               # verbosity flag
 track_changes: True        # tracks all changes since GUI startup by text highlighting (GUI only)
-warn_count: False          # show number of warnings and errors in table
+warn_count: False           # show number of warnings and errors in table
+min_sample: 3              # minimum samples for raising Warn/FAIL
 dt_thresh: [300, 1800]     # threshold (s) for timing delay colourisation (yellow/red)
 html_figures: True         # Create html figure directory and links
 reread_parameters: True    # reread parameters file (change parameters on runtime, not for itself/GUI refresh/datapath)
@@ -44,19 +44,33 @@ THRESHOLDS:
   high_volt: 14.8               # max voltage for over voltage warning
   unclassified: 5              # min voltage samples not classified for warning
 
+# ---------------------------------------- OPTIONAL PARAMETERS ---------------------------------------------------------
+
 # add links to html table with specified key as column and value as relative link, interpretable string parameters:
 # nw (e.g. 1Y), st (e.g. GR01A), nwst_id (e.g. 1Y.GR01A)
-# optional!
 add_links:
-  slmon: {"URL": "{nw}_{st}.html", "text": "show"}  # for example: slmon: {"URL": "path/{nw}_{st}.html", "text": "link"}
+  # for example: slmon: {"URL": "path/{nw}_{st}.html", "text": "link"}
+  slmon: {"URL": "../slmon/{nw}_{st}.html", "text": "show"}
+  24h-plot: {"URL": "../scheli/{nw}/{st}.png", "text": "plot"}
 
-# E-mail notifications (optional)
+# E-mail notifications
 EMAIL:
   mailserver: "localhost"
   addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"]  # list of mail addresses for info mails
   sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender
+  stations_blacklist: ['GR33']                     # do not send emails for specific stations
+  networks_blacklist: []                           # do not send emails for specific network
 
-# Factor for channel to SI-units (for plotting, optional)
+# names for plotting of the above defined parameter "channels" in the same order
+channel_names: ["Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"]
+# specify y-ticks (and ylims) giving, (ymin, ymax, step) for each of the above channels (0: default)
+CHANNEL_TICKS:
+  - [-10, 50, 10]
+  - [1, 5, 1]
+  - [1, 5, 1]
+  - [9, 15, 1]
+
+# Factor for channel to SI-units (for plotting)
 CHANNEL_UNITS:
   EX1: 1e-6
   EX2: 1e-6
@@ -64,7 +78,6 @@ CHANNEL_UNITS:
   VEI: 1e-3
 
 # Transform channel for plotting, perform arithmetic operations in given order, e.g.: PBox EX1 V to deg C: 20 * x -20
-# optional!
 CHANNEL_TRANSFORM:
   EX1:
     - ["*", 20]

stylesheets/desktop.css

@@ -0,0 +1,39 @@
+body {
+    background-color: #ffffff;
+    place-items: center;
+    text-align: center;
+}
+
+td {
+    border-radius: 4px;
+    padding: 0px;
+}
+
+th {
+    background-color: #999;
+    border-radius: 4px;
+    padding: 3px 1px;
+}
+
+a:link, a:visited {
+    background-color: #ccc;
+    color: #000;
+    text-decoration: none;
+    display: block;
+    border-radius: 4px;
+    border: 1px solid #bbb;
+}
+
+a:hover {
+    background-color: #aaa;
+    display: block;			  
+}
+
+.blink-bg {
+    animation: blinkingBackground 2s infinite;
+}
+@keyframes blinkingBackground{
+    0%		{ background-color: #ffcc00;}
+    50%		{ background-color: #ff3200;}
+    100%	{ background-color: #ffcc00;}
+}

stylesheets/mobile.css

@@ -0,0 +1,43 @@
+body {
+    background-color: #ffffff;
+    place-items: center;
+    text-align: center;
+}
+
+td {
+    border-radius: 4px;
+    padding: 10px 2px;
+}
+
+th {
+    background-color: #999;
+    border-radius: 4px;
+    padding: 10px, 2px;
+}
+
+a:link {
+    background-color: #ccc;
+    color: #000;
+    text-decoration: none;
+    display: block;
+    border-radius: 4px;
+    border: 1px solid #bbb;
+}
+
+a:hover {
+    background-color: #aaa;
+    display: block;			  
+}
+
+.hidden-mobile {
+    display: none;
+}
+
+.blink-bg {
+    animation: blinkingBackground 2s infinite;
+}
+@keyframes blinkingBackground{
+    0%		{ background-color: #ffee00;}
+    50%		{ background-color: #ff3200;}
+    100%	{ background-color: #ffee00;}
+}

survBot.py

@@ -19,7 +19,7 @@ from obspy.clients.filesystem.sds import Client
 
 from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str, \
     init_html_table, finish_html_table
-from utils import get_bg_color, modify_stream_for_plot, annotate_trace_axes
+from utils import get_bg_color, modify_stream_for_plot, trace_ylabels, trace_yticks
 
 try:
     import smtplib
@@ -157,7 +157,7 @@ class SurveillanceBot(object):
             if filename in self.filenames_read:
                 continue
             try:
-                st_new = read(filename)
+                st_new = read(filename, dtype=float)
                 # add file to read filenames to prevent re-reading in case it is not the current day (or end of
                 # previous day)
                 if not filename.endswith(f'{current_day:03}') and not (
@@ -167,7 +167,7 @@ class SurveillanceBot(object):
                 print(f'Could not read file {filename}:', e)
                 continue
             self.dataStream += st_new
-        self.dataStream.merge()
+        self.dataStream.merge(fill_value=np.nan)
 
         # organise data in dictionary with key for each station
         for trace in self.dataStream:
@@ -341,7 +341,8 @@ class SurveillanceBot(object):
             try:
                 st = modify_stream_for_plot(st, parameters=self.parameters)
                 st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full')
-                annotate_trace_axes(fig, self.parameters, self.verbosity)
+                trace_ylabels(fig, self.parameters, self.verbosity)
+                trace_yticks(fig, self.parameters, self.verbosity)
             except Exception as e:
                 print(f'Could not generate plot for {nwst_id}:')
                 print(traceback.format_exc())
@@ -349,10 +350,22 @@ class SurveillanceBot(object):
                 ax = fig.axes[0]
                 ax.set_title(f'Plot refreshed at (UTC) {UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S")}. '
                              f'Refreshed hourly or on FAIL status.')
+                for ax in fig.axes:
+                    ax.grid(True, alpha=0.1)
                 fig.savefig(fnout, dpi=150., bbox_inches='tight')
         plt.close(fig)
 
-    def write_html_table(self, default_color='#e6e6e6', default_header_color='#999'):
+    def write_html_table(self, default_color='#e6e6e6', default_header_color='#999', hide_keys_mobile=('other')):
+
+        def get_html_class(status=None, check_key=None):
+            """ helper function for html class if a certain condition is fulfilled """
+            html_class = None
+            if status and status.is_active:
+                html_class = 'blink-bg'
+            if check_key in hide_keys_mobile:
+                html_class = 'hidden-mobile'
+            return html_class
+
         self.check_html_dir()
         fnout = pjoin(self.outpath_html, 'survBot_out.html')
         if not fnout:
@@ -370,7 +383,8 @@ class SurveillanceBot(object):
                     header.insert(-1, key)
                 header_items = [dict(text='Station', color=default_header_color)]
                 for check_key in header:
-                    item = dict(text=check_key, color=default_header_color)
+                    html_class = get_html_class(check_key=check_key)
+                    item = dict(text=check_key, color=default_header_color, html_class=html_class)
                     header_items.append(item)
                 write_html_row(outfile, header_items, html_key='th')
 
@@ -397,8 +411,9 @@ class SurveillanceBot(object):
                                 if not type(message) in [str]:
                                     message = str(message) + deg_str
 
+                            html_class = get_html_class(status=status, check_key=check_key)
                             item = dict(text=str(message), tooltip=str(detailed_message), color=bg_color,
-                                        blink=status.is_active)
+                                        html_class=html_class)
                         elif check_key in self.add_links:
                             value = self.add_links.get(check_key).get('URL')
                             link_text = self.add_links.get(check_key).get('text')
@@ -566,11 +581,25 @@ class StationQC(object):
             if self.verbosity:
                 print('Mail functionality disabled. Return')
             return
+
         mail_params = self.parameters.get('EMAIL')
         if not mail_params:
             if self.verbosity:
                 print('parameter "EMAIL" not set in parameter file. Return')
             return
+
+        stations_blacklist = mail_params.get('stations_blacklist')
+        if stations_blacklist and self.station in stations_blacklist:
+            if self.verbosity:
+                print(f'Station {self.station} listed in blacklist. Return')
+            return
+
+        networks_blacklist = mail_params.get('networks_blacklist')
+        if networks_blacklist and self.network in networks_blacklist:
+            if self.verbosity:
+                print(f'Station {self.station} of network {self.network} listed in blacklist. Return')
+            return
+
         sender = mail_params.get('sender')
         addresses = mail_params.get('addresses')
         server = mail_params.get('mailserver')
@@ -705,9 +734,6 @@ class StationQC(object):
             self.status_ok(key, detailed_message=f'U={(voltage[-1])}V')
             return
 
-        n_overvolt = 0
-        n_undervolt = 0
-
         warn_message = f'Trace {trace.get_id()}:'
         if len(overvolt) > 0:
             # try calculate number of voltage peaks from gaps between indices
@@ -820,8 +846,7 @@ class StationQC(object):
                         self.status_ok(key)
                         continue
                     if volt_lvl > 1:
-                        # try calculate number of voltage peaks from gaps between indices
+                        n_occurrences = self.calc_occurrences(ind_array)
-                        n_occurrences = len(np.where(np.diff(ind_array) > 1)[0]) + 1
                         self.warn(key=key,
                                   detailed_message=f'Trace {trace.get_id()}: '
                                                    f'Found {n_occurrences} occurrence(s) of {volt_lvl}V: {key}: {message}'
@@ -832,6 +857,32 @@ class StationQC(object):
                     if volt_lvl == last_val or (volt_lvl == -1 and last_val < 1):
                         self.error(key, detailed_message=f'Last PowBox voltage state {last_val}V: {message}')
 
+    def calc_occurrences(self, ind_array):
+        # try calculate number of voltage peaks/plateaus from gaps between indices
+        if len(ind_array) == 0:
+            return 0
+        else:
+            # start index at 1 if there are gaps (n_peaks = n_gaps + 1)
+            n_occurrences = 1
+
+        min_samples = self.parameters.get('min_sample')
+        if not min_samples:
+            min_samples = 1
+
+        # calculated differences in index array, diff > 1: gap, diff == 1: within peak/plateau
+        diffs = np.diff(ind_array)
+        gap_start_inds = np.where(np.diff(ind_array) > 1)[0]
+        # iterate over all gaps and check "min_samples" before the gap
+        for gsi in gap_start_inds:
+            # right boundary index of peak (gap index - 1)
+            peak_rb_ind = gsi - 1
+            # left boundary index of peak
+            peak_lb_ind = max([0, peak_rb_ind - min_samples])
+            if all(diffs[peak_lb_ind: peak_rb_ind] == 1):
+                n_occurrences += 1
+
+        return n_occurrences
+
     def get_trace(self, stream, keys):
         if not type(keys) == list:
             keys = [keys]

survBotGUI.py

@@ -34,7 +34,7 @@ from obspy import UTCDateTime
 
 from survBot import SurveillanceBot
 from write_utils import *
-from utils import get_bg_color, modify_stream_for_plot, annotate_trace_axes
+from utils import get_bg_color, modify_stream_for_plot, trace_ylabels, trace_yticks
 
 try:
     from rest_api.utils import get_station_iccid
@@ -316,7 +316,8 @@ class MainWindow(QtWidgets.QMainWindow):
             self.plot_widget.setWindowTitle(nwst_id)
             st = modify_stream_for_plot(st, parameters=self.parameters)
             st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig)
-            annotate_trace_axes(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
+            trace_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
+            trace_yticks(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
             self.plot_widget.show()
 
     def notification(self, text):

utils.py

@@ -104,7 +104,7 @@ def transform_trace(data, transf):
     return data
 
 
-def annotate_trace_axes(fig, parameters, verbosity=0):
+def trace_ylabels(fig, parameters, verbosity=0):
     """
     Adds channel names to y-axis if defined in parameters.
     Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is
@@ -122,3 +122,24 @@ def annotate_trace_axes(fig, parameters, verbosity=0):
             ax.set_ylabel(channel_name)
 
 
+def trace_yticks(fig, parameters, verbosity=0):
+    """
+    Adds channel names to y-axis if defined in parameters.
+    Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is
+    difficult to assess the correct order from Obspy plotting routing.
+    """
+    ticks = parameters.get('CHANNEL_TICKS')
+    if not ticks:
+        return
+    if not len(ticks) == len(fig.axes):
+        if verbosity:
+            print('Mismatch in axis tick and label lengths. Not changing plot ticks.')
+        return
+    for ytick_tripple, ax in zip(ticks, fig.axes):
+        if not ytick_tripple:
+            continue
+        ymin, ymax, step = ytick_tripple
+
+        yticks = list(range(ymin, ymax + step, step))
+        ax.set_yticks(yticks)
+        ax.set_ylim(ymin - step, ymax + step)

write_utils.py

@@ -14,16 +14,13 @@ def write_html_header(fobj, refresh_rate=10):
     header = ['<!DOCTYPE html>',
               '<html>',
               '<head>',
-              '<link rel="stylesheet" href="stylesheet.css">',
+              '  <link rel="stylesheet" media="only screen and (max-width: 400px)" href="mobile.css" />',
+              '  <link rel="stylesheet" media="only screen and (min-width: 401px)" href="desktop.css" />',
               '</head>',
               f'<meta http-equiv="refresh" content="{refresh_rate}" >',
               '<meta charset="utf-8">',
+              '<meta name="viewport" content="width=device-width, initial-scale=1">',
               '<body>']
-    # style = ['<style>',
-    #           'table, th, td {',
-    #           'border:1px solid black;',
-    #           '}',
-    #           '</style>',]
     for item in header:
         fobj.write(item + '\n')
 
@@ -58,8 +55,9 @@ def write_html_row(fobj, items, html_key='td'):
         color = '#e6e6e6' if color == '#000000' else color
         hyperlink = item.get('hyperlink')
         image_str = f'<a href="{hyperlink}">' if hyperlink else ''
-        blink_str = f' class="blink-bg"' if item.get('blink') else ''
+        html_class = item.get('html_class')
-        fobj.write(2 * default_space + f'<{html_key}{blink_str} bgcolor="{color}" title="{tooltip}"> {image_str}'
+        class_str = f' class="{html_class}"' if html_class else ''
+        fobj.write(2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"> {image_str}'
                    + text + f'</{html_key}>\n')
     fobj.write(default_space + '</tr>\n')