Merge branch 'release0.2'

# Conflicts:
#	utils.py

.gitignore

@@ -212,3 +212,4 @@ flycheck_*.el
 
 
 /__simulate_fail.json
+/mailing_list.yaml

README.md

@@ -1,6 +1,6 @@
 # survBot
 
-version: 0.1
+version: 0.2
 
 survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output over SOH channels
  by analysing contents of a Seiscomp3 datapath.
@@ -40,8 +40,16 @@ The GUI can be loaded via
 python survBotGui.py
 ```
 
+## Version Changes
+- surveillance of mass, clock and gaps
+- individual mailing lists for different stations
+- html mail with recent status information
+- updated web page design
+- restructured parameter file
+- recognize if PBox is disconnected
+
 ## Staff
 
 Original author: M.Paffrath (marcel.paffrath@rub.de)
 
-November 2022
+June 2023

__init__.py

@@ -0,0 +1,3 @@
+# survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output
+# over SOH channels by analysing contents of a Seiscomp3 datapath.
+__version__ = "0.2"

parameters.yaml

@@ -1,17 +1,17 @@
 # Parameters file for Surveillance Bot
 datapath: "/data/SDS/"                    # SC3 Datapath
-networks: ["1Y", "HA"]                    # select networks, list or str
+networks: ["1Y", "HA", "MK"]              # select networks, list or str
 stations: "*"                             # select stations, list or str
 locations: "*"                            # select locations, list or str
-stations_blacklist: ["TEST", "EREA"]      # exclude these stations
+stations_blacklist: ["TEST", "EREA", "DOMV"]      # exclude these stations
 networks_blacklist: []                    # exclude these networks
 interval: 60               # Perform checks every x seconds
-n_track: 300               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
+n_track: 360               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
 timespan: 3                # Check data of the recent x days
 verbosity: 0               # verbosity flag
 track_changes: True        # tracks all changes since GUI startup by text highlighting (GUI only)
-warn_count: False           # show number of warnings and errors in table
-min_sample: 3              # minimum samples for raising Warn/FAIL
+warn_count: False          # show number of warnings and errors in table
+min_sample: 5              # minimum samples for raising Warn/FAIL
 dt_thresh: [300, 1800]     # threshold (s) for timing delay colourisation (yellow/red)
 html_figures: True         # Create html figure directory and links
 reread_parameters: True    # reread parameters file (change parameters on runtime, not for itself/GUI refresh/datapath)
@@ -72,12 +72,12 @@ CHANNELS:
   EX2:
     unit: 1e-6
     name: "PowBox 230V/12V (V)"
-    ticks: [1, 5, 1]
+    ticks: [0, 5, 1]
     warn: [2, 3, 4, 4.5, 5]
   EX3:
     unit: 1e-6
     name: "PowBox Router/Charger (V)"
-    ticks: [1, 5, 1]
+    ticks: [0, 5, 1]
     warn: [2, 2.5, 3, 4, 5]
   VEI:
     unit: 1e-3
@@ -122,6 +122,15 @@ add_links:
   slmon: {"URL": "../slmon/{nw}_{st}.html", "text": "show"}
   24h-plot: {"URL": "../scheli/{nw}/{st}.png", "text": "plot"}
 
+# add station-independent links below html table (list items separated with -)
+add_global_links:
+  # for example: - {"text": "our homepage", "URL": "https://www.rub.de"}
+  - {"text": "show recent events on map",
+     "URL": "https://fdsnws.geophysik.ruhr-uni-bochum.de/map/?lat=39.5&lon=21&zoom=7&baselayer=mapnik"}
+
+# html logo at page bottom (path relative to html directory)
+html_logo: "figures/Logo_RUB_BLAU_rgb.png"
+
 # E-mail notifications
 EMAIL:
   mailserver: "localhost"

stylesheets/desktop.css

@@ -2,26 +2,36 @@ body {
     background-color: #ffffff;
     place-items: center;
     text-align: center;
+    padding-bottom: 30px;
+    font-family: "Helvetica", "sans-serif";
+}
+
+table {
+    position: relative
 }
 
 td {
-    border-radius: 4px;
+    border-radius: 2px;
     padding: 0px;
+    white-space: nowrap;
 }
 
 th {
     background-color: #999;
-    border-radius: 4px;
+    color: #fff;
+    border-radius: 2px;
     padding: 3px 1px;
+    position: sticky;
+    top: 0;
 }
 
 a:link, a:visited {
-    background-color: #ccc;
+    background-color: #e8e8e8;
     color: #000;
     text-decoration: none;
     display: block;
     border-radius: 4px;
-    border: 1px solid #bbb;
+    border: 1px solid #ccc;
 }
 
 a:hover {
@@ -37,3 +47,12 @@ a:hover {
     50%		{ background-color: #ff3200;}
     100%	{ background-color: #ffcc00;}
 }
+
+.footer {
+   position: fixed;
+   left: 0;
+   bottom: 0;
+   width: 100%;
+   height: 50px;
+   text-align: center;
+}

stylesheets/mobile.css

@@ -2,26 +2,36 @@ body {
     background-color: #ffffff;
     place-items: center;
     text-align: center;
+    padding-bottom: 30px;
+    font-family: "Helvetica", "sans-serif";
+}
+
+table {
+    position: relative
 }
 
 td {
-    border-radius: 4px;
+    border-radius: 3px;
     padding: 10px 2px;
+    white-space: nowrap;
 }
 
 th {
     background-color: #999;
-    border-radius: 4px;
+    color: #fff;
+    border-radius: 3px;
     padding: 10px, 2px;
+    position: sticky;
+    top: 0;
 }
 
-a:link {
-    background-color: #ccc;
+a:link, a:visited {
+    background-color: #e8e8e8;
     color: #000;
     text-decoration: none;
     display: block;
-    border-radius: 4px;
-    border: 1px solid #bbb;
+    border-radius: 6px;
+    border: 1px solid #ccc;
 }
 
 a:hover {
@@ -37,7 +47,16 @@ a:hover {
     animation: blinkingBackground 2s infinite;
 }
 @keyframes blinkingBackground{
-    0%		{ background-color: #ffee00;}
+    0%		{ background-color: #ffcc00;}
     50%		{ background-color: #ff3200;}
-    100%	{ background-color: #ffee00;}
+    100%	{ background-color: #ffcc00;}
+}
+
+.footer {
+   position: fixed;
+   left: 0;
+   bottom: 0;
+   width: 100%;
+   height: 50px;
+   text-align: center;
 }

survBot.py

@@ -20,9 +20,9 @@ import matplotlib.pyplot as plt
 from obspy import read, UTCDateTime, Stream
 from obspy.clients.filesystem.sds import Client
 
-from write_utils import get_html_text, get_html_row, html_footer, get_html_header, get_print_title_str, \
+from write_utils import get_html_text, get_html_link, get_html_row, html_footer, get_html_header, get_print_title_str, \
     init_html_table, finish_html_table, get_mail_html_header, add_html_image
-from utils import get_bg_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds
+from utils import get_bg_color, get_font_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds
 
 try:
     import smtplib
@@ -120,9 +120,16 @@ class SurveillanceBot(object):
         self.networks_blacklist = self.parameters.get('networks_blacklist')
         self.refresh_period = self.parameters.get('interval')
         self.transform_parameters()
+
         add_links = self.parameters.get('add_links')
         self.add_links = add_links if add_links else {}
 
+        add_global_links = self.parameters.get('add_global_links')
+        # in case user forgets "-" in parameters file
+        if isinstance(add_global_links, dict):
+            add_global_links = [add_global_links]
+        self.add_global_links = add_global_links if add_global_links else []
+
     def transform_parameters(self):
         for key in ['networks', 'stations', 'locations', 'channels']:
             parameter = self.parameters.get(key)
@@ -434,7 +441,7 @@ class SurveillanceBot(object):
         fig_name = self.get_fig_path_rel(nwst_id)
         nwst_id_str = nwst_id.rstrip('.')
         col_items = [dict(text=nwst_id_str, color=default_color, hyperlink=fig_name if hyperlinks else None,
-                          bold=True, tooltip=f'Show plot of {nwst_id_str}')]
+                          bold=True, tooltip=f'Show plot of {nwst_id_str}', font_color='#000000')]
 
         for check_key in header:
             if check_key in self.keys:
@@ -446,6 +453,7 @@ class SurveillanceBot(object):
                 bg_color = get_bg_color(check_key, status, dt_thresh, hex=True)
                 if not bg_color:
                     bg_color = default_color
+                font_color = get_font_color(bg_color, hex=True)
 
                 # add degree sign for temp
                 if check_key == 'temp':
@@ -454,7 +462,7 @@ class SurveillanceBot(object):
 
                 html_class = self.get_html_class(hide_keys_mobile, status=status, check_key=check_key)
                 item = dict(text=str(message), tooltip=str(detailed_message), color=bg_color,
-                            html_class=html_class)
+                            html_class=html_class, font_color=font_color)
             elif check_key in self.add_links:
                 value = self.add_links.get(check_key).get('URL')
                 link_text = self.add_links.get(check_key).get('text')
@@ -496,8 +504,21 @@ class SurveillanceBot(object):
 
                 outfile.write(finish_html_table())
 
+                # add optional links below html table
+                for dct in self.add_global_links:
+                    link_str = get_html_link(dct.get('text'), dct.get('URL'))
+                    outfile.write(get_html_text(link_str))
+
+                # add status message
                 outfile.write(get_html_text(self.status_message))
-                outfile.write(html_footer())
+
+                # write footer with optional logo
+                logo_file = self.parameters.get('html_logo')
+                if not os.path.isfile(pjoin(self.outpath_html, logo_file)):
+                    print(f'Specified file {logo_file} not found.')
+                    logo_file = None
+
+                outfile.write(html_footer(footer_logo=logo_file))
 
         except Exception as e:
             print(f'Could not write HTML table to {fnout}:')
@@ -603,9 +624,11 @@ class StationQC(object):
         # current_status_message = '' if current_status_message in [None, 'OK', '-'] else current_status_message + ' | '
         # self.status_dict[key] = current_status_message + status_message
 
-    def error(self, key, detailed_message, last_occurrence=None, count=1):
+    def error(self, key, detailed_message, last_occurrence=None, count=1, disc=False):
         send_mail = False
         new_error = StatusError(count=count, show_count=self.parameters.get('warn_count'))
+        if disc:
+            new_error.set_disconnected()
         current_status = self.status_dict.get(key)
         if current_status.is_error:
             current_status.count += count
@@ -829,14 +852,24 @@ class StationQC(object):
             return max(endtimes)
 
     def check_for_inactive_message(self, key, dt_active):
+        """ send mail if station is inactive longer than dt_action and no FAIL status is present """
+
+        # check if any error is present in status_dict and not disconnected (in that case an email is sent already)
+        if self.check_for_any_error_no_dcn():
+            return
+
         dt_action = self.get_dt_for_action()
         interval = self.parameters.get('interval')
+
         if dt_action <= dt_active < dt_action + timedelta(seconds=interval):
             detailed_message = f'\n{self.nwst_id}\n\n'
             for key, status in self.status_dict.items():
                 detailed_message += f'{key}: {status.message}\n'
             self.send_mail(key, status_type='Inactive', additional_message=detailed_message)
 
+    def check_for_any_error_no_dcn(self):
+        return any([status.is_error and not status.connection_error for status in self.status_dict.values()])
+
     def start(self):
         self.analyse_channels()
 
@@ -1137,8 +1170,10 @@ class StationQC(object):
                                   count=n_occurrences,
                                   last_occurrence=self.get_last_occurrence(trace, ind_array))
                     # if last_val == current voltage (which is not 1) -> FAIL or last_val < 1: PBox no data
-                    if volt_lvl == last_val or (volt_lvl == -1 and last_val < 1):
+                    if volt_lvl == last_val:
                         self.error(key, detailed_message=f'Last PowBox voltage state {last_val}V: {message}')
+                    elif volt_lvl == -1 and last_val < 1:
+                        self.error(key, detailed_message=f'PowBox under 1V - connection error', disc=True)
 
     def gaps_analysis(self, key='gaps'):
         """ return gaps of a given nwst_id """
@@ -1272,19 +1307,23 @@ class StationQC(object):
 
 
 class Status(object):
+    """ Basic Status class. All status classes are derived from this class."""
     def __init__(self, message=None, detailed_messages=None, count: int = 0, last_occurrence=None, show_count=True):
         if message is None:
             message = '-'
         if detailed_messages is None:
             detailed_messages = []
+
         self.show_count = show_count
         self.message = message
         self.messages = [message]
         self.detailed_messages = detailed_messages
         self.count = count
         self.last_occurrence = last_occurrence
+
         self.is_warn = None
         self.is_error = None
+        self.connection_error = None
         self.is_other = False
         self.is_active = False
 
@@ -1331,6 +1370,15 @@ class StatusError(Status):
         super(StatusError, self).__init__(message=message, count=count, last_occurrence=last_occurence,
                                           detailed_messages=detailed_messages, show_count=show_count)
         self.set_error()
+        self.default_message = message
+
+    def set_disconnected(self, message='DCN'):
+        self.connection_error = True
+        self.message = message
+
+    def set_connected(self):
+        self.connection_error = False
+        self.message = self.default_message
 
 
 class StatusOther(Status):

utils.py

@@ -19,7 +19,10 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False):
         if status.is_warn:
             bg_color = get_warn_color(status.count)
         elif status.is_error:
-            bg_color = get_color('FAIL')
+            if status.connection_error:
+                bg_color = get_color('disc')
+            else:
+                bg_color = get_color('FAIL')
         else:
             bg_color = get_color(message)
     if not bg_color:
@@ -31,12 +34,19 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False):
 
 
 def get_color(key):
-    # some GUI default colors
-    colors_dict = {'FAIL': (255, 50, 0, 255),
+    # some old GUI default colors
+    # colors_dict = {'FAIL': (255, 85, 50, 255),
+    #                'NO DATA': (255, 255, 125, 255),
+    #                'WARN': (255, 255, 80, 255),
+    #                'OK': (173, 255, 133, 255),
+    #                'undefined': (230, 230, 230, 255),
+    #                'disc': (255, 160, 40, 255),}
+    colors_dict = {'FAIL': (195, 29, 14, 255),
                    'NO DATA': (255, 255, 125, 255),
-                   'WARN': (255, 255, 80, 255),
-                   'OK': (125, 255, 125, 255),
-                   'undefined': (230, 230, 230, 255)}
+                   'WARN': (250, 192, 63, 255),
+                   'OK': (185, 245, 145, 255),
+                   'undefined': (240, 240, 240, 255),
+                   'disc': (126, 127, 131, 255), }
     return colors_dict.get(key)
 
 
@@ -55,9 +65,11 @@ def get_time_delay_color(dt, dt_thresh):
     return get_color('FAIL')
 
 
-def get_warn_color(count):
-    color = (min([255, 200 + count ** 2]), 255, 80, 255)
-    return color
+def get_warn_color(count, n_colors=20):
+    if count >= n_colors:
+        count = -1
+    gradient = np.linspace((240, 245, 110, 255), (250, 192, 63, 255), n_colors, dtype=int)
+    return tuple(gradient[count])
 
 
 def get_mass_color(message):
@@ -77,6 +89,24 @@ def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
     return rgba
 
 
+def get_font_color(bg_color, hex=False):
+    if hex:
+        bg_color = matplotlib.colors.to_rgb(bg_color)
+    bg_color_hsv = matplotlib.colors.rgb_to_hsv(bg_color)
+    bg_color_hsl = hsv_to_hsl(bg_color_hsv)
+    font_color = (255, 255, 255, 255) if bg_color_hsl[2] < 0.6 else (0, 0, 0, 255)
+    if hex:
+        font_color = '#{:02x}{:02x}{:02x}'.format(*font_color[:3])
+    return font_color
+
+
+def hsv_to_hsl(hsv):
+    hue, saturation, value = hsv
+    lightness = value * (1 - saturation / 2)
+    saturation = 0 if lightness in (0, 1) else (value - lightness) / min(lightness, 1 - lightness)
+    return hue, saturation, lightness
+
+
 def modify_stream_for_plot(input_stream, parameters):
     """ copy (if necessary) and modify stream for plotting """
 
@@ -215,4 +245,4 @@ def plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs):
             if isinstance(warn_thresh, str):
                 warn_thresh = parameters.get('THRESHOLDS').get(warn_thresh)
             if type(warn_thresh in (float, int)):
-                ax.axhline(warn_thresh, **kwargs)
+                ax.axhline(warn_thresh, **kwargs)

write_utils.py

@@ -47,9 +47,15 @@ def finish_html_table():
     return '</table>\n'
 
 
-def html_footer():
-    footer = ['</body>',
-              '</html>']
+def html_footer(footer_logo=None):
+    footer = ['</body>']
+    if footer_logo:
+        logo_items = [f'<div class="footer">',
+                      f'  <img style="float: right; padding: 10px;" src="{footer_logo}" height=30px>',
+                      f'</div>']
+        footer += logo_items
+    footer.append('</html>\n')
+
     footer = _convert_to_textstring(footer)
     return footer
 
@@ -58,6 +64,10 @@ def add_html_image(img_data, img_format='png'):
     return f"""<br>\n<img width="100%" src="data:image/{img_format};base64, {b64encode(img_data).decode('ascii')}">"""
 
 
+def get_html_link(text, link):
+    return f'<a href="{link}"> {text} </a>'
+
+
 def get_html_row(items, html_key='td'):
     row_string = ''
     default_space = '  '
@@ -69,15 +79,14 @@ def get_html_row(items, html_key='td'):
         if item.get('italic'):
             text = '<i>' + text + '</i>'
         tooltip = item.get('tooltip')
-        color = item.get('color')
-        # check for black background of headers (shouldnt happen anymore)
-        color = '#e6e6e6' if color == '#000000' else color
+        font_color = item.get('font_color')
         hyperlink = item.get('hyperlink')
-        image_str = f'<a href="{hyperlink}">' if hyperlink else ''
+        color = 'transparent' if hyperlink else item.get('color')
+        text_str = get_html_link(text, hyperlink) if hyperlink else text
         html_class = item.get('html_class')
         class_str = f' class="{html_class}"' if html_class else ''
-        row_string += 2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"> {image_str}'\
-                    + text + f'</{html_key}>\n'
+        row_string += 2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"' \
+                      + f'style="color:{font_color}"> {text_str}</{html_key}>\n'
     row_string += default_space + '</tr>\n'
     return row_string