Merge pull request 'release/1.0' (#3) from release/1.0 into main

Reviewed-on: #3

README.md

@@ -26,12 +26,14 @@ to use the GUI:
 
 Configurations of *datapath*, *networks*, *stations* etc. can be done in the **parameters.yaml** input file.
 
-The main program is executed by entering
+The main program with html output is executed by entering
 
 ```shell script
-python survBot.py
+python survBot.py -html path_for_html_output
 ```
 
+There are example stylesheets in the folder *stylesheets* that can be copied into the path_for_html_output if desired.
+
 The GUI can be loaded via
 
 ```shell script

parameters.yaml

@@ -1,28 +1,34 @@
 # Parameters file for Surveillance Bot
-datapath: '/data/SDS/'      # SC3 Datapath
+datapath: "/data/SDS/"                    # SC3 Datapath
-networks: ['1Y', 'HA']
+networks: ["1Y", "HA"]                    # select networks, list or str
-stations: '*'
+stations: "*"                             # select stations, list or str
-locations: '*'
+locations: "*"                            # select locations, list or str
-channels: ['EX1', 'EX2', 'EX3', 'VEI']    # Specify SOH channels, currently supported EX[1-3] and VEI
+channels: ["EX1", "EX2", "EX3", "VEI"]    # Specify SOH channels, currently supported EX[1-3] and VEI
-stations_blacklist: ['TEST', 'EREA']
+stations_blacklist: ["TEST", "EREA"]      # exclude these stations
-networks_blacklist: []
+networks_blacklist: []                    # exclude these networks
 interval: 60               # Perform checks every x seconds
+n_track: 300               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
 timespan: 7                # Check data of the recent x days
-verbosity: 0
+verbosity: 0               # verbosity flag
-reread_parameters: True    # reread parameters file (change parameters on runtime, not for itself/GUI refresh/datapath)
 track_changes: True        # tracks all changes since GUI startup by text highlighting (GUI only)
+warn_count: False           # show number of warnings and errors in table
+min_sample: 3              # minimum samples for raising Warn/FAIL
 dt_thresh: [300, 1800]     # threshold (s) for timing delay colourisation (yellow/red)
+html_figures: True         # Create html figure directory and links
+reread_parameters: True    # reread parameters file (change parameters on runtime, not for itself/GUI refresh/datapath)
 
 POWBOX:
   pb_ok: 1  # Voltage for PowBox OK
   pb_SOH2: # PowBox channel 2 voltage translations
-    1: {"230V": 'OK', "12V": "OK"}
+    -1: {"230V": "PBox under 1V", "12V": "PBox under 1V"}
+    1: {"230V": "OK", "12V": "OK"}
     2: {"230V": "OFF", "12V": "OK"}
     3: {"230V": "OK", "12V": "overvoltage"}
     4: {"230V": "OK", "12V": "undervoltage"}
     4.5: {"230V": "OFF", "12V": "overvoltage"}
     5: {"230V": "OFF", "12V": "undervoltage"}
   pb_SOH3: # PowBox channel 3 voltage translations
+    -1: {"router": "PBox under 1V", "charger": "PBox under 1V"}
     1: {"router": "OK", "charger": "OK"}
     2: {"router": "OK", "charger": "0 < resets < 3"}
     2.5: {"router": "OK", "charger": "locked"}
@@ -30,10 +36,49 @@ POWBOX:
     4: {"router": "FAIL", "charger": "0 < resets < 3"}
     5: {"router": "FAIL", "charger": "locked"}
 
-
+# Thresholds for program warnings/voltage classifications
 THRESHOLDS:
   pb_thresh: 0.2               # Threshold for PowBox Voltage check +/- (V)
   max_temp: 50                 # max temperature for temperature warning
   low_volt: 12                 # min voltage for low voltage warning
   high_volt: 14.8               # max voltage for over voltage warning
   unclassified: 5              # min voltage samples not classified for warning
+
+# ---------------------------------------- OPTIONAL PARAMETERS ---------------------------------------------------------
+
+# add links to html table with specified key as column and value as relative link, interpretable string parameters:
+# nw (e.g. 1Y), st (e.g. GR01A), nwst_id (e.g. 1Y.GR01A)
+add_links:
+  # for example: slmon: {"URL": "path/{nw}_{st}.html", "text": "link"}
+  slmon: {"URL": "../slmon/{nw}_{st}.html", "text": "show"}
+  24h-plot: {"URL": "../scheli/{nw}/{st}.png", "text": "plot"}
+
+# E-mail notifications
+EMAIL:
+  mailserver: "localhost"
+  addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"]  # list of mail addresses for info mails
+  sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender
+  stations_blacklist: ['GR33']                     # do not send emails for specific stations
+  networks_blacklist: []                           # do not send emails for specific network
+
+# names for plotting of the above defined parameter "channels" in the same order
+channel_names: ["Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"]
+# specify y-ticks (and ylims) giving, (ymin, ymax, step) for each of the above channels (0: default)
+CHANNEL_TICKS:
+  - [-10, 50, 10]
+  - [1, 5, 1]
+  - [1, 5, 1]
+  - [9, 15, 1]
+
+# Factor for channel to SI-units (for plotting)
+CHANNEL_UNITS:
+  EX1: 1e-6
+  EX2: 1e-6
+  EX3: 1e-6
+  VEI: 1e-3
+
+# Transform channel for plotting, perform arithmetic operations in given order, e.g.: PBox EX1 V to deg C: 20 * x -20
+CHANNEL_TRANSFORM:
+  EX1:
+    - ["*", 20]
+    - ["-", 20]

stylesheets/desktop.css

@@ -0,0 +1,39 @@
+body {
+    background-color: #ffffff;
+    place-items: center;
+    text-align: center;
+}
+
+td {
+    border-radius: 4px;
+    padding: 0px;
+}
+
+th {
+    background-color: #999;
+    border-radius: 4px;
+    padding: 3px 1px;
+}
+
+a:link, a:visited {
+    background-color: #ccc;
+    color: #000;
+    text-decoration: none;
+    display: block;
+    border-radius: 4px;
+    border: 1px solid #bbb;
+}
+
+a:hover {
+    background-color: #aaa;
+    display: block;			  
+}
+
+.blink-bg {
+    animation: blinkingBackground 2s infinite;
+}
+@keyframes blinkingBackground{
+    0%		{ background-color: #ffcc00;}
+    50%		{ background-color: #ff3200;}
+    100%	{ background-color: #ffcc00;}
+}

stylesheets/mobile.css

@@ -0,0 +1,43 @@
+body {
+    background-color: #ffffff;
+    place-items: center;
+    text-align: center;
+}
+
+td {
+    border-radius: 4px;
+    padding: 10px 2px;
+}
+
+th {
+    background-color: #999;
+    border-radius: 4px;
+    padding: 10px, 2px;
+}
+
+a:link {
+    background-color: #ccc;
+    color: #000;
+    text-decoration: none;
+    display: block;
+    border-radius: 4px;
+    border: 1px solid #bbb;
+}
+
+a:hover {
+    background-color: #aaa;
+    display: block;			  
+}
+
+.hidden-mobile {
+    display: none;
+}
+
+.blink-bg {
+    animation: blinkingBackground 2s infinite;
+}
+@keyframes blinkingBackground{
+    0%		{ background-color: #ffee00;}
+    50%		{ background-color: #ff3200;}
+    100%	{ background-color: #ffee00;}
+}

submit_bot.sh

@@ -6,8 +6,7 @@ ulimit -s 8192
 #$ -cwd
 #$ -pe smp 1
 #$ -N survBot_bg
-
+#$ -l os=*stretch
-export PYTHONPATH="$PYTHONPATH:/home/marcel/git/code_base/"
 
 source /opt/anaconda3/etc/profile.d/conda.sh
 conda activate py37
@@ -17,4 +16,4 @@ export MKL_NUM_THREADS=1
 export NUMEXPR_NUM_THREADS=1
 export OMP_NUM_THREADS=1
 
-python survBot.py -html '/home/marcel/public_html/survBot_out.html'
+python survBot.py -html '/data/www/~marcel/'

survBotGUI.py

@@ -22,7 +22,6 @@ except ImportError:
         except ImportError:
             raise ImportError('Could import neither of PySide2, PySide6 or PyQt5')
 
-import matplotlib
 from matplotlib.figure import Figure
 
 if QtGui.__package__ in ['PySide2', 'PyQt5', 'PySide6']:
@@ -35,7 +34,7 @@ from obspy import UTCDateTime
 
 from survBot import SurveillanceBot
 from write_utils import *
-from utils import get_bg_color
+from utils import get_bg_color, modify_stream_for_plot, trace_ylabels, trace_yticks
 
 try:
     from rest_api.utils import get_station_iccid
@@ -130,10 +129,10 @@ class MainWindow(QtWidgets.QMainWindow):
         self.table.setRowCount(len(station_list))
         self.table.setHorizontalHeaderLabels(keys)
 
-        for index, st_id in enumerate(station_list):
+        for index, nwst_id in enumerate(station_list):
             item = QtWidgets.QTableWidgetItem()
-            item.setText(str(st_id.rstrip('.')))
+            item.setText(str(nwst_id.rstrip('.')))
-            item.setData(QtCore.Qt.UserRole, st_id)
+            item.setData(QtCore.Qt.UserRole, nwst_id)
             self.table.setVerticalHeaderItem(index, item)
 
         self.main_layout.addWidget(self.table)
@@ -180,38 +179,38 @@ class MainWindow(QtWidgets.QMainWindow):
         header_item = self.table.verticalHeaderItem(row_ind)
         if not header_item:
            return
-        st_id = header_item.data(QtCore.Qt.UserRole)
+        nwst_id = header_item.data(QtCore.Qt.UserRole)
 
         context_menu = QtWidgets.QMenu()
         read_sms = context_menu.addAction('Get last SMS')
         send_sms = context_menu.addAction('Send SMS')
         action = context_menu.exec_(self.mapToGlobal(self.last_mouse_loc))
         if action == read_sms:
-           self.read_sms(st_id)
+           self.read_sms(nwst_id)
         elif action == send_sms:
-           self.send_sms(st_id)
+           self.send_sms(nwst_id)
 
-    def read_sms(self, st_id):
+    def read_sms(self, nwst_id):
         """ Read recent SMS over rest_api using whereversim portal """
-        station = st_id.split('.')[1]
+        station = nwst_id.split('.')[1]
         iccid = get_station_iccid(station)
         if not iccid:
-            print('Could not find iccid for station', st_id)
+            print('Could not find iccid for station', nwst_id)
             return
         sms_widget = ReadSMSWidget(parent=self, iccid=iccid)
-        sms_widget.setWindowTitle(f'Recent SMS of station: {st_id}')
+        sms_widget.setWindowTitle(f'Recent SMS of station: {nwst_id}')
         if sms_widget.data:
             sms_widget.show()
         else:
             self.notification('No recent messages found.')
 
-    def send_sms(self, st_id):
+    def send_sms(self, nwst_id):
         """ Send SMS over rest_api using whereversim portal """
-        station = st_id.split('.')[1]
+        station = nwst_id.split('.')[1]
         iccid = get_station_iccid(station)
 
         sms_widget = SendSMSWidget(parent=self, iccid=iccid)
-        sms_widget.setWindowTitle(f'Send SMS to station: {st_id}')
+        sms_widget.setWindowTitle(f'Send SMS to station: {nwst_id}')
         sms_widget.show()
 
     def set_clear_on_refresh(self):
@@ -230,19 +229,19 @@ class MainWindow(QtWidgets.QMainWindow):
         self.fill_status_bar()
 
         for col_ind, check_key in enumerate(self.survBot.keys):
-            for row_ind, st_id in enumerate(self.survBot.station_list):
+            for row_ind, nwst_id in enumerate(self.survBot.station_list):
-                status_dict, detailed_dict = self.survBot.analysis_results.get(st_id)
+                status_dict = self.survBot.analysis_results.get(nwst_id)
                 status = status_dict.get(check_key)
-                detailed_message = detailed_dict.get(check_key)
+                message, detailed_message = status.get_status_str()
 
                 dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh]
                 bg_color = get_bg_color(check_key, status, dt_thresh)
                 if check_key == 'temp':
-                    if not type(status) in [str]:
+                    if not type(message) in [str]:
-                        status = str(status) + deg_str
+                        message = str(message) + deg_str
 
                 # Continue if nothing changed
-                text = str(status)
+                text = str(message)
                 cur_item = self.table.item(row_ind, col_ind)
                 if cur_item and text == cur_item.text():
                     if not self.parameters.get('track_changes') or self.clear_on_refresh:
@@ -253,9 +252,9 @@ class MainWindow(QtWidgets.QMainWindow):
 
                 # Create new data item
                 item = QtWidgets.QTableWidgetItem()
-                item.setText(str(status))
+                item.setText(str(message))
                 item.setTextAlignment(QtCore.Qt.AlignCenter)
-                item.setData(QtCore.Qt.UserRole, (st_id, check_key))
+                item.setData(QtCore.Qt.UserRole, (nwst_id, check_key))
 
                 # if text changed (known from above) set highlight color/font else (new init) set to default
                 cur_item = self.table.item(row_ind, col_ind)
@@ -310,12 +309,15 @@ class MainWindow(QtWidgets.QMainWindow):
             vheader.setSectionResizeMode(index, QtWidgets.QHeaderView.Stretch)
 
     def plot_stream(self, item):
-        st_id, check = item.data(QtCore.Qt.UserRole)
+        nwst_id, check = item.data(QtCore.Qt.UserRole)
-        st = self.survBot.data.get(st_id)
+        st = self.survBot.data.get(nwst_id)
         if st:
             self.plot_widget = PlotWidget(self)
-            self.plot_widget.setWindowTitle(st_id)
+            self.plot_widget.setWindowTitle(nwst_id)
+            st = modify_stream_for_plot(st, parameters=self.parameters)
             st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig)
+            trace_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
+            trace_yticks(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
             self.plot_widget.show()
 
     def notification(self, text):

utils.py

@@ -3,18 +3,20 @@
 
 import matplotlib
 
+
 def get_bg_color(check_key, status, dt_thresh=None, hex=False):
+    message = status.message
     if check_key == 'last active':
-        bg_color = get_time_delay_color(status, dt_thresh)
+        bg_color = get_time_delay_color(message, dt_thresh)
     elif check_key == 'temp':
-        bg_color = get_temp_color(status)
+        bg_color = get_temp_color(message)
     else:
-        statussplit = status.split(' ')
+        if status.is_warn:
-        if len(statussplit) > 1 and statussplit[0] == 'WARN':
+            bg_color = get_color('WARNX')(status.count)
-            x = int(status.split(' ')[-1].lstrip('(').rstrip(')'))
+        elif status.is_error:
-            bg_color = get_color('WARNX')(x)
+            bg_color = get_color('FAIL')
         else:
-            bg_color = get_color(status)
+            bg_color = get_color(message)
     if not bg_color:
         bg_color = get_color('undefined')
 
@@ -22,16 +24,18 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False):
         bg_color = '#{:02x}{:02x}{:02x}'.format(*bg_color[:3])
     return bg_color
 
+
 def get_color(key):
     # some GUI default colors
     colors_dict = {'FAIL': (255, 50, 0, 255),
                    'NO DATA': (255, 255, 125, 255),
-                   'WARN': (255, 255, 125, 255),
+                   'WARN': (255, 255, 80, 255),
-                   'WARNX': lambda x: (min([255, 200 + x ** 2]), 255, 125, 255),
+                   'WARNX': lambda x: (min([255, 200 + x ** 2]), 255, 80, 255),
                    'OK': (125, 255, 125, 255),
                    'undefined': (230, 230, 230, 255)}
     return colors_dict.get(key)
 
+
 def get_time_delay_color(dt, dt_thresh):
     """ Set color of time delay after thresholds specified in self.dt_thresh """
     if dt < dt_thresh[0]:
@@ -40,6 +44,7 @@ def get_time_delay_color(dt, dt_thresh):
         return get_color('WARN')
     return get_color('FAIL')
 
+
 def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
     """ Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """
     if type(temp) in [str]:
@@ -49,3 +54,92 @@ def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
     rgba = [int(255 * c) for c in cmap(val)]
     return rgba
 
+
+def modify_stream_for_plot(st, parameters):
+    """ copy (if necessary) and modify stream for plotting """
+    ch_units = parameters.get('CHANNEL_UNITS')
+    ch_transf = parameters.get('CHANNEL_TRANSFORM')
+
+    # if either of both are defined make copy
+    if ch_units or ch_transf:
+        st = st.copy()
+
+    # modify trace for plotting by multiplying unit factor (e.g. 1e-3 mV to V)
+    if ch_units:
+        for tr in st:
+            channel = tr.stats.channel
+            unit_factor = ch_units.get(channel)
+            if unit_factor:
+                tr.data = tr.data * float(unit_factor)
+    # modify trace for plotting by other arithmetic expressions
+    if ch_transf:
+        for tr in st:
+            channel = tr.stats.channel
+            transf = ch_transf.get(channel)
+            if transf:
+                tr.data = transform_trace(tr.data, transf)
+
+    return st
+
+
+def transform_trace(data, transf):
+    """
+    Transform trace with arithmetic operations in order, specified in transf
+    @param data: numpy array
+    @param transf: list of lists with arithmetic operations (e.g. [['*', '20'], ] -> multiply data by 20
+    """
+    # This looks a little bit hardcoded, however it is safer than using e.g. "eval"
+    for operator_str, val in transf:
+        if operator_str == '+':
+            data = data + val
+        elif operator_str == '-':
+            data = data - val
+        elif operator_str == '*':
+            data = data * val
+        elif operator_str == '/':
+            data = data / val
+        else:
+            raise IOError(f'Unknown arithmethic operator string: {operator_str}')
+
+    return data
+
+
+def trace_ylabels(fig, parameters, verbosity=0):
+    """
+    Adds channel names to y-axis if defined in parameters.
+    Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is
+    difficult to assess the correct order from Obspy plotting routing.
+    """
+    names = parameters.get('channel_names')
+    if not names: # or not len(st.traces):
+        return
+    if not len(names) == len(fig.axes):
+        if verbosity:
+            print('Mismatch in axis and label lengths. Not adding plot labels')
+        return
+    for channel_name, ax in zip(names, fig.axes):
+        if channel_name:
+            ax.set_ylabel(channel_name)
+
+
+def trace_yticks(fig, parameters, verbosity=0):
+    """
+    Adds channel names to y-axis if defined in parameters.
+    Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is
+    difficult to assess the correct order from Obspy plotting routing.
+    """
+    ticks = parameters.get('CHANNEL_TICKS')
+    if not ticks:
+        return
+    if not len(ticks) == len(fig.axes):
+        if verbosity:
+            print('Mismatch in axis tick and label lengths. Not changing plot ticks.')
+        return
+    for ytick_tripple, ax in zip(ticks, fig.axes):
+        if not ytick_tripple:
+            continue
+        ymin, ymax, step = ytick_tripple
+
+        yticks = list(range(ymin, ymax + step, step))
+        ax.set_yticks(yticks)
+        ax.set_ylim(ymin - step, ymax + step)

write_utils.py

@@ -1,52 +1,68 @@
 from datetime import timedelta
 
+
 def write_html_table_title(fobj, parameters):
     title = get_print_title_str(parameters)
     fobj.write(f'<h3>{title}</h3>\n')
 
+
 def write_html_text(fobj, text):
     fobj.write(f'<p>{text}</p>\n')
 
+
 def write_html_header(fobj, refresh_rate=10):
     header = ['<!DOCTYPE html>',
               '<html>',
+              '<head>',
+              '  <link rel="stylesheet" media="only screen and (max-width: 400px)" href="mobile.css" />',
+              '  <link rel="stylesheet" media="only screen and (min-width: 401px)" href="desktop.css" />',
+              '</head>',
               f'<meta http-equiv="refresh" content="{refresh_rate}" >',
               '<meta charset="utf-8">',
+              '<meta name="viewport" content="width=device-width, initial-scale=1">',
               '<body>']
-    # style = ['<style>',
-    #           'table, th, td {',
-    #           'border:1px solid black;',
-    #           '}',
-    #           '</style>',]
     for item in header:
         fobj.write(item + '\n')
 
+
 def init_html_table(fobj):
     fobj.write('<table style="width:100%">\n')
 
+
 def finish_html_table(fobj):
     fobj.write('</table>\n')
 
+
 def write_html_footer(fobj):
     footer = ['</body>',
               '</html>']
     for item in footer:
         fobj.write(item + '\n')
 
+
 def write_html_row(fobj, items, html_key='td'):
     default_space = '  '
     fobj.write(default_space + '<tr>\n')
     for item in items:
         text = item.get('text')
+        if item.get('bold'):
+            text = '<b>' + text + '</b>'
+        if item.get('italic'):
+            text = '<i>' + text + '</i>'
         tooltip = item.get('tooltip')
         color = item.get('color')
         # check for black background of headers (shouldnt happen anymore)
         color = '#e6e6e6' if color == '#000000' else color
-        fobj.write(2 * default_space + f'<{html_key} bgcolor="{color}" title="{tooltip}">' + text + f'</{html_key}>\n')
+        hyperlink = item.get('hyperlink')
+        image_str = f'<a href="{hyperlink}">' if hyperlink else ''
+        html_class = item.get('html_class')
+        class_str = f' class="{html_class}"' if html_class else ''
+        fobj.write(2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"> {image_str}'
+                   + text + f'</{html_key}>\n')
     fobj.write(default_space + '</tr>\n')
 
+
 def get_print_title_str(parameters):
     timespan = parameters.get('timespan') * 24 * 3600
-    tdelta_str = str(timedelta(seconds=int(timespan)))
+    tdelta_str = str(timedelta(seconds=int(timespan))).replace(', 0:00:00', '')
     return f'Analysis table of router quality within the last {tdelta_str}'
-