Merge branch 'release0.2'

# Conflicts:
#	utils.py

.gitignore

@@ -211,3 +211,5 @@ flycheck_*.el
 /network-security.data
 
 
+/__simulate_fail.json
+/mailing_list.yaml

README.md

@@ -1,6 +1,6 @@
 # survBot
 
-version: 0.1
+version: 0.2
 
 survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output over SOH channels
  by analysing contents of a Seiscomp3 datapath.
@@ -26,20 +26,30 @@ to use the GUI:
 
 Configurations of *datapath*, *networks*, *stations* etc. can be done in the **parameters.yaml** input file.
 
-The main program is executed by entering
+The main program with html output is executed by entering
 
 ```shell script
-python survBot.py
+python survBot.py -html path_for_html_output
 ```
 
+There are example stylesheets in the folder *stylesheets* that can be copied into the path_for_html_output if desired.
+
 The GUI can be loaded via
 
 ```shell script
 python survBotGui.py
 ```
 
+## Version Changes
+- surveillance of mass, clock and gaps
+- individual mailing lists for different stations
+- html mail with recent status information
+- updated web page design
+- restructured parameter file
+- recognize if PBox is disconnected
+
 ## Staff
 
 Original author: M.Paffrath (marcel.paffrath@rub.de)
 
-November 2022
+June 2023

__init__.py

@@ -0,0 +1,3 @@
+# survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output
+# over SOH channels by analysing contents of a Seiscomp3 datapath.
+__version__ = "0.2"

mailing_list.yaml

@@ -0,0 +1,9 @@
+# specify mail addresses and station network ids for which information mails shall be sent, e.g.:
+# "mail.address@provider.com, mail.address2@provider2.com":
+#   - 1Y.GR01
+#   - 1Y.GR02
+# "mail.address3@provder.com":
+#   - 1Y.GR03
+
+#"kasper.fischer@rub.de":
+#    - 1Y.GR01

parameters.yaml

@@ -1,27 +1,33 @@
 # Parameters file for Surveillance Bot
-datapath: '/data/SDS/'      # SC3 Datapath
+datapath: "/data/SDS/"                    # SC3 Datapath
-outpath_html: '/home/marcel/tmp/survBot_out.html' # output of HTML table
+networks: ["1Y", "HA", "MK"]              # select networks, list or str
-networks: ['1Y', 'HA']
+stations: "*"                             # select stations, list or str
-stations: '*'
+locations: "*"                            # select locations, list or str
-locations: '*'
+stations_blacklist: ["TEST", "EREA", "DOMV"]      # exclude these stations
-channels: ['EX1', 'EX2', 'EX3', 'VEI']    # Specify SOH channels, currently supported EX[1-3] and VEI
+networks_blacklist: []                    # exclude these networks
-stations_blacklist: ['TEST', 'EREA']
+interval: 60               # Perform checks every x seconds
-networks_blacklist: []
+n_track: 360               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
-interval: 20               # Perform checks every x seconds
+timespan: 3                # Check data of the recent x days
-timespan: 7                # Check data of the recent x days
+verbosity: 0               # verbosity flag
-verbosity: 0
 track_changes: True        # tracks all changes since GUI startup by text highlighting (GUI only)
+warn_count: False          # show number of warnings and errors in table
+min_sample: 5              # minimum samples for raising Warn/FAIL
+dt_thresh: [300, 1800]     # threshold (s) for timing delay colourisation (yellow/red)
+html_figures: True         # Create html figure directory and links
+reread_parameters: True    # reread parameters file (change parameters on runtime, not for itself/GUI refresh/datapath)
 
 POWBOX:
   pb_ok: 1  # Voltage for PowBox OK
   pb_SOH2: # PowBox channel 2 voltage translations
-    1: {"230V": 'OK', "12V": "OK"}
+    -1: {"230V": "PBox under 1V", "12V": "PBox under 1V"}
+    1: {"230V": "OK", "12V": "OK"}
     2: {"230V": "OFF", "12V": "OK"}
     3: {"230V": "OK", "12V": "overvoltage"}
     4: {"230V": "OK", "12V": "undervoltage"}
     4.5: {"230V": "OFF", "12V": "overvoltage"}
     5: {"230V": "OFF", "12V": "undervoltage"}
   pb_SOH3: # PowBox channel 3 voltage translations
+    -1: {"router": "PBox under 1V", "charger": "PBox under 1V"}
     1: {"router": "OK", "charger": "OK"}
     2: {"router": "OK", "charger": "0 < resets < 3"}
     2.5: {"router": "OK", "charger": "locked"}
@@ -29,10 +35,108 @@ POWBOX:
     4: {"router": "FAIL", "charger": "0 < resets < 3"}
     5: {"router": "FAIL", "charger": "locked"}
 
-
+# Thresholds for program warnings/voltage classifications
 THRESHOLDS:
   pb_thresh: 0.2               # Threshold for PowBox Voltage check +/- (V)
   max_temp: 50                 # max temperature for temperature warning
   low_volt: 12                 # min voltage for low voltage warning
   high_volt: 14.8              # max voltage for over voltage warning
   unclassified: 5              # min voltage samples not classified for warning
+  max_vm_warn: 1.5             # threshold for mass offset (warn), fail)
+  max_vm_fail: 2.5             # threshold for mass offset (warn), fail)
+  clockquality_warn: 90        # warn level - clock quality ranges from 0 % to 100 % with 100 % being the best level
+  clockquality_fail: 70        # fail level
+  min_gap: 0.1                 # minimum for gap declaration, should be > 0 [s]
+
+
+# ---------------------------------- Specification of input channels ---------------------------------------------------
+# Currently supported: EX[1-3], VEI, VM[1-3], LCQ
+#
+# For each channel a factor 'unit' for unit conversion (e.g. to SI) can be provided, as well as a 'name'
+# and 'ticks' [ymin, ymax, ystep] for plotting.
+# 'warn' and 'fail' plot horizontal lines in corresponding colors (can be str in TRESHOLDS, int/float or iterable)
+#
+# 'transform' can be provided for plotting to perform arithmetic operations in given order, e.g.:
+#  transform: - ["*", 20]
+#             - ["-", 20]
+#  --> PBox EX1 V to deg C: 20 * x -20
+CHANNELS:
+  EX1:
+    unit: 1e-6
+    name: "PowBox Temperature (°C)"
+    ticks: [-10, 50, 10]
+    transform:
+      - ["*", 20]
+      - ["-", 20]
+    warn: "max_temp"
+  EX2:
+    unit: 1e-6
+    name: "PowBox 230V/12V (V)"
+    ticks: [0, 5, 1]
+    warn: [2, 3, 4, 4.5, 5]
+  EX3:
+    unit: 1e-6
+    name: "PowBox Router/Charger (V)"
+    ticks: [0, 5, 1]
+    warn: [2, 2.5, 3, 4, 5]
+  VEI:
+    unit: 1e-3
+    name: "Datalogger (V)"
+    ticks: [9, 15, 1]
+    warn: ["low_volt", "high_volt"]
+    fail: 10.5
+  VM1:
+    unit: 1e-6
+    name: "Mass position W (V)"
+    ticks: [-2.5, 2.5, 1]
+    warn: [-1.5, 1.5]
+    fail: [-2.5, 2.5]
+  VM2:
+    unit: 1e-6
+    name: "Mass position V (V)"
+    ticks: [-2.5, 2.5, 1]
+    warn: [-1.5, 1.5]
+    fail: [-2.5, 2.5]
+  VM3:
+    unit: 1e-6
+    name: "Mass position U (V)"
+    ticks: [-2.5, 2.5, 1]
+    warn: [-1.5, 1.5]
+    fail: [-2.5, 2.5]
+  LCQ:
+    name: "Clock quality (%)"
+    ticks: [0, 100, 20]
+    warn: "clockquality_warn"
+    fail: "clockquality_fail"
+
+# specify data channels (can be additional to the above). From these channels only headers will be read
+data_channels: ["HHZ", "HHN", "HHE"]
+
+
+# ---------------------------------------- OPTIONAL PARAMETERS ---------------------------------------------------------
+
+# add links to html table with specified key as column and value as relative link, interpretable string parameters:
+# nw (e.g. 1Y), st (e.g. GR01A), nwst_id (e.g. 1Y.GR01A)
+add_links:
+  # for example: slmon: {"URL": "path/{nw}_{st}.html", "text": "link"}
+  slmon: {"URL": "../slmon/{nw}_{st}.html", "text": "show"}
+  24h-plot: {"URL": "../scheli/{nw}/{st}.png", "text": "plot"}
+
+# add station-independent links below html table (list items separated with -)
+add_global_links:
+  # for example: - {"text": "our homepage", "URL": "https://www.rub.de"}
+  - {"text": "show recent events on map",
+     "URL": "https://fdsnws.geophysik.ruhr-uni-bochum.de/map/?lat=39.5&lon=21&zoom=7&baselayer=mapnik"}
+
+# html logo at page bottom (path relative to html directory)
+html_logo: "figures/Logo_RUB_BLAU_rgb.png"
+
+# E-mail notifications
+EMAIL:
+  mailserver: "localhost"
+  addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"]  # list of mail addresses for info mails
+  sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender
+  stations_blacklist: ['GR33']                     # do not send emails for specific stations
+  networks_blacklist: []                           # do not send emails for specific network
+  # specify recipients for single stations in a yaml: key = email-address, val = station list (e.g. [1Y.GR01, 1Y.GR02])
+  external_mail_list: "mailing_list.yaml"

stylesheets/desktop.css

@@ -0,0 +1,58 @@
+body {
+    background-color: #ffffff;
+    place-items: center;
+    text-align: center;
+    padding-bottom: 30px;
+    font-family: "Helvetica", "sans-serif";
+}
+
+table {
+    position: relative
+}
+
+td {
+    border-radius: 2px;
+    padding: 0px;
+    white-space: nowrap;
+}
+
+th {
+    background-color: #999;
+    color: #fff;
+    border-radius: 2px;
+    padding: 3px 1px;
+    position: sticky;
+    top: 0;
+}
+
+a:link, a:visited {
+    background-color: #e8e8e8;
+    color: #000;
+    text-decoration: none;
+    display: block;
+    border-radius: 4px;
+    border: 1px solid #ccc;
+}
+
+a:hover {
+    background-color: #aaa;
+    display: block;			  
+}
+
+.blink-bg {
+    animation: blinkingBackground 2s infinite;
+}
+@keyframes blinkingBackground{
+    0%		{ background-color: #ffcc00;}
+    50%		{ background-color: #ff3200;}
+    100%	{ background-color: #ffcc00;}
+}
+
+.footer {
+   position: fixed;
+   left: 0;
+   bottom: 0;
+   width: 100%;
+   height: 50px;
+   text-align: center;
+}

stylesheets/mobile.css

@@ -0,0 +1,62 @@
+body {
+    background-color: #ffffff;
+    place-items: center;
+    text-align: center;
+    padding-bottom: 30px;
+    font-family: "Helvetica", "sans-serif";
+}
+
+table {
+    position: relative
+}
+
+td {
+    border-radius: 3px;
+    padding: 10px 2px;
+    white-space: nowrap;
+}
+
+th {
+    background-color: #999;
+    color: #fff;
+    border-radius: 3px;
+    padding: 10px, 2px;
+    position: sticky;
+    top: 0;
+}
+
+a:link, a:visited {
+    background-color: #e8e8e8;
+    color: #000;
+    text-decoration: none;
+    display: block;
+    border-radius: 6px;
+    border: 1px solid #ccc;
+}
+
+a:hover {
+    background-color: #aaa;
+    display: block;			  
+}
+
+.hidden-mobile {
+    display: none;
+}
+
+.blink-bg {
+    animation: blinkingBackground 2s infinite;
+}
+@keyframes blinkingBackground{
+    0%		{ background-color: #ffcc00;}
+    50%		{ background-color: #ff3200;}
+    100%	{ background-color: #ffcc00;}
+}
+
+.footer {
+   position: fixed;
+   left: 0;
+   bottom: 0;
+   width: 100%;
+   height: 50px;
+   text-align: center;
+}

submit_bot.sh

@@ -2,15 +2,18 @@
 ulimit -s 8192
 
 #$ -l low
-#$ -l os=*stretch
+#$ -l h_vmem=5G
 #$ -cwd
 #$ -pe smp 1
-##$ -q "*@minos15"
+#$ -N survBot_bg
-
+#$ -l os=*stretch
-export PYTHONPATH="$PYTHONPATH:/home/marcel/git/"
-export PYTHONPATH="$PYTHONPATH:/home/marcel/git/code_base/"
 
 source /opt/anaconda3/etc/profile.d/conda.sh
 conda activate py37
 
-python survBot.py
+# environment variables for numpy to prevent multi threading
+export MKL_NUM_THREADS=1
+export NUMEXPR_NUM_THREADS=1
+export OMP_NUM_THREADS=1
+
+python survBot.py -html '/data/www/~marcel/'

survBotGUI.py

@@ -1,4 +1,5 @@
-#! /usr/bin/env python
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
 """
 GUI overlay for the main survBot to show quality control of different stations specified in parameters.yaml file.
 """
@@ -9,7 +10,6 @@ __author__ = 'Marcel Paffrath'
 import os
 import sys
 import traceback
-import argparse
 
 try:
     from PySide2 import QtGui, QtCore, QtWidgets
@@ -22,7 +22,6 @@ except ImportError:
         except ImportError:
             raise ImportError('Could import neither of PySide2, PySide6 or PyQt5')
 
-import matplotlib
 from matplotlib.figure import Figure
 
 if QtGui.__package__ in ['PySide2', 'PyQt5', 'PySide6']:
@@ -35,6 +34,7 @@ from obspy import UTCDateTime
 
 from survBot import SurveillanceBot
 from write_utils import *
+from utils import get_bg_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds
 
 try:
     from rest_api.utils import get_station_iccid
@@ -77,24 +77,14 @@ class Thread(QtCore.QThread):
 
 
 class MainWindow(QtWidgets.QMainWindow):
-    def __init__(self, parameters='parameters.yaml', dt_thresh=(300, 1800)):
+    def __init__(self, parameters='parameters.yaml'):
         """
         Main window of survBot GUI.
         :param parameters: Parameters dictionary file (yaml format)
-        :param dt_thresh: threshold for timing delay colourisation (yellow/red)
         """
         super(MainWindow, self).__init__()
 
-        # some GUI default colors
-        self.colors_dict = {'FAIL': (255, 50, 0, 255),
-                            'NO DATA': (255, 255, 125, 255),
-                            'WARN': (255, 255, 125, 255),
-                            'WARNX': lambda x: (min([255, 200 + x**2]), 255, 125, 255),
-                            'OK': (125, 255, 125, 255),
-                            'undefined': (230, 230, 230, 255)}
-
         # init some attributes
-        self.dt_thresh = dt_thresh
         self.last_mouse_loc = None
         self.status_message = ''
         self.starttime = UTCDateTime()
@@ -109,6 +99,7 @@ class MainWindow(QtWidgets.QMainWindow):
         self.survBot = SurveillanceBot(parameter_path=parameters)
         self.parameters = self.survBot.parameters
         self.refresh_period = self.parameters.get('interval')
+        self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')]
 
         # create thread that is used to update
         self.thread = Thread(parent=self, runnable=self.survBot.execute_qc)
@@ -138,10 +129,10 @@ class MainWindow(QtWidgets.QMainWindow):
         self.table.setRowCount(len(station_list))
         self.table.setHorizontalHeaderLabels(keys)
 
-        for index, st_id in enumerate(station_list):
+        for index, nwst_id in enumerate(station_list):
             item = QtWidgets.QTableWidgetItem()
-            item.setText(str(st_id.rstrip('.')))
+            item.setText(str(nwst_id.rstrip('.')))
-            item.setData(QtCore.Qt.UserRole, st_id)
+            item.setData(QtCore.Qt.UserRole, nwst_id)
             self.table.setVerticalHeaderItem(index, item)
 
         self.main_layout.addWidget(self.table)
@@ -183,78 +174,43 @@ class MainWindow(QtWidgets.QMainWindow):
             self.last_mouse_loc = event.pos()
         return super(QtWidgets.QMainWindow, self).eventFilter(object, event)
 
-    def write_html_table(self):
-        fnout = self.parameters.get('outpath_html')
-        if not fnout:
-            return
-        try:
-            with open(fnout, 'w') as outfile:
-                write_html_header(outfile)
-                #write_html_table_title(outfile, self.parameters)
-                init_html_table(outfile)
-                nrows = self.table.rowCount()
-                ncolumns = self.table.columnCount()
-
-                # add header item 0 fix default black bg color for headers
-                station_header = QtWidgets.QTableWidgetItem(text='Station')
-                station_header.setText('Station')
-                header_items = [station_header]
-                for column in range(ncolumns):
-                    hheader = self.table.horizontalHeaderItem(column)
-                    header_items.append(hheader)
-                write_html_row(outfile, header_items, html_key='th')
-
-                for row in range(nrows):
-                    vheader = self.table.verticalHeaderItem(row)
-                    col_items = [vheader]
-                    for column in range(ncolumns):
-                        col_items.append(self.table.item(row, column))
-                    write_html_row(outfile, col_items)
-
-                finish_html_table(outfile)
-                write_html_text(outfile, self.status_message)
-                write_html_footer(outfile)
-        except Exception as e:
-            print(f'Could not write HTML table to {fnout}:')
-            print(e)
-
     def sms_context_menu(self, row_ind):
         """ Open a context menu when left-clicking vertical header item """
         header_item = self.table.verticalHeaderItem(row_ind)
         if not header_item:
            return
-        st_id = header_item.data(QtCore.Qt.UserRole)
+        nwst_id = header_item.data(QtCore.Qt.UserRole)
 
         context_menu = QtWidgets.QMenu()
         read_sms = context_menu.addAction('Get last SMS')
         send_sms = context_menu.addAction('Send SMS')
         action = context_menu.exec_(self.mapToGlobal(self.last_mouse_loc))
         if action == read_sms:
-           self.read_sms(st_id)
+           self.read_sms(nwst_id)
         elif action == send_sms:
-           self.send_sms(st_id)
+           self.send_sms(nwst_id)
 
-    def read_sms(self, st_id):
+    def read_sms(self, nwst_id):
         """ Read recent SMS over rest_api using whereversim portal """
-        station = st_id.split('.')[1]
+        station = nwst_id.split('.')[1]
         iccid = get_station_iccid(station)
         if not iccid:
-            print('Could not find iccid for station', st_id)
+            print('Could not find iccid for station', nwst_id)
             return
         sms_widget = ReadSMSWidget(parent=self, iccid=iccid)
-        sms_widget.setWindowTitle(f'Recent SMS of station: {st_id}')
+        sms_widget.setWindowTitle(f'Recent SMS of station: {nwst_id}')
         if sms_widget.data:
             sms_widget.show()
         else:
             self.notification('No recent messages found.')
 
-    def send_sms(self, st_id):
+    def send_sms(self, nwst_id):
         """ Send SMS over rest_api using whereversim portal """
-        station = st_id.split('.')[1]
+        station = nwst_id.split('.')[1]
         iccid = get_station_iccid(station)
 
         sms_widget = SendSMSWidget(parent=self, iccid=iccid)
-        sms_widget.setWindowTitle(f'Send SMS to station: {st_id}')
+        sms_widget.setWindowTitle(f'Send SMS to station: {nwst_id}')
         sms_widget.show()
 
     def set_clear_on_refresh(self):
@@ -262,39 +218,30 @@ class MainWindow(QtWidgets.QMainWindow):
 
     def fill_status_bar(self):
         """ Set status bar text """
-        timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
+        self.status_message = self.survBot.status_message
-        self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \
-                              f'Current time (UTC) {UTCDateTime().strftime("%Y-%m-%d %H:%M:%S")} | ' \
-                              f'Refresh period: {self.refresh_period}s | '\
-                              f'Showing data of last {timespan}'
         status_bar = self.statusBar()
         status_bar.showMessage(self.status_message)
 
     def fill_table(self):
         """ Fills the table with most recent information. Executed after execute_qc thread is done or on refresh. """
+
+        # fill status bar first with new time
+        self.fill_status_bar()
+
         for col_ind, check_key in enumerate(self.survBot.keys):
-            for row_ind, st_id in enumerate(self.survBot.station_list):
+            for row_ind, nwst_id in enumerate(self.survBot.station_list):
-                status_dict, detailed_dict = self.survBot.analysis_results.get(st_id)
+                status_dict = self.survBot.analysis_results.get(nwst_id)
                 status = status_dict.get(check_key)
-                detailed_message = detailed_dict.get(check_key)
+                message, detailed_message = status.get_status_str()
-                if check_key == 'last active':
+
-                    bg_color = self.get_time_delay_color(status)
+                dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh]
-                elif check_key == 'temp':
+                bg_color = get_bg_color(check_key, status, dt_thresh)
-                    bg_color = self.get_temp_color(status)
+                if check_key == 'temp':
-                    if not type(status) in [str]:
+                    if not type(message) in [str]:
-                        status = str(status) + deg_str
+                        message = str(message) + deg_str
-                else:
-                    statussplit = status.split(' ')
-                    if len(statussplit) > 1 and statussplit[0] == 'WARN':
-                        x = int(status.split(' ')[-1].lstrip('(').rstrip(')'))
-                        bg_color = self.colors_dict.get('WARNX')(x)
-                    else:
-                        bg_color = self.colors_dict.get(status)
-                if not bg_color:
-                    bg_color = self.colors_dict.get('undefined')
 
                 # Continue if nothing changed
-                text = str(status)
+                text = str(message)
                 cur_item = self.table.item(row_ind, col_ind)
                 if cur_item and text == cur_item.text():
                     if not self.parameters.get('track_changes') or self.clear_on_refresh:
@@ -305,9 +252,9 @@ class MainWindow(QtWidgets.QMainWindow):
 
                 # Create new data item
                 item = QtWidgets.QTableWidgetItem()
-                item.setText(str(status))
+                item.setText(str(message))
                 item.setTextAlignment(QtCore.Qt.AlignCenter)
-                item.setData(QtCore.Qt.UserRole, (st_id, check_key))
+                item.setData(QtCore.Qt.UserRole, (nwst_id, check_key))
 
                 # if text changed (known from above) set highlight color/font else (new init) set to default
                 cur_item = self.table.item(row_ind, col_ind)
@@ -330,26 +277,6 @@ class MainWindow(QtWidgets.QMainWindow):
 
         # table filling/refreshing done, set clear_on_refresh to False
         self.clear_on_refresh = False
-        # write html output if parameter is set
-        self.write_html_table()
-
-    def get_time_delay_color(self, dt):
-        """ Set color of time delay after thresholds specified in self.dt_thresh """
-        dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh]
-        if dt < dt_thresh[0]:
-            return self.colors_dict.get('OK')
-        elif dt_thresh[0] <= dt < dt_thresh[1]:
-            return self.colors_dict.get('WARN')
-        return self.colors_dict.get('FAIL')
-
-    def get_temp_color(self, temp, vmin=-10, vmax=60, cmap='coolwarm'):
-        """ Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """
-        if type(temp) in [str]:
-            return self.colors_dict.get('undefined')
-        cmap = matplotlib.cm.get_cmap(cmap)
-        val = (temp - vmin) / (vmax - vmin)
-        rgba = [int(255 * c) for c in cmap(val)]
-        return rgba
 
     def set_font_bold(self, item):
         """ Set item font bold """
@@ -382,12 +309,17 @@ class MainWindow(QtWidgets.QMainWindow):
             vheader.setSectionResizeMode(index, QtWidgets.QHeaderView.Stretch)
 
     def plot_stream(self, item):
-        st_id, check = item.data(QtCore.Qt.UserRole)
+        nwst_id, check = item.data(QtCore.Qt.UserRole)
-        st = self.survBot.data.get(st_id)
+        st = self.survBot.data.get(nwst_id)
         if st:
             self.plot_widget = PlotWidget(self)
-            self.plot_widget.setWindowTitle(st_id)
+            self.plot_widget.setWindowTitle(nwst_id)
+            st = modify_stream_for_plot(st, parameters=self.parameters)
             st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig)
+            # set_axis_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
+            set_axis_yticks(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
+            set_axis_color(fig=self.plot_widget.canvas.fig)
+            plot_axis_thresholds(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
             self.plot_widget.show()
 
     def notification(self, text):

survBotGUI.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-ulimit -s 8192
-
-#$ -l os=*stretch
-##$ -cwd
-#$ -pe smp 1
-##$ -q "*@minos15"
-
-export PYTHONPATH="$PYTHONPATH:/home/marcel/git/"
-
-source /opt/anaconda3/etc/profile.d/conda.sh
-conda activate py37
-
-python /home/marcel/git/survBot/survBotGUI.py

utils.py

@@ -0,0 +1,248 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import matplotlib
+import numpy as np
+
+from obspy import Stream
+
+
+def get_bg_color(check_key, status, dt_thresh=None, hex=False):
+    message = status.message
+    if check_key == 'last active':
+        bg_color = get_time_delay_color(message, dt_thresh)
+    elif check_key == 'temp':
+        bg_color = get_temp_color(message)
+    elif check_key == 'mass':
+        bg_color = get_mass_color(message)
+    else:
+        if status.is_warn:
+            bg_color = get_warn_color(status.count)
+        elif status.is_error:
+            if status.connection_error:
+                bg_color = get_color('disc')
+            else:
+                bg_color = get_color('FAIL')
+        else:
+            bg_color = get_color(message)
+    if not bg_color:
+        bg_color = get_color('undefined')
+
+    if hex:
+        bg_color = '#{:02x}{:02x}{:02x}'.format(*bg_color[:3])
+    return bg_color
+
+
+def get_color(key):
+    # some old GUI default colors
+    # colors_dict = {'FAIL': (255, 85, 50, 255),
+    #                'NO DATA': (255, 255, 125, 255),
+    #                'WARN': (255, 255, 80, 255),
+    #                'OK': (173, 255, 133, 255),
+    #                'undefined': (230, 230, 230, 255),
+    #                'disc': (255, 160, 40, 255),}
+    colors_dict = {'FAIL': (195, 29, 14, 255),
+                   'NO DATA': (255, 255, 125, 255),
+                   'WARN': (250, 192, 63, 255),
+                   'OK': (185, 245, 145, 255),
+                   'undefined': (240, 240, 240, 255),
+                   'disc': (126, 127, 131, 255), }
+    return colors_dict.get(key)
+
+
+def get_color_mpl(key):
+    color_tup = get_color(key)
+    return np.array([color/255. for color in color_tup])
+
+
+def get_time_delay_color(dt, dt_thresh):
+    """ Set color of time delay after thresholds specified in self.dt_thresh """
+    if isinstance(dt, type(dt_thresh[0])):
+        if dt < dt_thresh[0]:
+            return get_color('OK')
+        elif dt_thresh[0] <= dt < dt_thresh[1]:
+            return get_color('WARN')
+    return get_color('FAIL')
+
+
+def get_warn_color(count, n_colors=20):
+    if count >= n_colors:
+        count = -1
+    gradient = np.linspace((240, 245, 110, 255), (250, 192, 63, 255), n_colors, dtype=int)
+    return tuple(gradient[count])
+
+
+def get_mass_color(message):
+    # can change this to something else if wanted. This way it always returns get_color (without warn count)
+    if isinstance(message, (float, int)):
+        return get_color('OK')
+    return get_color(message)
+
+
+def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
+    """ Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """
+    if type(temp) in [str]:
+        return get_color('undefined')
+    cmap = matplotlib.cm.get_cmap(cmap)
+    val = (temp - vmin) / (vmax - vmin)
+    rgba = [int(255 * c) for c in cmap(val)]
+    return rgba
+
+
+def get_font_color(bg_color, hex=False):
+    if hex:
+        bg_color = matplotlib.colors.to_rgb(bg_color)
+    bg_color_hsv = matplotlib.colors.rgb_to_hsv(bg_color)
+    bg_color_hsl = hsv_to_hsl(bg_color_hsv)
+    font_color = (255, 255, 255, 255) if bg_color_hsl[2] < 0.6 else (0, 0, 0, 255)
+    if hex:
+        font_color = '#{:02x}{:02x}{:02x}'.format(*font_color[:3])
+    return font_color
+
+
+def hsv_to_hsl(hsv):
+    hue, saturation, value = hsv
+    lightness = value * (1 - saturation / 2)
+    saturation = 0 if lightness in (0, 1) else (value - lightness) / min(lightness, 1 - lightness)
+    return hue, saturation, lightness
+
+
+def modify_stream_for_plot(input_stream, parameters):
+    """ copy (if necessary) and modify stream for plotting """
+
+    # make a copy
+    st = Stream()
+
+    channels_dict = parameters.get('CHANNELS')
+
+    # iterate over all channels and put them to new stream in order
+    for index, ch_tup in enumerate(channels_dict.items()):
+        # unpack tuple from items
+        channel, channel_dict = ch_tup
+
+        # get correct channel from stream
+        st_sel = input_stream.select(channel=channel)
+        # in case there are != 1 there is ambiguity
+        if not len(st_sel) == 1:
+            continue
+
+        # make a copy to not modify original stream!
+        tr = st_sel[0].copy()
+
+        # multiply with conversion factor for unit
+        unit_factor = channel_dict.get('unit')
+        if unit_factor:
+            tr.data = tr.data * float(unit_factor)
+
+        # apply transformations if provided
+        transform = channel_dict.get('transform')
+        if transform:
+            tr.data = transform_trace(tr.data, transform)
+
+        # modify trace id to maintain plotting order
+        name = channel_dict.get('name')
+        tr.id = f'{index + 1}: {name} - {tr.id}'
+
+        st.append(tr)
+
+    return st
+
+
+def transform_trace(data, transf):
+    """
+    Transform trace with arithmetic operations in order, specified in transf
+    @param data: numpy array
+    @param transf: list of lists with arithmetic operations (e.g. [['*', '20'], ] -> multiply data by 20
+    """
+    # This looks a little bit hardcoded, however it is safer than using e.g. "eval"
+    for operator_str, val in transf:
+        if operator_str == '+':
+            data = data + val
+        elif operator_str == '-':
+            data = data - val
+        elif operator_str == '*':
+            data = data * val
+        elif operator_str == '/':
+            data = data / val
+        else:
+            raise IOError(f'Unknown arithmethic operator string: {operator_str}')
+
+    return data
+
+
+def set_axis_ylabels(fig, parameters, verbosity=0):
+    """
+    Adds channel names to y-axis if defined in parameters.
+    """
+    names = [channel.get('name') for channel in parameters.get('CHANNELS').values()]
+    if not names: # or not len(st.traces):
+        return
+    if not len(names) == len(fig.axes):
+        if verbosity:
+            print('Mismatch in axis and label lengths. Not adding plot labels')
+        return
+    for channel_name, ax in zip(names, fig.axes):
+        if channel_name:
+            ax.set_ylabel(channel_name)
+
+
+def set_axis_color(fig, color='0.8'):
+    """
+    Set all axes of figure to specific color
+    """
+    for ax in fig.axes:
+        for key in ['bottom', 'top', 'right', 'left']:
+            ax.spines[key].set_color(color)
+
+
+def set_axis_yticks(fig, parameters, verbosity=0):
+    """
+    Adds channel names to y-axis if defined in parameters.
+    """
+    ticks = [channel.get('ticks') for channel in parameters.get('CHANNELS').values()]
+    if not ticks:
+        return
+    if not len(ticks) == len(fig.axes):
+        if verbosity:
+            print('Mismatch in axis tick and label lengths. Not changing plot ticks.')
+        return
+    for ytick_tripple, ax in zip(ticks, fig.axes):
+        if not ytick_tripple:
+            continue
+        ymin, ymax, step = ytick_tripple
+
+        yticks = list(np.arange(ymin, ymax + step, step))
+        ax.set_yticks(yticks)
+        ax.set_ylim(ymin - 0.33 * step, ymax + 0.33 * step)
+
+
+def plot_axis_thresholds(fig, parameters, verbosity=0):
+    """
+    Adds channel thresholds (warn, fail) to y-axis if defined in parameters.
+    """
+    if verbosity > 0:
+        print('Plotting trace thresholds')
+
+    keys_colors = {'warn': dict(color=0.8 * get_color_mpl('WARN'), linestyle=(0, (5, 10)), alpha=0.5, linewidth=0.7),
+                   'fail': dict(color=0.8 * get_color_mpl('FAIL'), linestyle='solid', alpha=0.5, linewidth=0.7)}
+
+    for key, kwargs in keys_colors.items():
+        channel_threshold_list = [channel.get(key) for channel in parameters.get('CHANNELS').values()]
+        if not channel_threshold_list:
+            continue
+        plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs)
+
+
+def plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs):
+    for channel_thresholds, ax in zip(channel_threshold_list, fig.axes):
+        if not channel_thresholds:
+            continue
+
+        if not isinstance(channel_thresholds, (list, tuple)):
+            channel_thresholds = [channel_thresholds]
+
+        for warn_thresh in channel_thresholds:
+            if isinstance(warn_thresh, str):
+                warn_thresh = parameters.get('THRESHOLDS').get(warn_thresh)
+            if type(warn_thresh in (float, int)):
+                ax.axhline(warn_thresh, **kwargs)

write_utils.py

@@ -1,48 +1,97 @@
+from base64 import b64encode
 from datetime import timedelta
 
-def write_html_table_title(fobj, parameters):
+
+def _convert_to_textstring(lst):
+    return '\n'.join(lst)
+
+
+def get_html_table_title(parameters):
     title = get_print_title_str(parameters)
-    fobj.write(f'<h3>{title}</h3>\n')
+    return f'<h3>{title}</h3>\n'
 
-def write_html_text(fobj, text):
-    fobj.write(f'<p>{text}</p>\n')
 
-def write_html_header(fobj):
+def get_html_text(text):
+    return f'<p>{text}</p>\n'
+
+
+def get_html_header(refresh_rate=10):
     header = ['<!DOCTYPE html>',
               '<html>',
-              '<style>',
+              '<head>',
-              'table, th, td {',
+              '  <link rel="stylesheet" media="only screen and (max-width: 400px)" href="mobile.css" />',
-              'border:1px solid black;',
+              '  <link rel="stylesheet" media="only screen and (min-width: 401px)" href="desktop.css" />',
-              '}',
+              '</head>',
-              '</style>',
+              f'<meta http-equiv="refresh" content="{refresh_rate}" >',
+              '<meta charset="utf-8">',
+              '<meta name="viewport" content="width=device-width, initial-scale=1">',
               '<body>']
-    for item in header:
+    header = _convert_to_textstring(header)
-        fobj.write(item + '\n')
+    return header
 
-def init_html_table(fobj):
-    fobj.write('<table style="width:100%">\n')
 
-def finish_html_table(fobj):
+def get_mail_html_header():
-    fobj.write('</table>\n')
+    header = ['<html>',
+              '<head>',
+              '</head>',
+              '<body>']
+    header = _convert_to_textstring(header)
+    return header
 
-def write_html_footer(fobj):
-    footer = ['</body>',
-              '</html>']
-    for item in footer:
-        fobj.write(item + '\n')
 
-def write_html_row(fobj, items, html_key='td'):
+def init_html_table():
-    fobj.write('<tr>\n')
+    return '<table style="width:100%">\n'
+
+
+def finish_html_table():
+    return '</table>\n'
+
+
+def html_footer(footer_logo=None):
+    footer = ['</body>']
+    if footer_logo:
+        logo_items = [f'<div class="footer">',
+                      f'  <img style="float: right; padding: 10px;" src="{footer_logo}" height=30px>',
+                      f'</div>']
+        footer += logo_items
+    footer.append('</html>\n')
+
+    footer = _convert_to_textstring(footer)
+    return footer
+
+
+def add_html_image(img_data, img_format='png'):
+    return f"""<br>\n<img width="100%" src="data:image/{img_format};base64, {b64encode(img_data).decode('ascii')}">"""
+
+
+def get_html_link(text, link):
+    return f'<a href="{link}"> {text} </a>'
+
+
+def get_html_row(items, html_key='td'):
+    row_string = ''
+    default_space = '  '
+    row_string += default_space + '<tr>\n'
     for item in items:
-        text = item.text()
+        text = item.get('text')
-        color = item.backgroundColor().name()
+        if item.get('bold'):
-        # fix for black background of headers
+            text = '<b>' + text + '</b>'
-        color = '#e6e6e6' if color == '#000000' else color
+        if item.get('italic'):
-        fobj.write(f'<{html_key} bgcolor="{color}">' + text + f'</{html_key}>\n')
+            text = '<i>' + text + '</i>'
-    fobj.write('</tr>\n')
+        tooltip = item.get('tooltip')
+        font_color = item.get('font_color')
+        hyperlink = item.get('hyperlink')
+        color = 'transparent' if hyperlink else item.get('color')
+        text_str = get_html_link(text, hyperlink) if hyperlink else text
+        html_class = item.get('html_class')
+        class_str = f' class="{html_class}"' if html_class else ''
+        row_string += 2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"' \
+                      + f'style="color:{font_color}"> {text_str}</{html_key}>\n'
+    row_string += default_space + '</tr>\n'
+    return row_string
+
 
 def get_print_title_str(parameters):
     timespan = parameters.get('timespan') * 24 * 3600
-    tdelta_str = str(timedelta(seconds=int(timespan)))
+    tdelta_str = str(timedelta(seconds=int(timespan))).replace(', 0:00:00', '')
     return f'Analysis table of router quality within the last {tdelta_str}'
-