Update email parameters in parameters.yaml and modify SMTP connection in survBot.py

- Update mailserver, port, user, password, and sender in parameters.yaml
- Modify SMTP connection in survBot.py to support starttls connection if server is not "localhost"
- Read password from docker secret or environment variable if set to 'DOCKER' or 'ENV' respectively

.gitignore

@@ -211,3 +211,5 @@ flycheck_*.el
 /network-security.data
 
 
+/__simulate_fail.json
+/mailing_list.yaml

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+RUN apt update && apt install -y bind9-host iputils-ping
+
+COPY . .
+
+CMD [ "python", "./survBot.py", "-html", "www" ]

README.md

@@ -1,6 +1,6 @@
 # survBot
 
-version: 0.1
+version: 0.2
 
 survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output over SOH channels
  by analysing contents of a Seiscomp3 datapath.
@@ -40,8 +40,16 @@ The GUI can be loaded via
 python survBotGui.py
 ```
 
+## Version Changes
+- surveillance of mass, clock and gaps
+- individual mailing lists for different stations
+- html mail with recent status information
+- updated web page design
+- restructured parameter file
+- recognize if PBox is disconnected
+
 ## Staff
 
 Original author: M.Paffrath (marcel.paffrath@rub.de)
 
-November 2022
+June 2023

__init__.py

@@ -0,0 +1,3 @@
+# survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output
+# over SOH channels by analysing contents of a Seiscomp3 datapath.
+__version__ = "0.2-docker"

mailing_list.yaml

@@ -0,0 +1,9 @@
+# specify mail addresses and station network ids for which information mails shall be sent, e.g.:
+# "mail.address@provider.com, mail.address2@provider2.com":
+#   - 1Y.GR01
+#   - 1Y.GR02
+# "mail.address3@provider.com":
+#   - 1Y.GR03
+
+#"kasper.fischer@rub.de":
+#    - 1Y.GR01

parameters.yaml

@@ -1,19 +1,19 @@
 # Parameters file for Surveillance Bot
-datapath: "/data/SDS/"                    # SC3 Datapath
-networks: ["1Y", "HA"]                    # select networks, list or str
+datapath: "/data/SDS/"                    # path to SDS data archive
+networks: ["1Y", "HA", "MK"]              # select networks, list or str
 stations: "*"                             # select stations, list or str
 locations: "*"                            # select locations, list or str
-channels: ["EX1", "EX2", "EX3", "VEI", "LCQ"]    # Specify SOH channels, currently supported EX[1-3], VEI and LCQ
-stations_blacklist: ["TEST", "EREA"]      # exclude these stations
+stations_blacklist: ["ATHR", "DOMV", "EREA", "GR19", "GR27", "HAVD", "LAKA", "LFKM", "TEST"]      # exclude these stations
 networks_blacklist: []                    # exclude these networks
 interval: 60               # Perform checks every x seconds
-n_track: 300               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
+n_track: 360               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
 timespan: 7                # Check data of the recent x days
-verbosity: 0               # verbosity flag
+verbosity: 0               # verbosity flag for program console output (not logging)
+logging_level: WARN        # set logging level (info, warning, debug)
 track_changes: True        # tracks all changes since GUI startup by text highlighting (GUI only)
-warn_count: False           # show number of warnings and errors in table
-min_sample: 3              # minimum samples for raising Warn/FAIL
-dt_thresh: [300, 1800]     # threshold (s) for timing delay colourisation (yellow/red)
+warn_count: False          # show number of warnings and errors in table
+min_sample: 5              # minimum samples for raising Warn/FAIL
+dt_thresh: [300, 1800]     # threshold (s) for timing delay colorization (yellow/red)
 html_figures: True         # Create html figure directory and links
 reread_parameters: True    # reread parameters file (change parameters on runtime, not for itself/GUI refresh/datapath)
 
@@ -40,11 +40,81 @@ POWBOX:
 THRESHOLDS:
   pb_thresh: 0.2               # Threshold for PowBox Voltage check +/- (V)
   max_temp: 50                 # max temperature for temperature warning
+  critical_temp: 65            # max temperature for critical warning (fail)
   low_volt: 12                 # min voltage for low voltage warning
   high_volt: 14.8              # max voltage for over voltage warning
   unclassified: 5              # min voltage samples not classified for warning
-  clockquality_warn: 90        # clock quality ranges from 0 % to 100 % with 100 % being the best level
-  clockquality_fail: 70
+  max_vm_warn: 1.5             # threshold for mass offset (warn), fail)
+  max_vm_fail: 2.5             # threshold for mass offset (warn), fail)
+  clockquality_warn: 90        # warn level - clock quality ranges from 0 % to 100 % with 100 % being the best level
+  clockquality_fail: 70        # fail level
+  min_gap: 0.1                 # minimum for gap declaration, should be > 0 [s]
+
+
+# ---------------------------------- Specification of input channels ---------------------------------------------------
+# Currently supported: EX[1-3], VEI, VM[1-3], LCQ
+#
+# For each channel a factor 'unit' for unit conversion (e.g. to SI) can be provided, as well as a 'name'
+# and 'ticks' [ymin, ymax, ystep] for plotting.
+# 'warn' and 'fail' plot horizontal lines in corresponding colors (can be str in THRESHOLDS, int/float or iterable)
+# keyword "pb_SOH2" or "pb_SOH3" can be used to extract warning values from above POWBOX parameter definition
+#
+# 'transform' can be provided for plotting to perform arithmetic operations in given order, e.g.:
+#  transform: - ["*", 20]
+#             - ["-", 20]
+#  --> PBox EX1 V to deg C: 20 * x -20
+CHANNELS:
+  EX1:
+    unit: 1e-6
+    name: "PowBox Temperature (°C)"
+    ticks: [-10, 50, 10]
+    transform:
+      - ["*", 20]
+      - ["-", 20]
+    warn: "max_temp"
+  EX2:
+    unit: 1e-6
+    name: "PowBox 230V/12V (V)"
+    ticks: [0, 5, 1]
+    warn: "pb_SOH2"
+  EX3:
+    unit: 1e-6
+    name: "PowBox Router/Charger (V)"
+    ticks: [0, 5, 1]
+    warn: "pb_SOH3"
+  VEI:
+    unit: 1e-3
+    name: "Datalogger (V)"
+    ticks: [9, 15, 1]
+    warn: ["low_volt", "high_volt"]
+    fail: 10.5
+  VM1:
+    unit: 1e-6
+    name: "Mass position W (V)"
+    ticks: [-2.5, 2.5, 1]
+    warn: [-1.5, 1.5]
+    fail: [-2.5, 2.5]
+  VM2:
+    unit: 1e-6
+    name: "Mass position V (V)"
+    ticks: [-2.5, 2.5, 1]
+    warn: [-1.5, 1.5]
+    fail: [-2.5, 2.5]
+  VM3:
+    unit: 1e-6
+    name: "Mass position U (V)"
+    ticks: [-2.5, 2.5, 1]
+    warn: [-1.5, 1.5]
+    fail: [-2.5, 2.5]
+  LCQ:
+    name: "Clock quality (%)"
+    ticks: [0, 100, 20]
+    warn: "clockquality_warn"
+    fail: "clockquality_fail"
+
+# specify data channels (can be additional to the above). From these channels only headers will be read
+data_channels: ["HHZ", "HHN", "HHE"]
+
 
 # ---------------------------------------- OPTIONAL PARAMETERS ---------------------------------------------------------
 
@@ -54,34 +124,26 @@ add_links:
   # for example: slmon: {"URL": "path/{nw}_{st}.html", "text": "link"}
   slmon: {"URL": "../slmon/{nw}_{st}.html", "text": "show"}
   24h-plot: {"URL": "../scheli/{nw}/{st}.png", "text": "plot"}
+  ppsd: {"URL": "../ppsd/{nw}.{st}.html", "text": "show"}
+
+# add station-independent links below html table (list items separated with -)
+add_global_links:
+  # for example: - {"text": "our homepage", "URL": "https://www.rub.de"}
+  - {"text": "show recent events on map",
+     "URL": "https://fdsnws.geophysik.ruhr-uni-bochum.de/map/?lat=39.5&lon=21&zoom=7&baselayer=mapnik"}
+
+# html logo at page bottom (path relative to html directory)
+html_logo: "logo.png"
 
 # E-mail notifications
 EMAIL:
-  mailserver: "localhost"
+  mailserver: "smtp.rub.de"                   # mail server
+  port: 465                               # mail port
+  user: "DOCKER"                      # mail user, read from environment variable if set to "ENV" or from docker secret if set to "DOCKER"
+  password: "DOCKER"                  # mail password, read from environment variable if set to "ENV" or from docker secret if set to "DOCKER"
   addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"]  # list of mail addresses for info mails
-  sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender
-  stations_blacklist: ['GR33']                     # do not send emails for specific stations
+  sender: "RUB SeisObs <seisobs@ruhr-uni-bochum.de>" # mail sender
+  stations_blacklist: []                     # do not send emails for specific stations
   networks_blacklist: []                           # do not send emails for specific network
-
-# names for plotting of the above defined parameter "channels" in the same order
-channel_names: ["Clock Quality (%)", "Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"] # names for plotting (optional)
-# specify y-ticks (and ylims) giving, (ymin, ymax, step) for each of the above channels (0: default)
-CHANNEL_TICKS:
-  - [0, 100, 20]
-  - [-10, 50, 10]
-  - [1, 5, 1]
-  - [1, 5, 1]
-  - [9, 15, 1]
-
-# Factor for channel to SI-units (for plotting)
-CHANNEL_UNITS:
-  EX1: 1e-6
-  EX2: 1e-6
-  EX3: 1e-6
-  VEI: 1e-3
-
-# Transform channel for plotting, perform arithmetic operations in given order, e.g.: PBox EX1 V to deg C: 20 * x -20
-CHANNEL_TRANSFORM:
-  EX1:
-    - ["*", 20]
-    - ["-", 20]
+  # specify recipients for single stations in a yaml: key = email-address, val = station list (e.g. [1Y.GR01, 1Y.GR02])
+  external_mail_list: "mailing_list.yaml"

requirements.txt

@@ -0,0 +1,36 @@
+Brotli==1.1.0
+certifi==2025.1.31
+cffi==1.17.1
+charset-normalizer==3.4.1
+contourpy==1.3.1
+cycler==0.12.1
+decorator==5.2.1
+fonttools==4.56.0
+greenlet==3.1.1
+h2==4.2.0
+hpack==4.1.0
+hyperframe==6.1.0
+idna==3.10
+kiwisolver==1.4.8
+lxml==5.3.1
+matplotlib==3.8.4
+munkres==1.1.4
+numpy==1.26.4
+obspy==1.4.1
+packaging==24.2
+pillow==11.1.0
+pip==25.0.1
+pycparser==2.22
+pyparsing==3.2.1
+PySocks==1.7.1
+python-dateutil==2.9.0.post0
+PyYAML==6.0.2
+requests==2.32.3
+scipy==1.15.2
+setuptools==75.8.2
+six==1.17.0
+SQLAlchemy==1.4.54
+unicodedata2==16.0.0
+urllib3==2.3.0
+wheel==0.45.1
+zstandard==0.23.0

stylesheets/desktop.css

@@ -1,27 +1,36 @@
 body {
     background-color: #ffffff;
-    place-items: center;
     text-align: center;
+    padding-bottom: 30px;
+    font-family: "Helvetica", "sans-serif";
+}
+
+table {
+    position: relative
 }
 
 td {
-    border-radius: 4px;
+    border-radius: 2px;
     padding: 0px;
+    white-space: nowrap;
 }
 
 th {
-    background-color: #999;
-    border-radius: 4px;
+    background-color: #17365c;
+    color: #fff;
+    border-radius: 2px;
     padding: 3px 1px;
+    position: sticky;
+    top: 0;
 }
 
 a:link, a:visited {
-    background-color: #ccc;
+    background-color: #e8e8e8;
     color: #000;
     text-decoration: none;
     display: block;
     border-radius: 4px;
-    border: 1px solid #bbb;
+    border: 1px solid #ccc;
 }
 
 a:hover {
@@ -37,3 +46,12 @@ a:hover {
     50%		{ background-color: #ff3200;}
     100%	{ background-color: #ffcc00;}
 }
+
+.footer {
+   position: fixed;
+   left: 0;
+   bottom: 0;
+   width: 100%;
+   height: 50px;
+   text-align: center;
+}

stylesheets/mobile.css

@@ -1,27 +1,36 @@
 body {
     background-color: #ffffff;
-    place-items: center;
     text-align: center;
+    padding-bottom: 30px;
+    font-family: "Helvetica", "sans-serif";
+}
+
+table {
+    position: relative
 }
 
 td {
-    border-radius: 4px;
+    border-radius: 3px;
     padding: 10px 2px;
+    white-space: nowrap;
 }
 
 th {
-    background-color: #999;
-    border-radius: 4px;
+    background-color: #17365c;
+    color: #fff;
+    border-radius: 3px;
     padding: 10px, 2px;
+    position: sticky;
+    top: 0;
 }
 
-a:link {
-    background-color: #ccc;
+a:link, a:visited {
+    background-color: #e8e8e8;
     color: #000;
     text-decoration: none;
     display: block;
-    border-radius: 4px;
-    border: 1px solid #bbb;
+    border-radius: 6px;
+    border: 1px solid #ccc;
 }
 
 a:hover {
@@ -37,7 +46,16 @@ a:hover {
     animation: blinkingBackground 2s infinite;
 }
 @keyframes blinkingBackground{
-    0%		{ background-color: #ffee00;}
+    0%		{ background-color: #ffcc00;}
     50%		{ background-color: #ff3200;}
-    100%	{ background-color: #ffee00;}
+    100%	{ background-color: #ffcc00;}
+}
+
+.footer {
+   position: fixed;
+   left: 0;
+   bottom: 0;
+   width: 100%;
+   height: 50px;
+   text-align: center;
 }

submit_bot.sh

@@ -1,19 +1,24 @@
 #!/bin/bash
-ulimit -s 8192
 
 #$ -l low
-#$ -l h_vmem=5G
+#$ -l h_vmem=2.5G
+#$ -l mem=2.5G
+#$ -l h_stack=INFINITY
 #$ -cwd
 #$ -pe smp 1
-#$ -N survBot_bg
-#$ -l os=*stretch
+#$ -binding linear:1
+#$ -N survBot
+#$ -o /data/www/~kasper/survBot/survBot_bg.log
+#$ -e /data/www/~kasper/survBot/survBot_bg.err
+#$ -m e
+#$ -M kasper.fischer@rub.de
 
 source /opt/anaconda3/etc/profile.d/conda.sh
-conda activate py37
+conda activate survBot
 
 # environment variables for numpy to prevent multi threading
 export MKL_NUM_THREADS=1
 export NUMEXPR_NUM_THREADS=1
 export OMP_NUM_THREADS=1
 
-python survBot.py -html '/data/www/~marcel/'
+python survBot.py -html '/data/www/~kasper/survBot'

survBotGUI.py

@@ -1,439 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-GUI overlay for the main survBot to show quality control of different stations specified in parameters.yaml file.
-"""
-
-__version__ = '0.1'
-__author__ = 'Marcel Paffrath'
-
-import os
-import sys
-import traceback
-
-try:
-    from PySide2 import QtGui, QtCore, QtWidgets
-except ImportError:
-    try:
-        from PySide6 import QtGui, QtCore, QtWidgets
-    except ImportError:
-        try:
-            from PyQt5 import QtGui, QtCore, QtWidgets
-        except ImportError:
-            raise ImportError('Could import neither of PySide2, PySide6 or PyQt5')
-
-from matplotlib.figure import Figure
-
-if QtGui.__package__ in ['PySide2', 'PyQt5', 'PySide6']:
-    from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
-    from matplotlib.backends.backend_qt5agg import NavigationToolbar2QT
-else:
-    raise Exception('Not implemented')
-
-from obspy import UTCDateTime
-
-from survBot import SurveillanceBot
-from write_utils import *
-from utils import get_bg_color, modify_stream_for_plot, trace_ylabels, trace_yticks
-
-try:
-    from rest_api.utils import get_station_iccid
-    from rest_api.rest_api_utils import get_last_messages, send_message, get_default_params
-    sms_funcs = True
-except ImportError:
-    print('Could not load rest_api utils, SMS functionality disabled.')
-    sms_funcs = False
-
-deg_str = '\N{DEGREE SIGN}C'
-
-
-class Thread(QtCore.QThread):
-    """
-    A simple thread that runs outside of the main event loop. Executes the function "runnable" and prevents
-    freezing of the GUI. Run method is executed outside main event loop when called with thread.start().
-    """
-    update = QtCore.Signal()
-
-    def __init__(self, parent, runnable, verbosity=0):
-        super(Thread, self).__init__(parent=parent)
-        self.setParent(parent)
-        self.verbosity = verbosity
-        self.runnable = runnable
-        self.is_active = True
-
-    def run(self):
-        """ Try to run self.runnable and emit update signal, or print Exception if failed. """
-        try:
-            t0 = UTCDateTime()
-            self.runnable()
-            self.update.emit()
-        except Exception as e:
-            self.is_active = False
-            print(e)
-            print(traceback.format_exc())
-        finally:
-            if self.verbosity > 0:
-                print(f'Time for Thread execution: {UTCDateTime() - t0}')
-
-
-class MainWindow(QtWidgets.QMainWindow):
-    def __init__(self, parameters='parameters.yaml'):
-        """
-        Main window of survBot GUI.
-        :param parameters: Parameters dictionary file (yaml format)
-        """
-        super(MainWindow, self).__init__()
-
-        # init some attributes
-        self.last_mouse_loc = None
-        self.status_message = ''
-        self.starttime = UTCDateTime()
-
-        # setup main layout of the GUI
-        self.main_layout = QtWidgets.QVBoxLayout()
-        self.centralWidget = QtWidgets.QWidget()
-        self.centralWidget.setLayout(self.main_layout)
-        self.setCentralWidget(self.centralWidget)
-
-        # init new survBot instance, set parameters and refresh
-        self.survBot = SurveillanceBot(parameter_path=parameters)
-        self.parameters = self.survBot.parameters
-        self.refresh_period = self.parameters.get('interval')
-        self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')]
-
-        # create thread that is used to update
-        self.thread = Thread(parent=self, runnable=self.survBot.execute_qc)
-        self.thread.update.connect(self.fill_table)
-
-        self.init_table()
-        self.init_buttons()
-
-        # These filters were used to track current mouse position if an event (i.e. mouseclick) is triggered
-        self.table.installEventFilter(self)
-        self.installEventFilter(self)
-
-        # initiate clear_on_refresh flag and set status bar text
-        self.clear_on_refresh = False
-        self.fill_status_bar()
-
-        # start thread that executes qc at first initiation, then activate timer for further thread activation
-        self.thread.start()
-        self.run_refresh_timer()
-
-    def init_table(self):
-        self.table = QtWidgets.QTableWidget()
-        keys = self.survBot.keys
-        station_list = self.survBot.station_list
-
-        self.table.setColumnCount(len(keys))
-        self.table.setRowCount(len(station_list))
-        self.table.setHorizontalHeaderLabels(keys)
-
-        for index, nwst_id in enumerate(station_list):
-            item = QtWidgets.QTableWidgetItem()
-            item.setText(str(nwst_id.rstrip('.')))
-            item.setData(QtCore.Qt.UserRole, nwst_id)
-            self.table.setVerticalHeaderItem(index, item)
-
-        self.main_layout.addWidget(self.table)
-
-        self.table.itemDoubleClicked.connect(self.plot_stream)
-        self.table.setEditTriggers(QtWidgets.QTableWidget.NoEditTriggers)
-
-        if sms_funcs:
-            self.table.verticalHeader().sectionClicked.connect(self.sms_context_menu)
-
-        self.set_stretch()
-
-    def init_buttons(self):
-        if self.parameters.get('track_changes'):
-            button_text = 'Clear track and refresh'
-        else:
-            button_text = 'Refresh'
-        self.clear_button = QtWidgets.QPushButton(button_text)
-        self.clear_button.setToolTip('Reset track changes and refresh table')
-        self.clear_button.clicked.connect(self.refresh)
-        self.main_layout.addWidget(self.clear_button)
-
-    def refresh(self):
-        self.set_clear_on_refresh()
-        self.run_refresh_timer()
-        self.thread.start()
-
-    def run_refresh_timer(self):
-        self.timer = QtCore.QTimer()
-        self.timer.timeout.connect(self.thread.start)
-        self.timer.start(int(self.refresh_period * 1e3))
-
-    def eventFilter(self, object, event):
-        """
-        An event filter that stores last mouse position if an event is raised by the table. All events are passed
-        to the parent class of the Mainwindow afterwards.
-        """
-        if hasattr(event, 'pos'):
-            self.last_mouse_loc = event.pos()
-        return super(QtWidgets.QMainWindow, self).eventFilter(object, event)
-
-    def sms_context_menu(self, row_ind):
-        """ Open a context menu when left-clicking vertical header item """
-        header_item = self.table.verticalHeaderItem(row_ind)
-        if not header_item:
-           return
-        nwst_id = header_item.data(QtCore.Qt.UserRole)
-
-        context_menu = QtWidgets.QMenu()
-        read_sms = context_menu.addAction('Get last SMS')
-        send_sms = context_menu.addAction('Send SMS')
-        action = context_menu.exec_(self.mapToGlobal(self.last_mouse_loc))
-        if action == read_sms:
-           self.read_sms(nwst_id)
-        elif action == send_sms:
-           self.send_sms(nwst_id)
-
-    def read_sms(self, nwst_id):
-        """ Read recent SMS over rest_api using whereversim portal """
-        station = nwst_id.split('.')[1]
-        iccid = get_station_iccid(station)
-        if not iccid:
-            print('Could not find iccid for station', nwst_id)
-            return
-        sms_widget = ReadSMSWidget(parent=self, iccid=iccid)
-        sms_widget.setWindowTitle(f'Recent SMS of station: {nwst_id}')
-        if sms_widget.data:
-            sms_widget.show()
-        else:
-            self.notification('No recent messages found.')
-
-    def send_sms(self, nwst_id):
-        """ Send SMS over rest_api using whereversim portal """
-        station = nwst_id.split('.')[1]
-        iccid = get_station_iccid(station)
-
-        sms_widget = SendSMSWidget(parent=self, iccid=iccid)
-        sms_widget.setWindowTitle(f'Send SMS to station: {nwst_id}')
-        sms_widget.show()
-
-    def set_clear_on_refresh(self):
-        self.clear_on_refresh = True
-
-    def fill_status_bar(self):
-        """ Set status bar text """
-        self.status_message = self.survBot.status_message
-        status_bar = self.statusBar()
-        status_bar.showMessage(self.status_message)
-
-    def fill_table(self):
-        """ Fills the table with most recent information. Executed after execute_qc thread is done or on refresh. """
-
-        # fill status bar first with new time
-        self.fill_status_bar()
-
-        for col_ind, check_key in enumerate(self.survBot.keys):
-            for row_ind, nwst_id in enumerate(self.survBot.station_list):
-                status_dict = self.survBot.analysis_results.get(nwst_id)
-                status = status_dict.get(check_key)
-                message, detailed_message = status.get_status_str()
-
-                dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh]
-                bg_color = get_bg_color(check_key, status, dt_thresh)
-                if check_key == 'temp':
-                    if not type(message) in [str]:
-                        message = str(message) + deg_str
-
-                # Continue if nothing changed
-                text = str(message)
-                cur_item = self.table.item(row_ind, col_ind)
-                if cur_item and text == cur_item.text():
-                    if not self.parameters.get('track_changes') or self.clear_on_refresh:
-                        # set item to default color/font and continue
-                        self.set_font(cur_item)
-                        self.set_fg_color(cur_item)
-                    continue
-
-                # Create new data item
-                item = QtWidgets.QTableWidgetItem()
-                item.setText(str(message))
-                item.setTextAlignment(QtCore.Qt.AlignCenter)
-                item.setData(QtCore.Qt.UserRole, (nwst_id, check_key))
-
-                # if text changed (known from above) set highlight color/font else (new init) set to default
-                cur_item = self.table.item(row_ind, col_ind)
-                if cur_item and check_key != 'last active':
-                    self.set_fg_color(item, (0, 0, 0, 255))
-                    self.set_font_bold(item)
-                else:
-                    self.set_fg_color(item)
-                    self.set_font(item)
-
-                # set item tooltip
-                if detailed_message:
-                    item.setToolTip(str(detailed_message))
-
-                # set bg color corresponding to current text (OK/WARN/ERROR etc.)
-                self.set_bg_color(item, bg_color)
-
-                # insert new item
-                self.table.setItem(row_ind, col_ind, item)
-
-        # table filling/refreshing done, set clear_on_refresh to False
-        self.clear_on_refresh = False
-
-    def set_font_bold(self, item):
-        """ Set item font bold """
-        f = item.font()
-        f.setWeight(QtGui.QFont.Bold)
-        item.setFont(f)
-
-    def set_font(self, item):
-        """ Set item font normal """
-        f = item.font()
-        f.setWeight(QtGui.QFont.Normal)
-        item.setFont(f)
-
-    def set_bg_color(self, item, color):
-        """ Set background color of item, color is RGBA tuple """
-        color = QtGui.QColor(*color)
-        item.setBackground(color)
-
-    def set_fg_color(self, item, color=(20, 20, 20, 255)):
-        """ Set foreground (font) color of item, color is RGBA tuple """
-        color = QtGui.QColor(*color)
-        item.setForeground(color)
-
-    def set_stretch(self):
-        hheader = self.table.horizontalHeader()
-        for index in range(hheader.count()):
-            hheader.setSectionResizeMode(index, QtWidgets.QHeaderView.Stretch)
-        vheader = self.table.verticalHeader()
-        for index in range(vheader.count()):
-            vheader.setSectionResizeMode(index, QtWidgets.QHeaderView.Stretch)
-
-    def plot_stream(self, item):
-        nwst_id, check = item.data(QtCore.Qt.UserRole)
-        st = self.survBot.data.get(nwst_id)
-        if st:
-            self.plot_widget = PlotWidget(self)
-            self.plot_widget.setWindowTitle(nwst_id)
-            st = modify_stream_for_plot(st, parameters=self.parameters)
-            st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig)
-            trace_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
-            trace_yticks(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
-            self.plot_widget.show()
-
-    def notification(self, text):
-        mbox = QtWidgets.QMessageBox()
-        mbox.setWindowTitle('Notification')
-        #mbox.setDetailedText()
-        mbox.setText(text)
-        mbox.exec_()
-
-    def closeEvent(self, event):
-        self.thread.exit()
-        event.accept()
-
-
-class PlotCanvas(FigureCanvas):
-    def __init__(self, parent=None, width=10, height=8, dpi=100):
-        self.fig = Figure(figsize=(width, height), dpi=dpi)
-        FigureCanvas.__init__(self, self.fig)
-        self.setParent(parent)
-        FigureCanvas.setSizePolicy(self, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding)
-        FigureCanvas.updateGeometry(self)
-
-
-class PlotWidget(QtWidgets.QDialog):
-    def __init__(self, *args, **kwargs):
-        QtWidgets.QDialog.__init__(self, *args, **kwargs)
-        self.setLayout(QtWidgets.QVBoxLayout())
-        self.canvas = PlotCanvas(self, width=10, height=8)
-        self.toolbar = NavigationToolbar2QT(self.canvas, self)
-        self.layout().addWidget(self.toolbar)
-        self.layout().addWidget(self.canvas)
-
-
-class ReadSMSWidget(QtWidgets.QDialog):
-    def __init__(self, iccid, *args, **kwargs):
-        QtWidgets.QDialog.__init__(self, *args, **kwargs)
-        self.setLayout(QtWidgets.QVBoxLayout())
-        self.table = QtWidgets.QTableWidget()
-        self.layout().addWidget(self.table)
-        self.resize(1280, 400)
-
-        self.iccid = iccid
-        self.data = self.print_sms_table()
-        self.set_stretch()
-
-    def print_sms_table(self, n=5, ntextbreak=40):
-        messages = []
-        params = get_default_params(self.iccid)
-        for message in get_last_messages(params, n, only_delivered=False):
-            messages.append(message)
-        if not messages:
-            return
-        # pull dates to front
-        keys = ['dateSent', 'dateModified', 'dateReceived']
-        for item in messages[0].keys():
-            if not item in keys:
-                keys.append(item)
-        self.table.setRowCount(n)
-        self.table.setColumnCount(len(keys))
-        self.table.setHorizontalHeaderLabels(keys)
-        for row_index, message in enumerate(messages):
-            for col_index, key in enumerate(keys):
-                text = message.get(key)
-                if type(text) == str and len(text) > ntextbreak:
-                    textlist = list(text)
-                    for index in range(ntextbreak, len(text), ntextbreak):
-                        textlist.insert(index, '\n')
-                    text = ''.join(textlist)
-                item = QtWidgets.QTableWidgetItem()
-                item.setText(str(text))
-                self.table.setItem(row_index, col_index, item)
-        return True
-
-    def set_stretch(self):
-        hheader = self.table.horizontalHeader()
-        nheader = hheader.count()
-        for index in range(nheader):
-            if index < nheader - 1:
-                hheader.setSectionResizeMode(index, QtWidgets.QHeaderView.ResizeToContents)
-            else:
-                hheader.setSectionResizeMode(index, QtWidgets.QHeaderView.Stretch)
-        vheader = self.table.verticalHeader()
-        for index in range(vheader.count()):
-            vheader.setSectionResizeMode(index, QtWidgets.QHeaderView.Stretch)
-
-
-
-class SendSMSWidget(QtWidgets.QDialog):
-    def __init__(self, iccid, *args, **kwargs):
-        QtWidgets.QDialog.__init__(self, *args, **kwargs)
-        self.main_layout = QtWidgets.QVBoxLayout()
-        self.setLayout(self.main_layout)
-        self.resize(400, 100)
-
-        self.line_edit = QtWidgets.QLineEdit()
-        self.main_layout.addWidget(self.line_edit)
-
-        self.iccid = iccid
-
-        self.buttonBox = QtWidgets.QDialogButtonBox(QtWidgets.QDialogButtonBox.Ok |
-                                                    QtWidgets.QDialogButtonBox.Close)
-        self.main_layout.addWidget(self.buttonBox)
-        self.buttonBox.accepted.connect(self.send_sms)
-        self.buttonBox.rejected.connect(self.reject)
-
-    def send_sms(self):
-        text = self.line_edit.text()
-        params = get_default_params(self.iccid)
-        send_message(params, text)
-        self.close()
-
-
-if __name__ == '__main__':
-    program_path = sys.path[0]
-    parameters = os.path.join(program_path, 'parameters.yaml')
-    app = QtWidgets.QApplication([])
-    window = MainWindow(parameters=parameters)
-    window.showMaximized()
-    sys.exit(app.exec_())

utils.py

@@ -1,7 +1,20 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+import logging
+
 import matplotlib
+import numpy as np
+
+from obspy import Stream
+
+
+COLORS_DICT = {'FAIL': (195, 29, 14, 255),
+               'NO DATA': (255, 255, 125, 255),
+               'WARN': (250, 192, 63, 255),
+               'OK': (185, 245, 145, 255),
+               'undefined': (240, 240, 240, 255),
+               'disc': (126, 127, 131, 255), }
 
 
 def get_bg_color(check_key, status, dt_thresh=None, hex=False):
@@ -10,11 +23,16 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False):
         bg_color = get_time_delay_color(message, dt_thresh)
     elif check_key == 'temp':
         bg_color = get_temp_color(message)
+    elif check_key == 'mass':
+        bg_color = get_mass_color(message)
     else:
         if status.is_warn:
-            bg_color = get_color('WARNX')(status.count)
+            bg_color = get_warn_color(status.count)
         elif status.is_error:
-            bg_color = get_color('FAIL')
+            if status.connection_error:
+                bg_color = get_color('disc')
+            else:
+                bg_color = get_color('FAIL')
         else:
             bg_color = get_color(message)
     if not bg_color:
@@ -26,28 +44,52 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False):
 
 
 def get_color(key):
-    # some GUI default colors
-    colors_dict = {'FAIL': (255, 50, 0, 255),
-                   'NO DATA': (255, 255, 125, 255),
-                   'WARN': (255, 255, 80, 255),
-                   'WARNX': lambda x: (min([255, 200 + x ** 2]), 255, 80, 255),
-                   'OK': (125, 255, 125, 255),
-                   'undefined': (230, 230, 230, 255)}
-    return colors_dict.get(key)
+    # some old GUI default colors
+    # colors_dict = {'FAIL': (255, 85, 50, 255),
+    #                'NO DATA': (255, 255, 125, 255),
+    #                'WARN': (255, 255, 80, 255),
+    #                'OK': (173, 255, 133, 255),
+    #                'undefined': (230, 230, 230, 255),
+    #                'disc': (255, 160, 40, 255),}
+    if not key in COLORS_DICT.keys():
+        key = 'undefined'
+    return COLORS_DICT.get(key)
+
+
+def get_color_mpl(key):
+    color_tup = get_color(key)
+    return np.array([color/255. for color in color_tup])
 
 
 def get_time_delay_color(dt, dt_thresh):
     """ Set color of time delay after thresholds specified in self.dt_thresh """
-    if dt < dt_thresh[0]:
-        return get_color('OK')
-    elif dt_thresh[0] <= dt < dt_thresh[1]:
-        return get_color('WARN')
+    if isinstance(dt, type(dt_thresh[0])):
+        if dt < dt_thresh[0]:
+            return get_color('OK')
+        elif dt_thresh[0] <= dt < dt_thresh[1]:
+            return get_color('WARN')
     return get_color('FAIL')
 
 
+def get_warn_color(count, n_colors=20):
+    if count >= n_colors:
+        count = -1
+    gradient = np.linspace((240, 245, 110, 255), (250, 192, 63, 255), n_colors, dtype=int)
+    return tuple(gradient[count])
+
+
+def get_mass_color(message):
+    # can change this to something else if wanted. This way it always returns get_color (without warn count)
+    if isinstance(message, (float, int)):
+        return get_color('OK')
+    return get_color(message)
+
+
 def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
     """ Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """
     if type(temp) in [str]:
+        if temp in COLORS_DICT.keys():
+            return get_color(temp)
         return get_color('undefined')
     cmap = matplotlib.cm.get_cmap(cmap)
     val = (temp - vmin) / (vmax - vmin)
@@ -55,29 +97,61 @@ def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
     return rgba
 
 
-def modify_stream_for_plot(st, parameters):
+def get_font_color(bg_color, hex=False):
+    if hex:
+        bg_color = matplotlib.colors.to_rgb(bg_color)
+    bg_color_hsv = matplotlib.colors.rgb_to_hsv(bg_color)
+    bg_color_hsl = hsv_to_hsl(bg_color_hsv)
+    font_color = (255, 255, 255, 255) if bg_color_hsl[2] < 0.6 else (0, 0, 0, 255)
+    if hex:
+        font_color = '#{:02x}{:02x}{:02x}'.format(*font_color[:3])
+    return font_color
+
+
+def hsv_to_hsl(hsv):
+    hue, saturation, value = hsv
+    lightness = value * (1 - saturation / 2)
+    saturation = 0 if lightness in (0, 1) else (value - lightness) / min(lightness, 1 - lightness)
+    return hue, saturation, lightness
+
+
+def modify_stream_for_plot(input_stream, parameters):
     """ copy (if necessary) and modify stream for plotting """
-    ch_units = parameters.get('CHANNEL_UNITS')
-    ch_transf = parameters.get('CHANNEL_TRANSFORM')
 
-    # if either of both are defined make copy
-    if ch_units or ch_transf:
-        st = st.copy()
+    # make a copy
+    st = Stream()
 
-    # modify trace for plotting by multiplying unit factor (e.g. 1e-3 mV to V)
-    if ch_units:
-        for tr in st:
-            channel = tr.stats.channel
-            unit_factor = ch_units.get(channel)
-            if unit_factor:
-                tr.data = tr.data * float(unit_factor)
-    # modify trace for plotting by other arithmetic expressions
-    if ch_transf:
-        for tr in st:
-            channel = tr.stats.channel
-            transf = ch_transf.get(channel)
-            if transf:
-                tr.data = transform_trace(tr.data, transf)
+    channels_dict = parameters.get('CHANNELS')
+
+    # iterate over all channels and put them to new stream in order
+    for index, ch_tup in enumerate(channels_dict.items()):
+        # unpack tuple from items
+        channel, channel_dict = ch_tup
+
+        # get correct channel from stream
+        st_sel = input_stream.select(channel=channel)
+        # in case there are != 1 there is ambiguity
+        if not len(st_sel) == 1:
+            continue
+
+        # make a copy to not modify original stream!
+        tr = st_sel[0].copy()
+
+        # multiply with conversion factor for unit
+        unit_factor = channel_dict.get('unit')
+        if unit_factor:
+            tr.data = tr.data * float(unit_factor)
+
+        # apply transformations if provided
+        transform = channel_dict.get('transform')
+        if transform:
+            tr.data = transform_trace(tr.data, transform)
+
+        # modify trace id to maintain plotting order
+        name = channel_dict.get('name')
+        tr.id = f'{index + 1}: {name} - {tr.id}'
+
+        st.append(tr)
 
     return st
 
@@ -104,42 +178,104 @@ def transform_trace(data, transf):
     return data
 
 
-def trace_ylabels(fig, parameters, verbosity=0):
+def set_axis_ylabels(fig, parameters):
     """
     Adds channel names to y-axis if defined in parameters.
-    Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is
-    difficult to assess the correct order from Obspy plotting routing.
     """
-    names = parameters.get('channel_names')
+    names = [channel.get('name') for channel in parameters.get('CHANNELS').values()]
     if not names: # or not len(st.traces):
         return
     if not len(names) == len(fig.axes):
-        if verbosity:
-            print('Mismatch in axis and label lengths. Not adding plot labels')
+        logging.info('Mismatch in axis and label lengths. Not adding plot labels')
         return
     for channel_name, ax in zip(names, fig.axes):
         if channel_name:
             ax.set_ylabel(channel_name)
 
 
-def trace_yticks(fig, parameters, verbosity=0):
+def set_axis_color(fig, color='0.8', shade_color='0.95'):
+    """
+    Set all axes (frame) of figure to specific color. Shade every second axis.
+    """
+    for i, ax in enumerate(fig.axes):
+        for key in ['bottom', 'top', 'right', 'left']:
+            ax.spines[key].set_color(color)
+        if i % 2:
+            ax.set_facecolor(shade_color)
+
+
+def set_axis_yticks(fig, parameters):
     """
     Adds channel names to y-axis if defined in parameters.
-    Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is
-    difficult to assess the correct order from Obspy plotting routing.
     """
-    ticks = parameters.get('CHANNEL_TICKS')
+    ticks = [channel.get('ticks') for channel in parameters.get('CHANNELS').values()]
     if not ticks:
         return
     if not len(ticks) == len(fig.axes):
-        if verbosity:
-            print('Mismatch in axis tick and label lengths. Not changing plot ticks.')
+        logging.info('Mismatch in axis tick and label lengths. Not changing plot ticks.')
         return
     for ytick_tripple, ax in zip(ticks, fig.axes):
         if not ytick_tripple:
             continue
         ymin, ymax, step = ytick_tripple
 
-        yticks = list(range(ymin, ymax + step, step))
+        yticks = list(np.arange(ymin, ymax + step, step))
         ax.set_yticks(yticks)
-        ax.set_ylim(ymin - step, ymax + step)
+        ax.set_ylim(ymin - 0.33 * step, ymax + 0.33 * step)
+
+
+def plot_axis_thresholds(fig, parameters):
+    """
+    Adds channel thresholds (warn, fail) to y-axis if defined in parameters.
+    """
+    logging.info('Plotting trace thresholds')
+
+    keys_colors = {'warn': dict(color=0.8 * get_color_mpl('WARN'), linestyle=(0, (5, 10)), alpha=0.5, linewidth=0.7),
+                   'fail': dict(color=0.8 * get_color_mpl('FAIL'), linestyle='solid', alpha=0.5, linewidth=0.7)}
+
+    for key, kwargs in keys_colors.items():
+        channel_threshold_list = [channel.get(key) for channel in parameters.get('CHANNELS').values()]
+        if not channel_threshold_list:
+            continue
+        plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs)
+
+
+def plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs):
+    for channel_thresholds, ax in zip(channel_threshold_list, fig.axes):
+        if channel_thresholds in ['pb_SOH2', 'pb_SOH3']:
+            annotate_voltage_states(ax, parameters, channel_thresholds)
+            channel_thresholds = get_warn_states_pbox(channel_thresholds, parameters)
+
+        if not channel_thresholds:
+            continue
+
+        if not isinstance(channel_thresholds, (list, tuple)):
+            channel_thresholds = [channel_thresholds]
+
+        for warn_thresh in channel_thresholds:
+            if isinstance(warn_thresh, str):
+                warn_thresh = parameters.get('THRESHOLDS').get(warn_thresh)
+            if isinstance(warn_thresh, (float, int)):
+                ax.axhline(warn_thresh, **kwargs)
+
+
+def get_warn_states_pbox(soh_key: str, parameters: dict) -> list:
+    pb_dict = parameters.get('POWBOX').get(soh_key)
+    if not pb_dict:
+        return []
+    return [key for key in pb_dict.keys() if key > 1]
+
+
+def annotate_voltage_states(ax, parameters, pb_key, color='0.75'):
+    for voltage, voltage_dict in parameters.get('POWBOX').get(pb_key).items():
+        if float(voltage) < 1:
+            continue
+        out_string = ''
+        for key, val in voltage_dict.items():
+            if val != 'OK':
+                if out_string:
+                    out_string += ' | '
+                out_string += f'{key}: {val}'
+
+        ax.annotate(out_string, (ax.get_xlim()[-1], voltage), color=color, fontsize='xx-small',
+                    horizontalalignment='right')

write_utils.py

@@ -1,16 +1,21 @@
+from base64 import b64encode
 from datetime import timedelta
 
 
-def write_html_table_title(fobj, parameters):
+def _convert_to_textstring(lst):
+    return '\n'.join(lst)
+
+
+def get_html_table_title(parameters):
     title = get_print_title_str(parameters)
-    fobj.write(f'<h3>{title}</h3>\n')
+    return f'<h3>{title}</h3>\n'
 
 
-def write_html_text(fobj, text):
-    fobj.write(f'<p>{text}</p>\n')
+def get_html_text(text):
+    return f'<p>{text}</p>\n'
 
 
-def write_html_header(fobj, refresh_rate=10):
+def get_html_header(refresh_rate=10):
     header = ['<!DOCTYPE html>',
               '<html>',
               '<head>',
@@ -21,28 +26,52 @@ def write_html_header(fobj, refresh_rate=10):
               '<meta charset="utf-8">',
               '<meta name="viewport" content="width=device-width, initial-scale=1">',
               '<body>']
-    for item in header:
-        fobj.write(item + '\n')
+    header = _convert_to_textstring(header)
+    return header
 
 
-def init_html_table(fobj):
-    fobj.write('<table style="width:100%">\n')
+def get_mail_html_header():
+    header = ['<html>',
+              '<head>',
+              '</head>',
+              '<body>']
+    header = _convert_to_textstring(header)
+    return header
 
 
-def finish_html_table(fobj):
-    fobj.write('</table>\n')
+def init_html_table():
+    return '<table style="width:100%">\n'
 
 
-def write_html_footer(fobj):
-    footer = ['</body>',
-              '</html>']
-    for item in footer:
-        fobj.write(item + '\n')
+def finish_html_table():
+    return '</table>\n'
 
 
-def write_html_row(fobj, items, html_key='td'):
+def html_footer(footer_logo=None):
+    footer = ['</body>']
+    if footer_logo:
+        logo_items = [f'<div class="footer">',
+                      f'  <img style="float: right; padding: 10px;" src="{footer_logo}" height=30px>',
+                      f'</div>']
+        footer += logo_items
+    footer.append('</html>\n')
+
+    footer = _convert_to_textstring(footer)
+    return footer
+
+
+def add_html_image(img_data, img_format='png'):
+    return f"""<br>\n<img width="100%" src="data:image/{img_format};base64, {b64encode(img_data).decode('ascii')}">"""
+
+
+def get_html_link(text, link):
+    return f'<a href="{link}"> {text} </a>'
+
+
+def get_html_row(items, html_key='td'):
+    row_string = ''
     default_space = '  '
-    fobj.write(default_space + '<tr>\n')
+    row_string += default_space + '<tr>\n'
     for item in items:
         text = item.get('text')
         if item.get('bold'):
@@ -50,16 +79,16 @@ def write_html_row(fobj, items, html_key='td'):
         if item.get('italic'):
             text = '<i>' + text + '</i>'
         tooltip = item.get('tooltip')
-        color = item.get('color')
-        # check for black background of headers (shouldnt happen anymore)
-        color = '#e6e6e6' if color == '#000000' else color
+        font_color = item.get('font_color')
         hyperlink = item.get('hyperlink')
-        image_str = f'<a href="{hyperlink}">' if hyperlink else ''
+        color = 'transparent' if hyperlink else item.get('color')
+        text_str = get_html_link(text, hyperlink) if hyperlink else text
         html_class = item.get('html_class')
         class_str = f' class="{html_class}"' if html_class else ''
-        fobj.write(2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"> {image_str}'
-                   + text + f'</{html_key}>\n')
-    fobj.write(default_space + '</tr>\n')
+        row_string += 2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"' \
+                      + f'style="color:{font_color}"> {text_str}</{html_key}>\n'
+    row_string += default_space + '</tr>\n'
+    return row_string
 
 
 def get_print_title_str(parameters):