[bugfix] fixed footer output to produce valid html when using a logo

new parameters in for paramters.yaml file:
  - added parameter logo_alt_text
  - added parameter logo_height

.gitignore

@@ -211,3 +211,9 @@ flycheck_*.el
 /network-security.data
 
 
+/__simulate_fail.json
+/mailing_list.yaml
+
+.vscode/
+
+*.code-workspace

Dockerfile

@@ -0,0 +1,40 @@
+FROM python:3
+
+# metadata
+LABEL maintainer="Kasper D. Fischer <kasper.fischer@rub.de>"
+LABEL version="0.2-docker"
+LABEL description="Docker image for the survBot application"
+
+# install required system packages
+RUN apt update && apt install -y bind9-host iputils-ping
+
+# create user and group and home directory
+RUN groupadd -r survBot && useradd -r -g survBot survBot
+RUN mkdir -p /home/survBot && chown -R survBot:survBot /home/survBot
+
+# change working directory
+WORKDIR /usr/src/app
+
+# install required python packages
+RUN --mount=type=bind,source=requirements.txt,target=/tmp/requirements.txt \
+    pip install --no-cache-dir --requirement /tmp/requirements.txt
+
+# copy application files
+COPY survBot.py utils.py write_utils.py LICENSE README.md ./
+
+# copy configuration files
+VOLUME /usr/src/app/conf
+COPY parameters.yaml mailing_list.yam[l] simulate_fail.jso[n] conf/
+RUN ln -s conf/simulate_fail.json simulate_fail.json
+
+# copy www files
+VOLUME /usr/src/app/www
+COPY logo.pn[g] stylesheets/desktop.css stylesheets/mobile.css www/
+RUN ln -s www/survBot_out.html www/index.html
+
+# change ownership of working directory
+RUN chown -R survBot:survBot /usr/src/app
+
+# run the application as user survBot
+USER survBot
+CMD [ "python", "./survBot.py", "-html", "www", "-parfile", "conf/parameters.yaml" ]

README.md

@@ -1,9 +1,9 @@
 # survBot
 
-version: 0.1
+version: 0.2
 
 survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output over SOH channels
- by analysing contents of a Seiscomp3 datapath.
+ by analyzing contents of a Seiscomp data archive.
 
 ## Requirements
 
@@ -40,8 +40,53 @@ The GUI can be loaded via
 python survBotGui.py
 ```
 
+### Docker
+
+To run the program in a Docker container, first build the image:
+
+```shell script
+docker build -t survbot .
+```
+
+Then run the container:
+
+```shell script
+docker run -v /path/to/conf-dir:/usr/src/app/conf -v /path/to/output:/usr/src/app/www survbot
+```
+
+The directory `/path/to/conf-dir` should contain the `parameters.yaml` file, and the directory `/path/to/output` will contain the output HTML files.
+
+### Configuration of the e-mail server settings
+
+The e-mail server settings can be configured in the `parameters.yaml` file. The following settings are available:
+
+* `mailserver`: the address of the mail server
+* `auth_type`: the authentication type for the mail server (`None`, `SSL`, `TLS`)
+* `port`: the port of the mail server
+* `user`: the username for the mail server (if required)
+* `password`: the password for the mail server (if required)
+
+The `user` and `password` fields are optional, and can be left empty if the mail server does not require authentication. The `auth_type` field can be set to `None` if no authentication is required, `SSL` if the mail server requires SSL authentication, or `TLS` if the mail server requires TLS authentication. If the `user` or `password` fields are set to `Docker` ore `ENV` the program will try to read the values from the docker secrets `mail_user` and `mail_password` or environment variables `MAIL_USER` and `MAIL_PASSWORD` respectively. Docker secrets are only available in Docker Swarm mode, i.e. if the program is run as a service.
+
+## Version Changes
+
+### 0.2
+
+* surveillance of mass, clock and gaps
+* individual mailing lists for different stations
+* html mail with recent status information
+* updated web page design
+* restructured parameter file
+* recognize if PBox is disconnected
+
+### 0.2-docker
+
+* added Dockerfile for easy deployment
+* added more settings for connection to a mail server
+
 ## Staff
 
-Original author: M.Paffrath (marcel.paffrath@rub.de)
+Original author: M.Paffrath (<marcel.paffrath@rub.de>)
+Contributions: Kasper D. Fischer (<kasper.fischer@rub.de>)
 
-November 2022
+Jan 2025

__init__.py

@@ -0,0 +1,3 @@
+# survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output
+# over SOH channels by analysing contents of a Seiscomp3 datapath.
+__version__ = "0.2-docker"

mailing_list.yaml

@@ -0,0 +1,9 @@
+# specify mail addresses and station network ids for which information mails shall be sent, e.g.:
+# "mail.address@provider.com, mail.address2@provider2.com":
+#   - 1Y.GR01
+#   - 1Y.GR02
+# "mail.address3@provider.com":
+#   - 1Y.GR03
+
+#"kasper.fischer@rub.de":
+#    - 1Y.GR01

parameters.yaml

@@ -1,17 +1,18 @@
 # Parameters file for Surveillance Bot
-datapath: "/data/SDS/"                    # SC3 Datapath
-networks: ["1Y", "HA"]                    # select networks, list or str
+datapath: "/data/SDS/"                    # path to SDS data archive
+networks: ["1Y", "HA", "MK"]              # select networks, list or str
 stations: "*"                             # select stations, list or str
 locations: "*"                            # select locations, list or str
-stations_blacklist: ["TEST", "EREA"]      # exclude these stations
+stations_blacklist: ["ATHR", "DOMV", "EREA", "GR19", "GR27", "HAVD", "LAKA", "LFKM", "TEST"]      # exclude these stations
 networks_blacklist: []                    # exclude these networks
 interval: 60               # Perform checks every x seconds
-n_track: 300               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
+n_track: 360               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
 timespan: 7                # Check data of the recent x days
-verbosity: 0               # verbosity flag
+verbosity: 0               # verbosity flag for program console output (not logging)
+logging_level: WARN        # set logging level (info, warning, debug)
 track_changes: True        # tracks all changes since GUI startup by text highlighting (GUI only)
 warn_count: False          # show number of warnings and errors in table
-min_sample: 3              # minimum samples for raising Warn/FAIL
+min_sample: 5              # minimum samples for raising Warn/FAIL
 dt_thresh: [300, 1800]     # threshold (s) for timing delay colourisation (yellow/red)
 html_figures: True         # Create html figure directory and links
 reread_parameters: True    # reread parameters file (change parameters on runtime, not for itself/GUI refresh/datapath)
@@ -39,6 +40,7 @@ POWBOX:
 THRESHOLDS:
   pb_thresh: 0.2               # Threshold for PowBox Voltage check +/- (V)
   max_temp: 50                 # max temperature for temperature warning
+  critical_temp: 65            # max temperature for critical warning (fail)
   low_volt: 12                 # min voltage for low voltage warning
   high_volt: 14.8              # max voltage for over voltage warning
   unclassified: 5              # min voltage samples not classified for warning
@@ -54,7 +56,8 @@ THRESHOLDS:
 #
 # For each channel a factor 'unit' for unit conversion (e.g. to SI) can be provided, as well as a 'name'
 # and 'ticks' [ymin, ymax, ystep] for plotting.
-# 'warn' and 'fail' plot horizontal lines in corresponding colors (can be str in TRESHOLDS, int/float or iterable)
+# 'warn' and 'fail' plot horizontal lines in corresponding colors (can be str in THRESHOLDS, int/float or iterable)
+# keyword "pb_SOH2" or "pb_SOH3" can be used to extract warning values from above POWBOX parameter definition
 #
 # 'transform' can be provided for plotting to perform arithmetic operations in given order, e.g.:
 #  transform: - ["*", 20]
@@ -72,13 +75,13 @@ CHANNELS:
   EX2:
     unit: 1e-6
     name: "PowBox 230V/12V (V)"
-    ticks: [1, 5, 1]
-    warn: [2, 3, 4, 4.5, 5]
+    ticks: [0, 5, 1]
+    warn: "pb_SOH2"
   EX3:
     unit: 1e-6
     name: "PowBox Router/Charger (V)"
-    ticks: [1, 5, 1]
-    warn: [2, 2.5, 3, 4, 5]
+    ticks: [0, 5, 1]
+    warn: "pb_SOH3"
   VEI:
     unit: 1e-3
     name: "Datalogger (V)"
@@ -114,6 +117,11 @@ data_channels: ["HHZ", "HHN", "HHE"]
 
 
 # ---------------------------------------- OPTIONAL PARAMETERS ---------------------------------------------------------
+# deactivate powbox surveillance for stations (e.g. for solar powered), key: station, value: status message (abbr.)
+no_pbox_stations: {'GR33': 'SOL', 'GR27B': 'DCN', 'RP01': 'noPBox', 'RP02': 'noPBox', 'RP03': 'noPBox', 'RP04': 'noPBox', 'RP05B': 'noPBox', 'RP06': 'noPBox', 'RP07': 'noPBox', 'RP08': 'noPBox'}
+
+# deactivate mass position surveillance for stations without connected mass channels (e.g. STS-2 instruments), key: station, value: status message (abbr.)
+no_mass_stations: {'BIA': 'DCN', 'KNEZ': 'DCN', 'KRUS': 'DCN', 'OHR': 'DCN', 'OTOV': 'DCN', 'SKO': 'DCN', 'STIP': 'DCN', 'VAY': 'DCN', 'ZUPA': 'DCN'}
 
 # add links to html table with specified key as column and value as relative link, interpretable string parameters:
 # nw (e.g. 1Y), st (e.g. GR01A), nwst_id (e.g. 1Y.GR01A)
@@ -121,11 +129,31 @@ add_links:
   # for example: slmon: {"URL": "path/{nw}_{st}.html", "text": "link"}
   slmon: {"URL": "../slmon/{nw}_{st}.html", "text": "show"}
   24h-plot: {"URL": "../scheli/{nw}/{st}.png", "text": "plot"}
+  ppsd: {"URL": "../ppsd/{nw}.{st}.html", "text": "show"}
+
+# add station-independent links below html table (list items separated with -)
+add_global_links:
+  # for example: - {"text": "our homepage", "URL": "https://www.rub.de"}
+  - {"text": "show recent events on map",
+     "URL": "https://fdsnws.geophysik.ruhr-uni-bochum.de/map/?lat=39.5&lon=21&zoom=7&baselayer=osm"}
+
+# html logo at page bottom (path relative to html directory)
+html_logo: "logo.png"
 
 # E-mail notifications
 EMAIL:
-  mailserver: "localhost"
-  addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"]  # list of mail addresses for info mails
-  sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender
-  stations_blacklist: ['GR33']                     # do not send emails for specific stations
+  # specify mail server and credentials
+  # port, auth_type, user and password are only required if mailserver is not set to "localhost"
+  # user and password can be set to "ENV" or "DOCKER" to read from environment variables or docker secrets
+  mailserver: "smtp.example.org"      # mail server
+  auth_type: "SSL"                    # mail authentication type, can be "SSL", "TLS" or "None"
+  port: 465                           # mail port, default 465 for SSL, 587 for TLS
+  user: "DOCKER"                      # mail user, read from environment variable if set to "ENV" or from docker secret if set to "DOCKER"
+  password: "DOCKER"                  # mail password, read from environment variable if set to "ENV" or from docker secret if set to "DOCKER"
+  # specify mail recipients, sender and blacklists
+  addresses: ["test@example.org"]  # list of mail addresses for info mails
+  sender: "<test@example.org>"              # mail sender
+  stations_blacklist: []                                          # do not send emails for specific stations
   networks_blacklist: []                                          # do not send emails for specific network
+  # specify recipients for single stations in a yaml: key = email-address, val = station list (e.g. [1Y.GR01, 1Y.GR02])
+  external_mail_list: "conf/mailing_list.yaml"

requirements.txt

@@ -0,0 +1,36 @@
+Brotli==1.1.0
+certifi==2025.1.31
+cffi==1.17.1
+charset-normalizer==3.4.1
+contourpy==1.3.1
+cycler==0.12.1
+decorator==5.2.1
+fonttools==4.56.0
+greenlet==3.1.1
+h2==4.2.0
+hpack==4.1.0
+hyperframe==6.1.0
+idna==3.10
+kiwisolver==1.4.8
+lxml==5.3.1
+matplotlib==3.8.4
+munkres==1.1.4
+numpy==1.26.4
+obspy==1.4.1
+packaging==24.2
+pillow==11.1.0
+pip==25.0.1
+pycparser==2.22
+pyparsing==3.2.1
+PySocks==1.7.1
+python-dateutil==2.9.0.post0
+PyYAML==6.0.2
+requests==2.32.3
+scipy==1.15.2
+setuptools==75.8.2
+six==1.17.0
+SQLAlchemy==1.4.54
+unicodedata2==16.0.0
+urllib3==2.3.0
+wheel==0.45.1
+zstandard==0.23.0

stylesheets/desktop.css

@@ -1,27 +1,36 @@
 body {
     background-color: #ffffff;
-    place-items: center;
     text-align: center;
+    padding-bottom: 30px;
+    font-family: "Helvetica", "sans-serif";
+}
+
+table {
+    position: relative
 }
 
 td {
-    border-radius: 4px;
+    border-radius: 2px;
     padding: 0px;
+    white-space: nowrap;
 }
 
 th {
-    background-color: #999;
-    border-radius: 4px;
+    background-color: #17365c;
+    color: #fff;
+    border-radius: 2px;
     padding: 3px 1px;
+    position: sticky;
+    top: 0;
 }
 
 a:link, a:visited {
-    background-color: #ccc;
+    background-color: #e8e8e8;
     color: #000;
     text-decoration: none;
     display: block;
     border-radius: 4px;
-    border: 1px solid #bbb;
+    border: 1px solid #ccc;
 }
 
 a:hover {
@@ -37,3 +46,12 @@ a:hover {
     50%		{ background-color: #ff3200;}
     100%	{ background-color: #ffcc00;}
 }
+
+.footer {
+   position: fixed;
+   left: 0;
+   bottom: 0;
+   width: 100%;
+   height: 50px;
+   text-align: center;
+}

stylesheets/mobile.css

@@ -1,27 +1,36 @@
 body {
     background-color: #ffffff;
-    place-items: center;
     text-align: center;
+    padding-bottom: 30px;
+    font-family: "Helvetica", "sans-serif";
+}
+
+table {
+    position: relative
 }
 
 td {
-    border-radius: 4px;
+    border-radius: 3px;
     padding: 10px 2px;
+    white-space: nowrap;
 }
 
 th {
-    background-color: #999;
-    border-radius: 4px;
-    padding: 10px, 2px;
+    background-color: #17365c;
+    color: #fff;
+    border-radius: 3px;
+    padding: 10px 2px;
+    position: sticky;
+    top: 0;
 }
 
-a:link {
-    background-color: #ccc;
+a:link, a:visited {
+    background-color: #e8e8e8;
     color: #000;
     text-decoration: none;
     display: block;
-    border-radius: 4px;
-    border: 1px solid #bbb;
+    border-radius: 6px;
+    border: 1px solid #ccc;
 }
 
 a:hover {
@@ -37,7 +46,16 @@ a:hover {
     animation: blinkingBackground 2s infinite;
 }
 @keyframes blinkingBackground{
-    0%		{ background-color: #ffee00;}
+    0%		{ background-color: #ffcc00;}
     50%		{ background-color: #ff3200;}
-    100%	{ background-color: #ffee00;}
+    100%	{ background-color: #ffcc00;}
+}
+
+.footer {
+   position: fixed;
+   left: 0;
+   bottom: 0;
+   width: 100%;
+   height: 50px;
+   text-align: center;
 }

submit_bot.sh

@@ -1,19 +1,24 @@
 #!/bin/bash
-ulimit -s 8192
 
 #$ -l low
-#$ -l h_vmem=5G
+#$ -l h_vmem=2.5G
+#$ -l mem=2.5G
+#$ -l h_stack=INFINITY
 #$ -cwd
 #$ -pe smp 1
-#$ -N survBot_bg
-#$ -l os=*stretch
+#$ -binding linear:1
+#$ -N survBot
+#$ -o /data/www/~kasper/survBot/survBot_bg.log
+#$ -e /data/www/~kasper/survBot/survBot_bg.err
+#$ -m e
+#$ -M kasper.fischer@rub.de
 
 source /opt/anaconda3/etc/profile.d/conda.sh
-conda activate py37
+conda activate survBot
 
 # environment variables for numpy to prevent multi threading
 export MKL_NUM_THREADS=1
 export NUMEXPR_NUM_THREADS=1
 export OMP_NUM_THREADS=1
 
-python survBot.py -html '/data/www/~marcel/'
+python survBot.py -html '/data/www/~kasper/survBot'

survBot.py

@@ -1,13 +1,17 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-__version__ = '0.1'
-__author__ = 'Marcel Paffrath'
+__version__ = '0.3.1'
+__author__ = ['Marcel Paffrath <marcel.paffrath@rub.de>', 'Kasper D. Fischer <kasper.fischer@rub.de>']
 
 import os
+import io
+import copy
+import logging
 import traceback
 import yaml
 import argparse
+import json
 
 import time
 from datetime import timedelta
@@ -17,49 +21,78 @@ import matplotlib.pyplot as plt
 from obspy import read, UTCDateTime, Stream
 from obspy.clients.filesystem.sds import Client
 
-from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str, \
-    init_html_table, finish_html_table
-from utils import get_bg_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds
+from write_utils import get_html_text, get_html_link, get_html_row, html_footer, get_html_header, get_print_title_str, \
+    init_html_table, finish_html_table, get_mail_html_header, add_html_image
+from utils import get_bg_color, get_font_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds, \
+    connect_to_mail_server
 
 try:
     import smtplib
-    from email.mime.text import MIMEText
+    from email.message import EmailMessage
+    from email.utils import make_msgid
 
     mail_functionality = True
 except ImportError:
-    print('Could not import smtplib or mail. Disabled sending mails.')
+    logging.warning('Could not import smtplib or mail. Disabled sending mails.')
     mail_functionality = False
 
 pjoin = os.path.join
 UP = "\x1B[{length}A"
 CLR = "\x1B[0K"
-deg_str = '\N{DEGREE SIGN}C'
+DEG_STR = '\N{DEGREE SIGN}C'
 
 
-def read_yaml(file_path, n_read=3):
+def read_yaml(file_path: str, n_read: int = 3) -> dict:
     for index in range(n_read):
         try:
             with open(file_path, "r") as f:
                 params = yaml.safe_load(f)
+            set_logging_level(params)
         except Exception as e:
-            print(f'Could not read parameters file: {e}.\nWill try again {n_read - index - 1} time(s).')
+            logging.warning(f'Could not read parameters file: {e}.\nWill try again {n_read - index - 1} time(s).')
             time.sleep(10)
             continue
         return params
 
 
+def set_logging_level(params: dict) -> None:
+    logging_levels = {'info': logging.INFO,
+                      'warning': logging.WARNING,
+                      'warn': logging.WARNING,
+                      'debug': logging.DEBUG,
+                      'error': logging.ERROR,
+                      'critical': logging.CRITICAL}
+    logging_level_str = params.get('logging_level')
+    if not logging_level_str:
+        logging.warning('Could not set logging level. Parameter not set')
+        return
+    if not isinstance(logging_level_str, str):
+        logging.warning(
+            f'Could not set logging level. Parameter logging_level = {logging_level_str} could not be interpreted.')
+        return
+    logging.info(f'Setting logging level to {logging_level_str}')
+    logging_level = logging_levels.get(logging_level_str.lower())
+    logging.basicConfig(level=logging_level)
+
+
 def nsl_from_id(nwst_id):
+    nwst_id = get_full_seed_id(nwst_id)
     network, station, location = nwst_id.split('.')
     return dict(network=network, station=station, location=location)
 
 
+def get_full_seed_id(nwst_id):
+    seed_id = '{}.{}.{}'.format(*nwst_id.split('.'), '')
+    return seed_id
+
+
 def get_nwst_id(trace):
     stats = trace.stats
     return f'{stats.network}.{stats.station}.'  # {stats.location}'
 
 
 def fancy_timestr(dt, thresh=600, modif='+'):
-    if dt > timedelta(seconds=thresh):
+    if isinstance(dt, timedelta) and dt > timedelta(seconds=thresh):
         value = f'{modif} ' + str(dt) + f' {modif}'
     else:
         value = str(dt)
@@ -78,7 +111,7 @@ class SurveillanceBot(object):
         self.outpath_html = outpath_html
         self.filenames = []
         self.filenames_wf_data = []
-        self.filenames_read = []
+        self.filenames_read_last_modif = {}
         self.station_list = []
         self.analysis_print_list = []
         self.analysis_results = {}
@@ -90,6 +123,8 @@ class SurveillanceBot(object):
         self.status_message = ''
         self.html_fig_dir = 'figures'
 
+        self.active_figures = {}
+
         self.cl = Client(self.parameters.get('datapath'))  # TODO: Check if this has to be loaded again on update
         self.get_stations()
 
@@ -103,14 +138,20 @@ class SurveillanceBot(object):
         self.parameters['channels'] = channels
         self.reread_parameters = self.parameters.get('reread_parameters')
         self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')]
-        self.verbosity = self.parameters.get('verbosity')
         self.stations_blacklist = self.parameters.get('stations_blacklist')
         self.networks_blacklist = self.parameters.get('networks_blacklist')
         self.refresh_period = self.parameters.get('interval')
         self.transform_parameters()
+
         add_links = self.parameters.get('add_links')
         self.add_links = add_links if add_links else {}
 
+        add_global_links = self.parameters.get('add_global_links')
+        # in case user forgets "-" in parameters file
+        if isinstance(add_global_links, dict):
+            add_global_links = [add_global_links]
+        self.add_global_links = add_global_links if add_global_links else []
+
     def transform_parameters(self):
         for key in ['networks', 'stations', 'locations', 'channels']:
             parameter = self.parameters.get(key)
@@ -156,12 +197,11 @@ class SurveillanceBot(object):
                         if channel in channels_wf_data:
                             self.filenames_wf_data += fnames
 
-    def read_data(self, re_read_at_hour=1, daily_overlap=2):
+    def read_data(self, re_read_at_hour=1):
         '''
         read data method reads new data into self.stream
 
         :param re_read_at_hour: update archive at specified hour each day (hours up to 24)
-        :param daily_overlap: re-read data of previous day until specified hour (hours up to 24)
         '''
         self.data = {}
 
@@ -169,15 +209,16 @@ class SurveillanceBot(object):
         curr_time = UTCDateTime()
         current_day = curr_time.julday
         current_hour = curr_time.hour
-        yesterday = (curr_time - 24. * 3600.).julday
         if re_read_at_hour is not False and current_day != self.current_day and current_hour == re_read_at_hour:
-            self.filenames_read = []
+            self.filenames_read_last_modif = {}
             self.dataStream = Stream()
             self.current_day = current_day
 
         # add all data to current stream
         for filename in self.filenames:
-            if filename in self.filenames_read:
+            # if file already read and last modification time is the same as of last read operation: continue
+            if self.filenames_read_last_modif.get(filename) == os.path.getmtime(filename):
+                logging.info(f'Continue on file {filename}')
                 continue
             try:
                 # read only header of wf_data
@@ -185,19 +226,15 @@ class SurveillanceBot(object):
                     st_new = read(filename, headonly=True)
                 else:
                     st_new = read(filename, dtype=float)
-                # add file to read filenames to prevent re-reading in case it is not the current day (or end of
-                # previous day)
-                if not filename.endswith(f'{current_day:03}') and not (
-                        filename.endswith(f'{yesterday:03}') and current_hour <= daily_overlap):
-                    self.filenames_read.append(filename)
+                self.filenames_read_last_modif[filename] = os.path.getmtime(filename)
             except Exception as e:
-                print(f'Could not read file {filename}:', e)
+                logging.warning(f'Could not read file {filename}: {e}')
                 continue
             self.dataStream += st_new
         self.gaps = self.dataStream.get_gaps(min_gap=self.parameters['THRESHOLDS'].get('min_gap'))
         self.dataStream.merge()
 
-        # organise data in dictionary with key for each station
+        # organize data in dictionary with key for each station
         for trace in self.dataStream:
             nwst_id = get_nwst_id(trace)
             if not nwst_id in self.data.keys():
@@ -218,8 +255,7 @@ class SurveillanceBot(object):
             if stream:
                 nsl = nsl_from_id(nwst_id)
                 station_qc = StationQC(self, stream, nsl, self.parameters, self.keys, qc_starttime,
-                                       self.verbosity, print_func=self.print,
-                                       status_track=self.status_track.get(nwst_id))
+                                       print_func=self.print, status_track=self.status_track.get(nwst_id))
                 analysis_print_result = station_qc.return_print_analysis()
                 station_dict = station_qc.return_analysis()
             else:
@@ -315,7 +351,7 @@ class SurveillanceBot(object):
             first_exec = False
 
     def console_print(self, itemlist, str_len=21, sep='|', seplen=3):
-        assert len(sep) <= seplen, f'Make sure seperator has less than {seplen} characters'
+        assert len(sep) <= seplen, f'Make sure separator has less than {seplen} characters'
         sl = sep.ljust(seplen)
         sr = sep.rjust(seplen)
         string = sl
@@ -357,13 +393,13 @@ class SurveillanceBot(object):
         for nwst_id in self.station_list:
             self.write_html_figure(nwst_id)
 
-    def write_html_figure(self, nwst_id):
+    def write_html_figure(self, nwst_id, save_bytes=False):
         """ Write figure for html for specified station """
         self.check_fig_dir()
 
         fig = plt.figure(figsize=(16, 9))
-        fnout = self.get_fig_path_abs(nwst_id)
-        st = self.data.get(nwst_id)
+        fnames_out = [self.get_fig_path_abs(nwst_id), io.BytesIO()]
+        st = self.data.get(get_full_seed_id(nwst_id))
         if st:
             # TODO: this section failed once, adding try-except block for analysis and to prevent program from crashing
             try:
@@ -372,64 +408,66 @@ class SurveillanceBot(object):
                 st = modify_stream_for_plot(st, parameters=self.parameters)
                 st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full',
                         starttime=starttime, endtime=endtime)
-                # set_axis_ylabels(fig, self.parameters, self.verbosity)
-                set_axis_yticks(fig, self.parameters, self.verbosity)
+                # set_axis_ylabels(fig, self.parameters)
+                set_axis_yticks(fig, self.parameters)
                 set_axis_color(fig)
-                plot_axis_thresholds(fig, self.parameters, self.verbosity)
+                plot_axis_thresholds(fig, self.parameters)
             except Exception as e:
-                print(f'Could not generate plot for {nwst_id}:')
-                print(traceback.format_exc())
+                logging.error(f'Could not generate plot for {nwst_id}: {e}')
+                logging.error(traceback.format_exc())
             if len(fig.axes) > 0:
                 ax = fig.axes[0]
                 ax.set_title(f'Plot refreshed at (UTC) {UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S")}. '
                              f'Refreshed hourly or on FAIL status.')
                 for ax in fig.axes:
                     ax.grid(True, alpha=0.1)
+                for fnout in fnames_out:
+                    try:
                         fig.savefig(fnout, dpi=150., bbox_inches='tight')
+                    except IOError as e:
+                        logging.warning('Could not save figure with IO error. Disk quota exceeded?\nError message: {e}')
+                # if needed save figure as virtual object (e.g. for mailing)
+                if save_bytes:
+                    fnames_out[-1].seek(0)
+                    self.active_figures[nwst_id] = fnames_out[-1]
         plt.close(fig)
 
-    def write_html_table(self, default_color='#e6e6e6', default_header_color='#999', hide_keys_mobile=('other')):
-
-        def get_html_class(status=None, check_key=None):
+    def get_html_class(self, hide_keys_mobile=None, status=None, check_key=None):
         """ helper function for html class if a certain condition is fulfilled """
         html_class = None
         if status and status.is_active:
             html_class = 'blink-bg'
-            if check_key in hide_keys_mobile:
+        if hide_keys_mobile and check_key in hide_keys_mobile:
             html_class = 'hidden-mobile'
         return html_class
 
-        self.check_html_dir()
-        fnout = pjoin(self.outpath_html, 'survBot_out.html')
-        if not fnout:
-            return
-        try:
-            with open(fnout, 'w') as outfile:
-                write_html_header(outfile, self.refresh_period)
-                # write_html_table_title(outfile, self.parameters)
-                init_html_table(outfile)
-
+    def make_html_table_header(self, default_header_color, hide_keys_mobile=None, add_links=True):
         # First write header items
         header = self.keys.copy()
         # add columns for additional links
+        if add_links:
             for key in self.add_links:
                 header.insert(-1, key)
+
         header_items = [dict(text='Station', color=default_header_color)]
         for check_key in header:
-                    html_class = get_html_class(check_key=check_key)
+            html_class = self.get_html_class(hide_keys_mobile, check_key=check_key)
             item = dict(text=check_key, color=default_header_color, html_class=html_class)
             header_items.append(item)
-                write_html_row(outfile, header_items, html_key='th')
 
-                # Write all cells
-                for nwst_id in self.station_list:
+        return header, header_items
+
+    def get_html_row_items(self, status_dict, nwst_id, header, default_color, hide_keys_mobile=None,
+                           hyperlinks=True):
+        ''' create a html table row for the different keys '''
+
         fig_name = self.get_fig_path_rel(nwst_id)
         nwst_id_str = nwst_id.rstrip('.')
-                    col_items = [dict(text=nwst_id_str, color=default_color, hyperlink=fig_name,
-                                      bold=True, tooltip=f'Show plot of {nwst_id_str}')]
+        col_items = [dict(text=nwst_id_str, color=default_color, hyperlink=fig_name if hyperlinks else None,
+                          bold=True, tooltip=f'Show plot of {nwst_id_str}', font_color='#000000')]
+
         for check_key in header:
             if check_key in self.keys:
-                            status_dict = self.analysis_results.get(nwst_id)
                 status = status_dict.get(check_key)
                 message, detailed_message = status.get_status_str()
 
@@ -438,15 +476,16 @@ class SurveillanceBot(object):
                 bg_color = get_bg_color(check_key, status, dt_thresh, hex=True)
                 if not bg_color:
                     bg_color = default_color
+                font_color = get_font_color(bg_color, hex=True)
 
                 # add degree sign for temp
                 if check_key == 'temp':
                     if not type(message) in [str]:
-                                    message = str(message) + deg_str
+                        message = str(message) + DEG_STR
 
-                            html_class = get_html_class(status=status, check_key=check_key)
+                html_class = self.get_html_class(hide_keys_mobile, status=status, check_key=check_key)
                 item = dict(text=str(message), tooltip=str(detailed_message), color=bg_color,
-                                        html_class=html_class)
+                            html_class=html_class, font_color=font_color)
             elif check_key in self.add_links:
                 value = self.add_links.get(check_key).get('URL')
                 link_text = self.add_links.get(check_key).get('text')
@@ -455,24 +494,67 @@ class SurveillanceBot(object):
                 nw, st = nwst_id.split('.')[:2]
                 hyperlink_dict = dict(nw=nw, st=st, nwst_id=nwst_id)
                 link = value.format(**hyperlink_dict)
-                            item = dict(text=link_text, tooltip=link, hyperlink=link, color=default_color)
+                item = dict(text=link_text, tooltip=link, hyperlink=link if hyperlinks else None, color=default_color)
+            else:
+                item = dict(text='', tooltip='')
             col_items.append(item)
 
-                    write_html_row(outfile, col_items)
+        return col_items
+
+    def write_html_table(self, default_color='#e6e6e6', default_header_color='#999999', hide_keys_mobile=('other',)):
+        self.check_html_dir()
+        fnout = pjoin(self.outpath_html, 'survBot_out.html')
+        if not fnout:
+            return
+        try:
+            with open(fnout, 'w') as outfile:
+                outfile.write(get_html_header(self.refresh_period))
+
+                # write_html_table_title(self.parameters)
+                outfile.write(init_html_table())
+
+                # write html header row
+                header, header_items = self.make_html_table_header(default_header_color, hide_keys_mobile)
+                html_row = get_html_row(header_items, html_key='th')
+                outfile.write(html_row)
+
+                # Write all cells (row after row)
+                for nwst_id in self.station_list:
+                    # get list with column-wise items to write as a html row
+                    status_dict = self.analysis_results.get(nwst_id)
+                    col_items = self.get_html_row_items(status_dict, nwst_id, header, default_color, hide_keys_mobile)
+                    outfile.write(get_html_row(col_items))
+
+                outfile.write(finish_html_table())
+
+                # add optional links below html table
+                for dct in self.add_global_links:
+                    link_str = get_html_link(dct.get('text'), dct.get('URL'))
+                    outfile.write(get_html_text(link_str))
+
+                # add status message
+                outfile.write(get_html_text(self.status_message))
+
+                # write footer with optional logo
+                logo_file = self.parameters.get('html_logo')
+                logo_alt_text = self.parameters.get('logo_alt_text')
+                logo_height = self.parameters.get('logo_height')
+                if not os.path.isfile(pjoin(self.outpath_html, logo_file)):
+                    logging.info(f'Specified file {logo_file} not found.')
+                    logo_file = None
+
+                outfile.write(html_footer(footer_logo=logo_file, alt_text=logo_alt_text, height=logo_height))
 
-                finish_html_table(outfile)
-                write_html_text(outfile, self.status_message)
-                write_html_footer(outfile)
         except Exception as e:
-            print(f'Could not write HTML table to {fnout}:')
-            print(traceback.format_exc())
+            logging.info(f'Could not write HTML table to {fnout}:')
+            logging.debug(traceback.format_exc())
 
-        if self.verbosity:
-            print(f'Wrote html table to {fnout}')
+        logging.info(f'Wrote html table to {fnout}')
 
     def update_status_message(self):
         timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
-        self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \
+        self.status_message = f' Version {__version__} | ' \
+                              f'Program start time (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \
                               f'Current time (UTC) {UTCDateTime().strftime("%Y-%m-%d %H:%M:%S")} | ' \
                               f'Refresh period: {self.refresh_period}s | ' \
                               f'Showing data of last {timespan}'
@@ -483,7 +565,6 @@ class SurveillanceBot(object):
         string.replace('\n', clear_end)
         print(string, end=clear_end, **kwargs)
         self.print_count += n_nl + 1  # number of newlines + actual print with end='\n' (no check for kwargs end!)
-        # print('pc:', self.print_count)
 
     def clear_prints(self):
         print(UP.format(length=self.print_count), end='')
@@ -491,23 +572,21 @@ class SurveillanceBot(object):
 
 
 class StationQC(object):
-    def __init__(self, parent, stream, nsl, parameters, keys, starttime, verbosity, print_func, status_track=None):
+    def __init__(self, parent, stream, nsl, parameters, keys, starttime, print_func, status_track=None):
         """
         Station Quality Check class.
         :param nsl: dictionary containing network, station and location (key: str)
         :param parameters: parameters dictionary from parameters.yaml file
         """
-        if status_track is None:
-            status_track = {}
         self.parent = parent
         self.stream = stream
         self.nsl = nsl
         self.network = nsl.get('network')
         self.station = nsl.get('station')
         self.location = nsl.get('location')
-        self.parameters = parameters
+        # make a copy of parameters object to prevent accidental changes
+        self.parameters = copy.deepcopy(parameters)
         self.program_starttime = starttime
-        self.verbosity = verbosity
         self.last_active = False
         self.print = print_func
 
@@ -518,6 +597,9 @@ class StationQC(object):
             status_track = {}
         self.status_track = status_track
 
+        self.powbox_active = self.is_pbox_activated_check()
+        self.mass_active = self.is_mass_activated_check()
+
         self.start()
 
     @property
@@ -540,8 +622,7 @@ class StationQC(object):
         current_status = self.status_dict.get(key)
 
         # change this to something more useful, SMS/EMAIL/PUSH
-        if self.verbosity:
-            self.print(f'{UTCDateTime()}: {detailed_message}', flush=False)
+        logging.info(f'{UTCDateTime()}: {detailed_message}')
 
         # if error, do not overwrite with warning
         if current_status.is_error:
@@ -566,29 +647,35 @@ class StationQC(object):
         # current_status_message = '' if current_status_message in [None, 'OK', '-'] else current_status_message + ' | '
         # self.status_dict[key] = current_status_message + status_message
 
-    def error(self, key, detailed_message, last_occurrence=None, count=1):
+    def error(self, key, detailed_message, last_occurrence=None, count=1, disc=False):
+        send_mail = False
         new_error = StatusError(count=count, show_count=self.parameters.get('warn_count'))
+        if disc:
+            msg = disc if type(disc) == str else None
+            new_error.set_disconnected(msg)
         current_status = self.status_dict.get(key)
         if current_status.is_error:
             current_status.count += count
         else:
             current_status = new_error
             # if error is new and not on program-startup set active and refresh plot (using parent class)
-            if self.search_previous_errors(key, n_errors=1) is True:
-                self.parent.write_html_figure(self.nwst_id)
+            if self.status_track.get(key) and not self.status_track.get(key)[-1]:
+                self.parent.write_html_figure(self.nwst_id, save_bytes=True)
 
-        if self.verbosity:
-            self.print(f'{UTCDateTime()}: {detailed_message}', flush=False)
+        logging.info(f'{UTCDateTime()}: {detailed_message}')
 
         # do not send error mail if this is the first run (e.g. program startup) or state was already error (unchanged)
         if self.search_previous_errors(key) is True:
-            self.send_mail(key, status_type='FAIL', additional_message=detailed_message)
-            # set status to "inactive" after sending info mail
+            send_mail = True
+            # set status to "inactive" when info mail is sent
             current_status.is_active = False
         elif self.search_previous_errors(key) == 'active':
             current_status.is_active = True
 
+        # first update status, then send mail
         self._update_status(key, current_status, detailed_message, last_occurrence)
+        if send_mail:
+            self.send_mail(key, status_type='FAIL', additional_message=detailed_message)
 
     def search_previous_errors(self, key, n_errors=None):
         """
@@ -602,64 +689,139 @@ class StationQC(object):
         if n_errors is None:
             n_errors = self.parameters.get('n_track')
 
-        # +1 to check whether n_errors +1 was no error (error is new)
+        # +1 to check whether n_errors + 1 was no error (error is new)
         n_errors += 1
 
+        # simulate an error specified in json file (dictionary: {nwst_id: key} )
+        if self._simulated_error_check(key) is True:
+            logging.info(f'Simulating Error on {self.nwst_id}, {key}')
+            return True
+
         previous_errors = self.status_track.get(key)
         # only if error list is filled n_track times
         if previous_errors and len(previous_errors) == n_errors:
             # if first entry was no error but all others are, return True (-> new Fail n_track times)
             if not previous_errors[0] and all(previous_errors[1:]):
                 return True
-        # in case previous_errors exists, last item is error but not all items are error, error still active
-        elif previous_errors and previous_errors[-1] and not all(previous_errors):
+        # in case previous_errors exist, last item is error but not all items are error, error still active
+        if previous_errors and previous_errors[-1] and not all(previous_errors):
             return 'active'
         return False
 
+    def _simulated_error_check(self, key, fname='simulate_fail.json'):
+        if not os.path.isfile(fname):
+            return
+        with open(fname) as fid:
+            d = json.load(fid)
+        if d.get(self.nwst_id) == key:
+            return True
+
     def send_mail(self, key, status_type, additional_message=''):
         """ Send info mail using parameters specified in parameters file """
         if not mail_functionality:
-            if self.verbosity:
-                print('Mail functionality disabled. Return')
+            logging.info('Mail functionality disabled. Return')
             return
 
         mail_params = self.parameters.get('EMAIL')
         if not mail_params:
-            if self.verbosity:
-                print('parameter "EMAIL" not set in parameter file. Return')
+            logging.info('parameter "EMAIL" not set in parameter file. Return')
             return
 
         stations_blacklist = mail_params.get('stations_blacklist')
         if stations_blacklist and self.station in stations_blacklist:
-            if self.verbosity:
-                print(f'Station {self.station} listed in blacklist. Return')
+            logging.info(f'Station {self.station} listed in blacklist. Return')
             return
 
         networks_blacklist = mail_params.get('networks_blacklist')
         if networks_blacklist and self.network in networks_blacklist:
-            if self.verbosity:
-                print(f'Station {self.station} of network {self.network} listed in blacklist. Return')
+            logging.info(f'Station {self.station} of network {self.network} listed in blacklist. Return')
             return
 
         sender = mail_params.get('sender')
         addresses = mail_params.get('addresses')
-        server = mail_params.get('mailserver')
+        add_addresses = self.get_additional_mail_recipients(mail_params)
+        if add_addresses:
+            # create copy of addresses ( [:] ) to prevent changing original, general list with addresses
+            addresses = addresses[:] + list(add_addresses)
         if not sender or not addresses:
-            if self.verbosity:
-                print('Mail sender or addresses not correctly defined. Return')
+            logging.info('Mail sender or addresses not (correctly) defined. Return')
             return
         dt = self.get_dt_for_action()
         text = f'{key}: Status {status_type} longer than {dt}: ' + additional_message
-        msg = MIMEText(text)
+
+        msg = EmailMessage()
+
         msg['Subject'] = f'new message on station {self.nwst_id}'
         msg['From'] = sender
         msg['To'] = ', '.join(addresses)
 
-        # send message via SMTP server
-        s = smtplib.SMTP(server)
-        s.sendmail(sender, addresses, msg.as_string())
+        msg.set_content(text)
+
+        # html mail version
+        html_str = self.add_html_mail_body(text)
+        msg.add_alternative(html_str, subtype='html')
+
+        # connect to server, send mail and close connection
+        s = connect_to_mail_server(mail_params)
+        if not s: # if connection failed
+            return
+        s.send_message(msg)
         s.quit()
 
+    def add_html_mail_body(self, text, default_color='#e6e6e6'):
+        parent = self.parent
+
+        header, header_items = parent.make_html_table_header('#999999', add_links=False)
+        col_items = parent.get_html_row_items(self.status_dict, self.nwst_id, header, default_color, hyperlinks=False)
+
+        # set general status text
+        html_str = get_html_text(text)
+
+        # init html header and table
+        html_str += get_mail_html_header()
+        html_str += init_html_table()
+
+        # add table header and row of current station
+        html_str += get_html_row(header_items, html_key='th')
+        html_str += get_html_row(col_items)
+
+        html_str += finish_html_table()
+
+        if self.nwst_id in self.parent.active_figures.keys():
+            fid = self.parent.active_figures.pop(self.nwst_id)
+            html_str += add_html_image(img_data=fid.read())
+
+        html_str += html_footer()
+
+        return html_str
+
+    def get_additional_mail_recipients(self, mail_params):
+        """ return additional recipients from external mail list if this station (self.nwst_id) is specified """
+        eml_filename = mail_params.get('external_mail_list')
+        if eml_filename:
+            # try to open file
+            try:
+                with open(eml_filename, 'r') as fid:
+                    address_dict = yaml.safe_load(fid)
+
+                for address, nwst_ids in address_dict.items():
+                    if self.nwst_id in nwst_ids:
+                        yield address
+            # file not existing
+            except FileNotFoundError as e:
+                logging.warning(e)
+            # no dictionary
+            except AttributeError as e:
+                logging.warning(f'Could not read dictionary from file {eml_filename}: {e}')
+            # other exceptions
+            except Exception as e:
+                logging.warning(f'Could not open file {eml_filename}: {e}')
+        # no file specified
+        else:
+            logging.info('No external mail list set.')
+
+        return []
+
     def get_dt_for_action(self):
         n_track = self.parameters.get('n_track')
         interval = self.parameters.get('interval')
@@ -705,14 +867,24 @@ class StationQC(object):
             return max(endtimes)
 
     def check_for_inactive_message(self, key, dt_active):
+        """ send mail if station is inactive longer than dt_action and no FAIL status is present """
+
+        # check if any error is present in status_dict and not disconnected (in that case an email is sent already)
+        if self.check_for_any_error_no_dcn():
+            return
+
         dt_action = self.get_dt_for_action()
         interval = self.parameters.get('interval')
+
         if dt_action <= dt_active < dt_action + timedelta(seconds=interval):
             detailed_message = f'\n{self.nwst_id}\n\n'
             for key, status in self.status_dict.items():
                 detailed_message += f'{key}: {status.message}\n'
             self.send_mail(key, status_type='Inactive', additional_message=detailed_message)
 
+    def check_for_any_error_no_dcn(self):
+        return any([status.is_error and not status.connection_error for status in self.status_dict.values()])
+
     def start(self):
         self.analyse_channels()
 
@@ -720,9 +892,8 @@ class StationQC(object):
         timespan = self.parameters.get('timespan') * 24 * 3600
         self.analysis_starttime = self.program_starttime - timespan
 
-        if self.verbosity > 0:
-            self.print(150 * '#')
-            self.print('This is StationQT. Calculating quality for station'
+        logging.info(150 * '#')
+        logging.info('This is StationQC. Calculating quality for station'
                        ' {network}.{station}.{location}'.format(**self.nsl))
         self.voltage_analysis()
         self.pb_temp_analysis()
@@ -735,6 +906,13 @@ class StationQC(object):
         # activity check should be done last for useful status output (e.g. email)
         self.activity_check()
 
+        self._simulate_error()
+
+    def _simulate_error(self):
+        for key in self.keys:
+            if self._simulated_error_check(key):
+                self.error(key, 'SIMULATED ERROR')
+
     def return_print_analysis(self):
         items = [self.nwst_id]
         for key in self.keys:
@@ -743,7 +921,7 @@ class StationQC(object):
             if key == 'last active':
                 items.append(fancy_timestr(message))
             elif key == 'temp':
-                items.append(str(message) + deg_str)
+                items.append(str(message) + DEG_STR)
             else:
                 items.append(str(message))
         return items
@@ -782,9 +960,8 @@ class StationQC(object):
         clock_quality_warn_level = self.parameters.get('THRESHOLDS').get('clockquality_warn')
         clock_quality_fail_level = self.parameters.get('THRESHOLDS').get('clockquality_fail')
 
-        if self.verbosity > 1:
-            self.print(40 * '-')
-            self.print('Performing Clock Quality check', flush=False)
+        logging.info(40 * '-')
+        logging.info('Performing Clock Quality check')
 
         clockQuality_warn = np.where(clock_quality < clock_quality_warn_level)[0]
         clockQuality_fail = np.where(clock_quality < clock_quality_fail_level)[0]
@@ -828,9 +1005,8 @@ class StationQC(object):
         low_volt = self.parameters.get('THRESHOLDS').get('low_volt')
         high_volt = self.parameters.get('THRESHOLDS').get('high_volt')
 
-        if self.verbosity > 1:
-            self.print(40 * '-')
-            self.print('Performing Voltage check', flush=False)
+        logging.info(40 * '-')
+        logging.info('Performing Voltage check')
 
         overvolt = np.where(voltage > high_volt)[0]
         undervolt = np.where(voltage < low_volt)[0]
@@ -856,9 +1032,12 @@ class StationQC(object):
             self.warn(key, detailed_message=detailed_message, count=n_undervolt,
                       last_occurrence=self.get_last_occurrence(trace, undervolt))
 
-    def pb_temp_analysis(self, channel='EX1'):
+    def pb_temp_analysis(self, channel='EX1', t_max_default=50, t_crit_default=70):
         """ Analyse PowBox temperature output. """
         key = 'temp'
+        if not self.powbox_active:
+            self.set_pbox_inactive_error(key)
+            return
         st = self.stream.select(channel=channel)
         trace = self.get_trace(st, key)
         if not trace:
@@ -869,23 +1048,31 @@ class StationQC(object):
         # average temp
         timespan = min([self.parameters.get('timespan') * 24 * 3600, int(len(temp) / trace.stats.sampling_rate)])
         nsamp_av = int(trace.stats.sampling_rate) * timespan
-        av_temp_str = str(round(np.nanmean(temp[-nsamp_av:]), 1)) + deg_str
+        av_temp_str = str(round(np.nanmean(temp[-nsamp_av:]), 1)) + DEG_STR
         # dt of average
         dt_t_str = str(timedelta(seconds=int(timespan))).replace(', 0:00:00', '')
         # current temp
         cur_temp = round(temp[-1], 1)
-        if self.verbosity > 1:
-            self.print(40 * '-')
-            self.print('Performing PowBox temperature check (EX1)', flush=False)
-            self.print(f'Average temperature at {np.nanmean(temp)}\N{DEGREE SIGN}', flush=False)
-            self.print(f'Peak temperature at {max(temp)}\N{DEGREE SIGN}', flush=False)
-            self.print(f'Min temperature at {min(temp)}\N{DEGREE SIGN}', flush=False)
-        max_temp = thresholds.get('max_temp')
+        logging.info(40 * '-')
+        logging.info('Performing PowBox temperature check (EX1)')
+        logging.info(f'Average temperature at {np.nanmean(temp)}\N{DEGREE SIGN}')
+        logging.info(f'Peak temperature at {max(temp)}\N{DEGREE SIGN}')
+        logging.info(f'Min temperature at {min(temp)}\N{DEGREE SIGN}')
+        max_temp = thresholds.get('max_temp', t_max_default)
+        max_temp_crit = thresholds.get('critical_temp', t_crit_default)
         t_check = np.where(temp > max_temp)[0]
-        if len(t_check) > 0:
+        t_check_crit = np.where(temp > max_temp_crit)[0]
+        tcheck_message_template = ('Trace {id}: Temperature over {tmax}' + f'\N{DEGREE SIGN}'
+                                   + '! Current temperature: {temp}' + f'\N{DEGREE SIGN}')
+        if len(t_check_crit) > 0:
+            self.error(key=key,
+                      detailed_message=tcheck_message_template.format(id=trace.get_id(), tmax=max_temp, temp=cur_temp)
+                                       + self.get_last_occurrence_timestring(trace, t_check_crit),
+                      last_occurrence=self.get_last_occurrence(trace, t_check_crit))
+        elif len(t_check) > 0:
             self.warn(key=key,
-                      detailed_message=f'Trace {trace.get_id()}: '
-                                       f'Temperature over {max_temp}\N{DEGREE SIGN} at {trace.get_id()}!'
+                      detailed_message=tcheck_message_template.format(id=trace.get_id(), tmax=max_temp_crit,
+                                                                      temp=cur_temp)
                                        + self.get_last_occurrence_timestring(trace, t_check),
                       last_occurrence=self.get_last_occurrence(trace, t_check))
         else:
@@ -897,6 +1084,11 @@ class StationQC(object):
         """ Analyse datalogger mass channels. """
         key = 'mass'
 
+        # skip processing if mass is not active
+        if not self.mass_active:
+            self.set_mass_inactive_error(key)
+            return
+        
         # build stream with all channels
         st = Stream()
         for channel in channels:
@@ -934,23 +1126,26 @@ class StationQC(object):
             self.error(key=key,
                       detailed_message=f'Fail status for mass centering. Highest val (abs) {common_highest_val}V',)
 
-        if self.verbosity > 1:
-            self.print(40 * '-')
-            self.print('Performing mass position check', flush=False)
-            self.print(f'Average mass position at {common_highest_val}', flush=False)
+        logging.info(40 * '-')
+        logging.info('Performing mass position check')
+        logging.info(f'Average mass position at {common_highest_val}')
 
     def pb_power_analysis(self, channel='EX2', pb_dict_key='pb_SOH2'):
         """ Analyse EX2 channel of PowBox """
         keys = ['230V', '12V']
+        if not self.powbox_active:
+            for key in keys:
+                self.set_pbox_inactive_error(key)
+            return
+
         st = self.stream.select(channel=channel)
         trace = self.get_trace(st, keys)
         if not trace:
             return
 
         voltage = trace.data * self.get_unit_factor(channel)
-        if self.verbosity > 1:
-            self.print(40 * '-')
-            self.print('Performing PowBox 12V/230V check (EX2)', flush=False)
+        logging.info(40 * '-')
+        logging.info('Performing PowBox 12V/230V check (EX2)')
         voltage_check, voltage_dict, last_val = self.pb_voltage_ok(trace, voltage, pb_dict_key, channel=channel)
 
         if voltage_check:
@@ -964,16 +1159,19 @@ class StationQC(object):
     def pb_rout_charge_analysis(self, channel='EX3', pb_dict_key='pb_SOH3'):
         """ Analyse EX3 channel of PowBox """
         keys = ['router', 'charger']
-        pb_thresh = self.parameters.get('THRESHOLDS').get('pb_1v')
+        if not self.powbox_active:
+            for key in keys:
+                self.set_pbox_inactive_error(key)
+            return
+
         st = self.stream.select(channel=channel)
         trace = self.get_trace(st, keys)
         if not trace:
             return
 
         voltage = trace.data * self.get_unit_factor(channel)
-        if self.verbosity > 1:
-            self.print(40 * '-')
-            self.print('Performing PowBox Router/Charger check (EX3)', flush=False)
+        logging.info(40 * '-')
+        logging.info('Performing PowBox Router/Charger check (EX3)')
         voltage_check, voltage_dict, last_val = self.pb_voltage_ok(trace, voltage, pb_dict_key, channel=channel)
 
         if voltage_check:
@@ -997,7 +1195,7 @@ class StationQC(object):
                     if message == 'OK':
                         self.status_ok(key)
                         continue
-                    if volt_lvl > 1:
+                    if volt_lvl != 1:
                         n_occurrences = self.calc_occurrences(ind_array)
                         self.warn(key=key,
                                   detailed_message=f'Trace {trace.get_id()}: '
@@ -1006,8 +1204,10 @@ class StationQC(object):
                                   count=n_occurrences,
                                   last_occurrence=self.get_last_occurrence(trace, ind_array))
                     # if last_val == current voltage (which is not 1) -> FAIL or last_val < 1: PBox no data
-                    if volt_lvl == last_val or (volt_lvl == -1 and last_val < 1):
+                    if volt_lvl == last_val:
                         self.error(key, detailed_message=f'Last PowBox voltage state {last_val}V: {message}')
+                    elif volt_lvl == -1 and last_val < 1:
+                        self.error(key, detailed_message=f'PowBox under 1V - connection error', disc=True)
 
     def gaps_analysis(self, key='gaps'):
         """ return gaps of a given nwst_id """
@@ -1077,6 +1277,10 @@ class StationQC(object):
         with each voltage value associated to the different steps specified in POWBOX > pb_steps. Also raises
         self.warn in case there are unassociated voltage values recorded.
         """
+        
+        if not self.powbox_active:
+            return
+
         pb_thresh = self.parameters.get('THRESHOLDS').get('pb_thresh')
         pb_ok = self.parameters.get('POWBOX').get('pb_ok')
         # possible voltage levels are keys of pb voltage level dict
@@ -1104,7 +1308,7 @@ class StationQC(object):
 
         # Warn in case of voltage under OK-level (1V)
         if len(under) > 0:
-            # try calculate number of occurences from gaps between indices
+            # try calculate number of occurrences from gaps between indices
             n_occurrences = len(np.where(np.diff(under) > 1)[0]) + 1
             voltage_dict[-1] = under
             self.status_other(detailed_message=f'Trace {trace.get_id()}: '
@@ -1139,21 +1343,38 @@ class StationQC(object):
         """ get UTCDateTime from trace and index"""
         return trace.stats.starttime + trace.stats.delta * index
 
+    def is_pbox_activated_check(self):
+        return self.station not in self.parameters.get('no_pbox_stations', [])
+
+    def set_pbox_inactive_error(self, key):
+        msg = self.parameters.get('no_pbox_stations')[self.station]
+        self.error(key, detailed_message=f'PowBox not connected', disc=msg)
+
+    def is_mass_activated_check(self):
+        return self.station not in self.parameters.get('no_mass_stations', [])
+
+    def set_mass_inactive_error(self, key):
+        msg = self.parameters.get('no_mass_stations')[self.station]
+        self.error(key, detailed_message=f'Mass channels not connected', disc=msg)
 
 class Status(object):
+    """ Basic Status class. All status classes are derived from this class."""
     def __init__(self, message=None, detailed_messages=None, count: int = 0, last_occurrence=None, show_count=True):
         if message is None:
             message = '-'
         if detailed_messages is None:
             detailed_messages = []
+
         self.show_count = show_count
         self.message = message
         self.messages = [message]
         self.detailed_messages = detailed_messages
         self.count = count
         self.last_occurrence = last_occurrence
+
         self.is_warn = None
         self.is_error = None
+        self.connection_error = None
         self.is_other = False
         self.is_active = False
 
@@ -1189,22 +1410,33 @@ class StatusOK(Status):
 
 
 class StatusWarn(Status):
-    def __init__(self, message='WARN', count=1, last_occurence=None, detailed_messages=None, show_count=False):
-        super(StatusWarn, self).__init__(message=message, count=count, last_occurrence=last_occurence,
+    def __init__(self, message='WARN', count=1, last_occurrence=None, detailed_messages=None, show_count=False):
+        super(StatusWarn, self).__init__(message=message, count=count, last_occurrence=last_occurrence,
                                          detailed_messages=detailed_messages, show_count=show_count)
         self.set_warn()
 
 
 class StatusError(Status):
-    def __init__(self, message='FAIL', count=1, last_occurence=None, detailed_messages=None, show_count=False):
-        super(StatusError, self).__init__(message=message, count=count, last_occurrence=last_occurence,
+    def __init__(self, message='FAIL', count=1, last_occurrence=None, detailed_messages=None, show_count=False):
+        super(StatusError, self).__init__(message=message, count=count, last_occurrence=last_occurrence,
                                           detailed_messages=detailed_messages, show_count=show_count)
         self.set_error()
+        self.default_message = message
+
+    def set_disconnected(self, message=None):
+        self.connection_error = True
+        if not message:
+            message = 'DCN'
+        self.message = message
+
+    def set_connected(self):
+        self.connection_error = False
+        self.message = self.default_message
 
 
 class StatusOther(Status):
-    def __init__(self, messages=None, count=1, last_occurence=None, detailed_messages=None):
-        super(StatusOther, self).__init__(count=count, last_occurrence=last_occurence,
+    def __init__(self, messages=None, count=1, last_occurrence=None, detailed_messages=None):
+        super(StatusOther, self).__init__(count=count, last_occurrence=last_occurrence,
                                           detailed_messages=detailed_messages)
         if messages is None:
             messages = []
@@ -1233,7 +1465,9 @@ class StatusOther(Status):
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Call survBot')
     parser.add_argument('-html', dest='html_path', default=None, help='filepath for HTML output')
+    parser.add_argument('-parfile', dest='parfile', default='parameters.yaml',
+                        help='parameter file (default: parameters.yaml)')
     args = parser.parse_args()
 
-    survBot = SurveillanceBot(parameter_path='parameters.yaml', outpath_html=args.html_path)
+    survBot = SurveillanceBot(parameter_path=args.parfile, outpath_html=args.html_path)
     survBot.start()

survBotGUI.py

@@ -11,6 +11,8 @@ import os
 import sys
 import traceback
 
+import logging
+
 try:
     from PySide2 import QtGui, QtCore, QtWidgets
 except ImportError:
@@ -41,7 +43,7 @@ try:
     from rest_api.rest_api_utils import get_last_messages, send_message, get_default_params
     sms_funcs = True
 except ImportError:
-    print('Could not load rest_api utils, SMS functionality disabled.')
+    logging.warning('Could not load rest_api utils, SMS functionality disabled.')
     sms_funcs = False
 
 deg_str = '\N{DEGREE SIGN}C'
@@ -54,10 +56,9 @@ class Thread(QtCore.QThread):
     """
     update = QtCore.Signal()
 
-    def __init__(self, parent, runnable, verbosity=0):
+    def __init__(self, parent, runnable):
         super(Thread, self).__init__(parent=parent)
         self.setParent(parent)
-        self.verbosity = verbosity
         self.runnable = runnable
         self.is_active = True
 
@@ -69,11 +70,10 @@ class Thread(QtCore.QThread):
             self.update.emit()
         except Exception as e:
             self.is_active = False
-            print(e)
-            print(traceback.format_exc())
+            logging.error(e)
+            logging.debug(traceback.format_exc())
         finally:
-            if self.verbosity > 0:
-                print(f'Time for Thread execution: {UTCDateTime() - t0}')
+            logging.info(f'Time for Thread execution: {UTCDateTime() - t0}')
 
 
 class MainWindow(QtWidgets.QMainWindow):
@@ -195,7 +195,7 @@ class MainWindow(QtWidgets.QMainWindow):
         station = nwst_id.split('.')[1]
         iccid = get_station_iccid(station)
         if not iccid:
-            print('Could not find iccid for station', nwst_id)
+            logging.info(f'Could not find iccid for station: {nwst_id}')
             return
         sms_widget = ReadSMSWidget(parent=self, iccid=iccid)
         sms_widget.setWindowTitle(f'Recent SMS of station: {nwst_id}')

utils.py

@@ -1,12 +1,24 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+import logging
+
 import matplotlib
 import numpy as np
+import smtplib
+import os
 
 from obspy import Stream
 
 
+COLORS_DICT = {'FAIL': (195, 29, 14, 255),
+               'NO DATA': (255, 255, 125, 255),
+               'WARN': (250, 192, 63, 255),
+               'OK': (185, 245, 145, 255),
+               'undefined': (240, 240, 240, 255),
+               'disc': (126, 127, 131, 255), }
+
+
 def get_bg_color(check_key, status, dt_thresh=None, hex=False):
     message = status.message
     if check_key == 'last active':
@@ -19,6 +31,9 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False):
         if status.is_warn:
             bg_color = get_warn_color(status.count)
         elif status.is_error:
+            if status.connection_error:
+                bg_color = get_color('disc')
+            else:
                 bg_color = get_color('FAIL')
         else:
             bg_color = get_color(message)
@@ -31,13 +46,16 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False):
 
 
 def get_color(key):
-    # some GUI default colors
-    colors_dict = {'FAIL': (255, 50, 0, 255),
-                   'NO DATA': (255, 255, 125, 255),
-                   'WARN': (255, 255, 80, 255),
-                   'OK': (125, 255, 125, 255),
-                   'undefined': (230, 230, 230, 255)}
-    return colors_dict.get(key)
+    # some old GUI default colors
+    # colors_dict = {'FAIL': (255, 85, 50, 255),
+    #                'NO DATA': (255, 255, 125, 255),
+    #                'WARN': (255, 255, 80, 255),
+    #                'OK': (173, 255, 133, 255),
+    #                'undefined': (230, 230, 230, 255),
+    #                'disc': (255, 160, 40, 255),}
+    if not key in COLORS_DICT.keys():
+        key = 'undefined'
+    return COLORS_DICT.get(key)
 
 
 def get_color_mpl(key):
@@ -47,6 +65,7 @@ def get_color_mpl(key):
 
 def get_time_delay_color(dt, dt_thresh):
     """ Set color of time delay after thresholds specified in self.dt_thresh """
+    if isinstance(dt, type(dt_thresh[0])):
         if dt < dt_thresh[0]:
             return get_color('OK')
         elif dt_thresh[0] <= dt < dt_thresh[1]:
@@ -54,9 +73,11 @@ def get_time_delay_color(dt, dt_thresh):
     return get_color('FAIL')
 
 
-def get_warn_color(count):
-    color = (min([255, 200 + count ** 2]), 255, 80, 255)
-    return color
+def get_warn_color(count, n_colors=20):
+    if count >= n_colors:
+        count = -1
+    gradient = np.linspace((240, 245, 110, 255), (250, 192, 63, 255), n_colors, dtype=int)
+    return tuple(gradient[count])
 
 
 def get_mass_color(message):
@@ -69,6 +90,8 @@ def get_mass_color(message):
 def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
     """ Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """
     if type(temp) in [str]:
+        if temp in COLORS_DICT.keys():
+            return get_color(temp)
         return get_color('undefined')
     cmap = matplotlib.cm.get_cmap(cmap)
     val = (temp - vmin) / (vmax - vmin)
@@ -76,6 +99,24 @@ def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
     return rgba
 
 
+def get_font_color(bg_color, hex=False):
+    if hex:
+        bg_color = matplotlib.colors.to_rgb(bg_color)
+    bg_color_hsv = matplotlib.colors.rgb_to_hsv(bg_color)
+    bg_color_hsl = hsv_to_hsl(bg_color_hsv)
+    font_color = (255, 255, 255, 255) if bg_color_hsl[2] < 0.6 else (0, 0, 0, 255)
+    if hex:
+        font_color = '#{:02x}{:02x}{:02x}'.format(*font_color[:3])
+    return font_color
+
+
+def hsv_to_hsl(hsv):
+    hue, saturation, value = hsv
+    lightness = value * (1 - saturation / 2)
+    saturation = 0 if lightness in (0, 1) else (value - lightness) / min(lightness, 1 - lightness)
+    return hue, saturation, lightness
+
+
 def modify_stream_for_plot(input_stream, parameters):
     """ copy (if necessary) and modify stream for plotting """
 
@@ -134,12 +175,12 @@ def transform_trace(data, transf):
         elif operator_str == '/':
             data = data / val
         else:
-            raise IOError(f'Unknown arithmethic operator string: {operator_str}')
+            raise IOError(f'Unknown arithmetic operator string: {operator_str}')
 
     return data
 
 
-def set_axis_ylabels(fig, parameters, verbosity=0):
+def set_axis_ylabels(fig, parameters):
     """
     Adds channel names to y-axis if defined in parameters.
     """
@@ -147,24 +188,25 @@ def set_axis_ylabels(fig, parameters, verbosity=0):
     if not names: # or not len(st.traces):
         return
     if not len(names) == len(fig.axes):
-        if verbosity:
-            print('Mismatch in axis and label lengths. Not adding plot labels')
+        logging.info('Mismatch in axis and label lengths. Not adding plot labels')
         return
     for channel_name, ax in zip(names, fig.axes):
         if channel_name:
             ax.set_ylabel(channel_name)
 
 
-def set_axis_color(fig, color='grey'):
+def set_axis_color(fig, color='0.8', shade_color='0.95'):
     """
-    Set all axes of figure to specific color
+    Set all axes (frame) of figure to specific color. Shade every second axis.
     """
-    for ax in fig.axes:
+    for i, ax in enumerate(fig.axes):
         for key in ['bottom', 'top', 'right', 'left']:
             ax.spines[key].set_color(color)
+        if i % 2:
+            ax.set_facecolor(shade_color)
 
 
-def set_axis_yticks(fig, parameters, verbosity=0):
+def set_axis_yticks(fig, parameters):
     """
     Adds channel names to y-axis if defined in parameters.
     """
@@ -172,8 +214,7 @@ def set_axis_yticks(fig, parameters, verbosity=0):
     if not ticks:
         return
     if not len(ticks) == len(fig.axes):
-        if verbosity:
-            print('Mismatch in axis tick and label lengths. Not changing plot ticks.')
+        logging.info('Mismatch in axis tick and label lengths. Not changing plot ticks.')
         return
     for ytick_tripple, ax in zip(ticks, fig.axes):
         if not ytick_tripple:
@@ -185,12 +226,11 @@ def set_axis_yticks(fig, parameters, verbosity=0):
         ax.set_ylim(ymin - 0.33 * step, ymax + 0.33 * step)
 
 
-def plot_axis_thresholds(fig, parameters, verbosity=0):
+def plot_axis_thresholds(fig, parameters):
     """
     Adds channel thresholds (warn, fail) to y-axis if defined in parameters.
     """
-    if verbosity > 0:
-        print('Plotting trace thresholds')
+    logging.info('Plotting trace thresholds')
 
     keys_colors = {'warn': dict(color=0.8 * get_color_mpl('WARN'), linestyle=(0, (5, 10)), alpha=0.5, linewidth=0.7),
                    'fail': dict(color=0.8 * get_color_mpl('FAIL'), linestyle='solid', alpha=0.5, linewidth=0.7)}
@@ -204,6 +244,10 @@ def plot_axis_thresholds(fig, parameters, verbosity=0):
 
 def plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs):
     for channel_thresholds, ax in zip(channel_threshold_list, fig.axes):
+        if channel_thresholds in ['pb_SOH2', 'pb_SOH3']:
+            annotate_voltage_states(ax, parameters, channel_thresholds)
+            channel_thresholds = get_warn_states_pbox(channel_thresholds, parameters)
+
         if not channel_thresholds:
             continue
 
@@ -213,5 +257,74 @@ def plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs):
         for warn_thresh in channel_thresholds:
             if isinstance(warn_thresh, str):
                 warn_thresh = parameters.get('THRESHOLDS').get(warn_thresh)
-            if type(warn_thresh in (float, int)):
+            if isinstance(warn_thresh, (float, int)):
                 ax.axhline(warn_thresh, **kwargs)
+
+
+def get_warn_states_pbox(soh_key: str, parameters: dict) -> list:
+    pb_dict = parameters.get('POWBOX').get(soh_key)
+    if not pb_dict:
+        return []
+    return [key for key in pb_dict.keys() if key > 1]
+
+
+def annotate_voltage_states(ax, parameters, pb_key, color='0.75'):
+    for voltage, voltage_dict in parameters.get('POWBOX').get(pb_key).items():
+        if float(voltage) < 1:
+            continue
+        out_string = ''
+        for key, val in voltage_dict.items():
+            if val != 'OK':
+                if out_string:
+                    out_string += ' | '
+                out_string += f'{key}: {val}'
+
+        ax.annotate(out_string, (ax.get_xlim()[-1], voltage), color=color, fontsize='xx-small',
+                    horizontalalignment='right')
+
+def get_credential(source, param):
+    """
+    Retrieve a credential from a Docker secret or environment variable.
+    """
+    if source == 'DOCKER':
+        try:
+            with open('/run/secrets/'+param.lower(), 'r') as f:
+                return f.read().strip()
+        except FileNotFoundError as e:
+            logging.error(f'Could not read from Docker secret at /run/secrets/{param.lower()}')
+            logging.error(e)
+    elif source == 'ENV':
+        try:
+            return os.environ.get(param.upper())
+        except Exception as e:
+            logging.error(f'Could not read from environment variable {param.upper()}')
+            logging.error(e)
+    # return source if no credential was found
+    return source
+
+def connect_to_mail_server(mail_params):
+    """
+    Connect to mail server and return server object.
+    """
+    # get server from parameters
+    server = mail_params.get('mailserver')
+    # get auth_type from parameters
+    auth_type = mail_params.get('auth_type')
+    # set up connection to mail server
+    if auth_type == 'None':
+        s = smtplib.SMTP(server)
+    else:
+        # user and password from parameters, docker secret or environment variable
+        user = get_credential(mail_params.get('user'), 'mail_user')
+        password = get_credential(mail_params.get('password'), 'mail_password')
+        # create secure connection to server
+        if auth_type == 'SSL':
+            s = smtplib.SMTP_SSL(server, mail_params.get('port'))
+        elif auth_type == 'TLS':
+            s = smtplib.SMTP(server, mail_params.get('port'))
+            s.starttls()
+        else:
+            logging.error('Unknown authentication type. Mails can not be sent')
+            return
+        s.login(user, password)
+    return s

write_utils.py

@@ -1,48 +1,79 @@
+from base64 import b64encode
 from datetime import timedelta
 
 
-def write_html_table_title(fobj, parameters):
+def _convert_to_textstring(lst):
+    return '\n'.join(lst)
+
+
+def get_html_table_title(parameters):
     title = get_print_title_str(parameters)
-    fobj.write(f'<h3>{title}</h3>\n')
+    return f'<h3>{title}</h3>\n'
 
 
-def write_html_text(fobj, text):
-    fobj.write(f'<p>{text}</p>\n')
+def get_html_text(text):
+    return f'<p>{text}</p>\n'
 
 
-def write_html_header(fobj, refresh_rate=10):
+def get_html_header(refresh_rate=10):
     header = ['<!DOCTYPE html>',
-              '<html>',
+              '<html lang="en">',
+              '<head>',
+              '  <title>SurvBot status</title>',
+              '  <link rel="stylesheet" media="only screen and (max-width: 400px)" href="mobile.css">',
+              '  <link rel="stylesheet" media="only screen and (min-width: 401px)" href="desktop.css">',
+              f'  <meta http-equiv="refresh" content="{refresh_rate}">',
+              '  <meta charset="utf-8">',
+              '  <meta name="viewport" content="width=device-width, initial-scale=1">',
+              '</head>',
+              '<body>\n']
+    header = _convert_to_textstring(header)
+    return header
+
+
+def get_mail_html_header():
+    header = ['<html>',
               '<head>',
-              '  <link rel="stylesheet" media="only screen and (max-width: 400px)" href="mobile.css" />',
-              '  <link rel="stylesheet" media="only screen and (min-width: 401px)" href="desktop.css" />',
               '</head>',
-              f'<meta http-equiv="refresh" content="{refresh_rate}" >',
-              '<meta charset="utf-8">',
-              '<meta name="viewport" content="width=device-width, initial-scale=1">',
               '<body>']
-    for item in header:
-        fobj.write(item + '\n')
+    header = _convert_to_textstring(header)
+    return header
 
 
-def init_html_table(fobj):
-    fobj.write('<table style="width:100%">\n')
+def init_html_table():
+    return '<table style="width:100%">\n'
 
 
-def finish_html_table(fobj):
-    fobj.write('</table>\n')
+def finish_html_table():
+    return '</table>\n'
 
 
-def write_html_footer(fobj):
-    footer = ['</body>',
-              '</html>']
-    for item in footer:
-        fobj.write(item + '\n')
+def html_footer(footer_logo=None, alt_text='Logo', height=30):
+    if footer_logo:
+        footer = [f'<div class="footer">',
+                  f'  <img style="float: right; padding: 10px;" src="{footer_logo}" height={height} alt="{alt_text}">',
+                  f'</div>']
+    else:
+        footer = []
+    footer.append('</body>')
+    footer.append('</html>')
+
+    footer = _convert_to_textstring(footer)
+    return footer
 
 
-def write_html_row(fobj, items, html_key='td'):
+def add_html_image(img_data, img_format='png'):
+    return f"""<br>\n<img width="100%" src="data:image/{img_format};base64, {b64encode(img_data).decode('ascii')}">"""
+
+
+def get_html_link(text, link):
+    return f'<a href="{link}"> {text} </a>'
+
+
+def get_html_row(items, html_key='td'):
+    row_string = ''
     default_space = '  '
-    fobj.write(default_space + '<tr>\n')
+    row_string += default_space + '<tr>\n'
     for item in items:
         text = item.get('text')
         if item.get('bold'):
@@ -50,16 +81,22 @@ def write_html_row(fobj, items, html_key='td'):
         if item.get('italic'):
             text = '<i>' + text + '</i>'
         tooltip = item.get('tooltip')
-        color = item.get('color')
-        # check for black background of headers (shouldnt happen anymore)
-        color = '#e6e6e6' if color == '#000000' else color
+        font_color = item.get('font_color')
         hyperlink = item.get('hyperlink')
-        image_str = f'<a href="{hyperlink}">' if hyperlink else ''
+        color = 'transparent' if hyperlink else item.get('color')
+        text_str = get_html_link(text, hyperlink) if hyperlink else text
         html_class = item.get('html_class')
         class_str = f' class="{html_class}"' if html_class else ''
-        fobj.write(2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"> {image_str}'
-                   + text + f'</{html_key}>\n')
-    fobj.write(default_space + '</tr>\n')
+        row_string += 2 * default_space + f'<{html_key}{class_str} '
+        row_string += f'title="{tooltip}" ' if tooltip else ''
+        row_string += 'style="' if color or font_color else ''
+        row_string += f'background-color: {color};' if color else 'style="'
+        row_string += ' ' if font_color else ''
+        row_string += f'color: {font_color};' if font_color else ''
+        row_string += '" ' if color or font_color else '' 
+        row_string += f'>{text_str}</{html_key}>\n'
+    row_string += default_space + '</tr>\n'
+    return row_string
 
 
 def get_print_title_str(parameters):