Merge remote-tracking branch 'origin/develop' into develop

# Conflicts:
#	parameters.yaml

Dockerfile

@@ -0,0 +1,40 @@
+FROM python:3
+
+# metadata
+LABEL maintainer="Kasper D. Fischer <kasper.fischer@rub.de>"
+LABEL version="0.2-docker"
+LABEL description="Docker image for the survBot application"
+
+# install required system packages
+RUN apt update && apt install -y bind9-host iputils-ping
+
+# create user and group and home directory
+RUN groupadd -r survBot && useradd -r -g survBot survBot
+RUN mkdir -p /home/survBot && chown -R survBot:survBot /home/survBot
+
+# change working directory
+WORKDIR /usr/src/app
+
+# install required python packages
+RUN --mount=type=bind,source=requirements.txt,target=/tmp/requirements.txt \
+    pip install --no-cache-dir --requirement /tmp/requirements.txt
+
+# copy application files
+COPY survBot.py utils.py write_utils.py LICENSE README.md ./
+
+# copy configuration files
+VOLUME /usr/src/app/conf
+COPY parameters.yaml mailing_list.yam[l] simulate_fail.jso[n] conf/
+RUN ln -s conf/simulate_fail.json simulate_fail.json
+
+# copy www files
+VOLUME /usr/src/app/www
+COPY logo.pn[g] stylesheets/desktop.css stylesheets/mobile.css www/
+RUN ln -s www/survBot_out.html www/index.html
+
+# change ownership of working directory
+RUN chown -R survBot:survBot /usr/src/app
+
+# run the application as user survBot
+USER survBot
+CMD [ "python", "./survBot.py", "-html", "www", "-parfile", "conf/parameters.yaml" ]

README.md

@@ -3,7 +3,7 @@
 version: 0.2
 
 survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output over SOH channels
- by analysing contents of a Seiscomp3 datapath.
+ by analyzing contents of a Seiscomp data archive.
 
 ## Requirements
 
@@ -40,16 +40,53 @@ The GUI can be loaded via
 python survBotGui.py
 ```
 
+### Docker
+
+To run the program in a Docker container, first build the image:
+
+```shell script
+docker build -t survbot .
+```
+
+Then run the container:
+
+```shell script
+docker run -v /path/to/conf-dir:/usr/src/app/conf -v /path/to/output:/usr/src/app/www survbot
+```
+
+The directory `/path/to/conf-dir` should contain the `parameters.yaml` file, and the directory `/path/to/output` will contain the output HTML files.
+
+### Configuration of the e-mail server settings
+
+The e-mail server settings can be configured in the `parameters.yaml` file. The following settings are available:
+
+* `mailserver`: the address of the mail server
+* `auth_type`: the authentication type for the mail server (`None`, `SSL`, `TLS`)
+* `port`: the port of the mail server
+* `user`: the username for the mail server (if required)
+* `password`: the password for the mail server (if required)
+
+The `user` and `password` fields are optional, and can be left empty if the mail server does not require authentication. The `auth_type` field can be set to `None` if no authentication is required, `SSL` if the mail server requires SSL authentication, or `TLS` if the mail server requires TLS authentication. If the `user` or `password` fields are set to `Docker` ore `ENV` the program will try to read the values from the docker secrets `mail_user` and `mail_password` or environment variables `MAIL_USER` and `MAIL_PASSWORD` respectively. Docker secrets are only available in Docker Swarm mode, i.e. if the program is run as a service.
+
 ## Version Changes
-- surveillance of mass, clock and gaps
+
-- individual mailing lists for different stations
+### 0.2
-- html mail with recent status information
+
-- updated web page design
+* surveillance of mass, clock and gaps
-- restructured parameter file
+* individual mailing lists for different stations
-- recognize if PBox is disconnected
+* html mail with recent status information
+* updated web page design
+* restructured parameter file
+* recognize if PBox is disconnected
+
+### 0.2-docker
+
+* added Dockerfile for easy deployment
+* added more settings for connection to a mail server
 
 ## Staff
 
-Original author: M.Paffrath (marcel.paffrath@rub.de)
+Original author: M.Paffrath (<marcel.paffrath@rub.de>)
+Contributions: Kasper D. Fischer (<kasper.fischer@rub.de>)
 
-June 2023
+Jan 2025

__init__.py

@@ -1,3 +1,3 @@
 # survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output
 # over SOH channels by analysing contents of a Seiscomp3 datapath.
-__version__ = "0.2"
+__version__ = "0.2-docker"

mailing_list.yaml

@@ -2,7 +2,7 @@
 # "mail.address@provider.com, mail.address2@provider2.com":
 #   - 1Y.GR01
 #   - 1Y.GR02
-# "mail.address3@provder.com":
+# "mail.address3@provider.com":
 #   - 1Y.GR03
 
 #"kasper.fischer@rub.de":

parameters.yaml

@@ -1,9 +1,9 @@
 # Parameters file for Surveillance Bot
-datapath: "/data/SDS/"                    # SC3 Datapath
+datapath: "/data/SDS/"                    # path to SDS data archive
 networks: ["1Y", "HA", "MK"]              # select networks, list or str
 stations: "*"                             # select stations, list or str
 locations: "*"                            # select locations, list or str
-stations_blacklist: ["TEST", "EREA", "DOMV", "LFKM", "GR19", "LAKA", "ATHR", "HAVD"]      # exclude these stations
+stations_blacklist: ["ATHR", "DOMV", "EREA", "GR19", "GR27", "HAVD", "LAKA", "LFKM", "TEST"]      # exclude these stations
 networks_blacklist: []                    # exclude these networks
 interval: 60               # Perform checks every x seconds
 n_track: 360               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
@@ -56,7 +56,7 @@ THRESHOLDS:
 #
 # For each channel a factor 'unit' for unit conversion (e.g. to SI) can be provided, as well as a 'name'
 # and 'ticks' [ymin, ymax, ystep] for plotting.
-# 'warn' and 'fail' plot horizontal lines in corresponding colors (can be str in TRESHOLDS, int/float or iterable)
+# 'warn' and 'fail' plot horizontal lines in corresponding colors (can be str in THRESHOLDS, int/float or iterable)
 # keyword "pb_SOH2" or "pb_SOH3" can be used to extract warning values from above POWBOX parameter definition
 #
 # 'transform' can be provided for plotting to perform arithmetic operations in given order, e.g.:
@@ -135,14 +135,22 @@ add_global_links:
      "URL": "https://fdsnws.geophysik.ruhr-uni-bochum.de/map/?lat=39.5&lon=21&zoom=7&baselayer=mapnik"}
 
 # html logo at page bottom (path relative to html directory)
-html_logo: "figures/Logo_RUB_BLAU_rgb.png"
+html_logo: "logo.png"
 
 # E-mail notifications
 EMAIL:
-  mailserver: "localhost"
+  # specify mail server and credentials
+  # port, auth_type, user and password are only required if mailserver is not set to "localhost"
+  # user and password can be set to "ENV" or "DOCKER" to read from environment variables or docker secrets
+  mailserver: "smtp.rub.de"           # mail server
+  auth_type: "SSL"                    # mail authentication type, can be "SSL", "TLS" or "None"
+  port: 465                           # mail port, default 465 for SSL, 587 for TLS
+  user: "DOCKER"                      # mail user, read from environment variable if set to "ENV" or from docker secret if set to "DOCKER"
+  password: "DOCKER"                  # mail password, read from environment variable if set to "ENV" or from docker secret if set to "DOCKER"
+  # specify mail recipients, sender and blacklists
   addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"]  # list of mail addresses for info mails
-  sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender
+  sender: "RUB SeisObs <seisobs@ruhr-uni-bochum.de>"              # mail sender
-  stations_blacklist: []                     # do not send emails for specific stations
+  stations_blacklist: []                                          # do not send emails for specific stations
-  networks_blacklist: []                           # do not send emails for specific network
+  networks_blacklist: []                                          # do not send emails for specific network
   # specify recipients for single stations in a yaml: key = email-address, val = station list (e.g. [1Y.GR01, 1Y.GR02])
-  external_mail_list: "mailing_list.yaml"
+  external_mail_list: "conf/mailing_list.yaml"

requirements.txt

@@ -0,0 +1,36 @@
+Brotli==1.1.0
+certifi==2025.1.31
+cffi==1.17.1
+charset-normalizer==3.4.1
+contourpy==1.3.1
+cycler==0.12.1
+decorator==5.2.1
+fonttools==4.56.0
+greenlet==3.1.1
+h2==4.2.0
+hpack==4.1.0
+hyperframe==6.1.0
+idna==3.10
+kiwisolver==1.4.8
+lxml==5.3.1
+matplotlib==3.8.4
+munkres==1.1.4
+numpy==1.26.4
+obspy==1.4.1
+packaging==24.2
+pillow==11.1.0
+pip==25.0.1
+pycparser==2.22
+pyparsing==3.2.1
+PySocks==1.7.1
+python-dateutil==2.9.0.post0
+PyYAML==6.0.2
+requests==2.32.3
+scipy==1.15.2
+setuptools==75.8.2
+six==1.17.0
+SQLAlchemy==1.4.54
+unicodedata2==16.0.0
+urllib3==2.3.0
+wheel==0.45.1
+zstandard==0.23.0

stylesheets/desktop.css

@@ -16,7 +16,7 @@ td {
 }
 
 th {
-    background-color: #999;
+    background-color: #17365c;
     color: #fff;
     border-radius: 2px;
     padding: 3px 1px;

stylesheets/mobile.css

@@ -16,7 +16,7 @@ td {
 }
 
 th {
-    background-color: #999;
+    background-color: #17365c;
     color: #fff;
     border-radius: 3px;
     padding: 10px 2px;

submit_bot.sh

@@ -1,19 +1,24 @@
 #!/bin/bash
-ulimit -s 8192
 
 #$ -l low
-#$ -l h_vmem=5G
+#$ -l h_vmem=2.5G
+#$ -l mem=2.5G
+#$ -l h_stack=INFINITY
 #$ -cwd
 #$ -pe smp 1
-#$ -N survBot_bg
+#$ -binding linear:1
-#$ -l os=*stretch
+#$ -N survBot
+#$ -o /data/www/~kasper/survBot/survBot_bg.log
+#$ -e /data/www/~kasper/survBot/survBot_bg.err
+#$ -m e
+#$ -M kasper.fischer@rub.de
 
 source /opt/anaconda3/etc/profile.d/conda.sh
-conda activate py37
+conda activate survBot
 
 # environment variables for numpy to prevent multi threading
 export MKL_NUM_THREADS=1
 export NUMEXPR_NUM_THREADS=1
 export OMP_NUM_THREADS=1
 
-python survBot.py -html '/data/www/~marcel/'
+python survBot.py -html '/data/www/~kasper/survBot'

survBot.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-__version__ = '0.1'
+__version__ = '0.2-docker'
-__author__ = 'Marcel Paffrath'
+__author__ = 'Marcel Paffrath <marcel.paffrath@rub.de>'
 
 import os
 import io
@@ -23,7 +23,8 @@ from obspy.clients.filesystem.sds import Client
 
 from write_utils import get_html_text, get_html_link, get_html_row, html_footer, get_html_header, get_print_title_str, \
     init_html_table, finish_html_table, get_mail_html_header, add_html_image
-from utils import get_bg_color, get_font_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds
+from utils import get_bg_color, get_font_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds, \
+    connect_to_mail_server
 
 try:
     import smtplib
@@ -233,7 +234,7 @@ class SurveillanceBot(object):
         self.gaps = self.dataStream.get_gaps(min_gap=self.parameters['THRESHOLDS'].get('min_gap'))
         self.dataStream.merge()
 
-        # organise data in dictionary with key for each station
+        # organize data in dictionary with key for each station
         for trace in self.dataStream:
             nwst_id = get_nwst_id(trace)
             if not nwst_id in self.data.keys():
@@ -350,7 +351,7 @@ class SurveillanceBot(object):
             first_exec = False
 
     def console_print(self, itemlist, str_len=21, sep='|', seplen=3):
-        assert len(sep) <= seplen, f'Make sure seperator has less than {seplen} characters'
+        assert len(sep) <= seplen, f'Make sure separator has less than {seplen} characters'
         sl = sep.ljust(seplen)
         sr = sep.rjust(seplen)
         string = sl
@@ -738,7 +739,6 @@ class StationQC(object):
         if add_addresses:
             # create copy of addresses ( [:] ) to prevent changing original, general list with addresses
             addresses = addresses[:] + list(add_addresses)
-        server = mail_params.get('mailserver')
         if not sender or not addresses:
             logging.info('Mail sender or addresses not (correctly) defined. Return')
             return
@@ -757,8 +757,10 @@ class StationQC(object):
         html_str = self.add_html_mail_body(text)
         msg.add_alternative(html_str, subtype='html')
 
-        # send message via SMTP server
+        # connect to server, send mail and close connection
-        s = smtplib.SMTP(server)
+        s = connect_to_mail_server(mail_params)
+        if not s: # if connection failed
+            return
         s.send_message(msg)
         s.quit()
 
@@ -1297,7 +1299,7 @@ class StationQC(object):
 
         # Warn in case of voltage under OK-level (1V)
         if len(under) > 0:
-            # try calculate number of occurences from gaps between indices
+            # try calculate number of occurrences from gaps between indices
             n_occurrences = len(np.where(np.diff(under) > 1)[0]) + 1
             voltage_dict[-1] = under
             self.status_other(detailed_message=f'Trace {trace.get_id()}: '
@@ -1393,15 +1395,15 @@ class StatusOK(Status):
 
 
 class StatusWarn(Status):
-    def __init__(self, message='WARN', count=1, last_occurence=None, detailed_messages=None, show_count=False):
+    def __init__(self, message='WARN', count=1, last_occurrence=None, detailed_messages=None, show_count=False):
-        super(StatusWarn, self).__init__(message=message, count=count, last_occurrence=last_occurence,
+        super(StatusWarn, self).__init__(message=message, count=count, last_occurrence=last_occurrence,
                                          detailed_messages=detailed_messages, show_count=show_count)
         self.set_warn()
 
 
 class StatusError(Status):
-    def __init__(self, message='FAIL', count=1, last_occurence=None, detailed_messages=None, show_count=False):
+    def __init__(self, message='FAIL', count=1, last_occurrence=None, detailed_messages=None, show_count=False):
-        super(StatusError, self).__init__(message=message, count=count, last_occurrence=last_occurence,
+        super(StatusError, self).__init__(message=message, count=count, last_occurrence=last_occurrence,
                                           detailed_messages=detailed_messages, show_count=show_count)
         self.set_error()
         self.default_message = message
@@ -1418,8 +1420,8 @@ class StatusError(Status):
 
 
 class StatusOther(Status):
-    def __init__(self, messages=None, count=1, last_occurence=None, detailed_messages=None):
+    def __init__(self, messages=None, count=1, last_occurrence=None, detailed_messages=None):
-        super(StatusOther, self).__init__(count=count, last_occurrence=last_occurence,
+        super(StatusOther, self).__init__(count=count, last_occurrence=last_occurrence,
                                           detailed_messages=detailed_messages)
         if messages is None:
             messages = []

utils.py

@@ -5,6 +5,8 @@ import logging
 
 import matplotlib
 import numpy as np
+import smtplib
+import os
 
 from obspy import Stream
 
@@ -173,7 +175,7 @@ def transform_trace(data, transf):
         elif operator_str == '/':
             data = data / val
         else:
-            raise IOError(f'Unknown arithmethic operator string: {operator_str}')
+            raise IOError(f'Unknown arithmetic operator string: {operator_str}')
 
     return data
 
@@ -279,3 +281,50 @@ def annotate_voltage_states(ax, parameters, pb_key, color='0.75'):
 
         ax.annotate(out_string, (ax.get_xlim()[-1], voltage), color=color, fontsize='xx-small',
                     horizontalalignment='right')
+
+def get_credential(source, param):
+    """
+    Retrieve a credential from a Docker secret or environment variable.
+    """
+    if source == 'DOCKER':
+        try:
+            with open('/run/secrets/'+param.lower(), 'r') as f:
+                return f.read().strip()
+        except FileNotFoundError as e:
+            logging.error(f'Could not read from Docker secret at /run/secrets/{param.lower()}')
+            logging.error(e)
+    elif source == 'ENV':
+        try:
+            return os.environ.get(param.upper())
+        except Exception as e:
+            logging.error(f'Could not read from environment variable {param.upper()}')
+            logging.error(e)
+    # return source if no credential was found
+    return source
+
+def connect_to_mail_server(mail_params):
+    """
+    Connect to mail server and return server object.
+    """
+    # get server from parameters
+    server = mail_params.get('mailserver')
+    # get auth_type from parameters
+    auth_type = mail_params.get('auth_type')
+    # set up connection to mail server
+    if auth_type == 'None':
+        s = smtplib.SMTP(server)
+    else:
+        # user and password from parameters, docker secret or environment variable
+        user = get_credential(mail_params.get('user'), 'mail_user')
+        password = get_credential(mail_params.get('password'), 'mail_password')
+        # create secure connection to server
+        if auth_type == 'SSL':
+            s = smtplib.SMTP_SSL(server, mail_params.get('port'))
+        elif auth_type == 'TLS':
+            s = smtplib.SMTP(server, mail_params.get('port'))
+            s.starttls()
+        else:
+            logging.error('Unknown authentication type. Mails can not be sent')
+            return
+        s.login(user, password)
+    return s