diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6463563 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +FROM python:3 + +# metadata +LABEL maintainer="Kasper D. Fischer " +LABEL version="0.2-docker" +LABEL description="Docker image for the survBot application" + +# install required system packages +RUN apt update && apt install -y bind9-host iputils-ping + +# create user and group and home directory +RUN groupadd -r survBot && useradd -r -g survBot survBot +RUN mkdir -p /home/survBot && chown -R survBot:survBot /home/survBot + +# change working directory +WORKDIR /usr/src/app + +# install required python packages +RUN --mount=type=bind,source=requirements.txt,target=/tmp/requirements.txt \ + pip install --no-cache-dir --requirement /tmp/requirements.txt + +# copy application files +COPY survBot.py utils.py write_utils.py LICENSE README.md ./ + +# copy configuration files +VOLUME /usr/src/app/conf +COPY parameters.yaml mailing_list.yam[l] simulate_fail.jso[n] conf/ +RUN ln -s conf/simulate_fail.json simulate_fail.json + +# copy www files +VOLUME /usr/src/app/www +COPY logo.pn[g] stylesheets/desktop.css stylesheets/mobile.css www/ +RUN ln -s www/survBot_out.html www/index.html + +# change ownership of working directory +RUN chown -R survBot:survBot /usr/src/app + +# run the application as user survBot +USER survBot +CMD [ "python", "./survBot.py", "-html", "www", "-parfile", "conf/parameters.yaml" ] diff --git a/README.md b/README.md index 29b7733..3eb633f 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ version: 0.2 survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output over SOH channels - by analysing contents of a Seiscomp3 datapath. + by analyzing contents of a Seiscomp data archive. ## Requirements @@ -40,16 +40,53 @@ The GUI can be loaded via python survBotGui.py ``` +### Docker + +To run the program in a Docker container, first build the image: + +```shell script +docker build -t survbot . +``` + +Then run the container: + +```shell script +docker run -v /path/to/conf-dir:/usr/src/app/conf -v /path/to/output:/usr/src/app/www survbot +``` + +The directory `/path/to/conf-dir` should contain the `parameters.yaml` file, and the directory `/path/to/output` will contain the output HTML files. + +### Configuration of the e-mail server settings + +The e-mail server settings can be configured in the `parameters.yaml` file. The following settings are available: + +* `mailserver`: the address of the mail server +* `auth_type`: the authentication type for the mail server (`None`, `SSL`, `TLS`) +* `port`: the port of the mail server +* `user`: the username for the mail server (if required) +* `password`: the password for the mail server (if required) + +The `user` and `password` fields are optional, and can be left empty if the mail server does not require authentication. The `auth_type` field can be set to `None` if no authentication is required, `SSL` if the mail server requires SSL authentication, or `TLS` if the mail server requires TLS authentication. If the `user` or `password` fields are set to `Docker` ore `ENV` the program will try to read the values from the docker secrets `mail_user` and `mail_password` or environment variables `MAIL_USER` and `MAIL_PASSWORD` respectively. Docker secrets are only available in Docker Swarm mode, i.e. if the program is run as a service. + ## Version Changes -- surveillance of mass, clock and gaps -- individual mailing lists for different stations -- html mail with recent status information -- updated web page design -- restructured parameter file -- recognize if PBox is disconnected + +### 0.2 + +* surveillance of mass, clock and gaps +* individual mailing lists for different stations +* html mail with recent status information +* updated web page design +* restructured parameter file +* recognize if PBox is disconnected + +### 0.2-docker + +* added Dockerfile for easy deployment +* added more settings for connection to a mail server ## Staff -Original author: M.Paffrath (marcel.paffrath@rub.de) +Original author: M.Paffrath () +Contributions: Kasper D. Fischer () -June 2023 +Jan 2025 diff --git a/__init__.py b/__init__.py index 90d7001..7953f6e 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1,3 @@ # survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output # over SOH channels by analysing contents of a Seiscomp3 datapath. -__version__ = "0.2" +__version__ = "0.2-docker" diff --git a/mailing_list.yaml b/mailing_list.yaml index 5bae70a..4fd7be8 100644 --- a/mailing_list.yaml +++ b/mailing_list.yaml @@ -2,7 +2,7 @@ # "mail.address@provider.com, mail.address2@provider2.com": # - 1Y.GR01 # - 1Y.GR02 -# "mail.address3@provder.com": +# "mail.address3@provider.com": # - 1Y.GR03 #"kasper.fischer@rub.de": diff --git a/parameters.yaml b/parameters.yaml index 4b41238..d351b72 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -1,9 +1,9 @@ # Parameters file for Surveillance Bot -datapath: "/data/SDS/" # SC3 Datapath +datapath: "/data/SDS/" # path to SDS data archive networks: ["1Y", "HA", "MK"] # select networks, list or str stations: "*" # select stations, list or str locations: "*" # select locations, list or str -stations_blacklist: ["TEST", "EREA", "DOMV", "LFKM", "GR19", "LAKA", "ATHR", "HAVD"] # exclude these stations +stations_blacklist: ["ATHR", "DOMV", "EREA", "GR19", "GR27", "HAVD", "LAKA", "LFKM", "TEST"] # exclude these stations networks_blacklist: [] # exclude these networks interval: 60 # Perform checks every x seconds n_track: 360 # wait n_track * intervals before performing an action (i.e. send mail/end highlight status) @@ -56,7 +56,7 @@ THRESHOLDS: # # For each channel a factor 'unit' for unit conversion (e.g. to SI) can be provided, as well as a 'name' # and 'ticks' [ymin, ymax, ystep] for plotting. -# 'warn' and 'fail' plot horizontal lines in corresponding colors (can be str in TRESHOLDS, int/float or iterable) +# 'warn' and 'fail' plot horizontal lines in corresponding colors (can be str in THRESHOLDS, int/float or iterable) # keyword "pb_SOH2" or "pb_SOH3" can be used to extract warning values from above POWBOX parameter definition # # 'transform' can be provided for plotting to perform arithmetic operations in given order, e.g.: @@ -135,14 +135,22 @@ add_global_links: "URL": "https://fdsnws.geophysik.ruhr-uni-bochum.de/map/?lat=39.5&lon=21&zoom=7&baselayer=mapnik"} # html logo at page bottom (path relative to html directory) -html_logo: "figures/Logo_RUB_BLAU_rgb.png" +html_logo: "logo.png" # E-mail notifications EMAIL: - mailserver: "localhost" + # specify mail server and credentials + # port, auth_type, user and password are only required if mailserver is not set to "localhost" + # user and password can be set to "ENV" or "DOCKER" to read from environment variables or docker secrets + mailserver: "smtp.rub.de" # mail server + auth_type: "SSL" # mail authentication type, can be "SSL", "TLS" or "None" + port: 465 # mail port, default 465 for SSL, 587 for TLS + user: "DOCKER" # mail user, read from environment variable if set to "ENV" or from docker secret if set to "DOCKER" + password: "DOCKER" # mail password, read from environment variable if set to "ENV" or from docker secret if set to "DOCKER" + # specify mail recipients, sender and blacklists addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"] # list of mail addresses for info mails - sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender - stations_blacklist: [] # do not send emails for specific stations - networks_blacklist: [] # do not send emails for specific network + sender: "RUB SeisObs " # mail sender + stations_blacklist: [] # do not send emails for specific stations + networks_blacklist: [] # do not send emails for specific network # specify recipients for single stations in a yaml: key = email-address, val = station list (e.g. [1Y.GR01, 1Y.GR02]) - external_mail_list: "mailing_list.yaml" + external_mail_list: "conf/mailing_list.yaml" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1fb4a59 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,36 @@ +Brotli==1.1.0 +certifi==2025.1.31 +cffi==1.17.1 +charset-normalizer==3.4.1 +contourpy==1.3.1 +cycler==0.12.1 +decorator==5.2.1 +fonttools==4.56.0 +greenlet==3.1.1 +h2==4.2.0 +hpack==4.1.0 +hyperframe==6.1.0 +idna==3.10 +kiwisolver==1.4.8 +lxml==5.3.1 +matplotlib==3.8.4 +munkres==1.1.4 +numpy==1.26.4 +obspy==1.4.1 +packaging==24.2 +pillow==11.1.0 +pip==25.0.1 +pycparser==2.22 +pyparsing==3.2.1 +PySocks==1.7.1 +python-dateutil==2.9.0.post0 +PyYAML==6.0.2 +requests==2.32.3 +scipy==1.15.2 +setuptools==75.8.2 +six==1.17.0 +SQLAlchemy==1.4.54 +unicodedata2==16.0.0 +urllib3==2.3.0 +wheel==0.45.1 +zstandard==0.23.0 diff --git a/stylesheets/desktop.css b/stylesheets/desktop.css index ba59cfd..6ada37c 100644 --- a/stylesheets/desktop.css +++ b/stylesheets/desktop.css @@ -16,7 +16,7 @@ td { } th { - background-color: #999; + background-color: #17365c; color: #fff; border-radius: 2px; padding: 3px 1px; diff --git a/stylesheets/mobile.css b/stylesheets/mobile.css index aeeb416..b2a5d65 100644 --- a/stylesheets/mobile.css +++ b/stylesheets/mobile.css @@ -16,7 +16,7 @@ td { } th { - background-color: #999; + background-color: #17365c; color: #fff; border-radius: 3px; padding: 10px 2px; diff --git a/submit_bot.sh b/submit_bot.sh index 7b1f724..18a6cf1 100755 --- a/submit_bot.sh +++ b/submit_bot.sh @@ -1,19 +1,24 @@ #!/bin/bash -ulimit -s 8192 #$ -l low -#$ -l h_vmem=5G +#$ -l h_vmem=2.5G +#$ -l mem=2.5G +#$ -l h_stack=INFINITY #$ -cwd #$ -pe smp 1 -#$ -N survBot_bg -#$ -l os=*stretch +#$ -binding linear:1 +#$ -N survBot +#$ -o /data/www/~kasper/survBot/survBot_bg.log +#$ -e /data/www/~kasper/survBot/survBot_bg.err +#$ -m e +#$ -M kasper.fischer@rub.de source /opt/anaconda3/etc/profile.d/conda.sh -conda activate py37 +conda activate survBot # environment variables for numpy to prevent multi threading export MKL_NUM_THREADS=1 export NUMEXPR_NUM_THREADS=1 export OMP_NUM_THREADS=1 -python survBot.py -html '/data/www/~marcel/' +python survBot.py -html '/data/www/~kasper/survBot' diff --git a/survBot.py b/survBot.py index 775f437..bd4fe70 100755 --- a/survBot.py +++ b/survBot.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -__version__ = '0.1' -__author__ = 'Marcel Paffrath' +__version__ = '0.2-docker' +__author__ = 'Marcel Paffrath ' import os import io @@ -23,7 +23,8 @@ from obspy.clients.filesystem.sds import Client from write_utils import get_html_text, get_html_link, get_html_row, html_footer, get_html_header, get_print_title_str, \ init_html_table, finish_html_table, get_mail_html_header, add_html_image -from utils import get_bg_color, get_font_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds +from utils import get_bg_color, get_font_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds, \ + connect_to_mail_server try: import smtplib @@ -233,7 +234,7 @@ class SurveillanceBot(object): self.gaps = self.dataStream.get_gaps(min_gap=self.parameters['THRESHOLDS'].get('min_gap')) self.dataStream.merge() - # organise data in dictionary with key for each station + # organize data in dictionary with key for each station for trace in self.dataStream: nwst_id = get_nwst_id(trace) if not nwst_id in self.data.keys(): @@ -350,7 +351,7 @@ class SurveillanceBot(object): first_exec = False def console_print(self, itemlist, str_len=21, sep='|', seplen=3): - assert len(sep) <= seplen, f'Make sure seperator has less than {seplen} characters' + assert len(sep) <= seplen, f'Make sure separator has less than {seplen} characters' sl = sep.ljust(seplen) sr = sep.rjust(seplen) string = sl @@ -738,7 +739,6 @@ class StationQC(object): if add_addresses: # create copy of addresses ( [:] ) to prevent changing original, general list with addresses addresses = addresses[:] + list(add_addresses) - server = mail_params.get('mailserver') if not sender or not addresses: logging.info('Mail sender or addresses not (correctly) defined. Return') return @@ -757,8 +757,10 @@ class StationQC(object): html_str = self.add_html_mail_body(text) msg.add_alternative(html_str, subtype='html') - # send message via SMTP server - s = smtplib.SMTP(server) + # connect to server, send mail and close connection + s = connect_to_mail_server(mail_params) + if not s: # if connection failed + return s.send_message(msg) s.quit() @@ -1297,7 +1299,7 @@ class StationQC(object): # Warn in case of voltage under OK-level (1V) if len(under) > 0: - # try calculate number of occurences from gaps between indices + # try calculate number of occurrences from gaps between indices n_occurrences = len(np.where(np.diff(under) > 1)[0]) + 1 voltage_dict[-1] = under self.status_other(detailed_message=f'Trace {trace.get_id()}: ' @@ -1393,15 +1395,15 @@ class StatusOK(Status): class StatusWarn(Status): - def __init__(self, message='WARN', count=1, last_occurence=None, detailed_messages=None, show_count=False): - super(StatusWarn, self).__init__(message=message, count=count, last_occurrence=last_occurence, + def __init__(self, message='WARN', count=1, last_occurrence=None, detailed_messages=None, show_count=False): + super(StatusWarn, self).__init__(message=message, count=count, last_occurrence=last_occurrence, detailed_messages=detailed_messages, show_count=show_count) self.set_warn() class StatusError(Status): - def __init__(self, message='FAIL', count=1, last_occurence=None, detailed_messages=None, show_count=False): - super(StatusError, self).__init__(message=message, count=count, last_occurrence=last_occurence, + def __init__(self, message='FAIL', count=1, last_occurrence=None, detailed_messages=None, show_count=False): + super(StatusError, self).__init__(message=message, count=count, last_occurrence=last_occurrence, detailed_messages=detailed_messages, show_count=show_count) self.set_error() self.default_message = message @@ -1418,8 +1420,8 @@ class StatusError(Status): class StatusOther(Status): - def __init__(self, messages=None, count=1, last_occurence=None, detailed_messages=None): - super(StatusOther, self).__init__(count=count, last_occurrence=last_occurence, + def __init__(self, messages=None, count=1, last_occurrence=None, detailed_messages=None): + super(StatusOther, self).__init__(count=count, last_occurrence=last_occurrence, detailed_messages=detailed_messages) if messages is None: messages = [] diff --git a/utils.py b/utils.py index 2723635..057bc74 100644 --- a/utils.py +++ b/utils.py @@ -5,6 +5,8 @@ import logging import matplotlib import numpy as np +import smtplib +import os from obspy import Stream @@ -173,7 +175,7 @@ def transform_trace(data, transf): elif operator_str == '/': data = data / val else: - raise IOError(f'Unknown arithmethic operator string: {operator_str}') + raise IOError(f'Unknown arithmetic operator string: {operator_str}') return data @@ -279,3 +281,50 @@ def annotate_voltage_states(ax, parameters, pb_key, color='0.75'): ax.annotate(out_string, (ax.get_xlim()[-1], voltage), color=color, fontsize='xx-small', horizontalalignment='right') + +def get_credential(source, param): + """ + Retrieve a credential from a Docker secret or environment variable. + """ + if source == 'DOCKER': + try: + with open('/run/secrets/'+param.lower(), 'r') as f: + return f.read().strip() + except FileNotFoundError as e: + logging.error(f'Could not read from Docker secret at /run/secrets/{param.lower()}') + logging.error(e) + elif source == 'ENV': + try: + return os.environ.get(param.upper()) + except Exception as e: + logging.error(f'Could not read from environment variable {param.upper()}') + logging.error(e) + # return source if no credential was found + return source + +def connect_to_mail_server(mail_params): + """ + Connect to mail server and return server object. + """ + # get server from parameters + server = mail_params.get('mailserver') + # get auth_type from parameters + auth_type = mail_params.get('auth_type') + # set up connection to mail server + if auth_type == 'None': + s = smtplib.SMTP(server) + else: + # user and password from parameters, docker secret or environment variable + user = get_credential(mail_params.get('user'), 'mail_user') + password = get_credential(mail_params.get('password'), 'mail_password') + # create secure connection to server + if auth_type == 'SSL': + s = smtplib.SMTP_SSL(server, mail_params.get('port')) + elif auth_type == 'TLS': + s = smtplib.SMTP(server, mail_params.get('port')) + s.starttls() + else: + logging.error('Unknown authentication type. Mails can not be sent') + return + s.login(user, password) + return s