diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6463563 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +FROM python:3 + +# metadata +LABEL maintainer="Kasper D. Fischer " +LABEL version="0.2-docker" +LABEL description="Docker image for the survBot application" + +# install required system packages +RUN apt update && apt install -y bind9-host iputils-ping + +# create user and group and home directory +RUN groupadd -r survBot && useradd -r -g survBot survBot +RUN mkdir -p /home/survBot && chown -R survBot:survBot /home/survBot + +# change working directory +WORKDIR /usr/src/app + +# install required python packages +RUN --mount=type=bind,source=requirements.txt,target=/tmp/requirements.txt \ + pip install --no-cache-dir --requirement /tmp/requirements.txt + +# copy application files +COPY survBot.py utils.py write_utils.py LICENSE README.md ./ + +# copy configuration files +VOLUME /usr/src/app/conf +COPY parameters.yaml mailing_list.yam[l] simulate_fail.jso[n] conf/ +RUN ln -s conf/simulate_fail.json simulate_fail.json + +# copy www files +VOLUME /usr/src/app/www +COPY logo.pn[g] stylesheets/desktop.css stylesheets/mobile.css www/ +RUN ln -s www/survBot_out.html www/index.html + +# change ownership of working directory +RUN chown -R survBot:survBot /usr/src/app + +# run the application as user survBot +USER survBot +CMD [ "python", "./survBot.py", "-html", "www", "-parfile", "conf/parameters.yaml" ] diff --git a/README.md b/README.md index 29b7733..48bbc3c 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,38 @@ The GUI can be loaded via python survBotGui.py ``` +### Docker + +To run the program in a Docker container, first build the image: + +```shell script +docker build -t survbot . +``` + +Then run the container: + +```shell script +docker run -v /path/to/conf-dir:/usr/src/app/conf -v /path/to/output:/usr/src/app/www survbot +``` + +The directory `/path/to/conf-dir` should contain the `parameters.yaml` file, and the directory `/path/to/output` will contain the output HTML files. + +### Configuration of the e-mail server settings + +The e-mail server settings can be configured in the `parameters.yaml` file. The following settings are available: + +- `mailserver`: the address of the mail server +- `auth_type`: the authentication type for the mail server (`None`, `SSL`, `TLS`) +- `port`: the port of the mail server +- `user`: the username for the mail server (if required) +- `password`: the password for the mail server (if required) + +The `user` and `password` fields are optional, and can be left empty if the mail server does not require authentication. The `auth_type` field can be set to `None` if no authentication is required, `SSL` if the mail server requires SSL authentication, or `TLS` if the mail server requires TLS authentication. If the `user` or `password` fileds are set to `Docker` ore `ENV` the program will try to read the values from the docker secrets `mail_user` and `mail_password` or environment variables `MAIL_USER` and `MAIL_PASSWORD` respectively. Docker secrets are only available in Docker Swarm mode, i.e. if the program is run as a service. + ## Version Changes + +### 0.2 + - surveillance of mass, clock and gaps - individual mailing lists for different stations - html mail with recent status information @@ -48,8 +79,14 @@ python survBotGui.py - restructured parameter file - recognize if PBox is disconnected +### 0.2-docker + +- added Dockerfile for easy deployment +- added more settings for connection to a mail server + ## Staff Original author: M.Paffrath (marcel.paffrath@rub.de) +Contributions by: Kasper D. Fischer (kasper.fischer@rub.de) -June 2023 +Jan 2025 diff --git a/__init__.py b/__init__.py index 90d7001..7953f6e 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1,3 @@ # survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output # over SOH channels by analysing contents of a Seiscomp3 datapath. -__version__ = "0.2" +__version__ = "0.2-docker" diff --git a/mailing_list.yaml b/mailing_list.yaml index 5bae70a..4fd7be8 100644 --- a/mailing_list.yaml +++ b/mailing_list.yaml @@ -2,7 +2,7 @@ # "mail.address@provider.com, mail.address2@provider2.com": # - 1Y.GR01 # - 1Y.GR02 -# "mail.address3@provder.com": +# "mail.address3@provider.com": # - 1Y.GR03 #"kasper.fischer@rub.de": diff --git a/parameters.yaml b/parameters.yaml index eaab309..1be3d16 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -133,14 +133,22 @@ add_global_links: "URL": "https://fdsnws.geophysik.ruhr-uni-bochum.de/map/?lat=39.5&lon=21&zoom=7&baselayer=mapnik"} # html logo at page bottom (path relative to html directory) -html_logo: "figures/Logo_RUB_BLAU_rgb.png" +html_logo: "logo.png" # E-mail notifications EMAIL: - mailserver: "localhost" + # specify mail server and credentials + # port, auth_type, user and password are only required if mailserver is not set to "localhost" + # user and password can be set to "ENV" or "DOCKER" to read from environment variables or docker secrets + mailserver: "smtp.rub.de" # mail server + auth_type: "SSL" # mail authentication type, can be "SSL", "TLS" or "None" + port: 465 # mail port, default 465 for SSL, 587 for TLS + user: "DOCKER" # mail user, read from environment variable if set to "ENV" or from docker secret if set to "DOCKER" + password: "DOCKER" # mail password, read from environment variable if set to "ENV" or from docker secret if set to "DOCKER" + # specify mail recipients, sender and blacklists addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"] # list of mail addresses for info mails - sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender - stations_blacklist: [] # do not send emails for specific stations - networks_blacklist: [] # do not send emails for specific network + sender: "RUB SeisObs " # mail sender + stations_blacklist: [] # do not send emails for specific stations + networks_blacklist: [] # do not send emails for specific network # specify recipients for single stations in a yaml: key = email-address, val = station list (e.g. [1Y.GR01, 1Y.GR02]) - external_mail_list: "mailing_list.yaml" + external_mail_list: "conf/mailing_list.yaml" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1fb4a59 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,36 @@ +Brotli==1.1.0 +certifi==2025.1.31 +cffi==1.17.1 +charset-normalizer==3.4.1 +contourpy==1.3.1 +cycler==0.12.1 +decorator==5.2.1 +fonttools==4.56.0 +greenlet==3.1.1 +h2==4.2.0 +hpack==4.1.0 +hyperframe==6.1.0 +idna==3.10 +kiwisolver==1.4.8 +lxml==5.3.1 +matplotlib==3.8.4 +munkres==1.1.4 +numpy==1.26.4 +obspy==1.4.1 +packaging==24.2 +pillow==11.1.0 +pip==25.0.1 +pycparser==2.22 +pyparsing==3.2.1 +PySocks==1.7.1 +python-dateutil==2.9.0.post0 +PyYAML==6.0.2 +requests==2.32.3 +scipy==1.15.2 +setuptools==75.8.2 +six==1.17.0 +SQLAlchemy==1.4.54 +unicodedata2==16.0.0 +urllib3==2.3.0 +wheel==0.45.1 +zstandard==0.23.0 diff --git a/stylesheets/desktop.css b/stylesheets/desktop.css index f25fc28..6ada37c 100644 --- a/stylesheets/desktop.css +++ b/stylesheets/desktop.css @@ -1,6 +1,5 @@ body { background-color: #ffffff; - place-items: center; text-align: center; padding-bottom: 30px; font-family: "Helvetica", "sans-serif"; @@ -17,7 +16,7 @@ td { } th { - background-color: #999; + background-color: #17365c; color: #fff; border-radius: 2px; padding: 3px 1px; diff --git a/stylesheets/mobile.css b/stylesheets/mobile.css index 10701b6..cc3d828 100644 --- a/stylesheets/mobile.css +++ b/stylesheets/mobile.css @@ -1,6 +1,5 @@ body { background-color: #ffffff; - place-items: center; text-align: center; padding-bottom: 30px; font-family: "Helvetica", "sans-serif"; @@ -17,7 +16,7 @@ td { } th { - background-color: #999; + background-color: #17365c; color: #fff; border-radius: 3px; padding: 10px, 2px; diff --git a/submit_bot.sh b/submit_bot.sh index 7b1f724..18a6cf1 100755 --- a/submit_bot.sh +++ b/submit_bot.sh @@ -1,19 +1,24 @@ #!/bin/bash -ulimit -s 8192 #$ -l low -#$ -l h_vmem=5G +#$ -l h_vmem=2.5G +#$ -l mem=2.5G +#$ -l h_stack=INFINITY #$ -cwd #$ -pe smp 1 -#$ -N survBot_bg -#$ -l os=*stretch +#$ -binding linear:1 +#$ -N survBot +#$ -o /data/www/~kasper/survBot/survBot_bg.log +#$ -e /data/www/~kasper/survBot/survBot_bg.err +#$ -m e +#$ -M kasper.fischer@rub.de source /opt/anaconda3/etc/profile.d/conda.sh -conda activate py37 +conda activate survBot # environment variables for numpy to prevent multi threading export MKL_NUM_THREADS=1 export NUMEXPR_NUM_THREADS=1 export OMP_NUM_THREADS=1 -python survBot.py -html '/data/www/~marcel/' +python survBot.py -html '/data/www/~kasper/survBot' diff --git a/survBot.py b/survBot.py index 775f437..2d5fef5 100755 --- a/survBot.py +++ b/survBot.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -__version__ = '0.1' -__author__ = 'Marcel Paffrath' +__version__ = '0.2-docker' +__author__ = 'Marcel Paffrath ' import os import io @@ -23,7 +23,8 @@ from obspy.clients.filesystem.sds import Client from write_utils import get_html_text, get_html_link, get_html_row, html_footer, get_html_header, get_print_title_str, \ init_html_table, finish_html_table, get_mail_html_header, add_html_image -from utils import get_bg_color, get_font_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds +from utils import get_bg_color, get_font_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds, \ + connect_to_mail_server try: import smtplib @@ -738,7 +739,6 @@ class StationQC(object): if add_addresses: # create copy of addresses ( [:] ) to prevent changing original, general list with addresses addresses = addresses[:] + list(add_addresses) - server = mail_params.get('mailserver') if not sender or not addresses: logging.info('Mail sender or addresses not (correctly) defined. Return') return @@ -757,8 +757,10 @@ class StationQC(object): html_str = self.add_html_mail_body(text) msg.add_alternative(html_str, subtype='html') - # send message via SMTP server - s = smtplib.SMTP(server) + # connect to server, send mail and close connection + s = connect_to_mail_server(mail_params) + if not s: # if connection failed + return s.send_message(msg) s.quit() diff --git a/utils.py b/utils.py index 2723635..f8364ab 100644 --- a/utils.py +++ b/utils.py @@ -5,6 +5,7 @@ import logging import matplotlib import numpy as np +import smtplib from obspy import Stream @@ -279,3 +280,50 @@ def annotate_voltage_states(ax, parameters, pb_key, color='0.75'): ax.annotate(out_string, (ax.get_xlim()[-1], voltage), color=color, fontsize='xx-small', horizontalalignment='right') + +def get_credential(source, param): + """ + Retrieve a credential from a Docker secret or environment variable. + """ + if source == 'DOCKER': + try: + with open('/run/secrets/'+param.lower(), 'r') as f: + return f.read().strip() + except FileNotFoundError as e: + logging.error(f'Could not read from Docker secret at /run/secrets/{param.lower()}') + logging.error(e) + elif source == 'ENV': + try: + return os.environ.get(param.upper()) + except Exception as e: + logging.error(f'Could not read from environment variable {param.upper()}') + logging.error(e) + # return source if no credential was found + return source + +def connect_to_mail_server(self, mail_params): + """ + Connect to mail server and return server object. + """ + # get server from parameters + server = mail_params.get('mailserver') + # get auth_type from parameters + auth_type = mail_params.get('auth_type') + # set up connection to mail server + if auth_type == 'None': + s = smtplib.SMTP(server) + else: + # user and password from parameters, docker secret or environment variable + user = get_credential(mail_params.get('user'), 'mail_user') + password = get_credential(mail_params.get('password'), 'mail_password') + # create secure connection to server + if auth_type == 'SSL': + s = smtplib.SMTP_SSL(server, mail_params.get('port')) + elif auth_type == 'TLS': + s = smtplib.SMTP(server, mail_params.get('port')) + s.starttls() + else: + logging.error('Unknown authentication type. Mails can not be sent') + return + s.login(mail_params.get('user'), mail_params.get('password')) + return s