From 9626a4c88d04a0814fd61317ddb4f0d4ccbead0e Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Thu, 20 Mar 2025 10:00:25 +0100 Subject: [PATCH 01/15] [minor] add station GR27 to station blacklist in parameters.yaml - Updated stations_blacklist to exclude stations DOMV, EREA, GR19, GR27, LAKA, LFKM and TEST - Changed the description of datapath to clarify it as the path to SDS data archive. - Fixed some typos. --- parameters.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parameters.yaml b/parameters.yaml index 519fbaa..c1c4e20 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -1,9 +1,9 @@ # Parameters file for Surveillance Bot -datapath: "/data/SDS/" # SC3 Datapath +datapath: "/data/SDS/" # path to SDS data archive networks: ["1Y", "HA", "MK"] # select networks, list or str stations: "*" # select stations, list or str locations: "*" # select locations, list or str -stations_blacklist: ["TEST", "EREA", "DOMV", "LFKM", "GR19", "LAKA"] # exclude these stations +stations_blacklist: ["DOMV", "EREA", "GR19", "GR27", "LAKA", "LFKM", "TEST"] # exclude these stations networks_blacklist: [] # exclude these networks interval: 60 # Perform checks every x seconds n_track: 360 # wait n_track * intervals before performing an action (i.e. send mail/end highlight status) @@ -13,7 +13,7 @@ logging_level: WARN # set logging level (info, warning, debug) track_changes: True # tracks all changes since GUI startup by text highlighting (GUI only) warn_count: False # show number of warnings and errors in table min_sample: 5 # minimum samples for raising Warn/FAIL -dt_thresh: [300, 1800] # threshold (s) for timing delay colourisation (yellow/red) +dt_thresh: [300, 1800] # threshold (s) for timing delay colorization (yellow/red) html_figures: True # Create html figure directory and links reread_parameters: True # reread parameters file (change parameters on runtime, not for itself/GUI refresh/datapath) @@ -56,7 +56,7 @@ THRESHOLDS: # # For each channel a factor 'unit' for unit conversion (e.g. to SI) can be provided, as well as a 'name' # and 'ticks' [ymin, ymax, ystep] for plotting. -# 'warn' and 'fail' plot horizontal lines in corresponding colors (can be str in TRESHOLDS, int/float or iterable) +# 'warn' and 'fail' plot horizontal lines in corresponding colors (can be str in THRESHOLDS, int/float or iterable) # keyword "pb_SOH2" or "pb_SOH3" can be used to extract warning values from above POWBOX parameter definition # # 'transform' can be provided for plotting to perform arithmetic operations in given order, e.g.: From 3dbba37fe9d1c668a7295768be889eb15910e48c Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Thu, 20 Mar 2025 10:33:37 +0100 Subject: [PATCH 02/15] [minor] update stations_blacklist in parameters.yaml - Add "ATHR" and "HAVD" to the list of excluded stations - No changes to other parameters --- parameters.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parameters.yaml b/parameters.yaml index c1c4e20..eaab309 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -3,7 +3,7 @@ datapath: "/data/SDS/" # path to SDS data archive networks: ["1Y", "HA", "MK"] # select networks, list or str stations: "*" # select stations, list or str locations: "*" # select locations, list or str -stations_blacklist: ["DOMV", "EREA", "GR19", "GR27", "LAKA", "LFKM", "TEST"] # exclude these stations +stations_blacklist: ["ATHR", "DOMV", "EREA", "GR19", "GR27", "HAVD", "LAKA", "LFKM", "TEST"] # exclude these stations networks_blacklist: [] # exclude these networks interval: 60 # Perform checks every x seconds n_track: 360 # wait n_track * intervals before performing an action (i.e. send mail/end highlight status) From 0ce41e5654099f2780d345cbe3d01bdc5853d93e Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Thu, 20 Mar 2025 11:27:46 +0100 Subject: [PATCH 03/15] Change resource limits and logging paths in submit_bot.sh - Update h_vmem limit to 2.5G - Add mem limit of 2.5G - Set h_stack to INFINITY - Change output log path to /data/www/~kasper/survBot/survBot_bg.log - Change error log path to /data/www/~kasper/survBot/survBot_bg.err - Enable email notifications for errors - Update HTML output directory path --- submit_bot.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/submit_bot.sh b/submit_bot.sh index 7b1f724..18a6cf1 100755 --- a/submit_bot.sh +++ b/submit_bot.sh @@ -1,19 +1,24 @@ #!/bin/bash -ulimit -s 8192 #$ -l low -#$ -l h_vmem=5G +#$ -l h_vmem=2.5G +#$ -l mem=2.5G +#$ -l h_stack=INFINITY #$ -cwd #$ -pe smp 1 -#$ -N survBot_bg -#$ -l os=*stretch +#$ -binding linear:1 +#$ -N survBot +#$ -o /data/www/~kasper/survBot/survBot_bg.log +#$ -e /data/www/~kasper/survBot/survBot_bg.err +#$ -m e +#$ -M kasper.fischer@rub.de source /opt/anaconda3/etc/profile.d/conda.sh -conda activate py37 +conda activate survBot # environment variables for numpy to prevent multi threading export MKL_NUM_THREADS=1 export NUMEXPR_NUM_THREADS=1 export OMP_NUM_THREADS=1 -python survBot.py -html '/data/www/~marcel/' +python survBot.py -html '/data/www/~kasper/survBot' From f4605b146b0843f3eee32d00f7a824e79f21bd58 Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Thu, 20 Mar 2025 11:38:54 +0100 Subject: [PATCH 04/15] [feature] run survBot in Docker container - version 0.2-docker - add Dockerfile - update paramters.yaml to use logo.png - update stylesheet to reflect latest changes - removed survBotGUI.py which is not needed in Docker container --- Dockerfile | 16 ++ __init__.py | 2 +- mailing_list.yaml | 2 +- parameters.yaml | 2 +- stylesheets/desktop.css | 3 +- stylesheets/mobile.css | 3 +- survBot.py | 4 +- survBotGUI.py | 441 ---------------------------------------- 8 files changed, 23 insertions(+), 450 deletions(-) create mode 100644 Dockerfile delete mode 100755 survBotGUI.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7fd7f33 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3 + +WORKDIR /usr/src/app + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +RUN mkdir -p www +RUN ln -s www/survBot_out.html www/index.html +RUN cp stylesheets/*.css www/ +RUN touch logo.png +RUN cp logo.png www/logo.png + +CMD [ "python", "./survBot.py", "-html", "www" ] diff --git a/__init__.py b/__init__.py index 90d7001..7953f6e 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1,3 @@ # survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output # over SOH channels by analysing contents of a Seiscomp3 datapath. -__version__ = "0.2" +__version__ = "0.2-docker" diff --git a/mailing_list.yaml b/mailing_list.yaml index 5bae70a..4fd7be8 100644 --- a/mailing_list.yaml +++ b/mailing_list.yaml @@ -2,7 +2,7 @@ # "mail.address@provider.com, mail.address2@provider2.com": # - 1Y.GR01 # - 1Y.GR02 -# "mail.address3@provder.com": +# "mail.address3@provider.com": # - 1Y.GR03 #"kasper.fischer@rub.de": diff --git a/parameters.yaml b/parameters.yaml index eaab309..9917531 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -133,7 +133,7 @@ add_global_links: "URL": "https://fdsnws.geophysik.ruhr-uni-bochum.de/map/?lat=39.5&lon=21&zoom=7&baselayer=mapnik"} # html logo at page bottom (path relative to html directory) -html_logo: "figures/Logo_RUB_BLAU_rgb.png" +html_logo: "logo.png" # E-mail notifications EMAIL: diff --git a/stylesheets/desktop.css b/stylesheets/desktop.css index f25fc28..6ada37c 100644 --- a/stylesheets/desktop.css +++ b/stylesheets/desktop.css @@ -1,6 +1,5 @@ body { background-color: #ffffff; - place-items: center; text-align: center; padding-bottom: 30px; font-family: "Helvetica", "sans-serif"; @@ -17,7 +16,7 @@ td { } th { - background-color: #999; + background-color: #17365c; color: #fff; border-radius: 2px; padding: 3px 1px; diff --git a/stylesheets/mobile.css b/stylesheets/mobile.css index 10701b6..cc3d828 100644 --- a/stylesheets/mobile.css +++ b/stylesheets/mobile.css @@ -1,6 +1,5 @@ body { background-color: #ffffff; - place-items: center; text-align: center; padding-bottom: 30px; font-family: "Helvetica", "sans-serif"; @@ -17,7 +16,7 @@ td { } th { - background-color: #999; + background-color: #17365c; color: #fff; border-radius: 3px; padding: 10px, 2px; diff --git a/survBot.py b/survBot.py index 775f437..2117370 100755 --- a/survBot.py +++ b/survBot.py @@ -1,8 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -__version__ = '0.1' -__author__ = 'Marcel Paffrath' +__version__ = '0.2-docker' +__author__ = 'Marcel Paffrath ' import os import io diff --git a/survBotGUI.py b/survBotGUI.py deleted file mode 100755 index 79fc61c..0000000 --- a/survBotGUI.py +++ /dev/null @@ -1,441 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -GUI overlay for the main survBot to show quality control of different stations specified in parameters.yaml file. -""" - -__version__ = '0.1' -__author__ = 'Marcel Paffrath' - -import os -import sys -import traceback - -import logging - -try: - from PySide2 import QtGui, QtCore, QtWidgets -except ImportError: - try: - from PySide6 import QtGui, QtCore, QtWidgets - except ImportError: - try: - from PyQt5 import QtGui, QtCore, QtWidgets - except ImportError: - raise ImportError('Could import neither of PySide2, PySide6 or PyQt5') - -from matplotlib.figure import Figure - -if QtGui.__package__ in ['PySide2', 'PyQt5', 'PySide6']: - from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas - from matplotlib.backends.backend_qt5agg import NavigationToolbar2QT -else: - raise Exception('Not implemented') - -from obspy import UTCDateTime - -from survBot import SurveillanceBot -from write_utils import * -from utils import get_bg_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds - -try: - from rest_api.utils import get_station_iccid - from rest_api.rest_api_utils import get_last_messages, send_message, get_default_params - sms_funcs = True -except ImportError: - logging.warning('Could not load rest_api utils, SMS functionality disabled.') - sms_funcs = False - -deg_str = '\N{DEGREE SIGN}C' - - -class Thread(QtCore.QThread): - """ - A simple thread that runs outside of the main event loop. Executes the function "runnable" and prevents - freezing of the GUI. Run method is executed outside main event loop when called with thread.start(). - """ - update = QtCore.Signal() - - def __init__(self, parent, runnable): - super(Thread, self).__init__(parent=parent) - self.setParent(parent) - self.runnable = runnable - self.is_active = True - - def run(self): - """ Try to run self.runnable and emit update signal, or print Exception if failed. """ - try: - t0 = UTCDateTime() - self.runnable() - self.update.emit() - except Exception as e: - self.is_active = False - logging.error(e) - logging.debug(traceback.format_exc()) - finally: - logging.info(f'Time for Thread execution: {UTCDateTime() - t0}') - - -class MainWindow(QtWidgets.QMainWindow): - def __init__(self, parameters='parameters.yaml'): - """ - Main window of survBot GUI. - :param parameters: Parameters dictionary file (yaml format) - """ - super(MainWindow, self).__init__() - - # init some attributes - self.last_mouse_loc = None - self.status_message = '' - self.starttime = UTCDateTime() - - # setup main layout of the GUI - self.main_layout = QtWidgets.QVBoxLayout() - self.centralWidget = QtWidgets.QWidget() - self.centralWidget.setLayout(self.main_layout) - self.setCentralWidget(self.centralWidget) - - # init new survBot instance, set parameters and refresh - self.survBot = SurveillanceBot(parameter_path=parameters) - self.parameters = self.survBot.parameters - self.refresh_period = self.parameters.get('interval') - self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')] - - # create thread that is used to update - self.thread = Thread(parent=self, runnable=self.survBot.execute_qc) - self.thread.update.connect(self.fill_table) - - self.init_table() - self.init_buttons() - - # These filters were used to track current mouse position if an event (i.e. mouseclick) is triggered - self.table.installEventFilter(self) - self.installEventFilter(self) - - # initiate clear_on_refresh flag and set status bar text - self.clear_on_refresh = False - self.fill_status_bar() - - # start thread that executes qc at first initiation, then activate timer for further thread activation - self.thread.start() - self.run_refresh_timer() - - def init_table(self): - self.table = QtWidgets.QTableWidget() - keys = self.survBot.keys - station_list = self.survBot.station_list - - self.table.setColumnCount(len(keys)) - self.table.setRowCount(len(station_list)) - self.table.setHorizontalHeaderLabels(keys) - - for index, nwst_id in enumerate(station_list): - item = QtWidgets.QTableWidgetItem() - item.setText(str(nwst_id.rstrip('.'))) - item.setData(QtCore.Qt.UserRole, nwst_id) - self.table.setVerticalHeaderItem(index, item) - - self.main_layout.addWidget(self.table) - - self.table.itemDoubleClicked.connect(self.plot_stream) - self.table.setEditTriggers(QtWidgets.QTableWidget.NoEditTriggers) - - if sms_funcs: - self.table.verticalHeader().sectionClicked.connect(self.sms_context_menu) - - self.set_stretch() - - def init_buttons(self): - if self.parameters.get('track_changes'): - button_text = 'Clear track and refresh' - else: - button_text = 'Refresh' - self.clear_button = QtWidgets.QPushButton(button_text) - self.clear_button.setToolTip('Reset track changes and refresh table') - self.clear_button.clicked.connect(self.refresh) - self.main_layout.addWidget(self.clear_button) - - def refresh(self): - self.set_clear_on_refresh() - self.run_refresh_timer() - self.thread.start() - - def run_refresh_timer(self): - self.timer = QtCore.QTimer() - self.timer.timeout.connect(self.thread.start) - self.timer.start(int(self.refresh_period * 1e3)) - - def eventFilter(self, object, event): - """ - An event filter that stores last mouse position if an event is raised by the table. All events are passed - to the parent class of the Mainwindow afterwards. - """ - if hasattr(event, 'pos'): - self.last_mouse_loc = event.pos() - return super(QtWidgets.QMainWindow, self).eventFilter(object, event) - - def sms_context_menu(self, row_ind): - """ Open a context menu when left-clicking vertical header item """ - header_item = self.table.verticalHeaderItem(row_ind) - if not header_item: - return - nwst_id = header_item.data(QtCore.Qt.UserRole) - - context_menu = QtWidgets.QMenu() - read_sms = context_menu.addAction('Get last SMS') - send_sms = context_menu.addAction('Send SMS') - action = context_menu.exec_(self.mapToGlobal(self.last_mouse_loc)) - if action == read_sms: - self.read_sms(nwst_id) - elif action == send_sms: - self.send_sms(nwst_id) - - def read_sms(self, nwst_id): - """ Read recent SMS over rest_api using whereversim portal """ - station = nwst_id.split('.')[1] - iccid = get_station_iccid(station) - if not iccid: - logging.info(f'Could not find iccid for station: {nwst_id}') - return - sms_widget = ReadSMSWidget(parent=self, iccid=iccid) - sms_widget.setWindowTitle(f'Recent SMS of station: {nwst_id}') - if sms_widget.data: - sms_widget.show() - else: - self.notification('No recent messages found.') - - def send_sms(self, nwst_id): - """ Send SMS over rest_api using whereversim portal """ - station = nwst_id.split('.')[1] - iccid = get_station_iccid(station) - - sms_widget = SendSMSWidget(parent=self, iccid=iccid) - sms_widget.setWindowTitle(f'Send SMS to station: {nwst_id}') - sms_widget.show() - - def set_clear_on_refresh(self): - self.clear_on_refresh = True - - def fill_status_bar(self): - """ Set status bar text """ - self.status_message = self.survBot.status_message - status_bar = self.statusBar() - status_bar.showMessage(self.status_message) - - def fill_table(self): - """ Fills the table with most recent information. Executed after execute_qc thread is done or on refresh. """ - - # fill status bar first with new time - self.fill_status_bar() - - for col_ind, check_key in enumerate(self.survBot.keys): - for row_ind, nwst_id in enumerate(self.survBot.station_list): - status_dict = self.survBot.analysis_results.get(nwst_id) - status = status_dict.get(check_key) - message, detailed_message = status.get_status_str() - - dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh] - bg_color = get_bg_color(check_key, status, dt_thresh) - if check_key == 'temp': - if not type(message) in [str]: - message = str(message) + deg_str - - # Continue if nothing changed - text = str(message) - cur_item = self.table.item(row_ind, col_ind) - if cur_item and text == cur_item.text(): - if not self.parameters.get('track_changes') or self.clear_on_refresh: - # set item to default color/font and continue - self.set_font(cur_item) - self.set_fg_color(cur_item) - continue - - # Create new data item - item = QtWidgets.QTableWidgetItem() - item.setText(str(message)) - item.setTextAlignment(QtCore.Qt.AlignCenter) - item.setData(QtCore.Qt.UserRole, (nwst_id, check_key)) - - # if text changed (known from above) set highlight color/font else (new init) set to default - cur_item = self.table.item(row_ind, col_ind) - if cur_item and check_key != 'last active': - self.set_fg_color(item, (0, 0, 0, 255)) - self.set_font_bold(item) - else: - self.set_fg_color(item) - self.set_font(item) - - # set item tooltip - if detailed_message: - item.setToolTip(str(detailed_message)) - - # set bg color corresponding to current text (OK/WARN/ERROR etc.) - self.set_bg_color(item, bg_color) - - # insert new item - self.table.setItem(row_ind, col_ind, item) - - # table filling/refreshing done, set clear_on_refresh to False - self.clear_on_refresh = False - - def set_font_bold(self, item): - """ Set item font bold """ - f = item.font() - f.setWeight(QtGui.QFont.Bold) - item.setFont(f) - - def set_font(self, item): - """ Set item font normal """ - f = item.font() - f.setWeight(QtGui.QFont.Normal) - item.setFont(f) - - def set_bg_color(self, item, color): - """ Set background color of item, color is RGBA tuple """ - color = QtGui.QColor(*color) - item.setBackground(color) - - def set_fg_color(self, item, color=(20, 20, 20, 255)): - """ Set foreground (font) color of item, color is RGBA tuple """ - color = QtGui.QColor(*color) - item.setForeground(color) - - def set_stretch(self): - hheader = self.table.horizontalHeader() - for index in range(hheader.count()): - hheader.setSectionResizeMode(index, QtWidgets.QHeaderView.Stretch) - vheader = self.table.verticalHeader() - for index in range(vheader.count()): - vheader.setSectionResizeMode(index, QtWidgets.QHeaderView.Stretch) - - def plot_stream(self, item): - nwst_id, check = item.data(QtCore.Qt.UserRole) - st = self.survBot.data.get(nwst_id) - if st: - self.plot_widget = PlotWidget(self) - self.plot_widget.setWindowTitle(nwst_id) - st = modify_stream_for_plot(st, parameters=self.parameters) - st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig) - # set_axis_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters) - set_axis_yticks(fig=self.plot_widget.canvas.fig, parameters=self.parameters) - set_axis_color(fig=self.plot_widget.canvas.fig) - plot_axis_thresholds(fig=self.plot_widget.canvas.fig, parameters=self.parameters) - self.plot_widget.show() - - def notification(self, text): - mbox = QtWidgets.QMessageBox() - mbox.setWindowTitle('Notification') - #mbox.setDetailedText() - mbox.setText(text) - mbox.exec_() - - def closeEvent(self, event): - self.thread.exit() - event.accept() - - -class PlotCanvas(FigureCanvas): - def __init__(self, parent=None, width=10, height=8, dpi=100): - self.fig = Figure(figsize=(width, height), dpi=dpi) - FigureCanvas.__init__(self, self.fig) - self.setParent(parent) - FigureCanvas.setSizePolicy(self, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding) - FigureCanvas.updateGeometry(self) - - -class PlotWidget(QtWidgets.QDialog): - def __init__(self, *args, **kwargs): - QtWidgets.QDialog.__init__(self, *args, **kwargs) - self.setLayout(QtWidgets.QVBoxLayout()) - self.canvas = PlotCanvas(self, width=10, height=8) - self.toolbar = NavigationToolbar2QT(self.canvas, self) - self.layout().addWidget(self.toolbar) - self.layout().addWidget(self.canvas) - - -class ReadSMSWidget(QtWidgets.QDialog): - def __init__(self, iccid, *args, **kwargs): - QtWidgets.QDialog.__init__(self, *args, **kwargs) - self.setLayout(QtWidgets.QVBoxLayout()) - self.table = QtWidgets.QTableWidget() - self.layout().addWidget(self.table) - self.resize(1280, 400) - - self.iccid = iccid - self.data = self.print_sms_table() - self.set_stretch() - - def print_sms_table(self, n=5, ntextbreak=40): - messages = [] - params = get_default_params(self.iccid) - for message in get_last_messages(params, n, only_delivered=False): - messages.append(message) - if not messages: - return - # pull dates to front - keys = ['dateSent', 'dateModified', 'dateReceived'] - for item in messages[0].keys(): - if not item in keys: - keys.append(item) - self.table.setRowCount(n) - self.table.setColumnCount(len(keys)) - self.table.setHorizontalHeaderLabels(keys) - for row_index, message in enumerate(messages): - for col_index, key in enumerate(keys): - text = message.get(key) - if type(text) == str and len(text) > ntextbreak: - textlist = list(text) - for index in range(ntextbreak, len(text), ntextbreak): - textlist.insert(index, '\n') - text = ''.join(textlist) - item = QtWidgets.QTableWidgetItem() - item.setText(str(text)) - self.table.setItem(row_index, col_index, item) - return True - - def set_stretch(self): - hheader = self.table.horizontalHeader() - nheader = hheader.count() - for index in range(nheader): - if index < nheader - 1: - hheader.setSectionResizeMode(index, QtWidgets.QHeaderView.ResizeToContents) - else: - hheader.setSectionResizeMode(index, QtWidgets.QHeaderView.Stretch) - vheader = self.table.verticalHeader() - for index in range(vheader.count()): - vheader.setSectionResizeMode(index, QtWidgets.QHeaderView.Stretch) - - - -class SendSMSWidget(QtWidgets.QDialog): - def __init__(self, iccid, *args, **kwargs): - QtWidgets.QDialog.__init__(self, *args, **kwargs) - self.main_layout = QtWidgets.QVBoxLayout() - self.setLayout(self.main_layout) - self.resize(400, 100) - - self.line_edit = QtWidgets.QLineEdit() - self.main_layout.addWidget(self.line_edit) - - self.iccid = iccid - - self.buttonBox = QtWidgets.QDialogButtonBox(QtWidgets.QDialogButtonBox.Ok | - QtWidgets.QDialogButtonBox.Close) - self.main_layout.addWidget(self.buttonBox) - self.buttonBox.accepted.connect(self.send_sms) - self.buttonBox.rejected.connect(self.reject) - - def send_sms(self): - text = self.line_edit.text() - params = get_default_params(self.iccid) - send_message(params, text) - self.close() - - -if __name__ == '__main__': - program_path = sys.path[0] - parameters = os.path.join(program_path, 'parameters.yaml') - app = QtWidgets.QApplication([]) - window = MainWindow(parameters=parameters) - window.showMaximized() - sys.exit(app.exec_()) From 3e47f4275b7be32b001b8c870ed40e13566acca0 Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Thu, 20 Mar 2025 15:58:22 +0100 Subject: [PATCH 05/15] Add bind9-host and iputils-ping to Docker image requirements. - Install bind9-host and iputils-ping packages in the Docker image. - Create a new file "requirements.txt" with the specified package versions. --- Dockerfile | 7 +------ requirements.txt | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 6 deletions(-) create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index 7fd7f33..e37f11f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,13 +4,8 @@ WORKDIR /usr/src/app COPY requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt +RUN apt update && apt install -y bind9-host iputils-ping COPY . . -RUN mkdir -p www -RUN ln -s www/survBot_out.html www/index.html -RUN cp stylesheets/*.css www/ -RUN touch logo.png -RUN cp logo.png www/logo.png - CMD [ "python", "./survBot.py", "-html", "www" ] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1fb4a59 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,36 @@ +Brotli==1.1.0 +certifi==2025.1.31 +cffi==1.17.1 +charset-normalizer==3.4.1 +contourpy==1.3.1 +cycler==0.12.1 +decorator==5.2.1 +fonttools==4.56.0 +greenlet==3.1.1 +h2==4.2.0 +hpack==4.1.0 +hyperframe==6.1.0 +idna==3.10 +kiwisolver==1.4.8 +lxml==5.3.1 +matplotlib==3.8.4 +munkres==1.1.4 +numpy==1.26.4 +obspy==1.4.1 +packaging==24.2 +pillow==11.1.0 +pip==25.0.1 +pycparser==2.22 +pyparsing==3.2.1 +PySocks==1.7.1 +python-dateutil==2.9.0.post0 +PyYAML==6.0.2 +requests==2.32.3 +scipy==1.15.2 +setuptools==75.8.2 +six==1.17.0 +SQLAlchemy==1.4.54 +unicodedata2==16.0.0 +urllib3==2.3.0 +wheel==0.45.1 +zstandard==0.23.0 From 43912135e9849b3369de12b377d02771ef14af7a Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Thu, 20 Mar 2025 15:58:51 +0100 Subject: [PATCH 06/15] Update email parameters in parameters.yaml and modify SMTP connection in survBot.py - Update mailserver, port, user, password, and sender in parameters.yaml - Modify SMTP connection in survBot.py to support starttls connection if server is not "localhost" - Read password from docker secret or environment variable if set to 'DOCKER' or 'ENV' respectively --- parameters.yaml | 7 +++++-- survBot.py | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/parameters.yaml b/parameters.yaml index 9917531..fcf609f 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -137,9 +137,12 @@ html_logo: "logo.png" # E-mail notifications EMAIL: - mailserver: "localhost" + mailserver: "smtp.rub.de" # mail server + port: 465 # mail port + user: "DOCKER" # mail user, read from environment variable if set to "ENV" or from docker secret if set to "DOCKER" + password: "DOCKER" # mail password, read from environment variable if set to "ENV" or from docker secret if set to "DOCKER" addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"] # list of mail addresses for info mails - sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender + sender: "RUB SeisObs " # mail sender stations_blacklist: [] # do not send emails for specific stations networks_blacklist: [] # do not send emails for specific network # specify recipients for single stations in a yaml: key = email-address, val = station list (e.g. [1Y.GR01, 1Y.GR02]) diff --git a/survBot.py b/survBot.py index 2117370..3a9464c 100755 --- a/survBot.py +++ b/survBot.py @@ -758,7 +758,42 @@ class StationQC(object): msg.add_alternative(html_str, subtype='html') # send message via SMTP server - s = smtplib.SMTP(server) + # set up starttls connection if server is not "localhost" + if server == 'localhost': + # create connection to localhost + s = smtplib.SMTP(server) + else: + user = mail_params.get('user') + # read user from docker secret if it is set to 'DOCKER' or + # read user from environment variable if it is set to 'ENV' + if user == 'DOCKER': + try: + with open('/run/secrets/mail_user', 'r') as f: + user = f.read().strip() + except FileNotFoundError as e: + logging.error('Could not read mail user from docker secret') + logging.error(e) + elif user == 'ENV': + user = os.environ.get(mail_params.get('user_env')) + + password = mail_params.get('password') + # read password from docker secret if it is set to 'DOCKER' or + # read password from environment variable if it is set to 'ENV' + if password == 'DOCKER': + try: + with open('/run/secrets/mail_password', 'r') as f: + password = f.read().strip() + except FileNotFoundError as e: + logging.error('Could not read mail password from docker secret') + logging.error(e) + elif password == 'ENV': + password = os.environ.get(mail_params.get('password_env')) + + # create SSL connection to server + s = smtplib.SMTP_SSL(server, mail_params.get('port')) + s.login(mail_params.get('user'), mail_params.get('password')) + + # send mail and close connection s.send_message(msg) s.quit() From cf12500ec20d06ab9cab0ce712762631f8353d54 Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Fri, 21 Mar 2025 12:36:52 +0100 Subject: [PATCH 07/15] [minor] refactor mail server connection code - Added `connect_to_mail_server` function to handle mail server connection - Moved code for connecting to the mail server from `StationQC` class to `connect_to_mail_server` - Updated references to use `connect_to_mail_server` in `StationQC` class - Created new function `get_credential` to retrieve credentials from Docker secrets or environment variables --- survBot.py | 45 ++++++--------------------------------------- utils.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 39 deletions(-) diff --git a/survBot.py b/survBot.py index 3a9464c..2d5fef5 100755 --- a/survBot.py +++ b/survBot.py @@ -23,7 +23,8 @@ from obspy.clients.filesystem.sds import Client from write_utils import get_html_text, get_html_link, get_html_row, html_footer, get_html_header, get_print_title_str, \ init_html_table, finish_html_table, get_mail_html_header, add_html_image -from utils import get_bg_color, get_font_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds +from utils import get_bg_color, get_font_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds, \ + connect_to_mail_server try: import smtplib @@ -738,7 +739,6 @@ class StationQC(object): if add_addresses: # create copy of addresses ( [:] ) to prevent changing original, general list with addresses addresses = addresses[:] + list(add_addresses) - server = mail_params.get('mailserver') if not sender or not addresses: logging.info('Mail sender or addresses not (correctly) defined. Return') return @@ -757,43 +757,10 @@ class StationQC(object): html_str = self.add_html_mail_body(text) msg.add_alternative(html_str, subtype='html') - # send message via SMTP server - # set up starttls connection if server is not "localhost" - if server == 'localhost': - # create connection to localhost - s = smtplib.SMTP(server) - else: - user = mail_params.get('user') - # read user from docker secret if it is set to 'DOCKER' or - # read user from environment variable if it is set to 'ENV' - if user == 'DOCKER': - try: - with open('/run/secrets/mail_user', 'r') as f: - user = f.read().strip() - except FileNotFoundError as e: - logging.error('Could not read mail user from docker secret') - logging.error(e) - elif user == 'ENV': - user = os.environ.get(mail_params.get('user_env')) - - password = mail_params.get('password') - # read password from docker secret if it is set to 'DOCKER' or - # read password from environment variable if it is set to 'ENV' - if password == 'DOCKER': - try: - with open('/run/secrets/mail_password', 'r') as f: - password = f.read().strip() - except FileNotFoundError as e: - logging.error('Could not read mail password from docker secret') - logging.error(e) - elif password == 'ENV': - password = os.environ.get(mail_params.get('password_env')) - - # create SSL connection to server - s = smtplib.SMTP_SSL(server, mail_params.get('port')) - s.login(mail_params.get('user'), mail_params.get('password')) - - # send mail and close connection + # connect to server, send mail and close connection + s = connect_to_mail_server(mail_params) + if not s: # if connection failed + return s.send_message(msg) s.quit() diff --git a/utils.py b/utils.py index 2723635..f8364ab 100644 --- a/utils.py +++ b/utils.py @@ -5,6 +5,7 @@ import logging import matplotlib import numpy as np +import smtplib from obspy import Stream @@ -279,3 +280,50 @@ def annotate_voltage_states(ax, parameters, pb_key, color='0.75'): ax.annotate(out_string, (ax.get_xlim()[-1], voltage), color=color, fontsize='xx-small', horizontalalignment='right') + +def get_credential(source, param): + """ + Retrieve a credential from a Docker secret or environment variable. + """ + if source == 'DOCKER': + try: + with open('/run/secrets/'+param.lower(), 'r') as f: + return f.read().strip() + except FileNotFoundError as e: + logging.error(f'Could not read from Docker secret at /run/secrets/{param.lower()}') + logging.error(e) + elif source == 'ENV': + try: + return os.environ.get(param.upper()) + except Exception as e: + logging.error(f'Could not read from environment variable {param.upper()}') + logging.error(e) + # return source if no credential was found + return source + +def connect_to_mail_server(self, mail_params): + """ + Connect to mail server and return server object. + """ + # get server from parameters + server = mail_params.get('mailserver') + # get auth_type from parameters + auth_type = mail_params.get('auth_type') + # set up connection to mail server + if auth_type == 'None': + s = smtplib.SMTP(server) + else: + # user and password from parameters, docker secret or environment variable + user = get_credential(mail_params.get('user'), 'mail_user') + password = get_credential(mail_params.get('password'), 'mail_password') + # create secure connection to server + if auth_type == 'SSL': + s = smtplib.SMTP_SSL(server, mail_params.get('port')) + elif auth_type == 'TLS': + s = smtplib.SMTP(server, mail_params.get('port')) + s.starttls() + else: + logging.error('Unknown authentication type. Mails can not be sent') + return + s.login(mail_params.get('user'), mail_params.get('password')) + return s From 3f07b7bcd049ce7fe96966dd7a15b6744f9cdcff Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Fri, 21 Mar 2025 12:39:10 +0100 Subject: [PATCH 08/15] [minor] update Dockerfile and parameters.yaml - In the Dockerfile, added a new parameter "-parfile" with value "conf/parameters.yaml" to the CMD command in survBot.py. - In parameters.yaml, made changes to the EMAIL section: - Added comments explaining how to specify mail server and credentials. - Added auth_type field with value "SSL". - Updated port field to 465 for SSL. - Updated user and password fields to read from environment variables or docker secrets. - Added comments explaining how to specify mail recipients, sender, and blacklists. - Moved location of simulate_fail.json to conf/simulate_fail.json for easier Docker integration --- Dockerfile | 2 +- parameters.yaml | 17 +++++++++++------ survBot.py | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index e37f11f..14aa895 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,4 +8,4 @@ RUN apt update && apt install -y bind9-host iputils-ping COPY . . -CMD [ "python", "./survBot.py", "-html", "www" ] +CMD [ "python", "./survBot.py", "-html", "www", "-parfile", "conf/parameters.yaml" ] diff --git a/parameters.yaml b/parameters.yaml index fcf609f..1be3d16 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -137,13 +137,18 @@ html_logo: "logo.png" # E-mail notifications EMAIL: - mailserver: "smtp.rub.de" # mail server - port: 465 # mail port + # specify mail server and credentials + # port, auth_type, user and password are only required if mailserver is not set to "localhost" + # user and password can be set to "ENV" or "DOCKER" to read from environment variables or docker secrets + mailserver: "smtp.rub.de" # mail server + auth_type: "SSL" # mail authentication type, can be "SSL", "TLS" or "None" + port: 465 # mail port, default 465 for SSL, 587 for TLS user: "DOCKER" # mail user, read from environment variable if set to "ENV" or from docker secret if set to "DOCKER" password: "DOCKER" # mail password, read from environment variable if set to "ENV" or from docker secret if set to "DOCKER" + # specify mail recipients, sender and blacklists addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"] # list of mail addresses for info mails - sender: "RUB SeisObs " # mail sender - stations_blacklist: [] # do not send emails for specific stations - networks_blacklist: [] # do not send emails for specific network + sender: "RUB SeisObs " # mail sender + stations_blacklist: [] # do not send emails for specific stations + networks_blacklist: [] # do not send emails for specific network # specify recipients for single stations in a yaml: key = email-address, val = station list (e.g. [1Y.GR01, 1Y.GR02]) - external_mail_list: "mailing_list.yaml" + external_mail_list: "conf/mailing_list.yaml" diff --git a/survBot.py b/survBot.py index 2d5fef5..53357d7 100755 --- a/survBot.py +++ b/survBot.py @@ -704,7 +704,7 @@ class StationQC(object): return 'active' return False - def _simulated_error_check(self, key, fname='simulate_fail.json'): + def _simulated_error_check(self, key, fname='conf/simulate_fail.json'): if not os.path.isfile(fname): return with open(fname) as fid: From 8a7e402ec598d95a1dd7d145ac57b3b300b5eb3c Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Fri, 21 Mar 2025 14:08:00 +0100 Subject: [PATCH 09/15] reverted deletion of survBotGUI.py --- survBotGUI.py | 441 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 441 insertions(+) create mode 100755 survBotGUI.py diff --git a/survBotGUI.py b/survBotGUI.py new file mode 100755 index 0000000..79fc61c --- /dev/null +++ b/survBotGUI.py @@ -0,0 +1,441 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +GUI overlay for the main survBot to show quality control of different stations specified in parameters.yaml file. +""" + +__version__ = '0.1' +__author__ = 'Marcel Paffrath' + +import os +import sys +import traceback + +import logging + +try: + from PySide2 import QtGui, QtCore, QtWidgets +except ImportError: + try: + from PySide6 import QtGui, QtCore, QtWidgets + except ImportError: + try: + from PyQt5 import QtGui, QtCore, QtWidgets + except ImportError: + raise ImportError('Could import neither of PySide2, PySide6 or PyQt5') + +from matplotlib.figure import Figure + +if QtGui.__package__ in ['PySide2', 'PyQt5', 'PySide6']: + from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas + from matplotlib.backends.backend_qt5agg import NavigationToolbar2QT +else: + raise Exception('Not implemented') + +from obspy import UTCDateTime + +from survBot import SurveillanceBot +from write_utils import * +from utils import get_bg_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds + +try: + from rest_api.utils import get_station_iccid + from rest_api.rest_api_utils import get_last_messages, send_message, get_default_params + sms_funcs = True +except ImportError: + logging.warning('Could not load rest_api utils, SMS functionality disabled.') + sms_funcs = False + +deg_str = '\N{DEGREE SIGN}C' + + +class Thread(QtCore.QThread): + """ + A simple thread that runs outside of the main event loop. Executes the function "runnable" and prevents + freezing of the GUI. Run method is executed outside main event loop when called with thread.start(). + """ + update = QtCore.Signal() + + def __init__(self, parent, runnable): + super(Thread, self).__init__(parent=parent) + self.setParent(parent) + self.runnable = runnable + self.is_active = True + + def run(self): + """ Try to run self.runnable and emit update signal, or print Exception if failed. """ + try: + t0 = UTCDateTime() + self.runnable() + self.update.emit() + except Exception as e: + self.is_active = False + logging.error(e) + logging.debug(traceback.format_exc()) + finally: + logging.info(f'Time for Thread execution: {UTCDateTime() - t0}') + + +class MainWindow(QtWidgets.QMainWindow): + def __init__(self, parameters='parameters.yaml'): + """ + Main window of survBot GUI. + :param parameters: Parameters dictionary file (yaml format) + """ + super(MainWindow, self).__init__() + + # init some attributes + self.last_mouse_loc = None + self.status_message = '' + self.starttime = UTCDateTime() + + # setup main layout of the GUI + self.main_layout = QtWidgets.QVBoxLayout() + self.centralWidget = QtWidgets.QWidget() + self.centralWidget.setLayout(self.main_layout) + self.setCentralWidget(self.centralWidget) + + # init new survBot instance, set parameters and refresh + self.survBot = SurveillanceBot(parameter_path=parameters) + self.parameters = self.survBot.parameters + self.refresh_period = self.parameters.get('interval') + self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')] + + # create thread that is used to update + self.thread = Thread(parent=self, runnable=self.survBot.execute_qc) + self.thread.update.connect(self.fill_table) + + self.init_table() + self.init_buttons() + + # These filters were used to track current mouse position if an event (i.e. mouseclick) is triggered + self.table.installEventFilter(self) + self.installEventFilter(self) + + # initiate clear_on_refresh flag and set status bar text + self.clear_on_refresh = False + self.fill_status_bar() + + # start thread that executes qc at first initiation, then activate timer for further thread activation + self.thread.start() + self.run_refresh_timer() + + def init_table(self): + self.table = QtWidgets.QTableWidget() + keys = self.survBot.keys + station_list = self.survBot.station_list + + self.table.setColumnCount(len(keys)) + self.table.setRowCount(len(station_list)) + self.table.setHorizontalHeaderLabels(keys) + + for index, nwst_id in enumerate(station_list): + item = QtWidgets.QTableWidgetItem() + item.setText(str(nwst_id.rstrip('.'))) + item.setData(QtCore.Qt.UserRole, nwst_id) + self.table.setVerticalHeaderItem(index, item) + + self.main_layout.addWidget(self.table) + + self.table.itemDoubleClicked.connect(self.plot_stream) + self.table.setEditTriggers(QtWidgets.QTableWidget.NoEditTriggers) + + if sms_funcs: + self.table.verticalHeader().sectionClicked.connect(self.sms_context_menu) + + self.set_stretch() + + def init_buttons(self): + if self.parameters.get('track_changes'): + button_text = 'Clear track and refresh' + else: + button_text = 'Refresh' + self.clear_button = QtWidgets.QPushButton(button_text) + self.clear_button.setToolTip('Reset track changes and refresh table') + self.clear_button.clicked.connect(self.refresh) + self.main_layout.addWidget(self.clear_button) + + def refresh(self): + self.set_clear_on_refresh() + self.run_refresh_timer() + self.thread.start() + + def run_refresh_timer(self): + self.timer = QtCore.QTimer() + self.timer.timeout.connect(self.thread.start) + self.timer.start(int(self.refresh_period * 1e3)) + + def eventFilter(self, object, event): + """ + An event filter that stores last mouse position if an event is raised by the table. All events are passed + to the parent class of the Mainwindow afterwards. + """ + if hasattr(event, 'pos'): + self.last_mouse_loc = event.pos() + return super(QtWidgets.QMainWindow, self).eventFilter(object, event) + + def sms_context_menu(self, row_ind): + """ Open a context menu when left-clicking vertical header item """ + header_item = self.table.verticalHeaderItem(row_ind) + if not header_item: + return + nwst_id = header_item.data(QtCore.Qt.UserRole) + + context_menu = QtWidgets.QMenu() + read_sms = context_menu.addAction('Get last SMS') + send_sms = context_menu.addAction('Send SMS') + action = context_menu.exec_(self.mapToGlobal(self.last_mouse_loc)) + if action == read_sms: + self.read_sms(nwst_id) + elif action == send_sms: + self.send_sms(nwst_id) + + def read_sms(self, nwst_id): + """ Read recent SMS over rest_api using whereversim portal """ + station = nwst_id.split('.')[1] + iccid = get_station_iccid(station) + if not iccid: + logging.info(f'Could not find iccid for station: {nwst_id}') + return + sms_widget = ReadSMSWidget(parent=self, iccid=iccid) + sms_widget.setWindowTitle(f'Recent SMS of station: {nwst_id}') + if sms_widget.data: + sms_widget.show() + else: + self.notification('No recent messages found.') + + def send_sms(self, nwst_id): + """ Send SMS over rest_api using whereversim portal """ + station = nwst_id.split('.')[1] + iccid = get_station_iccid(station) + + sms_widget = SendSMSWidget(parent=self, iccid=iccid) + sms_widget.setWindowTitle(f'Send SMS to station: {nwst_id}') + sms_widget.show() + + def set_clear_on_refresh(self): + self.clear_on_refresh = True + + def fill_status_bar(self): + """ Set status bar text """ + self.status_message = self.survBot.status_message + status_bar = self.statusBar() + status_bar.showMessage(self.status_message) + + def fill_table(self): + """ Fills the table with most recent information. Executed after execute_qc thread is done or on refresh. """ + + # fill status bar first with new time + self.fill_status_bar() + + for col_ind, check_key in enumerate(self.survBot.keys): + for row_ind, nwst_id in enumerate(self.survBot.station_list): + status_dict = self.survBot.analysis_results.get(nwst_id) + status = status_dict.get(check_key) + message, detailed_message = status.get_status_str() + + dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh] + bg_color = get_bg_color(check_key, status, dt_thresh) + if check_key == 'temp': + if not type(message) in [str]: + message = str(message) + deg_str + + # Continue if nothing changed + text = str(message) + cur_item = self.table.item(row_ind, col_ind) + if cur_item and text == cur_item.text(): + if not self.parameters.get('track_changes') or self.clear_on_refresh: + # set item to default color/font and continue + self.set_font(cur_item) + self.set_fg_color(cur_item) + continue + + # Create new data item + item = QtWidgets.QTableWidgetItem() + item.setText(str(message)) + item.setTextAlignment(QtCore.Qt.AlignCenter) + item.setData(QtCore.Qt.UserRole, (nwst_id, check_key)) + + # if text changed (known from above) set highlight color/font else (new init) set to default + cur_item = self.table.item(row_ind, col_ind) + if cur_item and check_key != 'last active': + self.set_fg_color(item, (0, 0, 0, 255)) + self.set_font_bold(item) + else: + self.set_fg_color(item) + self.set_font(item) + + # set item tooltip + if detailed_message: + item.setToolTip(str(detailed_message)) + + # set bg color corresponding to current text (OK/WARN/ERROR etc.) + self.set_bg_color(item, bg_color) + + # insert new item + self.table.setItem(row_ind, col_ind, item) + + # table filling/refreshing done, set clear_on_refresh to False + self.clear_on_refresh = False + + def set_font_bold(self, item): + """ Set item font bold """ + f = item.font() + f.setWeight(QtGui.QFont.Bold) + item.setFont(f) + + def set_font(self, item): + """ Set item font normal """ + f = item.font() + f.setWeight(QtGui.QFont.Normal) + item.setFont(f) + + def set_bg_color(self, item, color): + """ Set background color of item, color is RGBA tuple """ + color = QtGui.QColor(*color) + item.setBackground(color) + + def set_fg_color(self, item, color=(20, 20, 20, 255)): + """ Set foreground (font) color of item, color is RGBA tuple """ + color = QtGui.QColor(*color) + item.setForeground(color) + + def set_stretch(self): + hheader = self.table.horizontalHeader() + for index in range(hheader.count()): + hheader.setSectionResizeMode(index, QtWidgets.QHeaderView.Stretch) + vheader = self.table.verticalHeader() + for index in range(vheader.count()): + vheader.setSectionResizeMode(index, QtWidgets.QHeaderView.Stretch) + + def plot_stream(self, item): + nwst_id, check = item.data(QtCore.Qt.UserRole) + st = self.survBot.data.get(nwst_id) + if st: + self.plot_widget = PlotWidget(self) + self.plot_widget.setWindowTitle(nwst_id) + st = modify_stream_for_plot(st, parameters=self.parameters) + st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig) + # set_axis_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters) + set_axis_yticks(fig=self.plot_widget.canvas.fig, parameters=self.parameters) + set_axis_color(fig=self.plot_widget.canvas.fig) + plot_axis_thresholds(fig=self.plot_widget.canvas.fig, parameters=self.parameters) + self.plot_widget.show() + + def notification(self, text): + mbox = QtWidgets.QMessageBox() + mbox.setWindowTitle('Notification') + #mbox.setDetailedText() + mbox.setText(text) + mbox.exec_() + + def closeEvent(self, event): + self.thread.exit() + event.accept() + + +class PlotCanvas(FigureCanvas): + def __init__(self, parent=None, width=10, height=8, dpi=100): + self.fig = Figure(figsize=(width, height), dpi=dpi) + FigureCanvas.__init__(self, self.fig) + self.setParent(parent) + FigureCanvas.setSizePolicy(self, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding) + FigureCanvas.updateGeometry(self) + + +class PlotWidget(QtWidgets.QDialog): + def __init__(self, *args, **kwargs): + QtWidgets.QDialog.__init__(self, *args, **kwargs) + self.setLayout(QtWidgets.QVBoxLayout()) + self.canvas = PlotCanvas(self, width=10, height=8) + self.toolbar = NavigationToolbar2QT(self.canvas, self) + self.layout().addWidget(self.toolbar) + self.layout().addWidget(self.canvas) + + +class ReadSMSWidget(QtWidgets.QDialog): + def __init__(self, iccid, *args, **kwargs): + QtWidgets.QDialog.__init__(self, *args, **kwargs) + self.setLayout(QtWidgets.QVBoxLayout()) + self.table = QtWidgets.QTableWidget() + self.layout().addWidget(self.table) + self.resize(1280, 400) + + self.iccid = iccid + self.data = self.print_sms_table() + self.set_stretch() + + def print_sms_table(self, n=5, ntextbreak=40): + messages = [] + params = get_default_params(self.iccid) + for message in get_last_messages(params, n, only_delivered=False): + messages.append(message) + if not messages: + return + # pull dates to front + keys = ['dateSent', 'dateModified', 'dateReceived'] + for item in messages[0].keys(): + if not item in keys: + keys.append(item) + self.table.setRowCount(n) + self.table.setColumnCount(len(keys)) + self.table.setHorizontalHeaderLabels(keys) + for row_index, message in enumerate(messages): + for col_index, key in enumerate(keys): + text = message.get(key) + if type(text) == str and len(text) > ntextbreak: + textlist = list(text) + for index in range(ntextbreak, len(text), ntextbreak): + textlist.insert(index, '\n') + text = ''.join(textlist) + item = QtWidgets.QTableWidgetItem() + item.setText(str(text)) + self.table.setItem(row_index, col_index, item) + return True + + def set_stretch(self): + hheader = self.table.horizontalHeader() + nheader = hheader.count() + for index in range(nheader): + if index < nheader - 1: + hheader.setSectionResizeMode(index, QtWidgets.QHeaderView.ResizeToContents) + else: + hheader.setSectionResizeMode(index, QtWidgets.QHeaderView.Stretch) + vheader = self.table.verticalHeader() + for index in range(vheader.count()): + vheader.setSectionResizeMode(index, QtWidgets.QHeaderView.Stretch) + + + +class SendSMSWidget(QtWidgets.QDialog): + def __init__(self, iccid, *args, **kwargs): + QtWidgets.QDialog.__init__(self, *args, **kwargs) + self.main_layout = QtWidgets.QVBoxLayout() + self.setLayout(self.main_layout) + self.resize(400, 100) + + self.line_edit = QtWidgets.QLineEdit() + self.main_layout.addWidget(self.line_edit) + + self.iccid = iccid + + self.buttonBox = QtWidgets.QDialogButtonBox(QtWidgets.QDialogButtonBox.Ok | + QtWidgets.QDialogButtonBox.Close) + self.main_layout.addWidget(self.buttonBox) + self.buttonBox.accepted.connect(self.send_sms) + self.buttonBox.rejected.connect(self.reject) + + def send_sms(self): + text = self.line_edit.text() + params = get_default_params(self.iccid) + send_message(params, text) + self.close() + + +if __name__ == '__main__': + program_path = sys.path[0] + parameters = os.path.join(program_path, 'parameters.yaml') + app = QtWidgets.QApplication([]) + window = MainWindow(parameters=parameters) + window.showMaximized() + sys.exit(app.exec_()) From 080e73c1db8a040347c2bc09e14588c2e3d811c9 Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Fri, 21 Mar 2025 14:22:59 +0100 Subject: [PATCH 10/15] [update] update Dockerfile - mojor update of Dockerfile - reverted location of simulate_fail.json file for simulating errors, e.g. to test sending e-mails --- Dockerfile | 35 +++++++++++++++++++++++++++++++---- survBot.py | 2 +- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 14aa895..4c5b95e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,38 @@ FROM python:3 -WORKDIR /usr/src/app +# metadata +LABEL maintainer="Kasper D. Fischer " +LABEL version="0.2-docker" +LABEL description="Docker image for the survBot application" -COPY requirements.txt ./ -RUN pip install --no-cache-dir -r requirements.txt +# install required system packages RUN apt update && apt install -y bind9-host iputils-ping -COPY . . +# create user and group +RUN groupadd -r survBot && useradd -r -g survBot survBot +# change working directory +WORKDIR /usr/src/app + +# install required python packages +RUN --mount=type=bind,source=requirements.txt,target=/tmp/requirements.txt \ + pip install --no-cache-dir --requirement /tmp/requirements.txt + +# switch to user survBot +USER survBot + +# copy application files +COPY survBot.py utils.py write_utils.py LICENSE README.md ./ + +# copy configuration files +VOLUME /usr/src/app/conf +COPY parameters.yaml mailing_list.yaml simulate_fail.json conf/ +RUN ln -s conf/simulate_fail.json simulate_fail.json + +# copy www files +VOLUME /usr/src/app/www +COPY logo.png stylesheets/desktop.css stylesheets/mobile.css www/ +RUN ln -s www/index.html survBot_out.html + +# run the application CMD [ "python", "./survBot.py", "-html", "www", "-parfile", "conf/parameters.yaml" ] diff --git a/survBot.py b/survBot.py index 53357d7..2d5fef5 100755 --- a/survBot.py +++ b/survBot.py @@ -704,7 +704,7 @@ class StationQC(object): return 'active' return False - def _simulated_error_check(self, key, fname='conf/simulate_fail.json'): + def _simulated_error_check(self, key, fname='simulate_fail.json'): if not os.path.isfile(fname): return with open(fname) as fid: From b46802a75e332a80dc646880e27112e44f7d8f53 Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Fri, 21 Mar 2025 14:59:03 +0100 Subject: [PATCH 11/15] [bugfix] updated ownership setting in Dockerfile --- Dockerfile | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4c5b95e..6463563 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,8 +8,9 @@ LABEL description="Docker image for the survBot application" # install required system packages RUN apt update && apt install -y bind9-host iputils-ping -# create user and group +# create user and group and home directory RUN groupadd -r survBot && useradd -r -g survBot survBot +RUN mkdir -p /home/survBot && chown -R survBot:survBot /home/survBot # change working directory WORKDIR /usr/src/app @@ -18,21 +19,22 @@ WORKDIR /usr/src/app RUN --mount=type=bind,source=requirements.txt,target=/tmp/requirements.txt \ pip install --no-cache-dir --requirement /tmp/requirements.txt -# switch to user survBot -USER survBot - # copy application files COPY survBot.py utils.py write_utils.py LICENSE README.md ./ # copy configuration files VOLUME /usr/src/app/conf -COPY parameters.yaml mailing_list.yaml simulate_fail.json conf/ +COPY parameters.yaml mailing_list.yam[l] simulate_fail.jso[n] conf/ RUN ln -s conf/simulate_fail.json simulate_fail.json # copy www files VOLUME /usr/src/app/www -COPY logo.png stylesheets/desktop.css stylesheets/mobile.css www/ -RUN ln -s www/index.html survBot_out.html +COPY logo.pn[g] stylesheets/desktop.css stylesheets/mobile.css www/ +RUN ln -s www/survBot_out.html www/index.html -# run the application +# change ownership of working directory +RUN chown -R survBot:survBot /usr/src/app + +# run the application as user survBot +USER survBot CMD [ "python", "./survBot.py", "-html", "www", "-parfile", "conf/parameters.yaml" ] From aaadff6306dc47f382a8b97a2611d2c67c4b5231 Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Fri, 21 Mar 2025 16:14:21 +0100 Subject: [PATCH 12/15] [update] README.md - add section on running survBot in a Docker container - add explaination of mal server settings in parameters.yaml --- README.md | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 29b7733..48bbc3c 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,38 @@ The GUI can be loaded via python survBotGui.py ``` +### Docker + +To run the program in a Docker container, first build the image: + +```shell script +docker build -t survbot . +``` + +Then run the container: + +```shell script +docker run -v /path/to/conf-dir:/usr/src/app/conf -v /path/to/output:/usr/src/app/www survbot +``` + +The directory `/path/to/conf-dir` should contain the `parameters.yaml` file, and the directory `/path/to/output` will contain the output HTML files. + +### Configuration of the e-mail server settings + +The e-mail server settings can be configured in the `parameters.yaml` file. The following settings are available: + +- `mailserver`: the address of the mail server +- `auth_type`: the authentication type for the mail server (`None`, `SSL`, `TLS`) +- `port`: the port of the mail server +- `user`: the username for the mail server (if required) +- `password`: the password for the mail server (if required) + +The `user` and `password` fields are optional, and can be left empty if the mail server does not require authentication. The `auth_type` field can be set to `None` if no authentication is required, `SSL` if the mail server requires SSL authentication, or `TLS` if the mail server requires TLS authentication. If the `user` or `password` fileds are set to `Docker` ore `ENV` the program will try to read the values from the docker secrets `mail_user` and `mail_password` or environment variables `MAIL_USER` and `MAIL_PASSWORD` respectively. Docker secrets are only available in Docker Swarm mode, i.e. if the program is run as a service. + ## Version Changes + +### 0.2 + - surveillance of mass, clock and gaps - individual mailing lists for different stations - html mail with recent status information @@ -48,8 +79,14 @@ python survBotGui.py - restructured parameter file - recognize if PBox is disconnected +### 0.2-docker + +- added Dockerfile for easy deployment +- added more settings for connection to a mail server + ## Staff Original author: M.Paffrath (marcel.paffrath@rub.de) +Contributions by: Kasper D. Fischer (kasper.fischer@rub.de) -June 2023 +Jan 2025 From b986da5fef7bc218f902eaef10b2c0e4e78b55ae Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Fri, 21 Mar 2025 17:43:41 +0100 Subject: [PATCH 13/15] [minor] fixed some typos and missing import --- README.md | 34 +++++++++++++++++----------------- survBot.py | 18 +++++++++--------- utils.py | 3 ++- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 48bbc3c..3eb633f 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ version: 0.2 survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output over SOH channels - by analysing contents of a Seiscomp3 datapath. + by analyzing contents of a Seiscomp data archive. ## Requirements @@ -60,33 +60,33 @@ The directory `/path/to/conf-dir` should contain the `parameters.yaml` file, and The e-mail server settings can be configured in the `parameters.yaml` file. The following settings are available: -- `mailserver`: the address of the mail server -- `auth_type`: the authentication type for the mail server (`None`, `SSL`, `TLS`) -- `port`: the port of the mail server -- `user`: the username for the mail server (if required) -- `password`: the password for the mail server (if required) +* `mailserver`: the address of the mail server +* `auth_type`: the authentication type for the mail server (`None`, `SSL`, `TLS`) +* `port`: the port of the mail server +* `user`: the username for the mail server (if required) +* `password`: the password for the mail server (if required) -The `user` and `password` fields are optional, and can be left empty if the mail server does not require authentication. The `auth_type` field can be set to `None` if no authentication is required, `SSL` if the mail server requires SSL authentication, or `TLS` if the mail server requires TLS authentication. If the `user` or `password` fileds are set to `Docker` ore `ENV` the program will try to read the values from the docker secrets `mail_user` and `mail_password` or environment variables `MAIL_USER` and `MAIL_PASSWORD` respectively. Docker secrets are only available in Docker Swarm mode, i.e. if the program is run as a service. +The `user` and `password` fields are optional, and can be left empty if the mail server does not require authentication. The `auth_type` field can be set to `None` if no authentication is required, `SSL` if the mail server requires SSL authentication, or `TLS` if the mail server requires TLS authentication. If the `user` or `password` fields are set to `Docker` ore `ENV` the program will try to read the values from the docker secrets `mail_user` and `mail_password` or environment variables `MAIL_USER` and `MAIL_PASSWORD` respectively. Docker secrets are only available in Docker Swarm mode, i.e. if the program is run as a service. ## Version Changes ### 0.2 -- surveillance of mass, clock and gaps -- individual mailing lists for different stations -- html mail with recent status information -- updated web page design -- restructured parameter file -- recognize if PBox is disconnected +* surveillance of mass, clock and gaps +* individual mailing lists for different stations +* html mail with recent status information +* updated web page design +* restructured parameter file +* recognize if PBox is disconnected ### 0.2-docker -- added Dockerfile for easy deployment -- added more settings for connection to a mail server +* added Dockerfile for easy deployment +* added more settings for connection to a mail server ## Staff -Original author: M.Paffrath (marcel.paffrath@rub.de) -Contributions by: Kasper D. Fischer (kasper.fischer@rub.de) +Original author: M.Paffrath () +Contributions: Kasper D. Fischer () Jan 2025 diff --git a/survBot.py b/survBot.py index 2d5fef5..bd4fe70 100755 --- a/survBot.py +++ b/survBot.py @@ -234,7 +234,7 @@ class SurveillanceBot(object): self.gaps = self.dataStream.get_gaps(min_gap=self.parameters['THRESHOLDS'].get('min_gap')) self.dataStream.merge() - # organise data in dictionary with key for each station + # organize data in dictionary with key for each station for trace in self.dataStream: nwst_id = get_nwst_id(trace) if not nwst_id in self.data.keys(): @@ -351,7 +351,7 @@ class SurveillanceBot(object): first_exec = False def console_print(self, itemlist, str_len=21, sep='|', seplen=3): - assert len(sep) <= seplen, f'Make sure seperator has less than {seplen} characters' + assert len(sep) <= seplen, f'Make sure separator has less than {seplen} characters' sl = sep.ljust(seplen) sr = sep.rjust(seplen) string = sl @@ -1299,7 +1299,7 @@ class StationQC(object): # Warn in case of voltage under OK-level (1V) if len(under) > 0: - # try calculate number of occurences from gaps between indices + # try calculate number of occurrences from gaps between indices n_occurrences = len(np.where(np.diff(under) > 1)[0]) + 1 voltage_dict[-1] = under self.status_other(detailed_message=f'Trace {trace.get_id()}: ' @@ -1395,15 +1395,15 @@ class StatusOK(Status): class StatusWarn(Status): - def __init__(self, message='WARN', count=1, last_occurence=None, detailed_messages=None, show_count=False): - super(StatusWarn, self).__init__(message=message, count=count, last_occurrence=last_occurence, + def __init__(self, message='WARN', count=1, last_occurrence=None, detailed_messages=None, show_count=False): + super(StatusWarn, self).__init__(message=message, count=count, last_occurrence=last_occurrence, detailed_messages=detailed_messages, show_count=show_count) self.set_warn() class StatusError(Status): - def __init__(self, message='FAIL', count=1, last_occurence=None, detailed_messages=None, show_count=False): - super(StatusError, self).__init__(message=message, count=count, last_occurrence=last_occurence, + def __init__(self, message='FAIL', count=1, last_occurrence=None, detailed_messages=None, show_count=False): + super(StatusError, self).__init__(message=message, count=count, last_occurrence=last_occurrence, detailed_messages=detailed_messages, show_count=show_count) self.set_error() self.default_message = message @@ -1420,8 +1420,8 @@ class StatusError(Status): class StatusOther(Status): - def __init__(self, messages=None, count=1, last_occurence=None, detailed_messages=None): - super(StatusOther, self).__init__(count=count, last_occurrence=last_occurence, + def __init__(self, messages=None, count=1, last_occurrence=None, detailed_messages=None): + super(StatusOther, self).__init__(count=count, last_occurrence=last_occurrence, detailed_messages=detailed_messages) if messages is None: messages = [] diff --git a/utils.py b/utils.py index f8364ab..65fe69e 100644 --- a/utils.py +++ b/utils.py @@ -6,6 +6,7 @@ import logging import matplotlib import numpy as np import smtplib +import os from obspy import Stream @@ -174,7 +175,7 @@ def transform_trace(data, transf): elif operator_str == '/': data = data / val else: - raise IOError(f'Unknown arithmethic operator string: {operator_str}') + raise IOError(f'Unknown arithmetic operator string: {operator_str}') return data From 16fbbde3d9c0e3938d5d40e58406ca88902af0ad Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Fri, 21 Mar 2025 17:48:49 +0100 Subject: [PATCH 14/15] [bugfix] fixed call of function connect_to_mail_server --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 65fe69e..3ad80f8 100644 --- a/utils.py +++ b/utils.py @@ -302,7 +302,7 @@ def get_credential(source, param): # return source if no credential was found return source -def connect_to_mail_server(self, mail_params): +def connect_to_mail_server(mail_params): """ Connect to mail server and return server object. """ From fcba73fcc57c1a2de9180d36fd3250dcb2be0bc5 Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Fri, 21 Mar 2025 22:42:23 +0100 Subject: [PATCH 15/15] [bugfix] fix bug in connect_to_mail_server function --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 3ad80f8..057bc74 100644 --- a/utils.py +++ b/utils.py @@ -326,5 +326,5 @@ def connect_to_mail_server(mail_params): else: logging.error('Unknown authentication type. Mails can not be sent') return - s.login(mail_params.get('user'), mail_params.get('password')) + s.login(user, password) return s