From c3f9ad0fd9a6ac8f6988cdc8c88173ca0bd23bd0 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2022 16:45:21 +0100 Subject: [PATCH] [update] moved html writing to survBot.py so that no GUI is needed --- parameters.yaml | 4 +- submit_bot.sh | 3 +- survBot.py | 115 +++++++++++++++++++++++++++++++++++++++--------- survBotGUI.py | 104 +++++-------------------------------------- survBotGUI.sh | 14 ------ utils.py | 51 +++++++++++++++++++++ write_utils.py | 8 ++-- 7 files changed, 163 insertions(+), 136 deletions(-) mode change 100644 => 100755 submit_bot.sh delete mode 100755 survBotGUI.sh create mode 100644 utils.py diff --git a/parameters.yaml b/parameters.yaml index a206927..19c1593 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -6,10 +6,12 @@ locations: '*' channels: ['EX1', 'EX2', 'EX3', 'VEI'] # Specify SOH channels, currently supported EX[1-3] and VEI stations_blacklist: ['TEST', 'EREA'] networks_blacklist: [] -interval: 20 # Perform checks every x seconds +interval: 60 # Perform checks every x seconds timespan: 7 # Check data of the recent x days verbosity: 0 +reread_parameters: True # reread parameters file (change parameters on runtime, not for itself/GUI refresh/datapath) track_changes: True # tracks all changes since GUI startup by text highlighting (GUI only) +dt_thresh: [300, 1800] # threshold (s) for timing delay colourisation (yellow/red) POWBOX: pb_ok: 1 # Voltage for PowBox OK diff --git a/submit_bot.sh b/submit_bot.sh old mode 100644 new mode 100755 index 4f1a918..02a6ec1 --- a/submit_bot.sh +++ b/submit_bot.sh @@ -17,5 +17,4 @@ export MKL_NUM_THREADS=1 export NUMEXPR_NUM_THREADS=1 export OMP_NUM_THREADS=1 - -python survBotGUI.py -html '/home/marcel/public_html/survBot_out.html' --background +python survBot.py -html '/home/marcel/public_html/survBot_out.html' diff --git a/survBot.py b/survBot.py index 7259e10..ba6f67a 100755 --- a/survBot.py +++ b/survBot.py @@ -5,7 +5,9 @@ __version__ = '0.1' __author__ = 'Marcel Paffrath' import os +import traceback import yaml +import argparse import time from datetime import timedelta @@ -14,7 +16,9 @@ import numpy as np from obspy import read, UTCDateTime, Stream from obspy.clients.filesystem.sds import Client -from write_utils import get_print_title_str +from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str,\ + init_html_table, finish_html_table +from utils import get_bg_color pjoin = os.path.join UP = "\x1B[{length}A" @@ -46,27 +50,35 @@ def fancy_timestr(dt, thresh=600, modif='+'): class SurveillanceBot(object): - def __init__(self, parameter_path): + def __init__(self, parameter_path, outpath_html=None): self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'temp', 'other'] - self.parameters = read_yaml(parameter_path) - self.transform_parameters() + self.parameter_path = parameter_path + self.update_parameters() self.starttime = UTCDateTime() - self.verbosity = self.parameters.get('verbosity') + self.outpath_html = outpath_html self.filenames = [] self.filenames_read = [] self.station_list = [] self.analysis_print_list = [] self.analysis_results = {} - self.stations_blacklist = self.parameters.get('stations_blacklist') - self.networks_blacklist = self.parameters.get('networks_blacklist') self.dataStream = Stream() self.data = {} self.print_count = 0 - self.refresh_period = 0 + self.status_message = '' self.cl = Client(self.parameters.get('datapath')) # TODO: Check if this has to be loaded again on update self.get_stations() + def update_parameters(self): + self.parameters = read_yaml(self.parameter_path) + self.reread_parameters = self.parameters.get('reread_parameters') + self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')] + self.verbosity = self.parameters.get('verbosity') + self.stations_blacklist = self.parameters.get('stations_blacklist') + self.networks_blacklist = self.parameters.get('networks_blacklist') + self.refresh_period = self.parameters.get('interval') + self.transform_parameters() + def transform_parameters(self): for key in ['networks', 'stations', 'locations', 'channels']: parameter = self.parameters.get(key) @@ -132,9 +144,11 @@ class SurveillanceBot(object): self.data[st_id].append(trace) def execute_qc(self): - self.starttime = UTCDateTime() + if self.reread_parameters: + self.update_parameters() self.get_filenames() self.read_data() + qc_starttime = UTCDateTime() self.analysis_print_list = [] self.analysis_results = {} @@ -142,7 +156,7 @@ class SurveillanceBot(object): stream = self.data.get(st_id) if stream: nsl = nsl_from_id(st_id) - station_qc = StationQC(stream, nsl, self.parameters, self.keys, self.starttime, self.verbosity, + station_qc = StationQC(stream, nsl, self.parameters, self.keys, qc_starttime, self.verbosity, print_func=self.print) analysis_print_result = station_qc.return_print_analysis() station_dict, warn_dict = station_qc.return_analysis() @@ -151,6 +165,8 @@ class SurveillanceBot(object): station_dict, warn_dict = self.get_no_data_station(st_id) self.analysis_print_list.append(analysis_print_result) self.analysis_results[st_id] = (station_dict, warn_dict) + + self.update_status_message() return 'ok' def get_no_data_station(self, st_id, no_data='-', to_print=False): @@ -187,10 +203,6 @@ class SurveillanceBot(object): if len(times) > 0: return min(times) - def print_analysis_html(self, filename): - with open(filename, 'w') as outfile: - pass - def print_analysis(self): self.print(200 * '+') title_str = get_print_title_str(self.parameters) @@ -202,19 +214,22 @@ class SurveillanceBot(object): for items in self.analysis_print_list: self.console_print(items) - def start(self, refresh_period=30): + def start(self): ''' Perform qc periodically. :param refresh_period: Update every x seconds :return: ''' - self.refresh_period = refresh_period status = 'ok' while status == 'ok' and self.refresh_period > 0: status = self.execute_qc() - self.print_analysis() + if self.outpath_html: + self.write_html_table() + else: + self.print_analysis() time.sleep(self.refresh_period) - self.clear_prints() + if not self.outpath_html: + self.clear_prints() def console_print(self, itemlist, str_len=21, sep='|', seplen=3): assert len(sep) <= seplen, f'Make sure seperator has less than {seplen} characters' @@ -225,6 +240,60 @@ class SurveillanceBot(object): string += item.center(str_len) + sr self.print(string, flush=False) + def write_html_table(self, default_color='#e6e6e6'): + fnout = self.outpath_html + if not fnout: + return + try: + with open(fnout, 'w') as outfile: + write_html_header(outfile, self.refresh_period) + #write_html_table_title(outfile, self.parameters) + init_html_table(outfile) + + # First write header items + header_items = [dict(text='Station', color=default_color)] + for check_key in self.keys: + item = dict(text=check_key, color=default_color) + header_items.append(item) + write_html_row(outfile, header_items, html_key='th') + + # Write all cells + for st_id in self.station_list: + col_items = [dict(text=st_id.rstrip('.'), color=default_color)] + for check_key in self.keys: + status_dict, detailed_dict = self.analysis_results.get(st_id) + status = status_dict.get(check_key) + + # get background color + dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh] + bg_color = get_bg_color(check_key, status, dt_thresh, hex=True) + if not bg_color: + bg_color = default_color + + # add degree sign for temp + if check_key == 'temp': + if not type(status) in [str]: + status = str(status) + deg_str + + item = dict(text=str(status), tooltip=str(detailed_dict.get(check_key)), + color=bg_color) + col_items.append(item) + write_html_row(outfile, col_items) + + finish_html_table(outfile) + write_html_text(outfile, self.status_message) + write_html_footer(outfile) + except Exception as e: + print(f'Could not write HTML table to {fnout}:') + print(traceback.format_exc()) + + def update_status_message(self): + timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600)) + self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \ + f'Current time (UTC) {UTCDateTime().strftime("%Y-%m-%d %H:%M:%S")} | ' \ + f'Refresh period: {self.refresh_period}s | '\ + f'Showing data of last {timespan}' + def print(self, string, **kwargs): clear_end = CLR + '\n' n_nl = string.count('\n') @@ -553,7 +622,7 @@ class StationQC(object): self.warn(key='other', detailed_message=f'Trace {trace.get_id()}: ' f'{n_unclassified}/{len(all_indices)} ' f'unclassified voltage values in channel {trace.get_id()}', - status_message=f'{channel}: {n_unclassified} u') + status_message=f'{channel}: {n_unclassified} uncl.') return False, voltage_dict, last_val @@ -563,5 +632,9 @@ class StationQC(object): if __name__ == '__main__': - survBot = SurveillanceBot(parameter_path='parameters.yaml') - survBot.start(refresh_period=30) + parser = argparse.ArgumentParser(description='Call survBot') + parser.add_argument('-html', dest='html_filename', default=None, help='filename for HTML output') + args = parser.parse_args() + + survBot = SurveillanceBot(parameter_path='parameters.yaml', outpath_html=args.html_filename) + survBot.start() diff --git a/survBotGUI.py b/survBotGUI.py index 9f645dd..3804047 100755 --- a/survBotGUI.py +++ b/survBotGUI.py @@ -10,7 +10,6 @@ __author__ = 'Marcel Paffrath' import os import sys import traceback -import argparse try: from PySide2 import QtGui, QtCore, QtWidgets @@ -36,6 +35,7 @@ from obspy import UTCDateTime from survBot import SurveillanceBot from write_utils import * +from utils import get_bg_color try: from rest_api.utils import get_station_iccid @@ -78,28 +78,17 @@ class Thread(QtCore.QThread): class MainWindow(QtWidgets.QMainWindow): - def __init__(self, parameters='parameters.yaml', outpath_html=None, dt_thresh=(300, 1800)): + def __init__(self, parameters='parameters.yaml'): """ Main window of survBot GUI. :param parameters: Parameters dictionary file (yaml format) - :param dt_thresh: threshold for timing delay colourisation (yellow/red) """ super(MainWindow, self).__init__() - # some GUI default colors - self.colors_dict = {'FAIL': (255, 50, 0, 255), - 'NO DATA': (255, 255, 125, 255), - 'WARN': (255, 255, 125, 255), - 'WARNX': lambda x: (min([255, 200 + x**2]), 255, 125, 255), - 'OK': (125, 255, 125, 255), - 'undefined': (230, 230, 230, 255)} - # init some attributes - self.dt_thresh = dt_thresh self.last_mouse_loc = None self.status_message = '' self.starttime = UTCDateTime() - self.outpath_html = outpath_html # setup main layout of the GUI self.main_layout = QtWidgets.QVBoxLayout() @@ -111,6 +100,7 @@ class MainWindow(QtWidgets.QMainWindow): self.survBot = SurveillanceBot(parameter_path=parameters) self.parameters = self.survBot.parameters self.refresh_period = self.parameters.get('interval') + self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')] # create thread that is used to update self.thread = Thread(parent=self, runnable=self.survBot.execute_qc) @@ -185,41 +175,6 @@ class MainWindow(QtWidgets.QMainWindow): self.last_mouse_loc = event.pos() return super(QtWidgets.QMainWindow, self).eventFilter(object, event) - def write_html_table(self): - fnout = self.outpath_html - if not fnout: - return - try: - with open(fnout, 'w') as outfile: - write_html_header(outfile, self.refresh_period) - #write_html_table_title(outfile, self.parameters) - init_html_table(outfile) - nrows = self.table.rowCount() - ncolumns = self.table.columnCount() - - # add header item 0 fix default black bg color for headers - station_header = QtWidgets.QTableWidgetItem(text='Station') - station_header.setText('Station') - header_items = [station_header] - for column in range(ncolumns): - hheader = self.table.horizontalHeaderItem(column) - header_items.append(hheader) - write_html_row(outfile, header_items, html_key='th') - - for row in range(nrows): - vheader = self.table.verticalHeaderItem(row) - col_items = [vheader] - for column in range(ncolumns): - col_items.append(self.table.item(row, column)) - write_html_row(outfile, col_items) - - finish_html_table(outfile) - write_html_text(outfile, self.status_message) - write_html_footer(outfile) - except Exception as e: - print(f'Could not write HTML table to {fnout}:') - print(e) - def sms_context_menu(self, row_ind): """ Open a context menu when left-clicking vertical header item """ header_item = self.table.verticalHeaderItem(row_ind) @@ -264,11 +219,7 @@ class MainWindow(QtWidgets.QMainWindow): def fill_status_bar(self): """ Set status bar text """ - timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600)) - self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \ - f'Current time (UTC) {UTCDateTime().strftime("%Y-%m-%d %H:%M:%S")} | ' \ - f'Refresh period: {self.refresh_period}s | '\ - f'Showing data of last {timespan}' + self.status_message = self.survBot.status_message status_bar = self.statusBar() status_bar.showMessage(self.status_message) @@ -283,21 +234,12 @@ class MainWindow(QtWidgets.QMainWindow): status_dict, detailed_dict = self.survBot.analysis_results.get(st_id) status = status_dict.get(check_key) detailed_message = detailed_dict.get(check_key) - if check_key == 'last active': - bg_color = self.get_time_delay_color(status) - elif check_key == 'temp': - bg_color = self.get_temp_color(status) + + dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh] + bg_color = get_bg_color(check_key, status, dt_thresh) + if check_key == 'temp': if not type(status) in [str]: status = str(status) + deg_str - else: - statussplit = status.split(' ') - if len(statussplit) > 1 and statussplit[0] == 'WARN': - x = int(status.split(' ')[-1].lstrip('(').rstrip(')')) - bg_color = self.colors_dict.get('WARNX')(x) - else: - bg_color = self.colors_dict.get(status) - if not bg_color: - bg_color = self.colors_dict.get('undefined') # Continue if nothing changed text = str(status) @@ -336,26 +278,6 @@ class MainWindow(QtWidgets.QMainWindow): # table filling/refreshing done, set clear_on_refresh to False self.clear_on_refresh = False - # write html output if parameter is set - self.write_html_table() - - def get_time_delay_color(self, dt): - """ Set color of time delay after thresholds specified in self.dt_thresh """ - dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh] - if dt < dt_thresh[0]: - return self.colors_dict.get('OK') - elif dt_thresh[0] <= dt < dt_thresh[1]: - return self.colors_dict.get('WARN') - return self.colors_dict.get('FAIL') - - def get_temp_color(self, temp, vmin=-10, vmax=60, cmap='coolwarm'): - """ Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """ - if type(temp) in [str]: - return self.colors_dict.get('undefined') - cmap = matplotlib.cm.get_cmap(cmap) - val = (temp - vmin) / (vmax - vmin) - rgba = [int(255 * c) for c in cmap(val)] - return rgba def set_font_bold(self, item): """ Set item font bold """ @@ -507,15 +429,9 @@ class SendSMSWidget(QtWidgets.QDialog): if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Call survBot GUI') - parser.add_argument('-html', dest='html_filename', default=None, help='filename for HTML output') - parser.add_argument('--background', dest='background', default=False, action='store_true', help='run in background') - args = parser.parse_args() - program_path = sys.path[0] parameters = os.path.join(program_path, 'parameters.yaml') app = QtWidgets.QApplication([]) - window = MainWindow(parameters=parameters, outpath_html=args.html_filename) - if not args.background: - window.showMaximized() + window = MainWindow(parameters=parameters) + window.showMaximized() sys.exit(app.exec_()) diff --git a/survBotGUI.sh b/survBotGUI.sh deleted file mode 100755 index 982f9e1..0000000 --- a/survBotGUI.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -ulimit -s 8192 - -#$ -l os=*stretch -##$ -cwd -#$ -pe smp 1 -##$ -q "*@minos15" - -export PYTHONPATH="$PYTHONPATH:/home/marcel/git/" - -source /opt/anaconda3/etc/profile.d/conda.sh -conda activate py37 - -python /home/marcel/git/survBot/survBotGUI.py diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..abd5708 --- /dev/null +++ b/utils.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import matplotlib + +def get_bg_color(check_key, status, dt_thresh=None, hex=False): + if check_key == 'last active': + bg_color = get_time_delay_color(status, dt_thresh) + elif check_key == 'temp': + bg_color = get_temp_color(status) + else: + statussplit = status.split(' ') + if len(statussplit) > 1 and statussplit[0] == 'WARN': + x = int(status.split(' ')[-1].lstrip('(').rstrip(')')) + bg_color = get_color('WARNX')(x) + else: + bg_color = get_color(status) + if not bg_color: + bg_color = get_color('undefined') + + if hex: + bg_color = '#{:02x}{:02x}{:02x}'.format(*bg_color[:3]) + return bg_color + +def get_color(key): + # some GUI default colors + colors_dict = {'FAIL': (255, 50, 0, 255), + 'NO DATA': (255, 255, 125, 255), + 'WARN': (255, 255, 125, 255), + 'WARNX': lambda x: (min([255, 200 + x ** 2]), 255, 125, 255), + 'OK': (125, 255, 125, 255), + 'undefined': (230, 230, 230, 255)} + return colors_dict.get(key) + +def get_time_delay_color(dt, dt_thresh): + """ Set color of time delay after thresholds specified in self.dt_thresh """ + if dt < dt_thresh[0]: + return get_color('OK') + elif dt_thresh[0] <= dt < dt_thresh[1]: + return get_color('WARN') + return get_color('FAIL') + +def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'): + """ Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """ + if type(temp) in [str]: + return get_color('undefined') + cmap = matplotlib.cm.get_cmap(cmap) + val = (temp - vmin) / (vmax - vmin) + rgba = [int(255 * c) for c in cmap(val)] + return rgba + diff --git a/write_utils.py b/write_utils.py index 51af04e..f7903b4 100644 --- a/write_utils.py +++ b/write_utils.py @@ -37,10 +37,10 @@ def write_html_row(fobj, items, html_key='td'): default_space = ' ' fobj.write(default_space + '\n') for item in items: - text = item.text() - tooltip = item.toolTip() - color = item.backgroundColor().name() - # fix for black background of headers + text = item.get('text') + tooltip = item.get('tooltip') + color = item.get('color') + # check for black background of headers (shouldnt happen anymore) color = '#e6e6e6' if color == '#000000' else color fobj.write(2 * default_space + f'<{html_key} bgcolor="{color}" title="{tooltip}">' + text + f'\n') fobj.write(default_space + '\n')