[update] moved html writing to survBot.py so that no GUI is needed
This commit is contained in:
parent
abc201c673
commit
c3f9ad0fd9
@ -6,10 +6,12 @@ locations: '*'
|
||||
channels: ['EX1', 'EX2', 'EX3', 'VEI'] # Specify SOH channels, currently supported EX[1-3] and VEI
|
||||
stations_blacklist: ['TEST', 'EREA']
|
||||
networks_blacklist: []
|
||||
interval: 20 # Perform checks every x seconds
|
||||
interval: 60 # Perform checks every x seconds
|
||||
timespan: 7 # Check data of the recent x days
|
||||
verbosity: 0
|
||||
reread_parameters: True # reread parameters file (change parameters on runtime, not for itself/GUI refresh/datapath)
|
||||
track_changes: True # tracks all changes since GUI startup by text highlighting (GUI only)
|
||||
dt_thresh: [300, 1800] # threshold (s) for timing delay colourisation (yellow/red)
|
||||
|
||||
POWBOX:
|
||||
pb_ok: 1 # Voltage for PowBox OK
|
||||
|
||||
3
submit_bot.sh
Normal file → Executable file
3
submit_bot.sh
Normal file → Executable file
@ -17,5 +17,4 @@ export MKL_NUM_THREADS=1
|
||||
export NUMEXPR_NUM_THREADS=1
|
||||
export OMP_NUM_THREADS=1
|
||||
|
||||
|
||||
python survBotGUI.py -html '/home/marcel/public_html/survBot_out.html' --background
|
||||
python survBot.py -html '/home/marcel/public_html/survBot_out.html'
|
||||
|
||||
115
survBot.py
115
survBot.py
@ -5,7 +5,9 @@ __version__ = '0.1'
|
||||
__author__ = 'Marcel Paffrath'
|
||||
|
||||
import os
|
||||
import traceback
|
||||
import yaml
|
||||
import argparse
|
||||
|
||||
import time
|
||||
from datetime import timedelta
|
||||
@ -14,7 +16,9 @@ import numpy as np
|
||||
from obspy import read, UTCDateTime, Stream
|
||||
from obspy.clients.filesystem.sds import Client
|
||||
|
||||
from write_utils import get_print_title_str
|
||||
from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str,\
|
||||
init_html_table, finish_html_table
|
||||
from utils import get_bg_color
|
||||
|
||||
pjoin = os.path.join
|
||||
UP = "\x1B[{length}A"
|
||||
@ -46,27 +50,35 @@ def fancy_timestr(dt, thresh=600, modif='+'):
|
||||
|
||||
|
||||
class SurveillanceBot(object):
|
||||
def __init__(self, parameter_path):
|
||||
def __init__(self, parameter_path, outpath_html=None):
|
||||
self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'temp', 'other']
|
||||
self.parameters = read_yaml(parameter_path)
|
||||
self.transform_parameters()
|
||||
self.parameter_path = parameter_path
|
||||
self.update_parameters()
|
||||
self.starttime = UTCDateTime()
|
||||
self.verbosity = self.parameters.get('verbosity')
|
||||
self.outpath_html = outpath_html
|
||||
self.filenames = []
|
||||
self.filenames_read = []
|
||||
self.station_list = []
|
||||
self.analysis_print_list = []
|
||||
self.analysis_results = {}
|
||||
self.stations_blacklist = self.parameters.get('stations_blacklist')
|
||||
self.networks_blacklist = self.parameters.get('networks_blacklist')
|
||||
self.dataStream = Stream()
|
||||
self.data = {}
|
||||
self.print_count = 0
|
||||
self.refresh_period = 0
|
||||
self.status_message = ''
|
||||
|
||||
self.cl = Client(self.parameters.get('datapath')) # TODO: Check if this has to be loaded again on update
|
||||
self.get_stations()
|
||||
|
||||
def update_parameters(self):
|
||||
self.parameters = read_yaml(self.parameter_path)
|
||||
self.reread_parameters = self.parameters.get('reread_parameters')
|
||||
self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')]
|
||||
self.verbosity = self.parameters.get('verbosity')
|
||||
self.stations_blacklist = self.parameters.get('stations_blacklist')
|
||||
self.networks_blacklist = self.parameters.get('networks_blacklist')
|
||||
self.refresh_period = self.parameters.get('interval')
|
||||
self.transform_parameters()
|
||||
|
||||
def transform_parameters(self):
|
||||
for key in ['networks', 'stations', 'locations', 'channels']:
|
||||
parameter = self.parameters.get(key)
|
||||
@ -132,9 +144,11 @@ class SurveillanceBot(object):
|
||||
self.data[st_id].append(trace)
|
||||
|
||||
def execute_qc(self):
|
||||
self.starttime = UTCDateTime()
|
||||
if self.reread_parameters:
|
||||
self.update_parameters()
|
||||
self.get_filenames()
|
||||
self.read_data()
|
||||
qc_starttime = UTCDateTime()
|
||||
|
||||
self.analysis_print_list = []
|
||||
self.analysis_results = {}
|
||||
@ -142,7 +156,7 @@ class SurveillanceBot(object):
|
||||
stream = self.data.get(st_id)
|
||||
if stream:
|
||||
nsl = nsl_from_id(st_id)
|
||||
station_qc = StationQC(stream, nsl, self.parameters, self.keys, self.starttime, self.verbosity,
|
||||
station_qc = StationQC(stream, nsl, self.parameters, self.keys, qc_starttime, self.verbosity,
|
||||
print_func=self.print)
|
||||
analysis_print_result = station_qc.return_print_analysis()
|
||||
station_dict, warn_dict = station_qc.return_analysis()
|
||||
@ -151,6 +165,8 @@ class SurveillanceBot(object):
|
||||
station_dict, warn_dict = self.get_no_data_station(st_id)
|
||||
self.analysis_print_list.append(analysis_print_result)
|
||||
self.analysis_results[st_id] = (station_dict, warn_dict)
|
||||
|
||||
self.update_status_message()
|
||||
return 'ok'
|
||||
|
||||
def get_no_data_station(self, st_id, no_data='-', to_print=False):
|
||||
@ -187,10 +203,6 @@ class SurveillanceBot(object):
|
||||
if len(times) > 0:
|
||||
return min(times)
|
||||
|
||||
def print_analysis_html(self, filename):
|
||||
with open(filename, 'w') as outfile:
|
||||
pass
|
||||
|
||||
def print_analysis(self):
|
||||
self.print(200 * '+')
|
||||
title_str = get_print_title_str(self.parameters)
|
||||
@ -202,19 +214,22 @@ class SurveillanceBot(object):
|
||||
for items in self.analysis_print_list:
|
||||
self.console_print(items)
|
||||
|
||||
def start(self, refresh_period=30):
|
||||
def start(self):
|
||||
'''
|
||||
Perform qc periodically.
|
||||
:param refresh_period: Update every x seconds
|
||||
:return:
|
||||
'''
|
||||
self.refresh_period = refresh_period
|
||||
status = 'ok'
|
||||
while status == 'ok' and self.refresh_period > 0:
|
||||
status = self.execute_qc()
|
||||
self.print_analysis()
|
||||
if self.outpath_html:
|
||||
self.write_html_table()
|
||||
else:
|
||||
self.print_analysis()
|
||||
time.sleep(self.refresh_period)
|
||||
self.clear_prints()
|
||||
if not self.outpath_html:
|
||||
self.clear_prints()
|
||||
|
||||
def console_print(self, itemlist, str_len=21, sep='|', seplen=3):
|
||||
assert len(sep) <= seplen, f'Make sure seperator has less than {seplen} characters'
|
||||
@ -225,6 +240,60 @@ class SurveillanceBot(object):
|
||||
string += item.center(str_len) + sr
|
||||
self.print(string, flush=False)
|
||||
|
||||
def write_html_table(self, default_color='#e6e6e6'):
|
||||
fnout = self.outpath_html
|
||||
if not fnout:
|
||||
return
|
||||
try:
|
||||
with open(fnout, 'w') as outfile:
|
||||
write_html_header(outfile, self.refresh_period)
|
||||
#write_html_table_title(outfile, self.parameters)
|
||||
init_html_table(outfile)
|
||||
|
||||
# First write header items
|
||||
header_items = [dict(text='Station', color=default_color)]
|
||||
for check_key in self.keys:
|
||||
item = dict(text=check_key, color=default_color)
|
||||
header_items.append(item)
|
||||
write_html_row(outfile, header_items, html_key='th')
|
||||
|
||||
# Write all cells
|
||||
for st_id in self.station_list:
|
||||
col_items = [dict(text=st_id.rstrip('.'), color=default_color)]
|
||||
for check_key in self.keys:
|
||||
status_dict, detailed_dict = self.analysis_results.get(st_id)
|
||||
status = status_dict.get(check_key)
|
||||
|
||||
# get background color
|
||||
dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh]
|
||||
bg_color = get_bg_color(check_key, status, dt_thresh, hex=True)
|
||||
if not bg_color:
|
||||
bg_color = default_color
|
||||
|
||||
# add degree sign for temp
|
||||
if check_key == 'temp':
|
||||
if not type(status) in [str]:
|
||||
status = str(status) + deg_str
|
||||
|
||||
item = dict(text=str(status), tooltip=str(detailed_dict.get(check_key)),
|
||||
color=bg_color)
|
||||
col_items.append(item)
|
||||
write_html_row(outfile, col_items)
|
||||
|
||||
finish_html_table(outfile)
|
||||
write_html_text(outfile, self.status_message)
|
||||
write_html_footer(outfile)
|
||||
except Exception as e:
|
||||
print(f'Could not write HTML table to {fnout}:')
|
||||
print(traceback.format_exc())
|
||||
|
||||
def update_status_message(self):
|
||||
timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
|
||||
self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \
|
||||
f'Current time (UTC) {UTCDateTime().strftime("%Y-%m-%d %H:%M:%S")} | ' \
|
||||
f'Refresh period: {self.refresh_period}s | '\
|
||||
f'Showing data of last {timespan}'
|
||||
|
||||
def print(self, string, **kwargs):
|
||||
clear_end = CLR + '\n'
|
||||
n_nl = string.count('\n')
|
||||
@ -553,7 +622,7 @@ class StationQC(object):
|
||||
self.warn(key='other', detailed_message=f'Trace {trace.get_id()}: '
|
||||
f'{n_unclassified}/{len(all_indices)} '
|
||||
f'unclassified voltage values in channel {trace.get_id()}',
|
||||
status_message=f'{channel}: {n_unclassified} u')
|
||||
status_message=f'{channel}: {n_unclassified} uncl.')
|
||||
|
||||
return False, voltage_dict, last_val
|
||||
|
||||
@ -563,5 +632,9 @@ class StationQC(object):
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
survBot = SurveillanceBot(parameter_path='parameters.yaml')
|
||||
survBot.start(refresh_period=30)
|
||||
parser = argparse.ArgumentParser(description='Call survBot')
|
||||
parser.add_argument('-html', dest='html_filename', default=None, help='filename for HTML output')
|
||||
args = parser.parse_args()
|
||||
|
||||
survBot = SurveillanceBot(parameter_path='parameters.yaml', outpath_html=args.html_filename)
|
||||
survBot.start()
|
||||
|
||||
104
survBotGUI.py
104
survBotGUI.py
@ -10,7 +10,6 @@ __author__ = 'Marcel Paffrath'
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
import argparse
|
||||
|
||||
try:
|
||||
from PySide2 import QtGui, QtCore, QtWidgets
|
||||
@ -36,6 +35,7 @@ from obspy import UTCDateTime
|
||||
|
||||
from survBot import SurveillanceBot
|
||||
from write_utils import *
|
||||
from utils import get_bg_color
|
||||
|
||||
try:
|
||||
from rest_api.utils import get_station_iccid
|
||||
@ -78,28 +78,17 @@ class Thread(QtCore.QThread):
|
||||
|
||||
|
||||
class MainWindow(QtWidgets.QMainWindow):
|
||||
def __init__(self, parameters='parameters.yaml', outpath_html=None, dt_thresh=(300, 1800)):
|
||||
def __init__(self, parameters='parameters.yaml'):
|
||||
"""
|
||||
Main window of survBot GUI.
|
||||
:param parameters: Parameters dictionary file (yaml format)
|
||||
:param dt_thresh: threshold for timing delay colourisation (yellow/red)
|
||||
"""
|
||||
super(MainWindow, self).__init__()
|
||||
|
||||
# some GUI default colors
|
||||
self.colors_dict = {'FAIL': (255, 50, 0, 255),
|
||||
'NO DATA': (255, 255, 125, 255),
|
||||
'WARN': (255, 255, 125, 255),
|
||||
'WARNX': lambda x: (min([255, 200 + x**2]), 255, 125, 255),
|
||||
'OK': (125, 255, 125, 255),
|
||||
'undefined': (230, 230, 230, 255)}
|
||||
|
||||
# init some attributes
|
||||
self.dt_thresh = dt_thresh
|
||||
self.last_mouse_loc = None
|
||||
self.status_message = ''
|
||||
self.starttime = UTCDateTime()
|
||||
self.outpath_html = outpath_html
|
||||
|
||||
# setup main layout of the GUI
|
||||
self.main_layout = QtWidgets.QVBoxLayout()
|
||||
@ -111,6 +100,7 @@ class MainWindow(QtWidgets.QMainWindow):
|
||||
self.survBot = SurveillanceBot(parameter_path=parameters)
|
||||
self.parameters = self.survBot.parameters
|
||||
self.refresh_period = self.parameters.get('interval')
|
||||
self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')]
|
||||
|
||||
# create thread that is used to update
|
||||
self.thread = Thread(parent=self, runnable=self.survBot.execute_qc)
|
||||
@ -185,41 +175,6 @@ class MainWindow(QtWidgets.QMainWindow):
|
||||
self.last_mouse_loc = event.pos()
|
||||
return super(QtWidgets.QMainWindow, self).eventFilter(object, event)
|
||||
|
||||
def write_html_table(self):
|
||||
fnout = self.outpath_html
|
||||
if not fnout:
|
||||
return
|
||||
try:
|
||||
with open(fnout, 'w') as outfile:
|
||||
write_html_header(outfile, self.refresh_period)
|
||||
#write_html_table_title(outfile, self.parameters)
|
||||
init_html_table(outfile)
|
||||
nrows = self.table.rowCount()
|
||||
ncolumns = self.table.columnCount()
|
||||
|
||||
# add header item 0 fix default black bg color for headers
|
||||
station_header = QtWidgets.QTableWidgetItem(text='Station')
|
||||
station_header.setText('Station')
|
||||
header_items = [station_header]
|
||||
for column in range(ncolumns):
|
||||
hheader = self.table.horizontalHeaderItem(column)
|
||||
header_items.append(hheader)
|
||||
write_html_row(outfile, header_items, html_key='th')
|
||||
|
||||
for row in range(nrows):
|
||||
vheader = self.table.verticalHeaderItem(row)
|
||||
col_items = [vheader]
|
||||
for column in range(ncolumns):
|
||||
col_items.append(self.table.item(row, column))
|
||||
write_html_row(outfile, col_items)
|
||||
|
||||
finish_html_table(outfile)
|
||||
write_html_text(outfile, self.status_message)
|
||||
write_html_footer(outfile)
|
||||
except Exception as e:
|
||||
print(f'Could not write HTML table to {fnout}:')
|
||||
print(e)
|
||||
|
||||
def sms_context_menu(self, row_ind):
|
||||
""" Open a context menu when left-clicking vertical header item """
|
||||
header_item = self.table.verticalHeaderItem(row_ind)
|
||||
@ -264,11 +219,7 @@ class MainWindow(QtWidgets.QMainWindow):
|
||||
|
||||
def fill_status_bar(self):
|
||||
""" Set status bar text """
|
||||
timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
|
||||
self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \
|
||||
f'Current time (UTC) {UTCDateTime().strftime("%Y-%m-%d %H:%M:%S")} | ' \
|
||||
f'Refresh period: {self.refresh_period}s | '\
|
||||
f'Showing data of last {timespan}'
|
||||
self.status_message = self.survBot.status_message
|
||||
status_bar = self.statusBar()
|
||||
status_bar.showMessage(self.status_message)
|
||||
|
||||
@ -283,21 +234,12 @@ class MainWindow(QtWidgets.QMainWindow):
|
||||
status_dict, detailed_dict = self.survBot.analysis_results.get(st_id)
|
||||
status = status_dict.get(check_key)
|
||||
detailed_message = detailed_dict.get(check_key)
|
||||
if check_key == 'last active':
|
||||
bg_color = self.get_time_delay_color(status)
|
||||
elif check_key == 'temp':
|
||||
bg_color = self.get_temp_color(status)
|
||||
|
||||
dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh]
|
||||
bg_color = get_bg_color(check_key, status, dt_thresh)
|
||||
if check_key == 'temp':
|
||||
if not type(status) in [str]:
|
||||
status = str(status) + deg_str
|
||||
else:
|
||||
statussplit = status.split(' ')
|
||||
if len(statussplit) > 1 and statussplit[0] == 'WARN':
|
||||
x = int(status.split(' ')[-1].lstrip('(').rstrip(')'))
|
||||
bg_color = self.colors_dict.get('WARNX')(x)
|
||||
else:
|
||||
bg_color = self.colors_dict.get(status)
|
||||
if not bg_color:
|
||||
bg_color = self.colors_dict.get('undefined')
|
||||
|
||||
# Continue if nothing changed
|
||||
text = str(status)
|
||||
@ -336,26 +278,6 @@ class MainWindow(QtWidgets.QMainWindow):
|
||||
|
||||
# table filling/refreshing done, set clear_on_refresh to False
|
||||
self.clear_on_refresh = False
|
||||
# write html output if parameter is set
|
||||
self.write_html_table()
|
||||
|
||||
def get_time_delay_color(self, dt):
|
||||
""" Set color of time delay after thresholds specified in self.dt_thresh """
|
||||
dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh]
|
||||
if dt < dt_thresh[0]:
|
||||
return self.colors_dict.get('OK')
|
||||
elif dt_thresh[0] <= dt < dt_thresh[1]:
|
||||
return self.colors_dict.get('WARN')
|
||||
return self.colors_dict.get('FAIL')
|
||||
|
||||
def get_temp_color(self, temp, vmin=-10, vmax=60, cmap='coolwarm'):
|
||||
""" Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """
|
||||
if type(temp) in [str]:
|
||||
return self.colors_dict.get('undefined')
|
||||
cmap = matplotlib.cm.get_cmap(cmap)
|
||||
val = (temp - vmin) / (vmax - vmin)
|
||||
rgba = [int(255 * c) for c in cmap(val)]
|
||||
return rgba
|
||||
|
||||
def set_font_bold(self, item):
|
||||
""" Set item font bold """
|
||||
@ -507,15 +429,9 @@ class SendSMSWidget(QtWidgets.QDialog):
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Call survBot GUI')
|
||||
parser.add_argument('-html', dest='html_filename', default=None, help='filename for HTML output')
|
||||
parser.add_argument('--background', dest='background', default=False, action='store_true', help='run in background')
|
||||
args = parser.parse_args()
|
||||
|
||||
program_path = sys.path[0]
|
||||
parameters = os.path.join(program_path, 'parameters.yaml')
|
||||
app = QtWidgets.QApplication([])
|
||||
window = MainWindow(parameters=parameters, outpath_html=args.html_filename)
|
||||
if not args.background:
|
||||
window.showMaximized()
|
||||
window = MainWindow(parameters=parameters)
|
||||
window.showMaximized()
|
||||
sys.exit(app.exec_())
|
||||
|
||||
@ -1,14 +0,0 @@
|
||||
#!/bin/bash
|
||||
ulimit -s 8192
|
||||
|
||||
#$ -l os=*stretch
|
||||
##$ -cwd
|
||||
#$ -pe smp 1
|
||||
##$ -q "*@minos15"
|
||||
|
||||
export PYTHONPATH="$PYTHONPATH:/home/marcel/git/"
|
||||
|
||||
source /opt/anaconda3/etc/profile.d/conda.sh
|
||||
conda activate py37
|
||||
|
||||
python /home/marcel/git/survBot/survBotGUI.py
|
||||
51
utils.py
Normal file
51
utils.py
Normal file
@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import matplotlib
|
||||
|
||||
def get_bg_color(check_key, status, dt_thresh=None, hex=False):
|
||||
if check_key == 'last active':
|
||||
bg_color = get_time_delay_color(status, dt_thresh)
|
||||
elif check_key == 'temp':
|
||||
bg_color = get_temp_color(status)
|
||||
else:
|
||||
statussplit = status.split(' ')
|
||||
if len(statussplit) > 1 and statussplit[0] == 'WARN':
|
||||
x = int(status.split(' ')[-1].lstrip('(').rstrip(')'))
|
||||
bg_color = get_color('WARNX')(x)
|
||||
else:
|
||||
bg_color = get_color(status)
|
||||
if not bg_color:
|
||||
bg_color = get_color('undefined')
|
||||
|
||||
if hex:
|
||||
bg_color = '#{:02x}{:02x}{:02x}'.format(*bg_color[:3])
|
||||
return bg_color
|
||||
|
||||
def get_color(key):
|
||||
# some GUI default colors
|
||||
colors_dict = {'FAIL': (255, 50, 0, 255),
|
||||
'NO DATA': (255, 255, 125, 255),
|
||||
'WARN': (255, 255, 125, 255),
|
||||
'WARNX': lambda x: (min([255, 200 + x ** 2]), 255, 125, 255),
|
||||
'OK': (125, 255, 125, 255),
|
||||
'undefined': (230, 230, 230, 255)}
|
||||
return colors_dict.get(key)
|
||||
|
||||
def get_time_delay_color(dt, dt_thresh):
|
||||
""" Set color of time delay after thresholds specified in self.dt_thresh """
|
||||
if dt < dt_thresh[0]:
|
||||
return get_color('OK')
|
||||
elif dt_thresh[0] <= dt < dt_thresh[1]:
|
||||
return get_color('WARN')
|
||||
return get_color('FAIL')
|
||||
|
||||
def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
|
||||
""" Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """
|
||||
if type(temp) in [str]:
|
||||
return get_color('undefined')
|
||||
cmap = matplotlib.cm.get_cmap(cmap)
|
||||
val = (temp - vmin) / (vmax - vmin)
|
||||
rgba = [int(255 * c) for c in cmap(val)]
|
||||
return rgba
|
||||
|
||||
@ -37,10 +37,10 @@ def write_html_row(fobj, items, html_key='td'):
|
||||
default_space = ' '
|
||||
fobj.write(default_space + '<tr>\n')
|
||||
for item in items:
|
||||
text = item.text()
|
||||
tooltip = item.toolTip()
|
||||
color = item.backgroundColor().name()
|
||||
# fix for black background of headers
|
||||
text = item.get('text')
|
||||
tooltip = item.get('tooltip')
|
||||
color = item.get('color')
|
||||
# check for black background of headers (shouldnt happen anymore)
|
||||
color = '#e6e6e6' if color == '#000000' else color
|
||||
fobj.write(2 * default_space + f'<{html_key} bgcolor="{color}" title="{tooltip}">' + text + f'</{html_key}>\n')
|
||||
fobj.write(default_space + '</tr>\n')
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user