marcel/survBot
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases 1 Wiki Activity

[update] first version write html output, todo: html parameters argparse?

Browse Source
This commit is contained in:
Marcel Paffrath 2022-11-07 17:56:41 +01:00
parent 64daf34941
commit 7a6072f8dd
5 changed files with 114 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
parameters.yaml
View File

@ -1,5 +1,6 @@
# Parameters file for Surveillance Bot
datapath: '/data/SDS/' # SC3 Datapath
outpath_html: '/home/marcel/tmp/survBot_out.html' # output of HTML table
networks: ['1Y', 'HA']
stations: '*'
locations: '*'

1
submit_bot.sh
View File

@ -8,6 +8,7 @@ ulimit -s 8192
##$ -q "*@minos15"
export PYTHONPATH="$PYTHONPATH:/home/marcel/git/"
export PYTHONPATH="$PYTHONPATH:/home/marcel/git/code_base/"
source /opt/anaconda3/etc/profile.d/conda.sh
conda activate py37

31
survBot.py
View File

@ -13,11 +13,14 @@ import numpy as np
from obspy import read, UTCDateTime, Stream
from obspy.clients.filesystem.sds import Client
from write_utils import get_print_title_str
pjoin = os.path.join
UP = "\x1B[{length}A"
CLR = "\x1B[0K"
deg_str = '\N{DEGREE SIGN}C'
def read_yaml(file_path):
with open(file_path, "r") as f:
return yaml.safe_load(f)
@ -30,7 +33,7 @@ def nsl_from_id(st_id):
def get_st_id(trace):
stats = trace.stats
return f'{stats.network}.{stats.station}.'#{stats.location}'
return f'{stats.network}.{stats.station}.' # {stats.location}'
def fancy_timestr(dt, thresh=600, modif='+'):
@ -60,7 +63,7 @@ class SurveillanceBot(object):
self.print_count = 0
self.refresh_period = 0
self.cl = Client(self.parameters.get('datapath')) #TODO: Check if this has to be loaded again on update
self.cl = Client(self.parameters.get('datapath')) # TODO: Check if this has to be loaded again on update
self.get_stations()
def transform_parameters(self):
@ -99,7 +102,7 @@ class SurveillanceBot(object):
for location in locations:
for channel in channels:
self.filenames += list(self.cl._get_filenames(network, station, location, channel,
starttime=t1, endtime=time_now))
starttime=t1, endtime=time_now))
def read_data(self):
self.data = {}
@ -183,11 +186,13 @@ class SurveillanceBot(object):
if len(times) > 0:
return min(times)
def print_analysis_html(self, filename):
with open(filename, 'w') as outfile:
pass
def print_analysis(self):
timespan = self.parameters.get('timespan') * 24 * 3600
self.print(200*'+')
tdelta_str = str(timedelta(seconds=int(timespan)))
title_str = f'Analysis table of router quality within the last {tdelta_str}'
self.print(200 * '+')
title_str = get_print_title_str(self.parameters)
self.print(title_str)
if self.refresh_period > 0:
self.print(f'Refreshing every {self.refresh_period}s.')
@ -224,8 +229,8 @@ class SurveillanceBot(object):
n_nl = string.count('\n')
string.replace('\n', clear_end)
print(string, end=clear_end, **kwargs)
self.print_count += n_nl + 1 #number of newlines + actual print with end='\n' (no check for kwargs end!)
#print('pc:', self.print_count)
self.print_count += n_nl + 1 # number of newlines + actual print with end='\n' (no check for kwargs end!)
# print('pc:', self.print_count)
def clear_prints(self):
print(UP.format(length=self.print_count), end='')
@ -300,7 +305,7 @@ class StationQC(object):
def analyse_channels(self):
if self.verbosity > 0:
self.print(150*'#')
self.print(150 * '#')
self.print('This is StationQT. Calculating quality for station'
' {network}.{station}.{location}'.format(**self.nsl))
self.voltage_analysis()
@ -333,7 +338,7 @@ class StationQC(object):
def voltage_analysis(self, channel='VEI'):
""" Analyse voltage channel for over/undervoltage """
key='voltage'
key = 'voltage'
st = self.stream.select(channel=channel)
trace = self.get_trace(st, key)
if not trace: return
@ -367,7 +372,7 @@ class StationQC(object):
def pb_temp_analysis(self, channel='EX1'):
""" Analyse PowBox temperature output. """
key='temp'
key = 'temp'
st = self.stream.select(channel=channel)
trace = self.get_trace(st, key)
if not trace: return
@ -375,7 +380,7 @@ class StationQC(object):
thresholds = self.parameters.get('THRESHOLDS')
temp = 20. * voltage - 20
# average temp
timespan = min([self.parameters.get('timespan') * 24 * 3600, int(len(temp)/trace.stats.sampling_rate)])
timespan = min([self.parameters.get('timespan') * 24 * 3600, int(len(temp) / trace.stats.sampling_rate)])
nsamp_av = int(trace.stats.sampling_rate) * timespan
av_temp_str = str(round(np.mean(temp[-nsamp_av:]), 1)) + deg_str
# dt of average

51
survBotGUI.py
View File

@ -9,7 +9,7 @@ __author__ = 'Marcel Paffrath'
import os
import sys
import traceback
from datetime import timedelta
import argparse
try:
from PySide2 import QtGui, QtCore, QtWidgets
@ -34,6 +34,7 @@ else:
from obspy import UTCDateTime
from survBot import SurveillanceBot
from write_utils import *
try:
from rest_api.utils import get_station_iccid
@ -95,6 +96,7 @@ class MainWindow(QtWidgets.QMainWindow):
# init some attributes
self.dt_thresh = dt_thresh
self.last_mouse_loc = None
self.status_message = ''
self.starttime = UTCDateTime()
# setup main layout of the GUI
@ -181,6 +183,41 @@ class MainWindow(QtWidgets.QMainWindow):
self.last_mouse_loc = event.pos()
return super(QtWidgets.QMainWindow, self).eventFilter(object, event)
def write_html_table(self):
fnout = self.parameters.get('outpath_html')
if not fnout:
return
try:
with open(fnout, 'w') as outfile:
write_html_header(outfile)
#write_html_table_title(outfile, self.parameters)
init_html_table(outfile)
nrows = self.table.rowCount()
ncolumns = self.table.columnCount()
# add header item 0 fix default black bg color for headers
station_header = QtWidgets.QTableWidgetItem(text='Station')
station_header.setText('Station')
header_items = [station_header]
for column in range(ncolumns):
hheader = self.table.horizontalHeaderItem(column)
header_items.append(hheader)
write_html_row(outfile, header_items, html_key='th')
for row in range(nrows):
vheader = self.table.verticalHeaderItem(row)
col_items = [vheader]
for column in range(ncolumns):
col_items.append(self.table.item(row, column))
write_html_row(outfile, col_items)
finish_html_table(outfile)
write_html_text(outfile, self.status_message)
write_html_footer(outfile)
except Exception as e:
print(f'Could not write HTML table to {fnout}:')
print(e)
def sms_context_menu(self, row_ind):
""" Open a context menu when left-clicking vertical header item """
header_item = self.table.verticalHeaderItem(row_ind)
@ -225,11 +262,13 @@ class MainWindow(QtWidgets.QMainWindow):
def fill_status_bar(self):
""" Set status bar text """
status_bar = self.statusBar()
timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
status_bar.showMessage(f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | '
f'Refresh period: {self.refresh_period}s | '
f'Showing data of last {timespan}')
self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \
f'Current time (UTC) {UTCDateTime().strftime("%Y-%m-%d %H:%M:%S")} | ' \
f'Refresh period: {self.refresh_period}s | '\
f'Showing data of last {timespan}'
status_bar = self.statusBar()
status_bar.showMessage(self.status_message)
def fill_table(self):
""" Fills the table with most recent information. Executed after execute_qc thread is done or on refresh. """
@ -291,6 +330,8 @@ class MainWindow(QtWidgets.QMainWindow):
# table filling/refreshing done, set clear_on_refresh to False
self.clear_on_refresh = False
# write html output if parameter is set
self.write_html_table()
def get_time_delay_color(self, dt):
""" Set color of time delay after thresholds specified in self.dt_thresh """

48
write_utils.py Normal file
View File

@ -0,0 +1,48 @@
from datetime import timedelta
def write_html_table_title(fobj, parameters):
title = get_print_title_str(parameters)
fobj.write(f'<h3>{title}</h3>\n')
def write_html_text(fobj, text):
fobj.write(f'<p>{text}</p>\n')
def write_html_header(fobj):
header = ['<!DOCTYPE html>',
'<html>',
'<style>',
'table, th, td {',
'border:1px solid black;',
'}',
'</style>',
'<body>']
for item in header:
fobj.write(item + '\n')
def init_html_table(fobj):
fobj.write('<table style="width:100%">\n')
def finish_html_table(fobj):
fobj.write('</table>\n')
def write_html_footer(fobj):
footer = ['</body>',
'</html>']
for item in footer:
fobj.write(item + '\n')
def write_html_row(fobj, items, html_key='td'):
fobj.write('<tr>\n')
for item in items:
text = item.text()
color = item.backgroundColor().name()
# fix for black background of headers
color = '#e6e6e6' if color == '#000000' else color
fobj.write(f'<{html_key} bgcolor="{color}">' + text + f'</{html_key}>\n')
fobj.write('</tr>\n')
def get_print_title_str(parameters):
timespan = parameters.get('timespan') * 24 * 3600
tdelta_str = str(timedelta(seconds=int(timespan)))
return f'Analysis table of router quality within the last {tdelta_str}'