marcel/survBot
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases 1 Wiki Activity

[update] added possibility to modify voltage level in plots from settings in parameters.yaml

Browse Source
This commit is contained in:
Marcel Paffrath 2022-11-21 15:31:32 +01:00
parent 3fe5fc48d1
commit a6d59c8c71
5 changed files with 119 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
parameters.yaml
View File

@ -55,3 +55,15 @@ EMAIL:
addresses: ['marcel.paffrath@rub.de', 'kasper.fischer@rub.de'] # list of mail addresses for info mails
sender: 'webmaster@geophysik.ruhr-uni-bochum.de' # mail sender
# Factor for channel to SI-units (for plotting)
CHANNEL_UNITS:
EX1: 1e-6
EX2: 1e-6
EX3: 1e-6
VEI: 1e-3
# Transform channel for plotting, perform arithmetic operations in given order, e.g.: PBox EX1 V to deg C: 20 * x -20
CHANNEL_TRANSFORM:
EX1:
- ['*', 20]
- ['-', 20]

87
survBot.py
View File

@ -17,13 +17,14 @@ import matplotlib.pyplot as plt
from obspy import read, UTCDateTime, Stream
from obspy.clients.filesystem.sds import Client
from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str,\
from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str, \
init_html_table, finish_html_table
from utils import get_bg_color
from utils import get_bg_color, modify_stream_for_plot
try:
import smtplib
from email.mime.text import MIMEText
mail_functionality = True
except ImportError:
print('Could not import smtplib or mail. Disabled sending mails.')
@ -188,7 +189,7 @@ class SurveillanceBot(object):
stream = self.data.get(nwst_id)
if stream:
nsl = nsl_from_id(nwst_id)
station_qc = StationQC(stream, nsl, self.parameters, self.keys, qc_starttime,
station_qc = StationQC(self, stream, nsl, self.parameters, self.keys, qc_starttime,
self.verbosity, print_func=self.print,
status_track=self.status_track.get(nwst_id))
analysis_print_result = station_qc.return_print_analysis()
@ -277,7 +278,7 @@ class SurveillanceBot(object):
if self.outpath_html:
self.write_html_table()
if self.parameters.get('html_figures'):
self.write_html_figures(check_plot_time=not(first_exec))
self.write_html_figures(check_plot_time=not (first_exec))
else:
self.print_analysis()
time.sleep(self.refresh_period)
@ -321,21 +322,28 @@ class SurveillanceBot(object):
os.mkdir(self.outpath_html)
def write_html_figures(self, check_plot_time=True):
""" Write figures for html, right now hardcoded hourly """
""" Write figures for html (e.g. hourly) """
if check_plot_time and not self.check_plot_hour():
return
self.check_fig_dir()
for nwst_id in self.station_list:
fig = plt.figure(figsize=(16, 9))
fnout = self.get_fig_path_abs(nwst_id)
st = self.data.get(nwst_id)
if st:
st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full')
ax = fig.axes[0]
ax.set_title(f'Hourly refreshed plot at (UTC) {UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S")}')
fig.savefig(fnout, dpi=150., bbox_inches='tight')
plt.close(fig)
self.write_html_figure(nwst_id)
def write_html_figure(self, nwst_id):
""" Write figure for html for specified station """
self.check_fig_dir()
fig = plt.figure(figsize=(16, 9))
fnout = self.get_fig_path_abs(nwst_id)
st = self.data.get(nwst_id)
if st:
st = modify_stream_for_plot(st, parameters=self.parameters)
st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full')
ax = fig.axes[0]
ax.set_title(f'Plot refreshed at (UTC) {UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S")}. '
f'Refreshed hourly or on FAIL status.')
fig.savefig(fnout, dpi=150., bbox_inches='tight')
plt.close(fig)
def write_html_table(self, default_color='#e6e6e6'):
self.check_html_dir()
@ -345,7 +353,7 @@ class SurveillanceBot(object):
try:
with open(fnout, 'w') as outfile:
write_html_header(outfile, self.refresh_period)
#write_html_table_title(outfile, self.parameters)
# write_html_table_title(outfile, self.parameters)
init_html_table(outfile)
# First write header items
@ -405,7 +413,7 @@ class SurveillanceBot(object):
timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \
f'Current time (UTC) {UTCDateTime().strftime("%Y-%m-%d %H:%M:%S")} | ' \
f'Refresh period: {self.refresh_period}s | '\
f'Refresh period: {self.refresh_period}s | ' \
f'Showing data of last {timespan}'
def print(self, string, **kwargs):
@ -422,12 +430,13 @@ class SurveillanceBot(object):
class StationQC(object):
def __init__(self, stream, nsl, parameters, keys, starttime, verbosity, print_func, status_track={}):
def __init__(self, parent, stream, nsl, parameters, keys, starttime, verbosity, print_func, status_track={}):
"""
Station Quality Check class.
:param nsl: dictionary containing network, station and location (key: str)
:param parameters: parameters dictionary from parameters.yaml file
"""
self.parent = parent
self.stream = stream
self.nsl = nsl
self.network = nsl.get('network')
@ -448,6 +457,10 @@ class StationQC(object):
self.start()
@property
def nwst_id(self):
return f'{self.network}.{self.station}'
def status_ok(self, key, detailed_message="Everything OK", status_message='OK', overwrite=False):
current_status = self.status_dict.get(key)
# do not overwrite existing warnings or errors
@ -497,6 +510,9 @@ class StationQC(object):
current_status.count += count
else:
current_status = new_error
# refresh plot (using parent class) if error is new and not on program-startup
if self.search_previous_errors(key, n_errors=1):
self.parent.write_html_figure(self.nwst_id)
self._update_status(key, current_status, detailed_message, last_occurrence)
@ -507,7 +523,7 @@ class StationQC(object):
if self.search_previous_errors(key):
self.send_mail(key, detailed_message)
def search_previous_errors(self, key):
def search_previous_errors(self, key, n_errors=None):
"""
Check n_track + 1 previous statuses for errors.
If first item in list is no error but all others are return True (first time n_track errors appeared --
@ -515,9 +531,12 @@ class StationQC(object):
In all other cases return True.
This also prevents sending status (e.g. mail) in case of program startup
"""
if n_errors is not None:
n_errors = self.parameters.get('n_track') + 1
previous_errors = self.status_track.get(key)
# only if error list is filled n_track times
if previous_errors and len(previous_errors) == self.parameters.get('n_track') + 1:
if previous_errors and len(previous_errors) == n_errors:
# if first entry was no error but all others are, return True (-> new Fail n_track times)
if not previous_errors[0] and all(previous_errors[1:]):
return True
@ -547,7 +566,7 @@ class StationQC(object):
dt = timedelta(seconds=n_track * interval)
text = f'{key} FAIL status longer than {dt}: ' + message
msg = MIMEText(text)
msg['Subject'] = f'new FAIL status on station {self.network}.{self.station}'
msg['Subject'] = f'new FAIL status on station {self.nwst_id}'
msg['From'] = sender
msg['To'] = ', '.join(addresses)
@ -556,7 +575,6 @@ class StationQC(object):
s.sendmail(sender, addresses, msg.as_string())
s.quit()
def status_other(self, detailed_message, status_message, last_occurrence=None, count=1):
key = 'other'
new_status = StatusOther(count=count, messages=[status_message])
@ -611,7 +629,7 @@ class StationQC(object):
self.pb_rout_charge_analysis()
def return_print_analysis(self):
items = [f'{self.network}.{self.station}']
items = [self.nwst_id]
for key in self.keys:
status = self.status_dict[key]
message = status.message
@ -670,11 +688,10 @@ class StationQC(object):
self.warn(key, detailed_message=detailed_message, count=n_overvolt,
last_occurrence=self.get_last_occurrence(trace, overvolt))
if len(undervolt) > 0:
# try calculate number of voltage peaks from gaps between indices
n_undervolt = len(np.where(np.diff(undervolt) > 1)[0]) + 1
detailed_message = warn_message + f' {n_undervolt}x Voltage under {low_volt}V '\
detailed_message = warn_message + f' {n_undervolt}x Voltage under {low_volt}V ' \
+ self.get_last_occurrence_timestring(trace, undervolt)
self.warn(key, detailed_message=detailed_message, count=n_undervolt,
last_occurrence=self.get_last_occurrence(trace, undervolt))
@ -693,7 +710,7 @@ class StationQC(object):
nsamp_av = int(trace.stats.sampling_rate) * timespan
av_temp_str = str(round(np.mean(temp[-nsamp_av:]), 1)) + deg_str
# dt of average
dt_t_str = str(timedelta(seconds=int(timespan)))
dt_t_str = str(timedelta(seconds=int(timespan))).replace(', 0:00:00', '')
# current temp
cur_temp = round(temp[-1], 1)
if self.verbosity > 1:
@ -778,7 +795,7 @@ class StationQC(object):
n_occurrences = len(np.where(np.diff(ind_array) > 1)[0]) + 1
self.warn(key=key,
detailed_message=f'Trace {trace.get_id()}: '
f'Found {n_occurrences} occurrence(s) of {volt_lvl}V: {key}: {message}'
f'Found {n_occurrences} occurrence(s) of {volt_lvl}V: {key}: {message}'
+ self.get_last_occurrence_timestring(trace, ind_array),
count=n_occurrences,
last_occurrence=self.get_last_occurrence(trace, ind_array))
@ -839,9 +856,9 @@ class StationQC(object):
n_occurrences = len(np.where(np.diff(under) > 1)[0]) + 1
voltage_dict[-1] = under
self.status_other(detailed_message=f'Trace {trace.get_id()}: '
f'Voltage below {pb_ok}V in {len(under)} samples, {n_occurrences} time(s). '
f'Mean voltage: {np.mean(voltage):.2}'
+ self.get_last_occurrence_timestring(trace, under),
f'Voltage below {pb_ok}V in {len(under)} samples, {n_occurrences} time(s). '
f'Mean voltage: {np.mean(voltage):.2}'
+ self.get_last_occurrence_timestring(trace, under),
status_message='under 1V ({})'.format(n_occurrences))
# classify last voltage values
@ -860,8 +877,8 @@ class StationQC(object):
max_uncl = self.parameters.get('THRESHOLDS').get('unclassified')
if max_uncl and n_unclassified > max_uncl:
self.status_other(detailed_message=f'Trace {trace.get_id()}: '
f'{n_unclassified}/{len(all_indices)} '
f'unclassified voltage values in channel {trace.get_id()}',
f'{n_unclassified}/{len(all_indices)} '
f'unclassified voltage values in channel {trace.get_id()}',
status_message=f'{channel}: {n_unclassified} uncl.')
return False, voltage_dict, last_val
@ -929,7 +946,7 @@ class StatusError(Status):
detailed_messages=detailed_messages, show_count=show_count)
self.set_error()
class StatusOther(Status):
def __init__(self, messages=None, count=1, last_occurence=None, detailed_messages=None):
super(StatusOther, self).__init__(count=count, last_occurrence=last_occurence,
@ -938,11 +955,11 @@ class StatusOther(Status):
messages = []
self.messages = messages
self.is_other = True
def get_status_str(self):
if self.messages == []:
return '-'
message = ''
for index, mes in enumerate(self.messages):
if index > 0:

4
survBotGUI.py
View File

@ -22,7 +22,6 @@ except ImportError:
except ImportError:
raise ImportError('Could import neither of PySide2, PySide6 or PyQt5')
import matplotlib
from matplotlib.figure import Figure
if QtGui.__package__ in ['PySide2', 'PyQt5', 'PySide6']:
@ -35,7 +34,7 @@ from obspy import UTCDateTime
from survBot import SurveillanceBot
from write_utils import *
from utils import get_bg_color
from utils import get_bg_color, modify_stream_for_plot
try:
from rest_api.utils import get_station_iccid
@ -315,6 +314,7 @@ class MainWindow(QtWidgets.QMainWindow):
if st:
self.plot_widget = PlotWidget(self)
self.plot_widget.setWindowTitle(nwst_id)
st = modify_stream_for_plot(st, parameters=self.parameters)
st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig)
self.plot_widget.show()

52
utils.py
View File

@ -3,6 +3,7 @@
import matplotlib
def get_bg_color(check_key, status, dt_thresh=None, hex=False):
message = status.message
if check_key == 'last active':
@ -23,6 +24,7 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False):
bg_color = '#{:02x}{:02x}{:02x}'.format(*bg_color[:3])
return bg_color
def get_color(key):
# some GUI default colors
colors_dict = {'FAIL': (255, 50, 0, 255),
@ -33,6 +35,7 @@ def get_color(key):
'undefined': (230, 230, 230, 255)}
return colors_dict.get(key)
def get_time_delay_color(dt, dt_thresh):
""" Set color of time delay after thresholds specified in self.dt_thresh """
if dt < dt_thresh[0]:
@ -41,6 +44,7 @@ def get_time_delay_color(dt, dt_thresh):
return get_color('WARN')
return get_color('FAIL')
def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
""" Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """
if type(temp) in [str]:
@ -50,3 +54,51 @@ def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
rgba = [int(255 * c) for c in cmap(val)]
return rgba
def modify_stream_for_plot(st, parameters):
""" copy (if necessary) and modify stream for plotting """
ch_units = parameters.get('CHANNEL_UNITS')
ch_transf = parameters.get('CHANNEL_TRANSFORM')
# if either of both are defined make copy
if ch_units or ch_transf:
st = st.copy()
# modify trace for plotting by multiplying unit factor (e.g. 1e-3 mV to V)
if ch_units:
for tr in st:
channel = tr.stats.channel
unit_factor = ch_units.get(channel)
if unit_factor:
tr.data = tr.data * float(unit_factor)
# modify trace for plotting by other arithmetic expressions
if ch_transf:
for tr in st:
channel = tr.stats.channel
transf = ch_transf.get(channel)
if transf:
tr.data = transform_trace(tr.data, transf)
return st
def transform_trace(data, transf):
"""
Transform trace with arithmetic operations in order, specified in transf
@param data: numpy array
@param transf: list of lists with arithmetic operations (e.g. [['*', '20'], ] -> multiply data by 20
"""
# This looks a little bit hardcoded, however it is safer than using e.g. "eval"
for operator_str, val in transf:
if operator_str == '+':
data = data + val
elif operator_str == '-':
data = data - val
elif operator_str == '*':
data = data * val
elif operator_str == '/':
data = data / val
else:
raise IOError(f'Unknown arithmethic operator string: {operator_str}')
return data

2
write_utils.py
View File

@ -53,6 +53,6 @@ def write_html_row(fobj, items, html_key='td'):
def get_print_title_str(parameters):
timespan = parameters.get('timespan') * 24 * 3600
tdelta_str = str(timedelta(seconds=int(timespan)))
tdelta_str = str(timedelta(seconds=int(timespan))).replace(', 0:00:00', '')
return f'Analysis table of router quality within the last {tdelta_str}'