[update] added possibility to modify voltage level in plots from settings in parameters.yaml
This commit is contained in:
parent
3fe5fc48d1
commit
a6d59c8c71
@ -55,3 +55,15 @@ EMAIL:
|
||||
addresses: ['marcel.paffrath@rub.de', 'kasper.fischer@rub.de'] # list of mail addresses for info mails
|
||||
sender: 'webmaster@geophysik.ruhr-uni-bochum.de' # mail sender
|
||||
|
||||
# Factor for channel to SI-units (for plotting)
|
||||
CHANNEL_UNITS:
|
||||
EX1: 1e-6
|
||||
EX2: 1e-6
|
||||
EX3: 1e-6
|
||||
VEI: 1e-3
|
||||
|
||||
# Transform channel for plotting, perform arithmetic operations in given order, e.g.: PBox EX1 V to deg C: 20 * x -20
|
||||
CHANNEL_TRANSFORM:
|
||||
EX1:
|
||||
- ['*', 20]
|
||||
- ['-', 20]
|
||||
81
survBot.py
81
survBot.py
@ -17,13 +17,14 @@ import matplotlib.pyplot as plt
|
||||
from obspy import read, UTCDateTime, Stream
|
||||
from obspy.clients.filesystem.sds import Client
|
||||
|
||||
from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str,\
|
||||
from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str, \
|
||||
init_html_table, finish_html_table
|
||||
from utils import get_bg_color
|
||||
from utils import get_bg_color, modify_stream_for_plot
|
||||
|
||||
try:
|
||||
import smtplib
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
mail_functionality = True
|
||||
except ImportError:
|
||||
print('Could not import smtplib or mail. Disabled sending mails.')
|
||||
@ -188,7 +189,7 @@ class SurveillanceBot(object):
|
||||
stream = self.data.get(nwst_id)
|
||||
if stream:
|
||||
nsl = nsl_from_id(nwst_id)
|
||||
station_qc = StationQC(stream, nsl, self.parameters, self.keys, qc_starttime,
|
||||
station_qc = StationQC(self, stream, nsl, self.parameters, self.keys, qc_starttime,
|
||||
self.verbosity, print_func=self.print,
|
||||
status_track=self.status_track.get(nwst_id))
|
||||
analysis_print_result = station_qc.return_print_analysis()
|
||||
@ -277,7 +278,7 @@ class SurveillanceBot(object):
|
||||
if self.outpath_html:
|
||||
self.write_html_table()
|
||||
if self.parameters.get('html_figures'):
|
||||
self.write_html_figures(check_plot_time=not(first_exec))
|
||||
self.write_html_figures(check_plot_time=not (first_exec))
|
||||
else:
|
||||
self.print_analysis()
|
||||
time.sleep(self.refresh_period)
|
||||
@ -321,21 +322,28 @@ class SurveillanceBot(object):
|
||||
os.mkdir(self.outpath_html)
|
||||
|
||||
def write_html_figures(self, check_plot_time=True):
|
||||
""" Write figures for html, right now hardcoded hourly """
|
||||
""" Write figures for html (e.g. hourly) """
|
||||
if check_plot_time and not self.check_plot_hour():
|
||||
return
|
||||
self.check_fig_dir()
|
||||
|
||||
for nwst_id in self.station_list:
|
||||
fig = plt.figure(figsize=(16, 9))
|
||||
fnout = self.get_fig_path_abs(nwst_id)
|
||||
st = self.data.get(nwst_id)
|
||||
if st:
|
||||
st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full')
|
||||
ax = fig.axes[0]
|
||||
ax.set_title(f'Hourly refreshed plot at (UTC) {UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S")}')
|
||||
fig.savefig(fnout, dpi=150., bbox_inches='tight')
|
||||
plt.close(fig)
|
||||
self.write_html_figure(nwst_id)
|
||||
|
||||
def write_html_figure(self, nwst_id):
|
||||
""" Write figure for html for specified station """
|
||||
self.check_fig_dir()
|
||||
|
||||
fig = plt.figure(figsize=(16, 9))
|
||||
fnout = self.get_fig_path_abs(nwst_id)
|
||||
st = self.data.get(nwst_id)
|
||||
if st:
|
||||
st = modify_stream_for_plot(st, parameters=self.parameters)
|
||||
st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full')
|
||||
ax = fig.axes[0]
|
||||
ax.set_title(f'Plot refreshed at (UTC) {UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S")}. '
|
||||
f'Refreshed hourly or on FAIL status.')
|
||||
fig.savefig(fnout, dpi=150., bbox_inches='tight')
|
||||
plt.close(fig)
|
||||
|
||||
def write_html_table(self, default_color='#e6e6e6'):
|
||||
self.check_html_dir()
|
||||
@ -345,7 +353,7 @@ class SurveillanceBot(object):
|
||||
try:
|
||||
with open(fnout, 'w') as outfile:
|
||||
write_html_header(outfile, self.refresh_period)
|
||||
#write_html_table_title(outfile, self.parameters)
|
||||
# write_html_table_title(outfile, self.parameters)
|
||||
init_html_table(outfile)
|
||||
|
||||
# First write header items
|
||||
@ -405,7 +413,7 @@ class SurveillanceBot(object):
|
||||
timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
|
||||
self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \
|
||||
f'Current time (UTC) {UTCDateTime().strftime("%Y-%m-%d %H:%M:%S")} | ' \
|
||||
f'Refresh period: {self.refresh_period}s | '\
|
||||
f'Refresh period: {self.refresh_period}s | ' \
|
||||
f'Showing data of last {timespan}'
|
||||
|
||||
def print(self, string, **kwargs):
|
||||
@ -422,12 +430,13 @@ class SurveillanceBot(object):
|
||||
|
||||
|
||||
class StationQC(object):
|
||||
def __init__(self, stream, nsl, parameters, keys, starttime, verbosity, print_func, status_track={}):
|
||||
def __init__(self, parent, stream, nsl, parameters, keys, starttime, verbosity, print_func, status_track={}):
|
||||
"""
|
||||
Station Quality Check class.
|
||||
:param nsl: dictionary containing network, station and location (key: str)
|
||||
:param parameters: parameters dictionary from parameters.yaml file
|
||||
"""
|
||||
self.parent = parent
|
||||
self.stream = stream
|
||||
self.nsl = nsl
|
||||
self.network = nsl.get('network')
|
||||
@ -448,6 +457,10 @@ class StationQC(object):
|
||||
|
||||
self.start()
|
||||
|
||||
@property
|
||||
def nwst_id(self):
|
||||
return f'{self.network}.{self.station}'
|
||||
|
||||
def status_ok(self, key, detailed_message="Everything OK", status_message='OK', overwrite=False):
|
||||
current_status = self.status_dict.get(key)
|
||||
# do not overwrite existing warnings or errors
|
||||
@ -497,6 +510,9 @@ class StationQC(object):
|
||||
current_status.count += count
|
||||
else:
|
||||
current_status = new_error
|
||||
# refresh plot (using parent class) if error is new and not on program-startup
|
||||
if self.search_previous_errors(key, n_errors=1):
|
||||
self.parent.write_html_figure(self.nwst_id)
|
||||
|
||||
self._update_status(key, current_status, detailed_message, last_occurrence)
|
||||
|
||||
@ -507,7 +523,7 @@ class StationQC(object):
|
||||
if self.search_previous_errors(key):
|
||||
self.send_mail(key, detailed_message)
|
||||
|
||||
def search_previous_errors(self, key):
|
||||
def search_previous_errors(self, key, n_errors=None):
|
||||
"""
|
||||
Check n_track + 1 previous statuses for errors.
|
||||
If first item in list is no error but all others are return True (first time n_track errors appeared --
|
||||
@ -515,9 +531,12 @@ class StationQC(object):
|
||||
In all other cases return True.
|
||||
This also prevents sending status (e.g. mail) in case of program startup
|
||||
"""
|
||||
if n_errors is not None:
|
||||
n_errors = self.parameters.get('n_track') + 1
|
||||
|
||||
previous_errors = self.status_track.get(key)
|
||||
# only if error list is filled n_track times
|
||||
if previous_errors and len(previous_errors) == self.parameters.get('n_track') + 1:
|
||||
if previous_errors and len(previous_errors) == n_errors:
|
||||
# if first entry was no error but all others are, return True (-> new Fail n_track times)
|
||||
if not previous_errors[0] and all(previous_errors[1:]):
|
||||
return True
|
||||
@ -547,7 +566,7 @@ class StationQC(object):
|
||||
dt = timedelta(seconds=n_track * interval)
|
||||
text = f'{key} FAIL status longer than {dt}: ' + message
|
||||
msg = MIMEText(text)
|
||||
msg['Subject'] = f'new FAIL status on station {self.network}.{self.station}'
|
||||
msg['Subject'] = f'new FAIL status on station {self.nwst_id}'
|
||||
msg['From'] = sender
|
||||
msg['To'] = ', '.join(addresses)
|
||||
|
||||
@ -556,7 +575,6 @@ class StationQC(object):
|
||||
s.sendmail(sender, addresses, msg.as_string())
|
||||
s.quit()
|
||||
|
||||
|
||||
def status_other(self, detailed_message, status_message, last_occurrence=None, count=1):
|
||||
key = 'other'
|
||||
new_status = StatusOther(count=count, messages=[status_message])
|
||||
@ -611,7 +629,7 @@ class StationQC(object):
|
||||
self.pb_rout_charge_analysis()
|
||||
|
||||
def return_print_analysis(self):
|
||||
items = [f'{self.network}.{self.station}']
|
||||
items = [self.nwst_id]
|
||||
for key in self.keys:
|
||||
status = self.status_dict[key]
|
||||
message = status.message
|
||||
@ -670,11 +688,10 @@ class StationQC(object):
|
||||
self.warn(key, detailed_message=detailed_message, count=n_overvolt,
|
||||
last_occurrence=self.get_last_occurrence(trace, overvolt))
|
||||
|
||||
|
||||
if len(undervolt) > 0:
|
||||
# try calculate number of voltage peaks from gaps between indices
|
||||
n_undervolt = len(np.where(np.diff(undervolt) > 1)[0]) + 1
|
||||
detailed_message = warn_message + f' {n_undervolt}x Voltage under {low_volt}V '\
|
||||
detailed_message = warn_message + f' {n_undervolt}x Voltage under {low_volt}V ' \
|
||||
+ self.get_last_occurrence_timestring(trace, undervolt)
|
||||
self.warn(key, detailed_message=detailed_message, count=n_undervolt,
|
||||
last_occurrence=self.get_last_occurrence(trace, undervolt))
|
||||
@ -693,7 +710,7 @@ class StationQC(object):
|
||||
nsamp_av = int(trace.stats.sampling_rate) * timespan
|
||||
av_temp_str = str(round(np.mean(temp[-nsamp_av:]), 1)) + deg_str
|
||||
# dt of average
|
||||
dt_t_str = str(timedelta(seconds=int(timespan)))
|
||||
dt_t_str = str(timedelta(seconds=int(timespan))).replace(', 0:00:00', '')
|
||||
# current temp
|
||||
cur_temp = round(temp[-1], 1)
|
||||
if self.verbosity > 1:
|
||||
@ -778,7 +795,7 @@ class StationQC(object):
|
||||
n_occurrences = len(np.where(np.diff(ind_array) > 1)[0]) + 1
|
||||
self.warn(key=key,
|
||||
detailed_message=f'Trace {trace.get_id()}: '
|
||||
f'Found {n_occurrences} occurrence(s) of {volt_lvl}V: {key}: {message}'
|
||||
f'Found {n_occurrences} occurrence(s) of {volt_lvl}V: {key}: {message}'
|
||||
+ self.get_last_occurrence_timestring(trace, ind_array),
|
||||
count=n_occurrences,
|
||||
last_occurrence=self.get_last_occurrence(trace, ind_array))
|
||||
@ -839,9 +856,9 @@ class StationQC(object):
|
||||
n_occurrences = len(np.where(np.diff(under) > 1)[0]) + 1
|
||||
voltage_dict[-1] = under
|
||||
self.status_other(detailed_message=f'Trace {trace.get_id()}: '
|
||||
f'Voltage below {pb_ok}V in {len(under)} samples, {n_occurrences} time(s). '
|
||||
f'Mean voltage: {np.mean(voltage):.2}'
|
||||
+ self.get_last_occurrence_timestring(trace, under),
|
||||
f'Voltage below {pb_ok}V in {len(under)} samples, {n_occurrences} time(s). '
|
||||
f'Mean voltage: {np.mean(voltage):.2}'
|
||||
+ self.get_last_occurrence_timestring(trace, under),
|
||||
status_message='under 1V ({})'.format(n_occurrences))
|
||||
|
||||
# classify last voltage values
|
||||
@ -860,8 +877,8 @@ class StationQC(object):
|
||||
max_uncl = self.parameters.get('THRESHOLDS').get('unclassified')
|
||||
if max_uncl and n_unclassified > max_uncl:
|
||||
self.status_other(detailed_message=f'Trace {trace.get_id()}: '
|
||||
f'{n_unclassified}/{len(all_indices)} '
|
||||
f'unclassified voltage values in channel {trace.get_id()}',
|
||||
f'{n_unclassified}/{len(all_indices)} '
|
||||
f'unclassified voltage values in channel {trace.get_id()}',
|
||||
status_message=f'{channel}: {n_unclassified} uncl.')
|
||||
|
||||
return False, voltage_dict, last_val
|
||||
|
||||
@ -22,7 +22,6 @@ except ImportError:
|
||||
except ImportError:
|
||||
raise ImportError('Could import neither of PySide2, PySide6 or PyQt5')
|
||||
|
||||
import matplotlib
|
||||
from matplotlib.figure import Figure
|
||||
|
||||
if QtGui.__package__ in ['PySide2', 'PyQt5', 'PySide6']:
|
||||
@ -35,7 +34,7 @@ from obspy import UTCDateTime
|
||||
|
||||
from survBot import SurveillanceBot
|
||||
from write_utils import *
|
||||
from utils import get_bg_color
|
||||
from utils import get_bg_color, modify_stream_for_plot
|
||||
|
||||
try:
|
||||
from rest_api.utils import get_station_iccid
|
||||
@ -315,6 +314,7 @@ class MainWindow(QtWidgets.QMainWindow):
|
||||
if st:
|
||||
self.plot_widget = PlotWidget(self)
|
||||
self.plot_widget.setWindowTitle(nwst_id)
|
||||
st = modify_stream_for_plot(st, parameters=self.parameters)
|
||||
st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig)
|
||||
self.plot_widget.show()
|
||||
|
||||
|
||||
52
utils.py
52
utils.py
@ -3,6 +3,7 @@
|
||||
|
||||
import matplotlib
|
||||
|
||||
|
||||
def get_bg_color(check_key, status, dt_thresh=None, hex=False):
|
||||
message = status.message
|
||||
if check_key == 'last active':
|
||||
@ -23,6 +24,7 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False):
|
||||
bg_color = '#{:02x}{:02x}{:02x}'.format(*bg_color[:3])
|
||||
return bg_color
|
||||
|
||||
|
||||
def get_color(key):
|
||||
# some GUI default colors
|
||||
colors_dict = {'FAIL': (255, 50, 0, 255),
|
||||
@ -33,6 +35,7 @@ def get_color(key):
|
||||
'undefined': (230, 230, 230, 255)}
|
||||
return colors_dict.get(key)
|
||||
|
||||
|
||||
def get_time_delay_color(dt, dt_thresh):
|
||||
""" Set color of time delay after thresholds specified in self.dt_thresh """
|
||||
if dt < dt_thresh[0]:
|
||||
@ -41,6 +44,7 @@ def get_time_delay_color(dt, dt_thresh):
|
||||
return get_color('WARN')
|
||||
return get_color('FAIL')
|
||||
|
||||
|
||||
def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
|
||||
""" Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """
|
||||
if type(temp) in [str]:
|
||||
@ -50,3 +54,51 @@ def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
|
||||
rgba = [int(255 * c) for c in cmap(val)]
|
||||
return rgba
|
||||
|
||||
|
||||
def modify_stream_for_plot(st, parameters):
|
||||
""" copy (if necessary) and modify stream for plotting """
|
||||
ch_units = parameters.get('CHANNEL_UNITS')
|
||||
ch_transf = parameters.get('CHANNEL_TRANSFORM')
|
||||
|
||||
# if either of both are defined make copy
|
||||
if ch_units or ch_transf:
|
||||
st = st.copy()
|
||||
|
||||
# modify trace for plotting by multiplying unit factor (e.g. 1e-3 mV to V)
|
||||
if ch_units:
|
||||
for tr in st:
|
||||
channel = tr.stats.channel
|
||||
unit_factor = ch_units.get(channel)
|
||||
if unit_factor:
|
||||
tr.data = tr.data * float(unit_factor)
|
||||
# modify trace for plotting by other arithmetic expressions
|
||||
if ch_transf:
|
||||
for tr in st:
|
||||
channel = tr.stats.channel
|
||||
transf = ch_transf.get(channel)
|
||||
if transf:
|
||||
tr.data = transform_trace(tr.data, transf)
|
||||
|
||||
return st
|
||||
|
||||
|
||||
def transform_trace(data, transf):
|
||||
"""
|
||||
Transform trace with arithmetic operations in order, specified in transf
|
||||
@param data: numpy array
|
||||
@param transf: list of lists with arithmetic operations (e.g. [['*', '20'], ] -> multiply data by 20
|
||||
"""
|
||||
# This looks a little bit hardcoded, however it is safer than using e.g. "eval"
|
||||
for operator_str, val in transf:
|
||||
if operator_str == '+':
|
||||
data = data + val
|
||||
elif operator_str == '-':
|
||||
data = data - val
|
||||
elif operator_str == '*':
|
||||
data = data * val
|
||||
elif operator_str == '/':
|
||||
data = data / val
|
||||
else:
|
||||
raise IOError(f'Unknown arithmethic operator string: {operator_str}')
|
||||
|
||||
return data
|
||||
|
||||
@ -53,6 +53,6 @@ def write_html_row(fobj, items, html_key='td'):
|
||||
|
||||
def get_print_title_str(parameters):
|
||||
timespan = parameters.get('timespan') * 24 * 3600
|
||||
tdelta_str = str(timedelta(seconds=int(timespan)))
|
||||
tdelta_str = str(timedelta(seconds=int(timespan))).replace(', 0:00:00', '')
|
||||
return f'Analysis table of router quality within the last {tdelta_str}'
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user