[update] re-worked channel definition in parameters.yaml, each channel now has its own dictionary with optional plotting flags
[WIP] clock quality work in progress, currently disabled
This commit is contained in:
parent
b17ee1288c
commit
7d5f9cf516
104
parameters.yaml
104
parameters.yaml
@ -3,8 +3,6 @@ datapath: "/data/SDS/" # SC3 Datapath
|
||||
networks: ["1Y", "HA"] # select networks, list or str
|
||||
stations: "*" # select stations, list or str
|
||||
locations: "*" # select locations, list or str
|
||||
channels: ["EX1", "EX2", "EX3", "VEI",
|
||||
"VM1", "VM2", "VM3", "LCQ"] # Specify SOH channels, currently supported EX[1-3], VEI and VM[1-3]
|
||||
stations_blacklist: ["TEST", "EREA"] # exclude these stations
|
||||
networks_blacklist: [] # exclude these networks
|
||||
interval: 60 # Perform checks every x seconds
|
||||
@ -44,10 +42,73 @@ THRESHOLDS:
|
||||
low_volt: 12 # min voltage for low voltage warning
|
||||
high_volt: 14.8 # max voltage for over voltage warning
|
||||
unclassified: 5 # min voltage samples not classified for warning
|
||||
max_vm: [1.5, 2.5] # thresholds for mass offset (warn, fail)
|
||||
max_vm_warn: 1.5 # threshold for mass offset (warn), fail)
|
||||
max_vm_fail: 2.5 # threshold for mass offset (warn), fail)
|
||||
clockquality_warn: 90 # clock quality ranges from 0 % to 100 % with 100 % being the best level
|
||||
clockquality_fail: 70
|
||||
|
||||
|
||||
# ---------------------------------- Specification of input channels ---------------------------------------------------
|
||||
# Currently supported: EX[1-3], VEI, VM[1-3], LCQ
|
||||
#
|
||||
# For each channel a factor 'unit' for unit conversion (e.g. to SI) can be provided, as well as a 'name'
|
||||
# and 'ticks' [ymin, ymax, ystep] for plotting.
|
||||
# 'warn' and 'fail' plot horizontal lines in corresponding colors (can be str in TRESHOLDS, int/float or iterable)
|
||||
#
|
||||
# 'transform' can be provided for plotting to perform arithmetic operations in given order, e.g.:
|
||||
# transform: - ["*", 20]
|
||||
# - ["-", 20]
|
||||
# --> PBox EX1 V to deg C: 20 * x -20
|
||||
CHANNELS:
|
||||
EX1:
|
||||
unit: 1e-6
|
||||
name: "Temperature (°C)"
|
||||
ticks: [-10, 50, 10]
|
||||
transform:
|
||||
- ["*", 20]
|
||||
- ["-", 20]
|
||||
warn: "max_temp"
|
||||
EX2:
|
||||
unit: 1e-6
|
||||
name: "230V/12V (V)"
|
||||
ticks: [1, 5, 1]
|
||||
warn: [2, 3, 4, 4.5, 5]
|
||||
EX3:
|
||||
unit: 1e-6
|
||||
name: "Rout/Charge (V)"
|
||||
ticks: [1, 5, 1]
|
||||
warn: [2, 2.5, 3, 4, 5]
|
||||
VEI:
|
||||
unit: 1e-3
|
||||
name: "Logger (V)"
|
||||
ticks: [9, 15, 1]
|
||||
warn: ["low_volt", "high_volt"]
|
||||
fail: 10.5
|
||||
VM1:
|
||||
unit: 1e-6
|
||||
name: "Mass 1 (V)"
|
||||
ticks: [-2.5, 2.5, 1]
|
||||
warn: [-1.5, 1.5]
|
||||
fail: [-2.5, 2.5]
|
||||
VM2:
|
||||
unit: 1e-6
|
||||
name: "Mass 2 (V)"
|
||||
ticks: [-2.5, 2.5, 1]
|
||||
warn: [-1.5, 1.5]
|
||||
fail: [-2.5, 2.5]
|
||||
VM3:
|
||||
unit: 1e-6
|
||||
name: "Mass 3 (V)"
|
||||
ticks: [-2.5, 2.5, 1]
|
||||
warn: [-1.5, 1.5]
|
||||
fail: [-2.5, 2.5]
|
||||
LCQ:
|
||||
name: "Clock Q (%)"
|
||||
ticks: [0, 100, 20]
|
||||
warn: "clockquality_warn"
|
||||
fail: "clockquality_fail"
|
||||
|
||||
|
||||
# ---------------------------------------- OPTIONAL PARAMETERS ---------------------------------------------------------
|
||||
|
||||
# add links to html table with specified key as column and value as relative link, interpretable string parameters:
|
||||
@ -64,40 +125,3 @@ EMAIL:
|
||||
sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender
|
||||
stations_blacklist: ['GR33'] # do not send emails for specific stations
|
||||
networks_blacklist: [] # do not send emails for specific network
|
||||
|
||||
# names for plotting of the above defined parameter "channels" in the same order
|
||||
channel_names: ["Temperature (°C)",
|
||||
"230V/12V (V)",
|
||||
"Rout/Charge (V)",
|
||||
"Logger (V)",
|
||||
"Mass 1 (V)",
|
||||
"Mass 2 (V)",
|
||||
"Mass 3 (V)",
|
||||
"Clock Q (%)"]
|
||||
|
||||
# specify y-ticks (and ylims) giving, (ymin, ymax, step) for each of the above channels (0: default)
|
||||
CHANNEL_TICKS:
|
||||
- [-10, 50, 10]
|
||||
- [1, 5, 1]
|
||||
- [1, 5, 1]
|
||||
- [9, 15, 1]
|
||||
- [-2, 2, 1]
|
||||
- [-2, 2, 1]
|
||||
- [-2, 2, 1]
|
||||
- [0, 100, 20]
|
||||
|
||||
# Factor for channel to SI-units (for plotting)
|
||||
CHANNEL_UNITS:
|
||||
EX1: 1e-6
|
||||
EX2: 1e-6
|
||||
EX3: 1e-6
|
||||
VEI: 1e-3
|
||||
VM1: 1e-6
|
||||
VM2: 1e-6
|
||||
VM3: 1e-6
|
||||
|
||||
# Transform channel for plotting, perform arithmetic operations in given order, e.g.: PBox EX1 V to deg C: 20 * x -20
|
||||
CHANNEL_TRANSFORM:
|
||||
EX1:
|
||||
- ["*", 20]
|
||||
- ["-", 20]
|
||||
26
survBot.py
26
survBot.py
@ -19,7 +19,7 @@ from obspy.clients.filesystem.sds import Client
|
||||
|
||||
from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str, \
|
||||
init_html_table, finish_html_table
|
||||
from utils import get_bg_color, modify_stream_for_plot, trace_ylabels, trace_yticks
|
||||
from utils import get_bg_color, modify_stream_for_plot, trace_yticks, trace_thresholds
|
||||
|
||||
try:
|
||||
import smtplib
|
||||
@ -68,7 +68,7 @@ def fancy_timestr(dt, thresh=600, modif='+'):
|
||||
|
||||
class SurveillanceBot(object):
|
||||
def __init__(self, parameter_path, outpath_html=None):
|
||||
self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'mass', 'clock', 'temp', 'other']
|
||||
self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'mass', 'temp', 'other']
|
||||
self.parameter_path = parameter_path
|
||||
self.update_parameters()
|
||||
self.starttime = UTCDateTime()
|
||||
@ -92,6 +92,8 @@ class SurveillanceBot(object):
|
||||
|
||||
def update_parameters(self):
|
||||
self.parameters = read_yaml(self.parameter_path)
|
||||
# add channels to list in parameters dicitonary
|
||||
self.parameters['channels'] = list(self.parameters.get('CHANNELS').keys())
|
||||
self.reread_parameters = self.parameters.get('reread_parameters')
|
||||
self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')]
|
||||
self.verbosity = self.parameters.get('verbosity')
|
||||
@ -348,8 +350,9 @@ class SurveillanceBot(object):
|
||||
try:
|
||||
st = modify_stream_for_plot(st, parameters=self.parameters)
|
||||
st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full')
|
||||
trace_ylabels(fig, self.parameters, self.verbosity)
|
||||
# trace_ylabels(fig, self.parameters, self.verbosity)
|
||||
trace_yticks(fig, self.parameters, self.verbosity)
|
||||
trace_thresholds(fig, self.parameters, self.verbosity)
|
||||
except Exception as e:
|
||||
print(f'Could not generate plot for {nwst_id}:')
|
||||
print(traceback.format_exc())
|
||||
@ -441,6 +444,9 @@ class SurveillanceBot(object):
|
||||
print(f'Could not write HTML table to {fnout}:')
|
||||
print(traceback.format_exc())
|
||||
|
||||
if self.verbosity:
|
||||
print(f'Wrote html table to {fnout}')
|
||||
|
||||
def update_status_message(self):
|
||||
timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
|
||||
self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \
|
||||
@ -693,7 +699,7 @@ class StationQC(object):
|
||||
self.pb_power_analysis()
|
||||
self.pb_rout_charge_analysis()
|
||||
self.mass_analysis()
|
||||
self.clock_quality_analysis()
|
||||
#self.clock_quality_analysis()
|
||||
|
||||
def return_print_analysis(self):
|
||||
items = [self.nwst_id]
|
||||
@ -860,17 +866,17 @@ class StationQC(object):
|
||||
common_highest_val = np.nanmax(abs(last_val_mean))
|
||||
common_highest_val = round(common_highest_val, 1)
|
||||
|
||||
# get thresholds for WARN (max_vm1) and FAIL (max_vm2)
|
||||
# get thresholds for WARN (max_vm_warn) and FAIL (max_vm_fail)
|
||||
thresholds = self.parameters.get('THRESHOLDS')
|
||||
max_vm = thresholds.get('max_vm')
|
||||
if not max_vm:
|
||||
max_vm_warn = thresholds.get('max_vm_warn')
|
||||
max_vm_fail = thresholds.get('max_vm_fail')
|
||||
if not max_vm_warn or not max_vm_fail:
|
||||
return
|
||||
max_vm1, max_vm2 = max_vm
|
||||
|
||||
# change status depending on common_highest_val
|
||||
if common_highest_val < max_vm1:
|
||||
if common_highest_val < max_vm_warn:
|
||||
self.status_ok(key, detailed_message=f'{common_highest_val}V')
|
||||
elif max_vm1 <= common_highest_val < max_vm2:
|
||||
elif max_vm_warn <= common_highest_val < max_vm_fail:
|
||||
self.warn(key=key,
|
||||
detailed_message=f'Warning raised for mass centering. Highest val {common_highest_val}V', )
|
||||
else:
|
||||
|
||||
@ -34,7 +34,7 @@ from obspy import UTCDateTime
|
||||
|
||||
from survBot import SurveillanceBot
|
||||
from write_utils import *
|
||||
from utils import get_bg_color, modify_stream_for_plot, trace_ylabels, trace_yticks
|
||||
from utils import get_bg_color, modify_stream_for_plot, trace_yticks, trace_thresholds
|
||||
|
||||
try:
|
||||
from rest_api.utils import get_station_iccid
|
||||
@ -316,8 +316,9 @@ class MainWindow(QtWidgets.QMainWindow):
|
||||
self.plot_widget.setWindowTitle(nwst_id)
|
||||
st = modify_stream_for_plot(st, parameters=self.parameters)
|
||||
st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig)
|
||||
trace_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
|
||||
# trace_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
|
||||
trace_yticks(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
|
||||
trace_thresholds(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
|
||||
self.plot_widget.show()
|
||||
|
||||
def notification(self, text):
|
||||
|
||||
102
utils.py
102
utils.py
@ -2,6 +2,9 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import matplotlib
|
||||
import numpy as np
|
||||
|
||||
from obspy import Stream
|
||||
|
||||
|
||||
def get_bg_color(check_key, status, dt_thresh=None, hex=False):
|
||||
@ -37,6 +40,11 @@ def get_color(key):
|
||||
return colors_dict.get(key)
|
||||
|
||||
|
||||
def get_color_mpl(key):
|
||||
color_tup = get_color(key)
|
||||
return np.array([color/255. for color in color_tup])
|
||||
|
||||
|
||||
def get_time_delay_color(dt, dt_thresh):
|
||||
""" Set color of time delay after thresholds specified in self.dt_thresh """
|
||||
if dt < dt_thresh[0]:
|
||||
@ -68,33 +76,43 @@ def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
|
||||
return rgba
|
||||
|
||||
|
||||
def modify_stream_for_plot(st, parameters):
|
||||
def modify_stream_for_plot(input_stream, parameters):
|
||||
""" copy (if necessary) and modify stream for plotting """
|
||||
ch_units = parameters.get('CHANNEL_UNITS')
|
||||
ch_transf = parameters.get('CHANNEL_TRANSFORM')
|
||||
|
||||
# make a copy
|
||||
st = st.copy()
|
||||
st = Stream()
|
||||
|
||||
# modify trace for plotting by multiplying unit factor (e.g. 1e-3 mV to V)
|
||||
if ch_units:
|
||||
for tr in st:
|
||||
channel = tr.stats.channel
|
||||
unit_factor = ch_units.get(channel)
|
||||
if unit_factor:
|
||||
tr.data = tr.data * float(unit_factor)
|
||||
channels_dict = parameters.get('CHANNELS')
|
||||
|
||||
# modify trace for plotting by other arithmetic expressions
|
||||
if ch_transf:
|
||||
for tr in st:
|
||||
channel = tr.stats.channel
|
||||
transf = ch_transf.get(channel)
|
||||
if transf:
|
||||
tr.data = transform_trace(tr.data, transf)
|
||||
# iterate over all channels and put them to new stream in order
|
||||
for index, ch_tup in enumerate(channels_dict.items()):
|
||||
# unpack tuple from items
|
||||
channel, channel_dict = ch_tup
|
||||
|
||||
# change channel IDs to prevent re-sorting in obspy routine
|
||||
for index, trace in enumerate(st):
|
||||
trace.id = f'trace {index + 1}: {trace.id}'
|
||||
# get correct channel from stream
|
||||
st_sel = input_stream.select(channel=channel)
|
||||
# in case there are != 1 there is ambiguity
|
||||
if not len(st_sel) == 1:
|
||||
continue
|
||||
|
||||
# make a copy to not modify original stream!
|
||||
tr = st_sel[0].copy()
|
||||
|
||||
# multiply with conversion factor for unit
|
||||
unit_factor = channel_dict.get('unit')
|
||||
if unit_factor:
|
||||
tr.data = tr.data * float(unit_factor)
|
||||
|
||||
# apply transformations if provided
|
||||
transform = channel_dict.get('transform')
|
||||
if transform:
|
||||
tr.data = transform_trace(tr.data, transform)
|
||||
|
||||
# modify trace id to maintain plotting order
|
||||
name = channel_dict.get('name')
|
||||
tr.id = f'trace {index + 1}: {name} - {tr.id}'
|
||||
|
||||
st.append(tr)
|
||||
|
||||
return st
|
||||
|
||||
@ -124,10 +142,8 @@ def transform_trace(data, transf):
|
||||
def trace_ylabels(fig, parameters, verbosity=0):
|
||||
"""
|
||||
Adds channel names to y-axis if defined in parameters.
|
||||
Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is
|
||||
difficult to assess the correct order from Obspy plotting routing.
|
||||
"""
|
||||
names = parameters.get('channel_names')
|
||||
names = [channel.get('name') for channel in parameters.get('CHANNELS').values()]
|
||||
if not names: # or not len(st.traces):
|
||||
return
|
||||
if not len(names) == len(fig.axes):
|
||||
@ -142,10 +158,8 @@ def trace_ylabels(fig, parameters, verbosity=0):
|
||||
def trace_yticks(fig, parameters, verbosity=0):
|
||||
"""
|
||||
Adds channel names to y-axis if defined in parameters.
|
||||
Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is
|
||||
difficult to assess the correct order from Obspy plotting routing.
|
||||
"""
|
||||
ticks = parameters.get('CHANNEL_TICKS')
|
||||
ticks = [channel.get('ticks') for channel in parameters.get('CHANNELS').values()]
|
||||
if not ticks:
|
||||
return
|
||||
if not len(ticks) == len(fig.axes):
|
||||
@ -157,6 +171,38 @@ def trace_yticks(fig, parameters, verbosity=0):
|
||||
continue
|
||||
ymin, ymax, step = ytick_tripple
|
||||
|
||||
yticks = list(range(ymin, ymax + step, step))
|
||||
yticks = list(np.arange(ymin, ymax + step, step))
|
||||
ax.set_yticks(yticks)
|
||||
ax.set_ylim(ymin - 0.33 * step, ymax + 0.33 * step)
|
||||
|
||||
|
||||
def trace_thresholds(fig, parameters, verbosity=0):
|
||||
"""
|
||||
Adds channel thresholds (warn, fail) to y-axis if defined in parameters.
|
||||
"""
|
||||
if verbosity > 0:
|
||||
print('Plotting trace thresholds')
|
||||
|
||||
keys_colors = {'warn': dict(color=0.8 * get_color_mpl('WARN'), linestyle=(0, (5, 10)), alpha=0.5, linewidth=0.7),
|
||||
'fail': dict(color=0.8 * get_color_mpl('FAIL'), linestyle='solid', alpha=0.5, linewidth=0.7)}
|
||||
|
||||
for key, kwargs in keys_colors.items():
|
||||
channel_threshold_list = [channel.get(key) for channel in parameters.get('CHANNELS').values()]
|
||||
if not channel_threshold_list:
|
||||
continue
|
||||
plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs)
|
||||
|
||||
|
||||
def plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs):
|
||||
for channel_thresholds, ax in zip(channel_threshold_list, fig.axes):
|
||||
if not channel_thresholds:
|
||||
continue
|
||||
|
||||
if not isinstance(channel_thresholds, (list, tuple)):
|
||||
channel_thresholds = [channel_thresholds]
|
||||
|
||||
for warn_thresh in channel_thresholds:
|
||||
if isinstance(warn_thresh, str):
|
||||
warn_thresh = parameters.get('THRESHOLDS').get(warn_thresh)
|
||||
if type(warn_thresh in (float, int)):
|
||||
ax.axhline(warn_thresh, **kwargs)
|
||||
Loading…
Reference in New Issue
Block a user