Compare commits
4 Commits
00d2d0119c
...
f41f24f626
Author | SHA1 | Date | |
---|---|---|---|
f41f24f626 | |||
c621b31f6e | |||
20b586f96b | |||
08b12aeb9d |
@ -8,7 +8,8 @@ networks_blacklist: [] # exclude these networks
|
|||||||
interval: 60 # Perform checks every x seconds
|
interval: 60 # Perform checks every x seconds
|
||||||
n_track: 360 # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
|
n_track: 360 # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
|
||||||
timespan: 3 # Check data of the recent x days
|
timespan: 3 # Check data of the recent x days
|
||||||
verbosity: 0 # verbosity flag
|
verbosity: 0 # verbosity flag for program console output (not logging)
|
||||||
|
logging_level: WARN # set logging level (info, warning, debug)
|
||||||
track_changes: True # tracks all changes since GUI startup by text highlighting (GUI only)
|
track_changes: True # tracks all changes since GUI startup by text highlighting (GUI only)
|
||||||
warn_count: False # show number of warnings and errors in table
|
warn_count: False # show number of warnings and errors in table
|
||||||
min_sample: 5 # minimum samples for raising Warn/FAIL
|
min_sample: 5 # minimum samples for raising Warn/FAIL
|
||||||
@ -55,6 +56,7 @@ THRESHOLDS:
|
|||||||
# For each channel a factor 'unit' for unit conversion (e.g. to SI) can be provided, as well as a 'name'
|
# For each channel a factor 'unit' for unit conversion (e.g. to SI) can be provided, as well as a 'name'
|
||||||
# and 'ticks' [ymin, ymax, ystep] for plotting.
|
# and 'ticks' [ymin, ymax, ystep] for plotting.
|
||||||
# 'warn' and 'fail' plot horizontal lines in corresponding colors (can be str in TRESHOLDS, int/float or iterable)
|
# 'warn' and 'fail' plot horizontal lines in corresponding colors (can be str in TRESHOLDS, int/float or iterable)
|
||||||
|
# keyword "pb_SOH2" or "pb_SOH3" can be used to extract warning values from above POWBOX parameter definition
|
||||||
#
|
#
|
||||||
# 'transform' can be provided for plotting to perform arithmetic operations in given order, e.g.:
|
# 'transform' can be provided for plotting to perform arithmetic operations in given order, e.g.:
|
||||||
# transform: - ["*", 20]
|
# transform: - ["*", 20]
|
||||||
@ -73,12 +75,12 @@ CHANNELS:
|
|||||||
unit: 1e-6
|
unit: 1e-6
|
||||||
name: "PowBox 230V/12V (V)"
|
name: "PowBox 230V/12V (V)"
|
||||||
ticks: [0, 5, 1]
|
ticks: [0, 5, 1]
|
||||||
warn: [2, 3, 4, 4.5, 5]
|
warn: "pb_SOH2"
|
||||||
EX3:
|
EX3:
|
||||||
unit: 1e-6
|
unit: 1e-6
|
||||||
name: "PowBox Router/Charger (V)"
|
name: "PowBox Router/Charger (V)"
|
||||||
ticks: [0, 5, 1]
|
ticks: [0, 5, 1]
|
||||||
warn: [2, 2.5, 3, 4, 5]
|
warn: "pb_SOH3"
|
||||||
VEI:
|
VEI:
|
||||||
unit: 1e-3
|
unit: 1e-3
|
||||||
name: "Datalogger (V)"
|
name: "Datalogger (V)"
|
||||||
@ -121,6 +123,7 @@ add_links:
|
|||||||
# for example: slmon: {"URL": "path/{nw}_{st}.html", "text": "link"}
|
# for example: slmon: {"URL": "path/{nw}_{st}.html", "text": "link"}
|
||||||
slmon: {"URL": "../slmon/{nw}_{st}.html", "text": "show"}
|
slmon: {"URL": "../slmon/{nw}_{st}.html", "text": "show"}
|
||||||
24h-plot: {"URL": "../scheli/{nw}/{st}.png", "text": "plot"}
|
24h-plot: {"URL": "../scheli/{nw}/{st}.png", "text": "plot"}
|
||||||
|
ppsd: {"URL": "../ppsd/{nw}.{st}.html", "text": "show"}
|
||||||
|
|
||||||
# add station-independent links below html table (list items separated with -)
|
# add station-independent links below html table (list items separated with -)
|
||||||
add_global_links:
|
add_global_links:
|
||||||
|
166
survBot.py
166
survBot.py
@ -7,6 +7,7 @@ __author__ = 'Marcel Paffrath'
|
|||||||
import os
|
import os
|
||||||
import io
|
import io
|
||||||
import copy
|
import copy
|
||||||
|
import logging
|
||||||
import traceback
|
import traceback
|
||||||
import yaml
|
import yaml
|
||||||
import argparse
|
import argparse
|
||||||
@ -31,27 +32,47 @@ try:
|
|||||||
|
|
||||||
mail_functionality = True
|
mail_functionality = True
|
||||||
except ImportError:
|
except ImportError:
|
||||||
print('Could not import smtplib or mail. Disabled sending mails.')
|
logging.warning('Could not import smtplib or mail. Disabled sending mails.')
|
||||||
mail_functionality = False
|
mail_functionality = False
|
||||||
|
|
||||||
pjoin = os.path.join
|
pjoin = os.path.join
|
||||||
UP = "\x1B[{length}A"
|
UP = "\x1B[{length}A"
|
||||||
CLR = "\x1B[0K"
|
CLR = "\x1B[0K"
|
||||||
deg_str = '\N{DEGREE SIGN}C'
|
DEG_STR = '\N{DEGREE SIGN}C'
|
||||||
|
|
||||||
|
|
||||||
def read_yaml(file_path, n_read=3):
|
def read_yaml(file_path: str, n_read: int = 3) -> dict:
|
||||||
for index in range(n_read):
|
for index in range(n_read):
|
||||||
try:
|
try:
|
||||||
with open(file_path, "r") as f:
|
with open(file_path, "r") as f:
|
||||||
params = yaml.safe_load(f)
|
params = yaml.safe_load(f)
|
||||||
|
set_logging_level(params)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f'Could not read parameters file: {e}.\nWill try again {n_read - index - 1} time(s).')
|
logging.warning(f'Could not read parameters file: {e}.\nWill try again {n_read - index - 1} time(s).')
|
||||||
time.sleep(10)
|
time.sleep(10)
|
||||||
continue
|
continue
|
||||||
return params
|
return params
|
||||||
|
|
||||||
|
|
||||||
|
def set_logging_level(params: dict) -> None:
|
||||||
|
logging_levels = {'info': logging.INFO,
|
||||||
|
'warning': logging.WARNING,
|
||||||
|
'warn': logging.WARNING,
|
||||||
|
'debug': logging.DEBUG,
|
||||||
|
'error': logging.ERROR,
|
||||||
|
'critical': logging.CRITICAL}
|
||||||
|
logging_level_str = params.get('logging_level')
|
||||||
|
if not logging_level_str:
|
||||||
|
logging.warning('Could not set logging level. Parameter not set')
|
||||||
|
return
|
||||||
|
if not isinstance(logging_level_str, str):
|
||||||
|
logging.warning(
|
||||||
|
f'Could not set logging level. Parameter logging_level = {logging_level_str} could not be interpreted.')
|
||||||
|
return
|
||||||
|
logging_level = logging_levels.get(logging_level_str.lower())
|
||||||
|
logging.basicConfig(level=logging_level)
|
||||||
|
|
||||||
|
|
||||||
def nsl_from_id(nwst_id):
|
def nsl_from_id(nwst_id):
|
||||||
nwst_id = get_full_seed_id(nwst_id)
|
nwst_id = get_full_seed_id(nwst_id)
|
||||||
network, station, location = nwst_id.split('.')
|
network, station, location = nwst_id.split('.')
|
||||||
@ -115,7 +136,6 @@ class SurveillanceBot(object):
|
|||||||
self.parameters['channels'] = channels
|
self.parameters['channels'] = channels
|
||||||
self.reread_parameters = self.parameters.get('reread_parameters')
|
self.reread_parameters = self.parameters.get('reread_parameters')
|
||||||
self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')]
|
self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')]
|
||||||
self.verbosity = self.parameters.get('verbosity')
|
|
||||||
self.stations_blacklist = self.parameters.get('stations_blacklist')
|
self.stations_blacklist = self.parameters.get('stations_blacklist')
|
||||||
self.networks_blacklist = self.parameters.get('networks_blacklist')
|
self.networks_blacklist = self.parameters.get('networks_blacklist')
|
||||||
self.refresh_period = self.parameters.get('interval')
|
self.refresh_period = self.parameters.get('interval')
|
||||||
@ -196,8 +216,7 @@ class SurveillanceBot(object):
|
|||||||
for filename in self.filenames:
|
for filename in self.filenames:
|
||||||
# if file already read and last modification time is the same as of last read operation: continue
|
# if file already read and last modification time is the same as of last read operation: continue
|
||||||
if self.filenames_read_last_modif.get(filename) == os.path.getmtime(filename):
|
if self.filenames_read_last_modif.get(filename) == os.path.getmtime(filename):
|
||||||
if self.verbosity > 0:
|
logging.info(f'Continue on file {filename}')
|
||||||
print('Continue on file', filename)
|
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
# read only header of wf_data
|
# read only header of wf_data
|
||||||
@ -207,7 +226,7 @@ class SurveillanceBot(object):
|
|||||||
st_new = read(filename, dtype=float)
|
st_new = read(filename, dtype=float)
|
||||||
self.filenames_read_last_modif[filename] = os.path.getmtime(filename)
|
self.filenames_read_last_modif[filename] = os.path.getmtime(filename)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f'Could not read file {filename}:', e)
|
logging.warning(f'Could not read file {filename}: {e}')
|
||||||
continue
|
continue
|
||||||
self.dataStream += st_new
|
self.dataStream += st_new
|
||||||
self.gaps = self.dataStream.get_gaps(min_gap=self.parameters['THRESHOLDS'].get('min_gap'))
|
self.gaps = self.dataStream.get_gaps(min_gap=self.parameters['THRESHOLDS'].get('min_gap'))
|
||||||
@ -234,8 +253,7 @@ class SurveillanceBot(object):
|
|||||||
if stream:
|
if stream:
|
||||||
nsl = nsl_from_id(nwst_id)
|
nsl = nsl_from_id(nwst_id)
|
||||||
station_qc = StationQC(self, stream, nsl, self.parameters, self.keys, qc_starttime,
|
station_qc = StationQC(self, stream, nsl, self.parameters, self.keys, qc_starttime,
|
||||||
self.verbosity, print_func=self.print,
|
print_func=self.print, status_track=self.status_track.get(nwst_id))
|
||||||
status_track=self.status_track.get(nwst_id))
|
|
||||||
analysis_print_result = station_qc.return_print_analysis()
|
analysis_print_result = station_qc.return_print_analysis()
|
||||||
station_dict = station_qc.return_analysis()
|
station_dict = station_qc.return_analysis()
|
||||||
else:
|
else:
|
||||||
@ -382,19 +400,19 @@ class SurveillanceBot(object):
|
|||||||
st = self.data.get(get_full_seed_id(nwst_id))
|
st = self.data.get(get_full_seed_id(nwst_id))
|
||||||
if st:
|
if st:
|
||||||
# TODO: this section failed once, adding try-except block for analysis and to prevent program from crashing
|
# TODO: this section failed once, adding try-except block for analysis and to prevent program from crashing
|
||||||
try:
|
#try:
|
||||||
endtime = UTCDateTime()
|
endtime = UTCDateTime()
|
||||||
starttime = endtime - self.parameters.get('timespan') * 24 * 3600
|
starttime = endtime - self.parameters.get('timespan') * 24 * 3600
|
||||||
st = modify_stream_for_plot(st, parameters=self.parameters)
|
st = modify_stream_for_plot(st, parameters=self.parameters)
|
||||||
st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full',
|
st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full',
|
||||||
starttime=starttime, endtime=endtime)
|
starttime=starttime, endtime=endtime)
|
||||||
# set_axis_ylabels(fig, self.parameters, self.verbosity)
|
# set_axis_ylabels(fig, self.parameters)
|
||||||
set_axis_yticks(fig, self.parameters, self.verbosity)
|
set_axis_yticks(fig, self.parameters)
|
||||||
set_axis_color(fig)
|
set_axis_color(fig)
|
||||||
plot_axis_thresholds(fig, self.parameters, self.verbosity)
|
plot_axis_thresholds(fig, self.parameters)
|
||||||
except Exception as e:
|
# except Exception as e:
|
||||||
print(f'Could not generate plot for {nwst_id}:')
|
# logging.error(f'Could not generate plot for {nwst_id}: {e}')
|
||||||
print(traceback.format_exc())
|
# logging.debug(traceback.format_exc())
|
||||||
if len(fig.axes) > 0:
|
if len(fig.axes) > 0:
|
||||||
ax = fig.axes[0]
|
ax = fig.axes[0]
|
||||||
ax.set_title(f'Plot refreshed at (UTC) {UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S")}. '
|
ax.set_title(f'Plot refreshed at (UTC) {UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S")}. '
|
||||||
@ -402,7 +420,10 @@ class SurveillanceBot(object):
|
|||||||
for ax in fig.axes:
|
for ax in fig.axes:
|
||||||
ax.grid(True, alpha=0.1)
|
ax.grid(True, alpha=0.1)
|
||||||
for fnout in fnames_out:
|
for fnout in fnames_out:
|
||||||
fig.savefig(fnout, dpi=150., bbox_inches='tight')
|
try:
|
||||||
|
fig.savefig(fnout, dpi=150., bbox_inches='tight')
|
||||||
|
except IOError as e:
|
||||||
|
logging.warning('Could not save figure with IO error. Disk quota exceeded?\nError message: {e}')
|
||||||
# if needed save figure as virtual object (e.g. for mailing)
|
# if needed save figure as virtual object (e.g. for mailing)
|
||||||
if save_bytes:
|
if save_bytes:
|
||||||
fnames_out[-1].seek(0)
|
fnames_out[-1].seek(0)
|
||||||
@ -458,7 +479,7 @@ class SurveillanceBot(object):
|
|||||||
# add degree sign for temp
|
# add degree sign for temp
|
||||||
if check_key == 'temp':
|
if check_key == 'temp':
|
||||||
if not type(message) in [str]:
|
if not type(message) in [str]:
|
||||||
message = str(message) + deg_str
|
message = str(message) + DEG_STR
|
||||||
|
|
||||||
html_class = self.get_html_class(hide_keys_mobile, status=status, check_key=check_key)
|
html_class = self.get_html_class(hide_keys_mobile, status=status, check_key=check_key)
|
||||||
item = dict(text=str(message), tooltip=str(detailed_message), color=bg_color,
|
item = dict(text=str(message), tooltip=str(detailed_message), color=bg_color,
|
||||||
@ -515,17 +536,16 @@ class SurveillanceBot(object):
|
|||||||
# write footer with optional logo
|
# write footer with optional logo
|
||||||
logo_file = self.parameters.get('html_logo')
|
logo_file = self.parameters.get('html_logo')
|
||||||
if not os.path.isfile(pjoin(self.outpath_html, logo_file)):
|
if not os.path.isfile(pjoin(self.outpath_html, logo_file)):
|
||||||
print(f'Specified file {logo_file} not found.')
|
logging.info(f'Specified file {logo_file} not found.')
|
||||||
logo_file = None
|
logo_file = None
|
||||||
|
|
||||||
outfile.write(html_footer(footer_logo=logo_file))
|
outfile.write(html_footer(footer_logo=logo_file))
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f'Could not write HTML table to {fnout}:')
|
logging.info(f'Could not write HTML table to {fnout}:')
|
||||||
print(traceback.format_exc())
|
logging.debug(traceback.format_exc())
|
||||||
|
|
||||||
if self.verbosity:
|
logging.info(f'Wrote html table to {fnout}')
|
||||||
print(f'Wrote html table to {fnout}')
|
|
||||||
|
|
||||||
def update_status_message(self):
|
def update_status_message(self):
|
||||||
timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
|
timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
|
||||||
@ -540,7 +560,6 @@ class SurveillanceBot(object):
|
|||||||
string.replace('\n', clear_end)
|
string.replace('\n', clear_end)
|
||||||
print(string, end=clear_end, **kwargs)
|
print(string, end=clear_end, **kwargs)
|
||||||
self.print_count += n_nl + 1 # number of newlines + actual print with end='\n' (no check for kwargs end!)
|
self.print_count += n_nl + 1 # number of newlines + actual print with end='\n' (no check for kwargs end!)
|
||||||
# print('pc:', self.print_count)
|
|
||||||
|
|
||||||
def clear_prints(self):
|
def clear_prints(self):
|
||||||
print(UP.format(length=self.print_count), end='')
|
print(UP.format(length=self.print_count), end='')
|
||||||
@ -548,7 +567,7 @@ class SurveillanceBot(object):
|
|||||||
|
|
||||||
|
|
||||||
class StationQC(object):
|
class StationQC(object):
|
||||||
def __init__(self, parent, stream, nsl, parameters, keys, starttime, verbosity, print_func, status_track=None):
|
def __init__(self, parent, stream, nsl, parameters, keys, starttime, print_func, status_track=None):
|
||||||
"""
|
"""
|
||||||
Station Quality Check class.
|
Station Quality Check class.
|
||||||
:param nsl: dictionary containing network, station and location (key: str)
|
:param nsl: dictionary containing network, station and location (key: str)
|
||||||
@ -565,7 +584,6 @@ class StationQC(object):
|
|||||||
# make a copy of parameters object to prevent accidental changes
|
# make a copy of parameters object to prevent accidental changes
|
||||||
self.parameters = copy.deepcopy(parameters)
|
self.parameters = copy.deepcopy(parameters)
|
||||||
self.program_starttime = starttime
|
self.program_starttime = starttime
|
||||||
self.verbosity = verbosity
|
|
||||||
self.last_active = False
|
self.last_active = False
|
||||||
self.print = print_func
|
self.print = print_func
|
||||||
|
|
||||||
@ -598,8 +616,7 @@ class StationQC(object):
|
|||||||
current_status = self.status_dict.get(key)
|
current_status = self.status_dict.get(key)
|
||||||
|
|
||||||
# change this to something more useful, SMS/EMAIL/PUSH
|
# change this to something more useful, SMS/EMAIL/PUSH
|
||||||
if self.verbosity:
|
logging.info(f'{UTCDateTime()}: {detailed_message}')
|
||||||
self.print(f'{UTCDateTime()}: {detailed_message}', flush=False)
|
|
||||||
|
|
||||||
# if error, do not overwrite with warning
|
# if error, do not overwrite with warning
|
||||||
if current_status.is_error:
|
if current_status.is_error:
|
||||||
@ -638,8 +655,7 @@ class StationQC(object):
|
|||||||
if self.status_track.get(key) and not self.status_track.get(key)[-1]:
|
if self.status_track.get(key) and not self.status_track.get(key)[-1]:
|
||||||
self.parent.write_html_figure(self.nwst_id, save_bytes=True)
|
self.parent.write_html_figure(self.nwst_id, save_bytes=True)
|
||||||
|
|
||||||
if self.verbosity:
|
logging.info(f'{UTCDateTime()}: {detailed_message}')
|
||||||
self.print(f'{UTCDateTime()}: {detailed_message}', flush=False)
|
|
||||||
|
|
||||||
# do not send error mail if this is the first run (e.g. program startup) or state was already error (unchanged)
|
# do not send error mail if this is the first run (e.g. program startup) or state was already error (unchanged)
|
||||||
if self.search_previous_errors(key) is True:
|
if self.search_previous_errors(key) is True:
|
||||||
@ -671,7 +687,7 @@ class StationQC(object):
|
|||||||
|
|
||||||
# simulate an error specified in json file (dictionary: {nwst_id: key} )
|
# simulate an error specified in json file (dictionary: {nwst_id: key} )
|
||||||
if self._simulated_error_check(key) is True:
|
if self._simulated_error_check(key) is True:
|
||||||
print(f'Simulating Error on {self.nwst_id}, {key}')
|
logging.info(f'Simulating Error on {self.nwst_id}, {key}')
|
||||||
return True
|
return True
|
||||||
|
|
||||||
previous_errors = self.status_track.get(key)
|
previous_errors = self.status_track.get(key)
|
||||||
@ -696,26 +712,22 @@ class StationQC(object):
|
|||||||
def send_mail(self, key, status_type, additional_message=''):
|
def send_mail(self, key, status_type, additional_message=''):
|
||||||
""" Send info mail using parameters specified in parameters file """
|
""" Send info mail using parameters specified in parameters file """
|
||||||
if not mail_functionality:
|
if not mail_functionality:
|
||||||
if self.verbosity:
|
logging.info('Mail functionality disabled. Return')
|
||||||
print('Mail functionality disabled. Return')
|
|
||||||
return
|
return
|
||||||
|
|
||||||
mail_params = self.parameters.get('EMAIL')
|
mail_params = self.parameters.get('EMAIL')
|
||||||
if not mail_params:
|
if not mail_params:
|
||||||
if self.verbosity:
|
logging.info('parameter "EMAIL" not set in parameter file. Return')
|
||||||
print('parameter "EMAIL" not set in parameter file. Return')
|
|
||||||
return
|
return
|
||||||
|
|
||||||
stations_blacklist = mail_params.get('stations_blacklist')
|
stations_blacklist = mail_params.get('stations_blacklist')
|
||||||
if stations_blacklist and self.station in stations_blacklist:
|
if stations_blacklist and self.station in stations_blacklist:
|
||||||
if self.verbosity:
|
logging.info(f'Station {self.station} listed in blacklist. Return')
|
||||||
print(f'Station {self.station} listed in blacklist. Return')
|
|
||||||
return
|
return
|
||||||
|
|
||||||
networks_blacklist = mail_params.get('networks_blacklist')
|
networks_blacklist = mail_params.get('networks_blacklist')
|
||||||
if networks_blacklist and self.network in networks_blacklist:
|
if networks_blacklist and self.network in networks_blacklist:
|
||||||
if self.verbosity:
|
logging.info(f'Station {self.station} of network {self.network} listed in blacklist. Return')
|
||||||
print(f'Station {self.station} of network {self.network} listed in blacklist. Return')
|
|
||||||
return
|
return
|
||||||
|
|
||||||
sender = mail_params.get('sender')
|
sender = mail_params.get('sender')
|
||||||
@ -726,8 +738,7 @@ class StationQC(object):
|
|||||||
addresses = addresses[:] + list(add_addresses)
|
addresses = addresses[:] + list(add_addresses)
|
||||||
server = mail_params.get('mailserver')
|
server = mail_params.get('mailserver')
|
||||||
if not sender or not addresses:
|
if not sender or not addresses:
|
||||||
if self.verbosity:
|
logging.info('Mail sender or addresses not (correctly) defined. Return')
|
||||||
print('Mail sender or addresses not (correctly) defined. Return')
|
|
||||||
return
|
return
|
||||||
dt = self.get_dt_for_action()
|
dt = self.get_dt_for_action()
|
||||||
text = f'{key}: Status {status_type} longer than {dt}: ' + additional_message
|
text = f'{key}: Status {status_type} longer than {dt}: ' + additional_message
|
||||||
@ -790,20 +801,16 @@ class StationQC(object):
|
|||||||
yield address
|
yield address
|
||||||
# file not existing
|
# file not existing
|
||||||
except FileNotFoundError as e:
|
except FileNotFoundError as e:
|
||||||
if self.verbosity:
|
logging.warning(e)
|
||||||
print(e)
|
|
||||||
# no dictionary
|
# no dictionary
|
||||||
except AttributeError as e:
|
except AttributeError as e:
|
||||||
if self.verbosity:
|
logging.warning(f'Could not read dictionary from file {eml_filename}: {e}')
|
||||||
print(f'Could not read dictionary from file {eml_filename}: {e}')
|
|
||||||
# other exceptions
|
# other exceptions
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
if self.verbosity:
|
logging.warning(f'Could not open file {eml_filename}: {e}')
|
||||||
print(f'Could not open file {eml_filename}: {e}')
|
|
||||||
# no file specified
|
# no file specified
|
||||||
else:
|
else:
|
||||||
if self.verbosity:
|
logging.info('No external mail list set.')
|
||||||
print('No external mail list set.')
|
|
||||||
|
|
||||||
return []
|
return []
|
||||||
|
|
||||||
@ -877,9 +884,8 @@ class StationQC(object):
|
|||||||
timespan = self.parameters.get('timespan') * 24 * 3600
|
timespan = self.parameters.get('timespan') * 24 * 3600
|
||||||
self.analysis_starttime = self.program_starttime - timespan
|
self.analysis_starttime = self.program_starttime - timespan
|
||||||
|
|
||||||
if self.verbosity > 0:
|
logging.info(150 * '#')
|
||||||
self.print(150 * '#')
|
logging.info('This is StationQC. Calculating quality for station'
|
||||||
self.print('This is StationQT. Calculating quality for station'
|
|
||||||
' {network}.{station}.{location}'.format(**self.nsl))
|
' {network}.{station}.{location}'.format(**self.nsl))
|
||||||
self.voltage_analysis()
|
self.voltage_analysis()
|
||||||
self.pb_temp_analysis()
|
self.pb_temp_analysis()
|
||||||
@ -907,7 +913,7 @@ class StationQC(object):
|
|||||||
if key == 'last active':
|
if key == 'last active':
|
||||||
items.append(fancy_timestr(message))
|
items.append(fancy_timestr(message))
|
||||||
elif key == 'temp':
|
elif key == 'temp':
|
||||||
items.append(str(message) + deg_str)
|
items.append(str(message) + DEG_STR)
|
||||||
else:
|
else:
|
||||||
items.append(str(message))
|
items.append(str(message))
|
||||||
return items
|
return items
|
||||||
@ -946,9 +952,8 @@ class StationQC(object):
|
|||||||
clock_quality_warn_level = self.parameters.get('THRESHOLDS').get('clockquality_warn')
|
clock_quality_warn_level = self.parameters.get('THRESHOLDS').get('clockquality_warn')
|
||||||
clock_quality_fail_level = self.parameters.get('THRESHOLDS').get('clockquality_fail')
|
clock_quality_fail_level = self.parameters.get('THRESHOLDS').get('clockquality_fail')
|
||||||
|
|
||||||
if self.verbosity > 1:
|
logging.info(40 * '-')
|
||||||
self.print(40 * '-')
|
logging.info('Performing Clock Quality check')
|
||||||
self.print('Performing Clock Quality check', flush=False)
|
|
||||||
|
|
||||||
clockQuality_warn = np.where(clock_quality < clock_quality_warn_level)[0]
|
clockQuality_warn = np.where(clock_quality < clock_quality_warn_level)[0]
|
||||||
clockQuality_fail = np.where(clock_quality < clock_quality_fail_level)[0]
|
clockQuality_fail = np.where(clock_quality < clock_quality_fail_level)[0]
|
||||||
@ -992,9 +997,8 @@ class StationQC(object):
|
|||||||
low_volt = self.parameters.get('THRESHOLDS').get('low_volt')
|
low_volt = self.parameters.get('THRESHOLDS').get('low_volt')
|
||||||
high_volt = self.parameters.get('THRESHOLDS').get('high_volt')
|
high_volt = self.parameters.get('THRESHOLDS').get('high_volt')
|
||||||
|
|
||||||
if self.verbosity > 1:
|
logging.info(40 * '-')
|
||||||
self.print(40 * '-')
|
logging.info('Performing Voltage check')
|
||||||
self.print('Performing Voltage check', flush=False)
|
|
||||||
|
|
||||||
overvolt = np.where(voltage > high_volt)[0]
|
overvolt = np.where(voltage > high_volt)[0]
|
||||||
undervolt = np.where(voltage < low_volt)[0]
|
undervolt = np.where(voltage < low_volt)[0]
|
||||||
@ -1033,17 +1037,16 @@ class StationQC(object):
|
|||||||
# average temp
|
# average temp
|
||||||
timespan = min([self.parameters.get('timespan') * 24 * 3600, int(len(temp) / trace.stats.sampling_rate)])
|
timespan = min([self.parameters.get('timespan') * 24 * 3600, int(len(temp) / trace.stats.sampling_rate)])
|
||||||
nsamp_av = int(trace.stats.sampling_rate) * timespan
|
nsamp_av = int(trace.stats.sampling_rate) * timespan
|
||||||
av_temp_str = str(round(np.nanmean(temp[-nsamp_av:]), 1)) + deg_str
|
av_temp_str = str(round(np.nanmean(temp[-nsamp_av:]), 1)) + DEG_STR
|
||||||
# dt of average
|
# dt of average
|
||||||
dt_t_str = str(timedelta(seconds=int(timespan))).replace(', 0:00:00', '')
|
dt_t_str = str(timedelta(seconds=int(timespan))).replace(', 0:00:00', '')
|
||||||
# current temp
|
# current temp
|
||||||
cur_temp = round(temp[-1], 1)
|
cur_temp = round(temp[-1], 1)
|
||||||
if self.verbosity > 1:
|
logging.info(40 * '-')
|
||||||
self.print(40 * '-')
|
logging.info('Performing PowBox temperature check (EX1)')
|
||||||
self.print('Performing PowBox temperature check (EX1)', flush=False)
|
logging.info(f'Average temperature at {np.nanmean(temp)}\N{DEGREE SIGN}')
|
||||||
self.print(f'Average temperature at {np.nanmean(temp)}\N{DEGREE SIGN}', flush=False)
|
logging.info(f'Peak temperature at {max(temp)}\N{DEGREE SIGN}')
|
||||||
self.print(f'Peak temperature at {max(temp)}\N{DEGREE SIGN}', flush=False)
|
logging.info(f'Min temperature at {min(temp)}\N{DEGREE SIGN}')
|
||||||
self.print(f'Min temperature at {min(temp)}\N{DEGREE SIGN}', flush=False)
|
|
||||||
max_temp = thresholds.get('max_temp')
|
max_temp = thresholds.get('max_temp')
|
||||||
t_check = np.where(temp > max_temp)[0]
|
t_check = np.where(temp > max_temp)[0]
|
||||||
if len(t_check) > 0:
|
if len(t_check) > 0:
|
||||||
@ -1098,10 +1101,9 @@ class StationQC(object):
|
|||||||
self.error(key=key,
|
self.error(key=key,
|
||||||
detailed_message=f'Fail status for mass centering. Highest val (abs) {common_highest_val}V',)
|
detailed_message=f'Fail status for mass centering. Highest val (abs) {common_highest_val}V',)
|
||||||
|
|
||||||
if self.verbosity > 1:
|
logging.info(40 * '-')
|
||||||
self.print(40 * '-')
|
logging.info('Performing mass position check')
|
||||||
self.print('Performing mass position check', flush=False)
|
logging.info(f'Average mass position at {common_highest_val}')
|
||||||
self.print(f'Average mass position at {common_highest_val}', flush=False)
|
|
||||||
|
|
||||||
def pb_power_analysis(self, channel='EX2', pb_dict_key='pb_SOH2'):
|
def pb_power_analysis(self, channel='EX2', pb_dict_key='pb_SOH2'):
|
||||||
""" Analyse EX2 channel of PowBox """
|
""" Analyse EX2 channel of PowBox """
|
||||||
@ -1112,9 +1114,8 @@ class StationQC(object):
|
|||||||
return
|
return
|
||||||
|
|
||||||
voltage = trace.data * self.get_unit_factor(channel)
|
voltage = trace.data * self.get_unit_factor(channel)
|
||||||
if self.verbosity > 1:
|
logging.info(40 * '-')
|
||||||
self.print(40 * '-')
|
logging.info('Performing PowBox 12V/230V check (EX2)')
|
||||||
self.print('Performing PowBox 12V/230V check (EX2)', flush=False)
|
|
||||||
voltage_check, voltage_dict, last_val = self.pb_voltage_ok(trace, voltage, pb_dict_key, channel=channel)
|
voltage_check, voltage_dict, last_val = self.pb_voltage_ok(trace, voltage, pb_dict_key, channel=channel)
|
||||||
|
|
||||||
if voltage_check:
|
if voltage_check:
|
||||||
@ -1135,9 +1136,8 @@ class StationQC(object):
|
|||||||
return
|
return
|
||||||
|
|
||||||
voltage = trace.data * self.get_unit_factor(channel)
|
voltage = trace.data * self.get_unit_factor(channel)
|
||||||
if self.verbosity > 1:
|
logging.info(40 * '-')
|
||||||
self.print(40 * '-')
|
logging.info('Performing PowBox Router/Charger check (EX3)')
|
||||||
self.print('Performing PowBox Router/Charger check (EX3)', flush=False)
|
|
||||||
voltage_check, voltage_dict, last_val = self.pb_voltage_ok(trace, voltage, pb_dict_key, channel=channel)
|
voltage_check, voltage_dict, last_val = self.pb_voltage_ok(trace, voltage, pb_dict_key, channel=channel)
|
||||||
|
|
||||||
if voltage_check:
|
if voltage_check:
|
||||||
|
@ -11,6 +11,8 @@ import os
|
|||||||
import sys
|
import sys
|
||||||
import traceback
|
import traceback
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from PySide2 import QtGui, QtCore, QtWidgets
|
from PySide2 import QtGui, QtCore, QtWidgets
|
||||||
except ImportError:
|
except ImportError:
|
||||||
@ -41,7 +43,7 @@ try:
|
|||||||
from rest_api.rest_api_utils import get_last_messages, send_message, get_default_params
|
from rest_api.rest_api_utils import get_last_messages, send_message, get_default_params
|
||||||
sms_funcs = True
|
sms_funcs = True
|
||||||
except ImportError:
|
except ImportError:
|
||||||
print('Could not load rest_api utils, SMS functionality disabled.')
|
logging.warning('Could not load rest_api utils, SMS functionality disabled.')
|
||||||
sms_funcs = False
|
sms_funcs = False
|
||||||
|
|
||||||
deg_str = '\N{DEGREE SIGN}C'
|
deg_str = '\N{DEGREE SIGN}C'
|
||||||
@ -54,10 +56,9 @@ class Thread(QtCore.QThread):
|
|||||||
"""
|
"""
|
||||||
update = QtCore.Signal()
|
update = QtCore.Signal()
|
||||||
|
|
||||||
def __init__(self, parent, runnable, verbosity=0):
|
def __init__(self, parent, runnable):
|
||||||
super(Thread, self).__init__(parent=parent)
|
super(Thread, self).__init__(parent=parent)
|
||||||
self.setParent(parent)
|
self.setParent(parent)
|
||||||
self.verbosity = verbosity
|
|
||||||
self.runnable = runnable
|
self.runnable = runnable
|
||||||
self.is_active = True
|
self.is_active = True
|
||||||
|
|
||||||
@ -69,11 +70,10 @@ class Thread(QtCore.QThread):
|
|||||||
self.update.emit()
|
self.update.emit()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.is_active = False
|
self.is_active = False
|
||||||
print(e)
|
logging.error(e)
|
||||||
print(traceback.format_exc())
|
logging.debug(traceback.format_exc())
|
||||||
finally:
|
finally:
|
||||||
if self.verbosity > 0:
|
logging.info(f'Time for Thread execution: {UTCDateTime() - t0}')
|
||||||
print(f'Time for Thread execution: {UTCDateTime() - t0}')
|
|
||||||
|
|
||||||
|
|
||||||
class MainWindow(QtWidgets.QMainWindow):
|
class MainWindow(QtWidgets.QMainWindow):
|
||||||
@ -195,7 +195,7 @@ class MainWindow(QtWidgets.QMainWindow):
|
|||||||
station = nwst_id.split('.')[1]
|
station = nwst_id.split('.')[1]
|
||||||
iccid = get_station_iccid(station)
|
iccid = get_station_iccid(station)
|
||||||
if not iccid:
|
if not iccid:
|
||||||
print('Could not find iccid for station', nwst_id)
|
logging.info(f'Could not find iccid for station: {nwst_id}')
|
||||||
return
|
return
|
||||||
sms_widget = ReadSMSWidget(parent=self, iccid=iccid)
|
sms_widget = ReadSMSWidget(parent=self, iccid=iccid)
|
||||||
sms_widget.setWindowTitle(f'Recent SMS of station: {nwst_id}')
|
sms_widget.setWindowTitle(f'Recent SMS of station: {nwst_id}')
|
||||||
|
45
utils.py
45
utils.py
@ -1,6 +1,8 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
import matplotlib
|
import matplotlib
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
@ -170,7 +172,7 @@ def transform_trace(data, transf):
|
|||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
def set_axis_ylabels(fig, parameters, verbosity=0):
|
def set_axis_ylabels(fig, parameters):
|
||||||
"""
|
"""
|
||||||
Adds channel names to y-axis if defined in parameters.
|
Adds channel names to y-axis if defined in parameters.
|
||||||
"""
|
"""
|
||||||
@ -178,8 +180,7 @@ def set_axis_ylabels(fig, parameters, verbosity=0):
|
|||||||
if not names: # or not len(st.traces):
|
if not names: # or not len(st.traces):
|
||||||
return
|
return
|
||||||
if not len(names) == len(fig.axes):
|
if not len(names) == len(fig.axes):
|
||||||
if verbosity:
|
logging.info('Mismatch in axis and label lengths. Not adding plot labels')
|
||||||
print('Mismatch in axis and label lengths. Not adding plot labels')
|
|
||||||
return
|
return
|
||||||
for channel_name, ax in zip(names, fig.axes):
|
for channel_name, ax in zip(names, fig.axes):
|
||||||
if channel_name:
|
if channel_name:
|
||||||
@ -195,7 +196,7 @@ def set_axis_color(fig, color='0.8'):
|
|||||||
ax.spines[key].set_color(color)
|
ax.spines[key].set_color(color)
|
||||||
|
|
||||||
|
|
||||||
def set_axis_yticks(fig, parameters, verbosity=0):
|
def set_axis_yticks(fig, parameters):
|
||||||
"""
|
"""
|
||||||
Adds channel names to y-axis if defined in parameters.
|
Adds channel names to y-axis if defined in parameters.
|
||||||
"""
|
"""
|
||||||
@ -203,8 +204,7 @@ def set_axis_yticks(fig, parameters, verbosity=0):
|
|||||||
if not ticks:
|
if not ticks:
|
||||||
return
|
return
|
||||||
if not len(ticks) == len(fig.axes):
|
if not len(ticks) == len(fig.axes):
|
||||||
if verbosity:
|
logging.info('Mismatch in axis tick and label lengths. Not changing plot ticks.')
|
||||||
print('Mismatch in axis tick and label lengths. Not changing plot ticks.')
|
|
||||||
return
|
return
|
||||||
for ytick_tripple, ax in zip(ticks, fig.axes):
|
for ytick_tripple, ax in zip(ticks, fig.axes):
|
||||||
if not ytick_tripple:
|
if not ytick_tripple:
|
||||||
@ -216,12 +216,11 @@ def set_axis_yticks(fig, parameters, verbosity=0):
|
|||||||
ax.set_ylim(ymin - 0.33 * step, ymax + 0.33 * step)
|
ax.set_ylim(ymin - 0.33 * step, ymax + 0.33 * step)
|
||||||
|
|
||||||
|
|
||||||
def plot_axis_thresholds(fig, parameters, verbosity=0):
|
def plot_axis_thresholds(fig, parameters):
|
||||||
"""
|
"""
|
||||||
Adds channel thresholds (warn, fail) to y-axis if defined in parameters.
|
Adds channel thresholds (warn, fail) to y-axis if defined in parameters.
|
||||||
"""
|
"""
|
||||||
if verbosity > 0:
|
logging.info('Plotting trace thresholds')
|
||||||
print('Plotting trace thresholds')
|
|
||||||
|
|
||||||
keys_colors = {'warn': dict(color=0.8 * get_color_mpl('WARN'), linestyle=(0, (5, 10)), alpha=0.5, linewidth=0.7),
|
keys_colors = {'warn': dict(color=0.8 * get_color_mpl('WARN'), linestyle=(0, (5, 10)), alpha=0.5, linewidth=0.7),
|
||||||
'fail': dict(color=0.8 * get_color_mpl('FAIL'), linestyle='solid', alpha=0.5, linewidth=0.7)}
|
'fail': dict(color=0.8 * get_color_mpl('FAIL'), linestyle='solid', alpha=0.5, linewidth=0.7)}
|
||||||
@ -235,6 +234,10 @@ def plot_axis_thresholds(fig, parameters, verbosity=0):
|
|||||||
|
|
||||||
def plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs):
|
def plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs):
|
||||||
for channel_thresholds, ax in zip(channel_threshold_list, fig.axes):
|
for channel_thresholds, ax in zip(channel_threshold_list, fig.axes):
|
||||||
|
if channel_thresholds in ['pb_SOH2', 'pb_SOH3']:
|
||||||
|
annotate_voltage_states(ax, parameters, channel_thresholds)
|
||||||
|
channel_thresholds = get_warn_states_pbox(channel_thresholds, parameters)
|
||||||
|
|
||||||
if not channel_thresholds:
|
if not channel_thresholds:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@ -244,5 +247,27 @@ def plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs):
|
|||||||
for warn_thresh in channel_thresholds:
|
for warn_thresh in channel_thresholds:
|
||||||
if isinstance(warn_thresh, str):
|
if isinstance(warn_thresh, str):
|
||||||
warn_thresh = parameters.get('THRESHOLDS').get(warn_thresh)
|
warn_thresh = parameters.get('THRESHOLDS').get(warn_thresh)
|
||||||
if type(warn_thresh in (float, int)):
|
if isinstance(warn_thresh, (float, int)):
|
||||||
ax.axhline(warn_thresh, **kwargs)
|
ax.axhline(warn_thresh, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def get_warn_states_pbox(soh_key: str, parameters: dict) -> list:
|
||||||
|
pb_dict = parameters.get('POWBOX').get(soh_key)
|
||||||
|
if not pb_dict:
|
||||||
|
return []
|
||||||
|
return [key for key in pb_dict.keys() if key > 1]
|
||||||
|
|
||||||
|
|
||||||
|
def annotate_voltage_states(ax, parameters, pb_key):
|
||||||
|
for voltage, voltage_dict in parameters.get('POWBOX').get(pb_key).items():
|
||||||
|
if float(voltage) < 1:
|
||||||
|
continue
|
||||||
|
out_string = ''
|
||||||
|
for key, val in voltage_dict.items():
|
||||||
|
if val != 'OK':
|
||||||
|
if out_string:
|
||||||
|
out_string += ' | '
|
||||||
|
out_string += f'{key}: {val}'
|
||||||
|
|
||||||
|
ax.annotate(out_string, (ax.get_xlim()[-1], voltage), color='0.8', fontsize='xx-small',
|
||||||
|
horizontalalignment='right')
|
||||||
|
Loading…
Reference in New Issue
Block a user