From cb3623e4a90d05418cf9f54c5220aaf56524ddc8 Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Tue, 6 Dec 2022 15:43:13 +0100 Subject: [PATCH 01/40] Add quality check for clock quality (LCQ) --- parameters.yaml | 14 ++++++++------ survBot.py | 44 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/parameters.yaml b/parameters.yaml index 916992c..ce6486a 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -1,10 +1,10 @@ # Parameters file for Surveillance Bot -datapath: "/data/SDS/" # SC3 Datapath -networks: ["1Y", "HA"] -stations: "*" -locations: "*" -channels: ["EX1", "EX2", "EX3", "VEI"] # Specify SOH channels, currently supported EX[1-3] and VEI -channel_names: ["Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"] # names for plotting (optional) +datapath: "/data/SDS/" # SC3 Datapath +networks: ["1Y", "HA"] # select networks, list or str +stations: "*" # select stations, list or str +locations: "*" # select locations, list or str +channels: ["EX1", "EX2", "EX3", "VEI", "LCQ"] # Specify SOH channels, currently supported EX[1-3], VEI, LCQ +channel_names: ["Clock Quality (%)", "Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"] # names for plotting (optional) stations_blacklist: ["TEST", "EREA"] networks_blacklist: [] interval: 60 # Perform checks every x seconds @@ -43,6 +43,8 @@ THRESHOLDS: low_volt: 12 # min voltage for low voltage warning high_volt: 14.8 # max voltage for over voltage warning unclassified: 5 # min voltage samples not classified for warning + clockquality_warn: 90 # clock quality ranges from 0 % to 100 % with 100 % being the best level + clockquality_fail: 70 # add links to html table with specified key as column and value as relative link, interpretable string parameters: # nw (e.g. 1Y), st (e.g. GR01A), nwst_id (e.g. 1Y.GR01A) diff --git a/survBot.py b/survBot.py index 62fd9c3..763a93c 100755 --- a/survBot.py +++ b/survBot.py @@ -61,7 +61,7 @@ def fancy_timestr(dt, thresh=600, modif='+'): class SurveillanceBot(object): def __init__(self, parameter_path, outpath_html=None): - self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'temp', 'other'] + self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'clock', 'temp', 'other'] self.parameter_path = parameter_path self.update_parameters() self.starttime = UTCDateTime() @@ -668,6 +668,7 @@ class StationQC(object): self.pb_temp_analysis() self.pb_power_analysis() self.pb_rout_charge_analysis() + self.clock_quality_analysis() def return_print_analysis(self): items = [self.nwst_id] @@ -696,6 +697,47 @@ class StationQC(object): def get_last_occurrence(self, trace, indices): return self.get_time(trace, indices[-1]) + def clock_quality_analysis(self, channel='LCQ'): + """ Analyse clock quality """ + key = 'clock' + st = self.stream.select(channel=channel) + trace = self.get_trace(st, key) + if not trace: return + clockQuality = trace.data + clockQuality_warn_level = self.parameters.get('THRESHOLDS').get('clockquality_warn') + clockQuality_fail_level = self.parameters.get('THRESHOLDS').get('clockquality_fail') + + if self.verbosity > 1: + self.print(40 * '-') + self.print('Performing Clock Quality check', flush=False) + + clockQuality_warn = np.where(clockQuality < clockQuality_warn_level)[0] + clockQuality_fail = np.where(clockQuality < clockQuality_fail_level)[0] + + if len(clockQuality_warn) == 0 and len(clockQuality_fail) == 0: + self.status_ok(key, detailed_message=f'ClockQuality={(clockQuality[-1])}') + return + + n_qc_warn = 0 + n_qc_fail = 0 + + warn_message = f'Trace {trace.get_id()}:' + if len(clockQuality_warn) > 0: + # try calculate number of warn peaks from gaps between indices + n_qc_warn = len(np.where(np.diff(clockQuality_warn) > 1)[0]) + 1 + detailed_message = warn_message + f' {n_qc_warn}x Qlock Quality less then {clockQuality_warn_level}' \ + + self.get_last_occurrence_timestring(trace, clockQuality_warn) + self.warn(key, detailed_message=detailed_message, count=n_qc_warn, + last_occurrence=self.get_last_occurrence(trace, clockQuality_warn)) + + if len(clockQuality_fail) > 0: + # try calculate number of fail peaks from gaps between indices + n_qc_fail = len(np.where(np.diff(clockQuality_fail) > 1)[0]) + 1 + detailed_message = warn_message + f' {n_qc_fail}x Qlock Quality less then {clockQuality_fail_level}V ' \ + + self.get_last_occurrence_timestring(trace, clockQuality_fail) + self.error(key, detailed_message=detailed_message, count=n_qc_fail, + last_occurrence=self.get_last_occurrence(trace, clockQuality_fail)) + def voltage_analysis(self, channel='VEI'): """ Analyse voltage channel for over/undervoltage """ key = 'voltage' From 69412dc5fe6ba733315790413e4883cf37e0b147 Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Tue, 6 Dec 2022 15:51:24 +0100 Subject: [PATCH 02/40] use function calc_occurences --- survBot.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/survBot.py b/survBot.py index 77c1a1d..6ab6fc5 100755 --- a/survBot.py +++ b/survBot.py @@ -721,13 +721,10 @@ class StationQC(object): self.status_ok(key, detailed_message=f'ClockQuality={(clockQuality[-1])}') return - n_qc_warn = 0 - n_qc_fail = 0 - warn_message = f'Trace {trace.get_id()}:' if len(clockQuality_warn) > 0: # try calculate number of warn peaks from gaps between indices - n_qc_warn = len(np.where(np.diff(clockQuality_warn) > 1)[0]) + 1 + n_qc_warn = self.calc_occurrences(clockQuality_warn) detailed_message = warn_message + f' {n_qc_warn}x Qlock Quality less then {clockQuality_warn_level}' \ + self.get_last_occurrence_timestring(trace, clockQuality_warn) self.warn(key, detailed_message=detailed_message, count=n_qc_warn, @@ -735,7 +732,7 @@ class StationQC(object): if len(clockQuality_fail) > 0: # try calculate number of fail peaks from gaps between indices - n_qc_fail = len(np.where(np.diff(clockQuality_fail) > 1)[0]) + 1 + n_qc_fail = self.calc_occurrences(clockQuality_fail) detailed_message = warn_message + f' {n_qc_fail}x Qlock Quality less then {clockQuality_fail_level}V ' \ + self.get_last_occurrence_timestring(trace, clockQuality_fail) self.error(key, detailed_message=detailed_message, count=n_qc_fail, From ac9f83d8561a5275871b96d9ca34e4b27463d4bc Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Tue, 6 Dec 2022 16:46:19 +0100 Subject: [PATCH 03/40] fixed paramter file --- parameters.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/parameters.yaml b/parameters.yaml index 0c5ff74..690d8a5 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -4,7 +4,6 @@ networks: ["1Y", "HA"] # select networks, list or str stations: "*" # select stations, list or str locations: "*" # select locations, list or str channels: ["EX1", "EX2", "EX3", "VEI", "LCQ"] # Specify SOH channels, currently supported EX[1-3], VEI, LCQ -channel_names: ["Clock Quality (%)", "Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"] # names for plotting (optional) stations_blacklist: ["TEST", "EREA"] networks_blacklist: [] interval: 60 # Perform checks every x seconds @@ -42,7 +41,7 @@ THRESHOLDS: pb_thresh: 0.2 # Threshold for PowBox Voltage check +/- (V) max_temp: 50 # max temperature for temperature warning low_volt: 12 # min voltage for low voltage warning - high_volt: 14.8 # max voltage for over voltage warning + high_volt: 14.8 # max voltage for over voltage warning unclassified: 5 # min voltage samples not classified for warning clockquality_warn: 90 # clock quality ranges from 0 % to 100 % with 100 % being the best level clockquality_fail: 70 @@ -63,9 +62,10 @@ EMAIL: sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender # names for plotting of the above defined parameter "channels" in the same order -channel_names: ["Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"] +channel_names: ["Clock Quality (%)", "Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"] # names for plotting (optional) # specify y-ticks (and ylims) giving, (ymin, ymax, step) for each of the above channels (0: default) CHANNEL_TICKS: + - [0, 100, 20] - [-10, 50, 10] - [1, 5, 1] - [1, 5, 1] From 541815d81ffd62922dec8d11cd23f01c84691f3e Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Tue, 6 Dec 2022 15:43:13 +0100 Subject: [PATCH 04/40] Add quality check for clock quality (LCQ) --- parameters.yaml | 6 ++++-- survBot.py | 44 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/parameters.yaml b/parameters.yaml index 0fdf7c6..5208559 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -3,7 +3,7 @@ datapath: "/data/SDS/" # SC3 Datapath networks: ["1Y", "HA"] # select networks, list or str stations: "*" # select stations, list or str locations: "*" # select locations, list or str -channels: ["EX1", "EX2", "EX3", "VEI"] # Specify SOH channels, currently supported EX[1-3] and VEI +channels: ["EX1", "EX2", "EX3", "VEI", "LCQ"] # Specify SOH channels, currently supported EX[1-3], VEI, LCQ stations_blacklist: ["TEST", "EREA"] # exclude these stations networks_blacklist: [] # exclude these networks interval: 60 # Perform checks every x seconds @@ -43,6 +43,8 @@ THRESHOLDS: low_volt: 12 # min voltage for low voltage warning high_volt: 14.8 # max voltage for over voltage warning unclassified: 5 # min voltage samples not classified for warning + clockquality_warn: 90 # clock quality ranges from 0 % to 100 % with 100 % being the best level + clockquality_fail: 70 # ---------------------------------------- OPTIONAL PARAMETERS --------------------------------------------------------- @@ -62,7 +64,7 @@ EMAIL: networks_blacklist: [] # do not send emails for specific network # names for plotting of the above defined parameter "channels" in the same order -channel_names: ["Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"] +channel_names: ["Clock Quality (%)", "Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"] # names for plotting (optional) # specify y-ticks (and ylims) giving, (ymin, ymax, step) for each of the above channels (0: default) CHANNEL_TICKS: - [-10, 50, 10] diff --git a/survBot.py b/survBot.py index 10dfabf..0de3f7f 100755 --- a/survBot.py +++ b/survBot.py @@ -61,7 +61,7 @@ def fancy_timestr(dt, thresh=600, modif='+'): class SurveillanceBot(object): def __init__(self, parameter_path, outpath_html=None): - self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'temp', 'other'] + self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'clock', 'temp', 'other'] self.parameter_path = parameter_path self.update_parameters() self.starttime = UTCDateTime() @@ -685,6 +685,7 @@ class StationQC(object): self.pb_temp_analysis() self.pb_power_analysis() self.pb_rout_charge_analysis() + self.clock_quality_analysis() def return_print_analysis(self): items = [self.nwst_id] @@ -713,6 +714,47 @@ class StationQC(object): def get_last_occurrence(self, trace, indices): return self.get_time(trace, indices[-1]) + def clock_quality_analysis(self, channel='LCQ'): + """ Analyse clock quality """ + key = 'clock' + st = self.stream.select(channel=channel) + trace = self.get_trace(st, key) + if not trace: return + clockQuality = trace.data + clockQuality_warn_level = self.parameters.get('THRESHOLDS').get('clockquality_warn') + clockQuality_fail_level = self.parameters.get('THRESHOLDS').get('clockquality_fail') + + if self.verbosity > 1: + self.print(40 * '-') + self.print('Performing Clock Quality check', flush=False) + + clockQuality_warn = np.where(clockQuality < clockQuality_warn_level)[0] + clockQuality_fail = np.where(clockQuality < clockQuality_fail_level)[0] + + if len(clockQuality_warn) == 0 and len(clockQuality_fail) == 0: + self.status_ok(key, detailed_message=f'ClockQuality={(clockQuality[-1])}') + return + + n_qc_warn = 0 + n_qc_fail = 0 + + warn_message = f'Trace {trace.get_id()}:' + if len(clockQuality_warn) > 0: + # try calculate number of warn peaks from gaps between indices + n_qc_warn = len(np.where(np.diff(clockQuality_warn) > 1)[0]) + 1 + detailed_message = warn_message + f' {n_qc_warn}x Qlock Quality less then {clockQuality_warn_level}' \ + + self.get_last_occurrence_timestring(trace, clockQuality_warn) + self.warn(key, detailed_message=detailed_message, count=n_qc_warn, + last_occurrence=self.get_last_occurrence(trace, clockQuality_warn)) + + if len(clockQuality_fail) > 0: + # try calculate number of fail peaks from gaps between indices + n_qc_fail = len(np.where(np.diff(clockQuality_fail) > 1)[0]) + 1 + detailed_message = warn_message + f' {n_qc_fail}x Qlock Quality less then {clockQuality_fail_level}V ' \ + + self.get_last_occurrence_timestring(trace, clockQuality_fail) + self.error(key, detailed_message=detailed_message, count=n_qc_fail, + last_occurrence=self.get_last_occurrence(trace, clockQuality_fail)) + def voltage_analysis(self, channel='VEI'): """ Analyse voltage channel for over/undervoltage """ key = 'voltage' From 735abac249359b45c33f3cdcab047ccded905960 Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Tue, 6 Dec 2022 15:51:24 +0100 Subject: [PATCH 05/40] use function calc_occurences --- survBot.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/survBot.py b/survBot.py index 0de3f7f..e6e6ebf 100755 --- a/survBot.py +++ b/survBot.py @@ -735,13 +735,10 @@ class StationQC(object): self.status_ok(key, detailed_message=f'ClockQuality={(clockQuality[-1])}') return - n_qc_warn = 0 - n_qc_fail = 0 - warn_message = f'Trace {trace.get_id()}:' if len(clockQuality_warn) > 0: # try calculate number of warn peaks from gaps between indices - n_qc_warn = len(np.where(np.diff(clockQuality_warn) > 1)[0]) + 1 + n_qc_warn = self.calc_occurrences(clockQuality_warn) detailed_message = warn_message + f' {n_qc_warn}x Qlock Quality less then {clockQuality_warn_level}' \ + self.get_last_occurrence_timestring(trace, clockQuality_warn) self.warn(key, detailed_message=detailed_message, count=n_qc_warn, @@ -749,7 +746,7 @@ class StationQC(object): if len(clockQuality_fail) > 0: # try calculate number of fail peaks from gaps between indices - n_qc_fail = len(np.where(np.diff(clockQuality_fail) > 1)[0]) + 1 + n_qc_fail = self.calc_occurrences(clockQuality_fail) detailed_message = warn_message + f' {n_qc_fail}x Qlock Quality less then {clockQuality_fail_level}V ' \ + self.get_last_occurrence_timestring(trace, clockQuality_fail) self.error(key, detailed_message=detailed_message, count=n_qc_fail, From f2e322230ef06d18a2f2d62faec336b30692bbd0 Mon Sep 17 00:00:00 2001 From: "Kasper D. Fischer" Date: Tue, 6 Dec 2022 16:46:19 +0100 Subject: [PATCH 06/40] fixed paramter file --- parameters.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parameters.yaml b/parameters.yaml index 5208559..3123f07 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -41,7 +41,7 @@ THRESHOLDS: pb_thresh: 0.2 # Threshold for PowBox Voltage check +/- (V) max_temp: 50 # max temperature for temperature warning low_volt: 12 # min voltage for low voltage warning - high_volt: 14.8 # max voltage for over voltage warning + high_volt: 14.8 # max voltage for over voltage warning unclassified: 5 # min voltage samples not classified for warning clockquality_warn: 90 # clock quality ranges from 0 % to 100 % with 100 % being the best level clockquality_fail: 70 @@ -67,6 +67,7 @@ EMAIL: channel_names: ["Clock Quality (%)", "Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"] # names for plotting (optional) # specify y-ticks (and ylims) giving, (ymin, ymax, step) for each of the above channels (0: default) CHANNEL_TICKS: + - [0, 100, 20] - [-10, 50, 10] - [1, 5, 1] - [1, 5, 1] From 174a8e0823909e085ba30ad5224d110299ba88ef Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 20 Dec 2022 16:54:27 +0100 Subject: [PATCH 07/40] [new] mass channel surveillance added --- parameters.yaml | 19 ++++++++++-- survBot.py | 78 +++++++++++++++++++++++++++++++++++++++++++++---- utils.py | 29 ++++++++++++++---- 3 files changed, 112 insertions(+), 14 deletions(-) diff --git a/parameters.yaml b/parameters.yaml index 0fdf7c6..8718b4f 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -3,7 +3,8 @@ datapath: "/data/SDS/" # SC3 Datapath networks: ["1Y", "HA"] # select networks, list or str stations: "*" # select stations, list or str locations: "*" # select locations, list or str -channels: ["EX1", "EX2", "EX3", "VEI"] # Specify SOH channels, currently supported EX[1-3] and VEI +channels: ["EX1", "EX2", "EX3", "VEI", + "VM1", "VM2", "VM3"] # Specify SOH channels, currently supported EX[1-3], VEI and VM[1-3] stations_blacklist: ["TEST", "EREA"] # exclude these stations networks_blacklist: [] # exclude these networks interval: 60 # Perform checks every x seconds @@ -43,6 +44,7 @@ THRESHOLDS: low_volt: 12 # min voltage for low voltage warning high_volt: 14.8 # max voltage for over voltage warning unclassified: 5 # min voltage samples not classified for warning + max_vm: [1.5, 2.5] # thresholds for mass offset (warn, fail) # ---------------------------------------- OPTIONAL PARAMETERS --------------------------------------------------------- @@ -62,13 +64,23 @@ EMAIL: networks_blacklist: [] # do not send emails for specific network # names for plotting of the above defined parameter "channels" in the same order -channel_names: ["Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"] +channel_names: ["Temperature (°C)", + "230V/12V (V)", + "Rout/Charge (V)", + "Logger (V)", + "Mass 1 (V)", + "Mass 2 (V)", + "Mass 3 (V)"] + # specify y-ticks (and ylims) giving, (ymin, ymax, step) for each of the above channels (0: default) CHANNEL_TICKS: - [-10, 50, 10] - [1, 5, 1] - [1, 5, 1] - [9, 15, 1] + - [-2, 2, 1] + - [-2, 2, 1] + - [-2, 2, 1] # Factor for channel to SI-units (for plotting) CHANNEL_UNITS: @@ -76,6 +88,9 @@ CHANNEL_UNITS: EX2: 1e-6 EX3: 1e-6 VEI: 1e-3 + VM1: 1e-6 + VM2: 1e-6 + VM3: 1e-6 # Transform channel for plotting, perform arithmetic operations in given order, e.g.: PBox EX1 V to deg C: 20 * x -20 CHANNEL_TRANSFORM: diff --git a/survBot.py b/survBot.py index 10dfabf..bb1f828 100755 --- a/survBot.py +++ b/survBot.py @@ -61,7 +61,7 @@ def fancy_timestr(dt, thresh=600, modif='+'): class SurveillanceBot(object): def __init__(self, parameter_path, outpath_html=None): - self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'temp', 'other'] + self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'mass', 'temp', 'other'] self.parameter_path = parameter_path self.update_parameters() self.starttime = UTCDateTime() @@ -242,7 +242,7 @@ class SurveillanceBot(object): def get_station_delay(self, nwst_id): """ try to get station delay from SDS archive using client""" locations = ['', '0', '00'] - channels = ['HHZ', 'HHE', 'HHN', 'VEI', 'EX1', 'EX2', 'EX3'] + channels = ['HHZ', 'HHE', 'HHN'] + self.parameters.get('channels') network, station = nwst_id.split('.')[:2] times = [] @@ -685,6 +685,7 @@ class StationQC(object): self.pb_temp_analysis() self.pb_power_analysis() self.pb_rout_charge_analysis() + self.mass_analysis() def return_print_analysis(self): items = [self.nwst_id] @@ -718,7 +719,8 @@ class StationQC(object): key = 'voltage' st = self.stream.select(channel=channel) trace = self.get_trace(st, key) - if not trace: return + if not trace: + return voltage = trace.data * 1e-3 low_volt = self.parameters.get('THRESHOLDS').get('low_volt') high_volt = self.parameters.get('THRESHOLDS').get('high_volt') @@ -756,14 +758,15 @@ class StationQC(object): key = 'temp' st = self.stream.select(channel=channel) trace = self.get_trace(st, key) - if not trace: return + if not trace: + return voltage = trace.data * 1e-6 thresholds = self.parameters.get('THRESHOLDS') temp = 20. * voltage - 20 # average temp timespan = min([self.parameters.get('timespan') * 24 * 3600, int(len(temp) / trace.stats.sampling_rate)]) nsamp_av = int(trace.stats.sampling_rate) * timespan - av_temp_str = str(round(np.mean(temp[-nsamp_av:]), 1)) + deg_str + av_temp_str = str(round(np.nanmean(temp[-nsamp_av:]), 1)) + deg_str # dt of average dt_t_str = str(timedelta(seconds=int(timespan))).replace(', 0:00:00', '') # current temp @@ -771,7 +774,7 @@ class StationQC(object): if self.verbosity > 1: self.print(40 * '-') self.print('Performing PowBox temperature check (EX1)', flush=False) - self.print(f'Average temperature at {np.mean(temp)}\N{DEGREE SIGN}', flush=False) + self.print(f'Average temperature at {np.nanmean(temp)}\N{DEGREE SIGN}', flush=False) self.print(f'Peak temperature at {max(temp)}\N{DEGREE SIGN}', flush=False) self.print(f'Min temperature at {min(temp)}\N{DEGREE SIGN}', flush=False) max_temp = thresholds.get('max_temp') @@ -787,6 +790,52 @@ class StationQC(object): status_message=cur_temp, detailed_message=f'Average temperature of last {dt_t_str}: {av_temp_str}') + def mass_analysis(self, channels=('VM1', 'VM2', 'VM3'), n_samp_mean=10): + """ Analyse datalogger mass channels. """ + key = 'mass' + + # build stream with all channels + st = Stream() + for channel in channels: + st += self.stream.select(channel=channel).copy() + st.merge() + + # return if there are no three components + if not len(st) == 3: + return + + # correct for channel unit + for trace in st: + trace.data = trace.data * 1e-6 # hardcoded, change this? + + # calculate average of absolute maximum of mass offset of last n_samp_mean + last_values = np.array([trace.data[-n_samp_mean:] for trace in st]) + last_val_mean = np.nanmean(last_values, axis=1) + common_highest_val = np.nanmax(abs(last_val_mean)) + common_highest_val = round(common_highest_val, 1) + + # get thresholds for WARN (max_vm1) and FAIL (max_vm2) + thresholds = self.parameters.get('THRESHOLDS') + max_vm = thresholds.get('max_vm') + if not max_vm: + return + max_vm1, max_vm2 = max_vm + + # change status depending on common_highest_val + if common_highest_val < max_vm1: + self.status_ok(key, detailed_message=f'{common_highest_val}V') + elif max_vm1 <= common_highest_val < max_vm2: + self.warn(key=key, + detailed_message=f'Warning raised for mass centering. Highest val {common_highest_val}V', ) + else: + self.error(key=key, + detailed_message=f'Fail status for mass centering. Highest val {common_highest_val}V',) + + if self.verbosity > 1: + self.print(40 * '-') + self.print('Performing mass position check', flush=False) + self.print(f'Average mass position at {common_highest_val}', flush=False) + def pb_power_analysis(self, channel='EX2', pb_dict_key='pb_SOH2'): """ Analyse EX2 channel of PowBox """ keys = ['230V', '12V'] @@ -1058,6 +1107,23 @@ class StatusOther(Status): return message, detailed_message +def common_mass_trace(st): + traces = st.traces + if not len(traces) == 3: + return + check_keys = ['sampling_rate', 'starttime', 'endtime', 'npts'] + # check if values of the above keys are identical for all traces + for c_key in check_keys: + if not traces[0].stats.get(c_key) == traces[1].stats.get(c_key) == traces[2].stats.get(c_key): + return + max_1_2 = np.fmax(abs(traces[0]), abs(traces[1])) + abs_max = np.fmax(max_1_2, abs(traces[2])) + return_trace = traces[0].copy() + return_trace.data = abs_max + return return_trace + + + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Call survBot') parser.add_argument('-html', dest='html_path', default=None, help='filepath for HTML output') diff --git a/utils.py b/utils.py index b9fb78d..105143d 100644 --- a/utils.py +++ b/utils.py @@ -10,9 +10,11 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False): bg_color = get_time_delay_color(message, dt_thresh) elif check_key == 'temp': bg_color = get_temp_color(message) + elif check_key == 'mass': + bg_color = get_mass_color(message) else: if status.is_warn: - bg_color = get_color('WARNX')(status.count) + bg_color = get_warn_color(status.count) elif status.is_error: bg_color = get_color('FAIL') else: @@ -30,7 +32,6 @@ def get_color(key): colors_dict = {'FAIL': (255, 50, 0, 255), 'NO DATA': (255, 255, 125, 255), 'WARN': (255, 255, 80, 255), - 'WARNX': lambda x: (min([255, 200 + x ** 2]), 255, 80, 255), 'OK': (125, 255, 125, 255), 'undefined': (230, 230, 230, 255)} return colors_dict.get(key) @@ -45,6 +46,18 @@ def get_time_delay_color(dt, dt_thresh): return get_color('FAIL') +def get_warn_color(count): + color = (min([255, 200 + count ** 2]), 255, 80, 255) + return color + + +def get_mass_color(message): + # can change this to something else if wanted. This way it always returns get_color (without warn count) + if isinstance(message, (float, int)): + return get_color('OK') + return get_color(message) + + def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'): """ Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """ if type(temp) in [str]: @@ -60,9 +73,8 @@ def modify_stream_for_plot(st, parameters): ch_units = parameters.get('CHANNEL_UNITS') ch_transf = parameters.get('CHANNEL_TRANSFORM') - # if either of both are defined make copy - if ch_units or ch_transf: - st = st.copy() + # make a copy + st = st.copy() # modify trace for plotting by multiplying unit factor (e.g. 1e-3 mV to V) if ch_units: @@ -71,6 +83,7 @@ def modify_stream_for_plot(st, parameters): unit_factor = ch_units.get(channel) if unit_factor: tr.data = tr.data * float(unit_factor) + # modify trace for plotting by other arithmetic expressions if ch_transf: for tr in st: @@ -79,6 +92,10 @@ def modify_stream_for_plot(st, parameters): if transf: tr.data = transform_trace(tr.data, transf) + # change channel IDs to prevent re-sorting in obspy routine + for index, trace in enumerate(st): + trace.id = f'trace {index + 1}: {trace.id}' + return st @@ -142,4 +159,4 @@ def trace_yticks(fig, parameters, verbosity=0): yticks = list(range(ymin, ymax + step, step)) ax.set_yticks(yticks) - ax.set_ylim(ymin - step, ymax + step) \ No newline at end of file + ax.set_ylim(ymin - 0.33 * step, ymax + 0.33 * step) \ No newline at end of file From b17ee1288ca3b221b5baeb096d6f3b0de9d85809 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 21 Dec 2022 11:57:37 +0100 Subject: [PATCH 08/40] [minor] try re-read yaml in case it failed --- survBot.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/survBot.py b/survBot.py index 17a8c83..ee4c211 100755 --- a/survBot.py +++ b/survBot.py @@ -36,9 +36,16 @@ CLR = "\x1B[0K" deg_str = '\N{DEGREE SIGN}C' -def read_yaml(file_path): - with open(file_path, "r") as f: - return yaml.safe_load(f) +def read_yaml(file_path, n_read=3): + for index in range(n_read): + try: + with open(file_path, "r") as f: + params = yaml.safe_load(f) + except Exception as e: + print(f'Could not read parameters file: {e}.\nWill try again {n_read - index - 1} time(s).') + time.sleep(10) + continue + return params def nsl_from_id(nwst_id): From 7d5f9cf516394fb208d0283217756833b32ef273 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 21 Dec 2022 12:51:01 +0100 Subject: [PATCH 09/40] [update] re-worked channel definition in parameters.yaml, each channel now has its own dictionary with optional plotting flags [WIP] clock quality work in progress, currently disabled --- parameters.yaml | 104 +++++++++++++++++++++++++++++------------------- survBot.py | 26 +++++++----- survBotGUI.py | 5 ++- utils.py | 104 ++++++++++++++++++++++++++++++++++-------------- 4 files changed, 158 insertions(+), 81 deletions(-) diff --git a/parameters.yaml b/parameters.yaml index ca98506..a33dbfe 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -3,8 +3,6 @@ datapath: "/data/SDS/" # SC3 Datapath networks: ["1Y", "HA"] # select networks, list or str stations: "*" # select stations, list or str locations: "*" # select locations, list or str -channels: ["EX1", "EX2", "EX3", "VEI", - "VM1", "VM2", "VM3", "LCQ"] # Specify SOH channels, currently supported EX[1-3], VEI and VM[1-3] stations_blacklist: ["TEST", "EREA"] # exclude these stations networks_blacklist: [] # exclude these networks interval: 60 # Perform checks every x seconds @@ -44,10 +42,73 @@ THRESHOLDS: low_volt: 12 # min voltage for low voltage warning high_volt: 14.8 # max voltage for over voltage warning unclassified: 5 # min voltage samples not classified for warning - max_vm: [1.5, 2.5] # thresholds for mass offset (warn, fail) + max_vm_warn: 1.5 # threshold for mass offset (warn), fail) + max_vm_fail: 2.5 # threshold for mass offset (warn), fail) clockquality_warn: 90 # clock quality ranges from 0 % to 100 % with 100 % being the best level clockquality_fail: 70 + +# ---------------------------------- Specification of input channels --------------------------------------------------- +# Currently supported: EX[1-3], VEI, VM[1-3], LCQ +# +# For each channel a factor 'unit' for unit conversion (e.g. to SI) can be provided, as well as a 'name' +# and 'ticks' [ymin, ymax, ystep] for plotting. +# 'warn' and 'fail' plot horizontal lines in corresponding colors (can be str in TRESHOLDS, int/float or iterable) +# +# 'transform' can be provided for plotting to perform arithmetic operations in given order, e.g.: +# transform: - ["*", 20] +# - ["-", 20] +# --> PBox EX1 V to deg C: 20 * x -20 +CHANNELS: + EX1: + unit: 1e-6 + name: "Temperature (°C)" + ticks: [-10, 50, 10] + transform: + - ["*", 20] + - ["-", 20] + warn: "max_temp" + EX2: + unit: 1e-6 + name: "230V/12V (V)" + ticks: [1, 5, 1] + warn: [2, 3, 4, 4.5, 5] + EX3: + unit: 1e-6 + name: "Rout/Charge (V)" + ticks: [1, 5, 1] + warn: [2, 2.5, 3, 4, 5] + VEI: + unit: 1e-3 + name: "Logger (V)" + ticks: [9, 15, 1] + warn: ["low_volt", "high_volt"] + fail: 10.5 + VM1: + unit: 1e-6 + name: "Mass 1 (V)" + ticks: [-2.5, 2.5, 1] + warn: [-1.5, 1.5] + fail: [-2.5, 2.5] + VM2: + unit: 1e-6 + name: "Mass 2 (V)" + ticks: [-2.5, 2.5, 1] + warn: [-1.5, 1.5] + fail: [-2.5, 2.5] + VM3: + unit: 1e-6 + name: "Mass 3 (V)" + ticks: [-2.5, 2.5, 1] + warn: [-1.5, 1.5] + fail: [-2.5, 2.5] + LCQ: + name: "Clock Q (%)" + ticks: [0, 100, 20] + warn: "clockquality_warn" + fail: "clockquality_fail" + + # ---------------------------------------- OPTIONAL PARAMETERS --------------------------------------------------------- # add links to html table with specified key as column and value as relative link, interpretable string parameters: @@ -64,40 +125,3 @@ EMAIL: sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender stations_blacklist: ['GR33'] # do not send emails for specific stations networks_blacklist: [] # do not send emails for specific network - -# names for plotting of the above defined parameter "channels" in the same order -channel_names: ["Temperature (°C)", - "230V/12V (V)", - "Rout/Charge (V)", - "Logger (V)", - "Mass 1 (V)", - "Mass 2 (V)", - "Mass 3 (V)", - "Clock Q (%)"] - -# specify y-ticks (and ylims) giving, (ymin, ymax, step) for each of the above channels (0: default) -CHANNEL_TICKS: - - [-10, 50, 10] - - [1, 5, 1] - - [1, 5, 1] - - [9, 15, 1] - - [-2, 2, 1] - - [-2, 2, 1] - - [-2, 2, 1] - - [0, 100, 20] - -# Factor for channel to SI-units (for plotting) -CHANNEL_UNITS: - EX1: 1e-6 - EX2: 1e-6 - EX3: 1e-6 - VEI: 1e-3 - VM1: 1e-6 - VM2: 1e-6 - VM3: 1e-6 - -# Transform channel for plotting, perform arithmetic operations in given order, e.g.: PBox EX1 V to deg C: 20 * x -20 -CHANNEL_TRANSFORM: - EX1: - - ["*", 20] - - ["-", 20] \ No newline at end of file diff --git a/survBot.py b/survBot.py index ee4c211..c9721ac 100755 --- a/survBot.py +++ b/survBot.py @@ -19,7 +19,7 @@ from obspy.clients.filesystem.sds import Client from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str, \ init_html_table, finish_html_table -from utils import get_bg_color, modify_stream_for_plot, trace_ylabels, trace_yticks +from utils import get_bg_color, modify_stream_for_plot, trace_yticks, trace_thresholds try: import smtplib @@ -68,7 +68,7 @@ def fancy_timestr(dt, thresh=600, modif='+'): class SurveillanceBot(object): def __init__(self, parameter_path, outpath_html=None): - self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'mass', 'clock', 'temp', 'other'] + self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'mass', 'temp', 'other'] self.parameter_path = parameter_path self.update_parameters() self.starttime = UTCDateTime() @@ -92,6 +92,8 @@ class SurveillanceBot(object): def update_parameters(self): self.parameters = read_yaml(self.parameter_path) + # add channels to list in parameters dicitonary + self.parameters['channels'] = list(self.parameters.get('CHANNELS').keys()) self.reread_parameters = self.parameters.get('reread_parameters') self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')] self.verbosity = self.parameters.get('verbosity') @@ -348,8 +350,9 @@ class SurveillanceBot(object): try: st = modify_stream_for_plot(st, parameters=self.parameters) st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full') - trace_ylabels(fig, self.parameters, self.verbosity) + # trace_ylabels(fig, self.parameters, self.verbosity) trace_yticks(fig, self.parameters, self.verbosity) + trace_thresholds(fig, self.parameters, self.verbosity) except Exception as e: print(f'Could not generate plot for {nwst_id}:') print(traceback.format_exc()) @@ -441,6 +444,9 @@ class SurveillanceBot(object): print(f'Could not write HTML table to {fnout}:') print(traceback.format_exc()) + if self.verbosity: + print(f'Wrote html table to {fnout}') + def update_status_message(self): timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600)) self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \ @@ -693,7 +699,7 @@ class StationQC(object): self.pb_power_analysis() self.pb_rout_charge_analysis() self.mass_analysis() - self.clock_quality_analysis() + #self.clock_quality_analysis() def return_print_analysis(self): items = [self.nwst_id] @@ -860,17 +866,17 @@ class StationQC(object): common_highest_val = np.nanmax(abs(last_val_mean)) common_highest_val = round(common_highest_val, 1) - # get thresholds for WARN (max_vm1) and FAIL (max_vm2) + # get thresholds for WARN (max_vm_warn) and FAIL (max_vm_fail) thresholds = self.parameters.get('THRESHOLDS') - max_vm = thresholds.get('max_vm') - if not max_vm: + max_vm_warn = thresholds.get('max_vm_warn') + max_vm_fail = thresholds.get('max_vm_fail') + if not max_vm_warn or not max_vm_fail: return - max_vm1, max_vm2 = max_vm # change status depending on common_highest_val - if common_highest_val < max_vm1: + if common_highest_val < max_vm_warn: self.status_ok(key, detailed_message=f'{common_highest_val}V') - elif max_vm1 <= common_highest_val < max_vm2: + elif max_vm_warn <= common_highest_val < max_vm_fail: self.warn(key=key, detailed_message=f'Warning raised for mass centering. Highest val {common_highest_val}V', ) else: diff --git a/survBotGUI.py b/survBotGUI.py index 2f93e6b..e0e8fee 100755 --- a/survBotGUI.py +++ b/survBotGUI.py @@ -34,7 +34,7 @@ from obspy import UTCDateTime from survBot import SurveillanceBot from write_utils import * -from utils import get_bg_color, modify_stream_for_plot, trace_ylabels, trace_yticks +from utils import get_bg_color, modify_stream_for_plot, trace_yticks, trace_thresholds try: from rest_api.utils import get_station_iccid @@ -316,8 +316,9 @@ class MainWindow(QtWidgets.QMainWindow): self.plot_widget.setWindowTitle(nwst_id) st = modify_stream_for_plot(st, parameters=self.parameters) st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig) - trace_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters) + # trace_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters) trace_yticks(fig=self.plot_widget.canvas.fig, parameters=self.parameters) + trace_thresholds(fig=self.plot_widget.canvas.fig, parameters=self.parameters) self.plot_widget.show() def notification(self, text): diff --git a/utils.py b/utils.py index 105143d..53bd027 100644 --- a/utils.py +++ b/utils.py @@ -2,6 +2,9 @@ # -*- coding: utf-8 -*- import matplotlib +import numpy as np + +from obspy import Stream def get_bg_color(check_key, status, dt_thresh=None, hex=False): @@ -37,6 +40,11 @@ def get_color(key): return colors_dict.get(key) +def get_color_mpl(key): + color_tup = get_color(key) + return np.array([color/255. for color in color_tup]) + + def get_time_delay_color(dt, dt_thresh): """ Set color of time delay after thresholds specified in self.dt_thresh """ if dt < dt_thresh[0]: @@ -68,33 +76,43 @@ def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'): return rgba -def modify_stream_for_plot(st, parameters): +def modify_stream_for_plot(input_stream, parameters): """ copy (if necessary) and modify stream for plotting """ - ch_units = parameters.get('CHANNEL_UNITS') - ch_transf = parameters.get('CHANNEL_TRANSFORM') # make a copy - st = st.copy() + st = Stream() - # modify trace for plotting by multiplying unit factor (e.g. 1e-3 mV to V) - if ch_units: - for tr in st: - channel = tr.stats.channel - unit_factor = ch_units.get(channel) - if unit_factor: - tr.data = tr.data * float(unit_factor) + channels_dict = parameters.get('CHANNELS') - # modify trace for plotting by other arithmetic expressions - if ch_transf: - for tr in st: - channel = tr.stats.channel - transf = ch_transf.get(channel) - if transf: - tr.data = transform_trace(tr.data, transf) + # iterate over all channels and put them to new stream in order + for index, ch_tup in enumerate(channels_dict.items()): + # unpack tuple from items + channel, channel_dict = ch_tup - # change channel IDs to prevent re-sorting in obspy routine - for index, trace in enumerate(st): - trace.id = f'trace {index + 1}: {trace.id}' + # get correct channel from stream + st_sel = input_stream.select(channel=channel) + # in case there are != 1 there is ambiguity + if not len(st_sel) == 1: + continue + + # make a copy to not modify original stream! + tr = st_sel[0].copy() + + # multiply with conversion factor for unit + unit_factor = channel_dict.get('unit') + if unit_factor: + tr.data = tr.data * float(unit_factor) + + # apply transformations if provided + transform = channel_dict.get('transform') + if transform: + tr.data = transform_trace(tr.data, transform) + + # modify trace id to maintain plotting order + name = channel_dict.get('name') + tr.id = f'trace {index + 1}: {name} - {tr.id}' + + st.append(tr) return st @@ -124,10 +142,8 @@ def transform_trace(data, transf): def trace_ylabels(fig, parameters, verbosity=0): """ Adds channel names to y-axis if defined in parameters. - Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is - difficult to assess the correct order from Obspy plotting routing. """ - names = parameters.get('channel_names') + names = [channel.get('name') for channel in parameters.get('CHANNELS').values()] if not names: # or not len(st.traces): return if not len(names) == len(fig.axes): @@ -142,10 +158,8 @@ def trace_ylabels(fig, parameters, verbosity=0): def trace_yticks(fig, parameters, verbosity=0): """ Adds channel names to y-axis if defined in parameters. - Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is - difficult to assess the correct order from Obspy plotting routing. """ - ticks = parameters.get('CHANNEL_TICKS') + ticks = [channel.get('ticks') for channel in parameters.get('CHANNELS').values()] if not ticks: return if not len(ticks) == len(fig.axes): @@ -157,6 +171,38 @@ def trace_yticks(fig, parameters, verbosity=0): continue ymin, ymax, step = ytick_tripple - yticks = list(range(ymin, ymax + step, step)) + yticks = list(np.arange(ymin, ymax + step, step)) ax.set_yticks(yticks) - ax.set_ylim(ymin - 0.33 * step, ymax + 0.33 * step) \ No newline at end of file + ax.set_ylim(ymin - 0.33 * step, ymax + 0.33 * step) + + +def trace_thresholds(fig, parameters, verbosity=0): + """ + Adds channel thresholds (warn, fail) to y-axis if defined in parameters. + """ + if verbosity > 0: + print('Plotting trace thresholds') + + keys_colors = {'warn': dict(color=0.8 * get_color_mpl('WARN'), linestyle=(0, (5, 10)), alpha=0.5, linewidth=0.7), + 'fail': dict(color=0.8 * get_color_mpl('FAIL'), linestyle='solid', alpha=0.5, linewidth=0.7)} + + for key, kwargs in keys_colors.items(): + channel_threshold_list = [channel.get(key) for channel in parameters.get('CHANNELS').values()] + if not channel_threshold_list: + continue + plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs) + + +def plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs): + for channel_thresholds, ax in zip(channel_threshold_list, fig.axes): + if not channel_thresholds: + continue + + if not isinstance(channel_thresholds, (list, tuple)): + channel_thresholds = [channel_thresholds] + + for warn_thresh in channel_thresholds: + if isinstance(warn_thresh, str): + warn_thresh = parameters.get('THRESHOLDS').get(warn_thresh) + if type(warn_thresh in (float, int)): + ax.axhline(warn_thresh, **kwargs) \ No newline at end of file From c90b430fa84676235d34273adc9af11c8178b172 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 21 Dec 2022 15:48:18 +0100 Subject: [PATCH 10/40] [bugfix] corrected error check (plot on new FAIL status) [minor] message output --- survBot.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/survBot.py b/survBot.py index c9721ac..fea08c4 100755 --- a/survBot.py +++ b/survBot.py @@ -574,8 +574,11 @@ class StationQC(object): In all other cases return False. This also prevents sending status (e.g. mail) in case of program startup """ - if n_errors is not None: - n_errors = self.parameters.get('n_track') + 1 + if n_errors is None: + n_errors = self.parameters.get('n_track') + + # +1 to check whether n_errors +1 was no error (error is new) + n_errors += 1 previous_errors = self.status_track.get(key) # only if error list is filled n_track times @@ -878,10 +881,10 @@ class StationQC(object): self.status_ok(key, detailed_message=f'{common_highest_val}V') elif max_vm_warn <= common_highest_val < max_vm_fail: self.warn(key=key, - detailed_message=f'Warning raised for mass centering. Highest val {common_highest_val}V', ) + detailed_message=f'Warning raised for mass centering. Highest val (abs) {common_highest_val}V', ) else: self.error(key=key, - detailed_message=f'Fail status for mass centering. Highest val {common_highest_val}V',) + detailed_message=f'Fail status for mass centering. Highest val (abs) {common_highest_val}V',) if self.verbosity > 1: self.print(40 * '-') From bf000fe04263e871b52d1d2953823824a95fbc53 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 21 Dec 2022 16:03:10 +0100 Subject: [PATCH 11/40] [minor] visual tweaks --- parameters.yaml | 16 ++++++++-------- survBot.py | 9 +++++---- survBotGUI.py | 9 +++++---- utils.py | 17 +++++++++++++---- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/parameters.yaml b/parameters.yaml index a33dbfe..cfce588 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -62,7 +62,7 @@ THRESHOLDS: CHANNELS: EX1: unit: 1e-6 - name: "Temperature (°C)" + name: "PowBox Temperature (°C)" ticks: [-10, 50, 10] transform: - ["*", 20] @@ -70,40 +70,40 @@ CHANNELS: warn: "max_temp" EX2: unit: 1e-6 - name: "230V/12V (V)" + name: "PowBox 230V/12V (V)" ticks: [1, 5, 1] warn: [2, 3, 4, 4.5, 5] EX3: unit: 1e-6 - name: "Rout/Charge (V)" + name: "PowBox Router/Charger (V)" ticks: [1, 5, 1] warn: [2, 2.5, 3, 4, 5] VEI: unit: 1e-3 - name: "Logger (V)" + name: "Datalogger (V)" ticks: [9, 15, 1] warn: ["low_volt", "high_volt"] fail: 10.5 VM1: unit: 1e-6 - name: "Mass 1 (V)" + name: "Mass position W (V)" ticks: [-2.5, 2.5, 1] warn: [-1.5, 1.5] fail: [-2.5, 2.5] VM2: unit: 1e-6 - name: "Mass 2 (V)" + name: "Mass position V (V)" ticks: [-2.5, 2.5, 1] warn: [-1.5, 1.5] fail: [-2.5, 2.5] VM3: unit: 1e-6 - name: "Mass 3 (V)" + name: "Mass position U (V)" ticks: [-2.5, 2.5, 1] warn: [-1.5, 1.5] fail: [-2.5, 2.5] LCQ: - name: "Clock Q (%)" + name: "Clock quality (%)" ticks: [0, 100, 20] warn: "clockquality_warn" fail: "clockquality_fail" diff --git a/survBot.py b/survBot.py index fea08c4..d73019d 100755 --- a/survBot.py +++ b/survBot.py @@ -19,7 +19,7 @@ from obspy.clients.filesystem.sds import Client from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str, \ init_html_table, finish_html_table -from utils import get_bg_color, modify_stream_for_plot, trace_yticks, trace_thresholds +from utils import get_bg_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds try: import smtplib @@ -350,9 +350,10 @@ class SurveillanceBot(object): try: st = modify_stream_for_plot(st, parameters=self.parameters) st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full') - # trace_ylabels(fig, self.parameters, self.verbosity) - trace_yticks(fig, self.parameters, self.verbosity) - trace_thresholds(fig, self.parameters, self.verbosity) + # set_axis_ylabels(fig, self.parameters, self.verbosity) + set_axis_yticks(fig, self.parameters, self.verbosity) + set_axis_color(fig) + plot_axis_thresholds(fig, self.parameters, self.verbosity) except Exception as e: print(f'Could not generate plot for {nwst_id}:') print(traceback.format_exc()) diff --git a/survBotGUI.py b/survBotGUI.py index e0e8fee..3a21c97 100755 --- a/survBotGUI.py +++ b/survBotGUI.py @@ -34,7 +34,7 @@ from obspy import UTCDateTime from survBot import SurveillanceBot from write_utils import * -from utils import get_bg_color, modify_stream_for_plot, trace_yticks, trace_thresholds +from utils import get_bg_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds try: from rest_api.utils import get_station_iccid @@ -316,9 +316,10 @@ class MainWindow(QtWidgets.QMainWindow): self.plot_widget.setWindowTitle(nwst_id) st = modify_stream_for_plot(st, parameters=self.parameters) st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig) - # trace_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters) - trace_yticks(fig=self.plot_widget.canvas.fig, parameters=self.parameters) - trace_thresholds(fig=self.plot_widget.canvas.fig, parameters=self.parameters) + # set_axis_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters) + set_axis_yticks(fig=self.plot_widget.canvas.fig, parameters=self.parameters) + set_axis_color(fig=self.plot_widget.canvas.fig) + plot_axis_thresholds(fig=self.plot_widget.canvas.fig, parameters=self.parameters) self.plot_widget.show() def notification(self, text): diff --git a/utils.py b/utils.py index 53bd027..a88f158 100644 --- a/utils.py +++ b/utils.py @@ -110,7 +110,7 @@ def modify_stream_for_plot(input_stream, parameters): # modify trace id to maintain plotting order name = channel_dict.get('name') - tr.id = f'trace {index + 1}: {name} - {tr.id}' + tr.id = f'{index + 1}: {name} - {tr.id}' st.append(tr) @@ -139,7 +139,7 @@ def transform_trace(data, transf): return data -def trace_ylabels(fig, parameters, verbosity=0): +def set_axis_ylabels(fig, parameters, verbosity=0): """ Adds channel names to y-axis if defined in parameters. """ @@ -155,7 +155,16 @@ def trace_ylabels(fig, parameters, verbosity=0): ax.set_ylabel(channel_name) -def trace_yticks(fig, parameters, verbosity=0): +def set_axis_color(fig, color='grey'): + """ + Set all axes of figure to specific color + """ + for ax in fig.axes: + for key in ['bottom', 'top', 'right', 'left']: + ax.spines[key].set_color(color) + + +def set_axis_yticks(fig, parameters, verbosity=0): """ Adds channel names to y-axis if defined in parameters. """ @@ -176,7 +185,7 @@ def trace_yticks(fig, parameters, verbosity=0): ax.set_ylim(ymin - 0.33 * step, ymax + 0.33 * step) -def trace_thresholds(fig, parameters, verbosity=0): +def plot_axis_thresholds(fig, parameters, verbosity=0): """ Adds channel thresholds (warn, fail) to y-axis if defined in parameters. """ From 03616a2b7b64c94a8c1303de177b7f7d7dfdfb4d Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 22 Dec 2022 15:36:55 +0100 Subject: [PATCH 12/40] [update] refined and enabled clock quality check --- survBot.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/survBot.py b/survBot.py index d73019d..55089c8 100755 --- a/survBot.py +++ b/survBot.py @@ -68,7 +68,7 @@ def fancy_timestr(dt, thresh=600, modif='+'): class SurveillanceBot(object): def __init__(self, parameter_path, outpath_html=None): - self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'mass', 'temp', 'other'] + self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'mass', 'clock', 'temp', 'other'] self.parameter_path = parameter_path self.update_parameters() self.starttime = UTCDateTime() @@ -703,7 +703,7 @@ class StationQC(object): self.pb_power_analysis() self.pb_rout_charge_analysis() self.mass_analysis() - #self.clock_quality_analysis() + self.clock_quality_analysis() def return_print_analysis(self): items = [self.nwst_id] @@ -732,12 +732,13 @@ class StationQC(object): def get_last_occurrence(self, trace, indices): return self.get_time(trace, indices[-1]) - def clock_quality_analysis(self, channel='LCQ'): + def clock_quality_analysis(self, channel='LCQ', n_sample_average=10): """ Analyse clock quality """ key = 'clock' st = self.stream.select(channel=channel) trace = self.get_trace(st, key) - if not trace: return + if not trace: + return clockQuality = trace.data clockQuality_warn_level = self.parameters.get('THRESHOLDS').get('clockquality_warn') clockQuality_fail_level = self.parameters.get('THRESHOLDS').get('clockquality_fail') @@ -753,23 +754,30 @@ class StationQC(object): self.status_ok(key, detailed_message=f'ClockQuality={(clockQuality[-1])}') return + last_val_average = np.nanmean(clockQuality[-n_sample_average:]) + + # keep OK status if there are only minor warnings (lower warn level) warn_message = f'Trace {trace.get_id()}:' if len(clockQuality_warn) > 0: # try calculate number of warn peaks from gaps between indices n_qc_warn = self.calc_occurrences(clockQuality_warn) - detailed_message = warn_message + f' {n_qc_warn}x Qlock Quality less then {clockQuality_warn_level}%' \ + detailed_message = warn_message + f' {n_qc_warn}x Clock quality less then {clockQuality_warn_level}%' \ + self.get_last_occurrence_timestring(trace, clockQuality_warn) - self.warn(key, detailed_message=detailed_message, count=n_qc_warn, - last_occurrence=self.get_last_occurrence(trace, clockQuality_warn)) + self.status_ok(key, detailed_message=detailed_message) + # set WARN status for sever warnings in the past if len(clockQuality_fail) > 0: # try calculate number of fail peaks from gaps between indices n_qc_fail = self.calc_occurrences(clockQuality_fail) - detailed_message = warn_message + f' {n_qc_fail}x Qlock Quality less then {clockQuality_fail_level}%' \ + detailed_message = warn_message + f' {n_qc_fail}x Clock quality less then {clockQuality_fail_level}%' \ + self.get_last_occurrence_timestring(trace, clockQuality_fail) - self.error(key, detailed_message=detailed_message, count=n_qc_fail, + self.warn(key, detailed_message=detailed_message, count=n_qc_fail, last_occurrence=self.get_last_occurrence(trace, clockQuality_fail)) + # set FAIL state if last value is less than fail level + if last_val_average < clockQuality_fail_level: + self.error(key, detailed_message=f'ClockQuality={(clockQuality[-1])}') + def voltage_analysis(self, channel='VEI'): """ Analyse voltage channel for over/undervoltage """ key = 'voltage' @@ -862,7 +870,7 @@ class StationQC(object): # correct for channel unit for trace in st: - trace.data = trace.data * 1e-6 # hardcoded, change this? + trace.data = trace.data * 1e-6 # TODO: Here and elsewhere: hardcoded, change this? # calculate average of absolute maximum of mass offset of last n_samp_mean last_values = np.array([trace.data[-n_samp_mean:] for trace in st]) From bc70dc08160d53fcf55cf8b161151d9138f955c8 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 22 Dec 2022 15:56:32 +0100 Subject: [PATCH 13/40] [minor] soft-coded unit factor for channel analysis --- survBot.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/survBot.py b/survBot.py index 55089c8..6a9bcff 100755 --- a/survBot.py +++ b/survBot.py @@ -721,6 +721,15 @@ class StationQC(object): def return_analysis(self): return self.status_dict + def get_unit_factor(self, channel): + """ Get channel multiplier for unit from parameters. If none is specified return 1 """ + channel_params = self.parameters.get('CHANNELS').get(channel) + if channel_params: + multiplier = channel_params.get('unit') + if multiplier: + return float(multiplier) + return 1 + def get_last_occurrence_timestring(self, trace, indices): """ returns a nicely formatted string of the timedelta since program starttime and occurrence and abs time""" last_occur = self.get_last_occurrence(trace, indices) @@ -765,7 +774,7 @@ class StationQC(object): + self.get_last_occurrence_timestring(trace, clockQuality_warn) self.status_ok(key, detailed_message=detailed_message) - # set WARN status for sever warnings in the past + # set WARN status for severe warnings in the past if len(clockQuality_fail) > 0: # try calculate number of fail peaks from gaps between indices n_qc_fail = self.calc_occurrences(clockQuality_fail) @@ -785,7 +794,7 @@ class StationQC(object): trace = self.get_trace(st, key) if not trace: return - voltage = trace.data * 1e-3 + voltage = trace.data * self.get_unit_factor(channel) low_volt = self.parameters.get('THRESHOLDS').get('low_volt') high_volt = self.parameters.get('THRESHOLDS').get('high_volt') @@ -824,7 +833,7 @@ class StationQC(object): trace = self.get_trace(st, key) if not trace: return - voltage = trace.data * 1e-6 + voltage = trace.data * self.get_unit_factor(channel) thresholds = self.parameters.get('THRESHOLDS') temp = 20. * voltage - 20 # average temp @@ -870,7 +879,7 @@ class StationQC(object): # correct for channel unit for trace in st: - trace.data = trace.data * 1e-6 # TODO: Here and elsewhere: hardcoded, change this? + trace.data = trace.data * self.get_unit_factor(trace.stats.channel) # calculate average of absolute maximum of mass offset of last n_samp_mean last_values = np.array([trace.data[-n_samp_mean:] for trace in st]) @@ -908,7 +917,7 @@ class StationQC(object): if not trace: return - voltage = trace.data * 1e-6 + voltage = trace.data * self.get_unit_factor(channel) if self.verbosity > 1: self.print(40 * '-') self.print('Performing PowBox 12V/230V check (EX2)', flush=False) @@ -931,7 +940,7 @@ class StationQC(object): if not trace: return - voltage = trace.data * 1e-6 + voltage = trace.data * self.get_unit_factor(channel) if self.verbosity > 1: self.print(40 * '-') self.print('Performing PowBox Router/Charger check (EX3)', flush=False) From 24d15d9d55d95af8a56a43b9ab4445f52d06242d Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 22 Dec 2022 16:00:07 +0100 Subject: [PATCH 14/40] [fix] mutable default argument --- survBot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/survBot.py b/survBot.py index 6a9bcff..f4809a7 100755 --- a/survBot.py +++ b/survBot.py @@ -469,12 +469,14 @@ class SurveillanceBot(object): class StationQC(object): - def __init__(self, parent, stream, nsl, parameters, keys, starttime, verbosity, print_func, status_track={}): + def __init__(self, parent, stream, nsl, parameters, keys, starttime, verbosity, print_func, status_track=None): """ Station Quality Check class. :param nsl: dictionary containing network, station and location (key: str) :param parameters: parameters dictionary from parameters.yaml file """ + if status_track is None: + status_track = {} self.parent = parent self.stream = stream self.nsl = nsl From a89ea1b06d58e647a7f03fe3e3d5ed3fd36712bd Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 22 Dec 2022 16:00:37 +0100 Subject: [PATCH 15/40] [refactor] PEP8 naming convention --- survBot.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/survBot.py b/survBot.py index f4809a7..6c270ba 100755 --- a/survBot.py +++ b/survBot.py @@ -750,29 +750,29 @@ class StationQC(object): trace = self.get_trace(st, key) if not trace: return - clockQuality = trace.data - clockQuality_warn_level = self.parameters.get('THRESHOLDS').get('clockquality_warn') - clockQuality_fail_level = self.parameters.get('THRESHOLDS').get('clockquality_fail') + clock_quality = trace.data + clock_quality_warn_level = self.parameters.get('THRESHOLDS').get('clockquality_warn') + clock_quality_fail_level = self.parameters.get('THRESHOLDS').get('clockquality_fail') if self.verbosity > 1: self.print(40 * '-') self.print('Performing Clock Quality check', flush=False) - clockQuality_warn = np.where(clockQuality < clockQuality_warn_level)[0] - clockQuality_fail = np.where(clockQuality < clockQuality_fail_level)[0] + clockQuality_warn = np.where(clock_quality < clock_quality_warn_level)[0] + clockQuality_fail = np.where(clock_quality < clock_quality_fail_level)[0] if len(clockQuality_warn) == 0 and len(clockQuality_fail) == 0: - self.status_ok(key, detailed_message=f'ClockQuality={(clockQuality[-1])}') + self.status_ok(key, detailed_message=f'ClockQuality={(clock_quality[-1])}') return - last_val_average = np.nanmean(clockQuality[-n_sample_average:]) + last_val_average = np.nanmean(clock_quality[-n_sample_average:]) # keep OK status if there are only minor warnings (lower warn level) warn_message = f'Trace {trace.get_id()}:' if len(clockQuality_warn) > 0: # try calculate number of warn peaks from gaps between indices n_qc_warn = self.calc_occurrences(clockQuality_warn) - detailed_message = warn_message + f' {n_qc_warn}x Clock quality less then {clockQuality_warn_level}%' \ + detailed_message = warn_message + f' {n_qc_warn}x Clock quality less then {clock_quality_warn_level}%' \ + self.get_last_occurrence_timestring(trace, clockQuality_warn) self.status_ok(key, detailed_message=detailed_message) @@ -780,14 +780,14 @@ class StationQC(object): if len(clockQuality_fail) > 0: # try calculate number of fail peaks from gaps between indices n_qc_fail = self.calc_occurrences(clockQuality_fail) - detailed_message = warn_message + f' {n_qc_fail}x Clock quality less then {clockQuality_fail_level}%' \ + detailed_message = warn_message + f' {n_qc_fail}x Clock quality less then {clock_quality_fail_level}%' \ + self.get_last_occurrence_timestring(trace, clockQuality_fail) self.warn(key, detailed_message=detailed_message, count=n_qc_fail, last_occurrence=self.get_last_occurrence(trace, clockQuality_fail)) # set FAIL state if last value is less than fail level - if last_val_average < clockQuality_fail_level: - self.error(key, detailed_message=f'ClockQuality={(clockQuality[-1])}') + if last_val_average < clock_quality_fail_level: + self.error(key, detailed_message=f'ClockQuality={(clock_quality[-1])}') def voltage_analysis(self, channel='VEI'): """ Analyse voltage channel for over/undervoltage """ From a5486e19aa6cba6f81c883a092fa71bf3c69470f Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 3 Jan 2023 17:59:42 +0100 Subject: [PATCH 16/40] [update] first working version of gap check, testing needed [minor] soft-coded data channels --- parameters.yaml | 8 +++++-- survBot.py | 56 ++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/parameters.yaml b/parameters.yaml index cfce588..6ca41b4 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -44,8 +44,9 @@ THRESHOLDS: unclassified: 5 # min voltage samples not classified for warning max_vm_warn: 1.5 # threshold for mass offset (warn), fail) max_vm_fail: 2.5 # threshold for mass offset (warn), fail) - clockquality_warn: 90 # clock quality ranges from 0 % to 100 % with 100 % being the best level - clockquality_fail: 70 + clockquality_warn: 90 # warn level - clock quality ranges from 0 % to 100 % with 100 % being the best level + clockquality_fail: 70 # fail level + min_gap: 0.1 # minimum for gap declaration, should be > 0 [s] # ---------------------------------- Specification of input channels --------------------------------------------------- @@ -108,6 +109,9 @@ CHANNELS: warn: "clockquality_warn" fail: "clockquality_fail" +# specify data channels (can be additional to the above). From these channels only headers will be read +data_channels: ["HHZ", "HHN", "HHE"] + # ---------------------------------------- OPTIONAL PARAMETERS --------------------------------------------------------- diff --git a/survBot.py b/survBot.py index 6c270ba..2c7f9ee 100755 --- a/survBot.py +++ b/survBot.py @@ -68,7 +68,8 @@ def fancy_timestr(dt, thresh=600, modif='+'): class SurveillanceBot(object): def __init__(self, parameter_path, outpath_html=None): - self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'mass', 'clock', 'temp', 'other'] + self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'mass', 'clock', 'gaps', 'temp', + 'other'] self.parameter_path = parameter_path self.update_parameters() self.starttime = UTCDateTime() @@ -76,6 +77,7 @@ class SurveillanceBot(object): self.current_day = self.starttime.julday self.outpath_html = outpath_html self.filenames = [] + self.filenames_wf_data = [] self.filenames_read = [] self.station_list = [] self.analysis_print_list = [] @@ -83,6 +85,7 @@ class SurveillanceBot(object): self.status_track = {} self.dataStream = Stream() self.data = {} + self.gaps = [] self.print_count = 0 self.status_message = '' self.html_fig_dir = 'figures' @@ -92,8 +95,12 @@ class SurveillanceBot(object): def update_parameters(self): self.parameters = read_yaml(self.parameter_path) - # add channels to list in parameters dicitonary - self.parameters['channels'] = list(self.parameters.get('CHANNELS').keys()) + # add channels to list in parameters dictionary, also add data channels + channels = list(self.parameters.get('CHANNELS').keys()) + for channel in self.parameters.get('data_channels'): + if not channel in channels: + channels.append(channel) + self.parameters['channels'] = channels self.reread_parameters = self.parameters.get('reread_parameters') self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')] self.verbosity = self.parameters.get('verbosity') @@ -129,18 +136,25 @@ class SurveillanceBot(object): def get_filenames(self): self.filenames = [] + self.filenames_wf_data = [] time_now = UTCDateTime() t1 = time_now - self.parameters.get('timespan') * 24 * 3600 networks = self.parameters.get('networks') stations = self.parameters.get('stations') locations = self.parameters.get('locations') channels = self.parameters.get('channels') + channels_wf_data = self.parameters.get('data_channels') for network in networks: for station in stations: for location in locations: for channel in channels: - self.filenames += list(self.cl._get_filenames(network, station, location, channel, - starttime=t1, endtime=time_now)) + fnames = list(self.cl._get_filenames(network, station, location, channel, + starttime=t1, endtime=time_now)) + self.filenames += fnames + + # keep track of filenames with wf data (only read headers later) + if channel in channels_wf_data: + self.filenames_wf_data += fnames def read_data(self, re_read_at_hour=1, daily_overlap=2): ''' @@ -166,7 +180,11 @@ class SurveillanceBot(object): if filename in self.filenames_read: continue try: - st_new = read(filename, dtype=float) + # read only header of wf_data + if filename in self.filenames_wf_data: + st_new = read(filename, headonly=True) + else: + st_new = read(filename, dtype=float) # add file to read filenames to prevent re-reading in case it is not the current day (or end of # previous day) if not filename.endswith(f'{current_day:03}') and not ( @@ -176,7 +194,8 @@ class SurveillanceBot(object): print(f'Could not read file {filename}:', e) continue self.dataStream += st_new - self.dataStream.merge(fill_value=np.nan) + self.gaps = self.dataStream.get_gaps(min_gap=self.parameters['THRESHOLDS'].get('min_gap')) + self.dataStream.merge() # organise data in dictionary with key for each station for trace in self.dataStream: @@ -251,7 +270,7 @@ class SurveillanceBot(object): def get_station_delay(self, nwst_id): """ try to get station delay from SDS archive using client""" locations = ['', '0', '00'] - channels = ['HHZ', 'HHE', 'HHN'] + self.parameters.get('channels') + channels = self.parameters.get('channels') + self.parameters.get('data_channels') network, station = nwst_id.split('.')[:2] times = [] @@ -706,6 +725,7 @@ class StationQC(object): self.pb_rout_charge_analysis() self.mass_analysis() self.clock_quality_analysis() + self.gaps_analysis() def return_print_analysis(self): items = [self.nwst_id] @@ -981,6 +1001,26 @@ class StationQC(object): if volt_lvl == last_val or (volt_lvl == -1 and last_val < 1): self.error(key, detailed_message=f'Last PowBox voltage state {last_val}V: {message}') + def gaps_analysis(self, key='gaps'): + """ return gaps of a given nwst_id """ + + gaps = [] + for gap_list in self.parent.gaps: + nw_gap, st_gap = gap_list[:2] + if nw_gap == self.network and st_gap == self.station: + gaps.append(gap_list) + + if not gaps: + self.status_ok(key=key) + return + + detailed_message = '' + for gap_list in gaps: + text = '{}.{}.{}.{}: last sample - {}, next sample - {}, delta {}, samples {}\n'.format(*gap_list) + detailed_message += text + + self.warn(key=key, detailed_message=detailed_message, count=len(gaps)) + def calc_occurrences(self, ind_array): # try calculate number of voltage peaks/plateaus from gaps between indices if len(ind_array) == 0: From 908535fcc865bc25a9869120330b4f6a564fea4b Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 6 Jan 2023 10:38:07 +0100 Subject: [PATCH 17/40] [minor] modified email message for activity check, plot figures in same time intervals --- survBot.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/survBot.py b/survBot.py index 2c7f9ee..6921e93 100755 --- a/survBot.py +++ b/survBot.py @@ -365,10 +365,13 @@ class SurveillanceBot(object): fnout = self.get_fig_path_abs(nwst_id) st = self.data.get(nwst_id) if st: - # TODO: this section might fail, adding try-except block for analysis and to prevent program from crashing + # TODO: this section failed once, adding try-except block for analysis and to prevent program from crashing try: + endtime = UTCDateTime() + starttime = endtime - self.parameters.get('timespan') * 24 * 3600 st = modify_stream_for_plot(st, parameters=self.parameters) - st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full') + st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full', + starttime=starttime, endtime=endtime) # set_axis_ylabels(fig, self.parameters, self.verbosity) set_axis_yticks(fig, self.parameters, self.verbosity) set_axis_color(fig) @@ -705,7 +708,10 @@ class StationQC(object): dt_action = self.get_dt_for_action() interval = self.parameters.get('interval') if dt_action <= dt_active < dt_action + timedelta(seconds=interval): - self.send_mail(key, status_type='Inactive') + detailed_message = f'\n{self.nwst_id}\n\n' + for key, status in self.status_dict.items(): + detailed_message += f'{key}: {status.message}\n' + self.send_mail(key, status_type='Inactive', additional_message=detailed_message) def start(self): self.analyse_channels() @@ -718,7 +724,6 @@ class StationQC(object): self.print(150 * '#') self.print('This is StationQT. Calculating quality for station' ' {network}.{station}.{location}'.format(**self.nsl)) - self.activity_check() self.voltage_analysis() self.pb_temp_analysis() self.pb_power_analysis() @@ -727,6 +732,9 @@ class StationQC(object): self.clock_quality_analysis() self.gaps_analysis() + # activity check should be done last for useful status output (e.g. email) + self.activity_check() + def return_print_analysis(self): items = [self.nwst_id] for key in self.keys: From c7360488119ed51b20251327d7018b6a58242838 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 31 Jan 2023 16:10:57 +0100 Subject: [PATCH 18/40] [update] re-read only modified files --- survBot.py | 18 +++++++----------- utils.py | 2 +- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/survBot.py b/survBot.py index 6921e93..4799425 100755 --- a/survBot.py +++ b/survBot.py @@ -78,7 +78,7 @@ class SurveillanceBot(object): self.outpath_html = outpath_html self.filenames = [] self.filenames_wf_data = [] - self.filenames_read = [] + self.filenames_read_last_modif = {} self.station_list = [] self.analysis_print_list = [] self.analysis_results = {} @@ -156,12 +156,11 @@ class SurveillanceBot(object): if channel in channels_wf_data: self.filenames_wf_data += fnames - def read_data(self, re_read_at_hour=1, daily_overlap=2): + def read_data(self, re_read_at_hour=1): ''' read data method reads new data into self.stream :param re_read_at_hour: update archive at specified hour each day (hours up to 24) - :param daily_overlap: re-read data of previous day until specified hour (hours up to 24) ''' self.data = {} @@ -169,15 +168,16 @@ class SurveillanceBot(object): curr_time = UTCDateTime() current_day = curr_time.julday current_hour = curr_time.hour - yesterday = (curr_time - 24. * 3600.).julday if re_read_at_hour is not False and current_day != self.current_day and current_hour == re_read_at_hour: - self.filenames_read = [] + self.filenames_read_last_modif = {} self.dataStream = Stream() self.current_day = current_day # add all data to current stream for filename in self.filenames: - if filename in self.filenames_read: + # if file already read and last modification time is the same as of last read operation: continue + if self.filenames_read_last_modif.get(filename) == os.path.getmtime(filename): + print('Continue on file', filename) continue try: # read only header of wf_data @@ -185,11 +185,7 @@ class SurveillanceBot(object): st_new = read(filename, headonly=True) else: st_new = read(filename, dtype=float) - # add file to read filenames to prevent re-reading in case it is not the current day (or end of - # previous day) - if not filename.endswith(f'{current_day:03}') and not ( - filename.endswith(f'{yesterday:03}') and current_hour <= daily_overlap): - self.filenames_read.append(filename) + self.filenames_read_last_modif[filename] = os.path.getmtime(filename) except Exception as e: print(f'Could not read file {filename}:', e) continue diff --git a/utils.py b/utils.py index a88f158..f433d9c 100644 --- a/utils.py +++ b/utils.py @@ -155,7 +155,7 @@ def set_axis_ylabels(fig, parameters, verbosity=0): ax.set_ylabel(channel_name) -def set_axis_color(fig, color='grey'): +def set_axis_color(fig, color='0.8'): """ Set all axes of figure to specific color """ From 3a384fd7b59a5cf4f3fddc464f9fe4a44def6860 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 31 Jan 2023 16:12:07 +0100 Subject: [PATCH 19/40] [update] add external mail list for detailed specification of info mail recipients --- parameters.yaml | 2 ++ survBot.py | 30 +++++++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/parameters.yaml b/parameters.yaml index 6ca41b4..1ce4213 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -129,3 +129,5 @@ EMAIL: sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender stations_blacklist: ['GR33'] # do not send emails for specific stations networks_blacklist: [] # do not send emails for specific network + # specify recipients for single stations in a yaml: key = email-address, val = station list (e.g. [1Y.GR01, 1Y.GR02]) + external_mail_list: "mailing_list.yaml" diff --git a/survBot.py b/survBot.py index 4799425..8c0f4ae 100755 --- a/survBot.py +++ b/survBot.py @@ -5,6 +5,7 @@ __version__ = '0.1' __author__ = 'Marcel Paffrath' import os +import copy import traceback import yaml import argparse @@ -501,7 +502,8 @@ class StationQC(object): self.network = nsl.get('network') self.station = nsl.get('station') self.location = nsl.get('location') - self.parameters = parameters + # make a copy of parameters object to prevent accidental changes + self.parameters = copy.deepcopy(parameters) self.program_starttime = starttime self.verbosity = verbosity self.last_active = False @@ -639,10 +641,14 @@ class StationQC(object): sender = mail_params.get('sender') addresses = mail_params.get('addresses') + add_addresses = self.get_additional_mail_recipients(mail_params) + if add_addresses: + # create copy of addresses ( [:] ) to prevent changing original, general list with addresses + addresses = addresses[:] + list(add_addresses) server = mail_params.get('mailserver') if not sender or not addresses: if self.verbosity: - print('Mail sender or addresses not correctly defined. Return') + print('Mail sender or addresses not (correctly) defined. Return') return dt = self.get_dt_for_action() text = f'{key}: Status {status_type} longer than {dt}: ' + additional_message @@ -653,9 +659,27 @@ class StationQC(object): # send message via SMTP server s = smtplib.SMTP(server) - s.sendmail(sender, addresses, msg.as_string()) + s.send_message(msg) s.quit() + def get_additional_mail_recipients(self, mail_params): + """ return additional recipients from external mail list if this station (self.nwst_id) is specified """ + eml_filename = mail_params.get('external_mail_list') + if not eml_filename: + return [] + try: + with open(eml_filename) as fid: + address_dict = yaml.safe_load(fid) + except FileNotFoundError as e: + if self.verbosity: + print(e) + if not isinstance(address_dict, dict): + if self.verbosity: + print(f'Could not read dictionary from file {eml_filename}') + for address, nwst_ids in address_dict.items(): + if self.nwst_id in nwst_ids: + yield address + def get_dt_for_action(self): n_track = self.parameters.get('n_track') interval = self.parameters.get('interval') From d7cbbe6876943db9fda920b6ab6c17c5013c73f3 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 31 Jan 2023 17:37:17 +0100 Subject: [PATCH 20/40] [minor] add missing verbosity flag --- survBot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/survBot.py b/survBot.py index 8c0f4ae..367c0bf 100755 --- a/survBot.py +++ b/survBot.py @@ -178,7 +178,8 @@ class SurveillanceBot(object): for filename in self.filenames: # if file already read and last modification time is the same as of last read operation: continue if self.filenames_read_last_modif.get(filename) == os.path.getmtime(filename): - print('Continue on file', filename) + if self.verbosity > 0: + print('Continue on file', filename) continue try: # read only header of wf_data From 5632bab07b9321b46a0011dd5a6e460d98d856de Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 1 Feb 2023 14:49:26 +0100 Subject: [PATCH 21/40] [update] re-wrote html part, increased flexibility, can now send html via mail incl. figure --- parameters.yaml | 4 +- simulate_fail.json | 2 + survBot.py | 280 +++++++++++++++++++++++++++++++-------------- write_utils.py | 58 +++++++--- 4 files changed, 238 insertions(+), 106 deletions(-) create mode 100644 simulate_fail.json diff --git a/parameters.yaml b/parameters.yaml index 1ce4213..4d12005 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -7,7 +7,7 @@ stations_blacklist: ["TEST", "EREA"] # exclude these stations networks_blacklist: [] # exclude these networks interval: 60 # Perform checks every x seconds n_track: 300 # wait n_track * intervals before performing an action (i.e. send mail/end highlight status) -timespan: 7 # Check data of the recent x days +timespan: 1 # Check data of the recent x days verbosity: 0 # verbosity flag track_changes: True # tracks all changes since GUI startup by text highlighting (GUI only) warn_count: False # show number of warnings and errors in table @@ -125,7 +125,7 @@ add_links: # E-mail notifications EMAIL: mailserver: "localhost" - addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"] # list of mail addresses for info mails + addresses: ["marcel.paffrath@rub.de",]# "kasper.fischer@rub.de"] # list of mail addresses for info mails sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender stations_blacklist: ['GR33'] # do not send emails for specific stations networks_blacklist: [] # do not send emails for specific network diff --git a/simulate_fail.json b/simulate_fail.json new file mode 100644 index 0000000..33bd01e --- /dev/null +++ b/simulate_fail.json @@ -0,0 +1,2 @@ +{"1Y.GR01": "230V", +"1Y.GR03": "charger"} \ No newline at end of file diff --git a/survBot.py b/survBot.py index 367c0bf..9cd5be1 100755 --- a/survBot.py +++ b/survBot.py @@ -5,10 +5,12 @@ __version__ = '0.1' __author__ = 'Marcel Paffrath' import os +import io import copy import traceback import yaml import argparse +import json import time from datetime import timedelta @@ -18,13 +20,14 @@ import matplotlib.pyplot as plt from obspy import read, UTCDateTime, Stream from obspy.clients.filesystem.sds import Client -from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str, \ - init_html_table, finish_html_table +from write_utils import get_html_text, get_html_row, html_footer, get_html_header, get_print_title_str, \ + init_html_table, finish_html_table, get_mail_html_header, add_html_image from utils import get_bg_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds try: import smtplib - from email.mime.text import MIMEText + from email.message import EmailMessage + from email.utils import make_msgid mail_functionality = True except ImportError: @@ -50,10 +53,16 @@ def read_yaml(file_path, n_read=3): def nsl_from_id(nwst_id): + nwst_id = get_full_seed_id(nwst_id) network, station, location = nwst_id.split('.') return dict(network=network, station=station, location=location) +def get_full_seed_id(nwst_id): + seed_id = '{}.{}.{}'.format(*nwst_id.split('.'), '') + return seed_id + + def get_nwst_id(trace): stats = trace.stats return f'{stats.network}.{stats.station}.' # {stats.location}' @@ -91,6 +100,8 @@ class SurveillanceBot(object): self.status_message = '' self.html_fig_dir = 'figures' + self.active_figures = {} + self.cl = Client(self.parameters.get('datapath')) # TODO: Check if this has to be loaded again on update self.get_stations() @@ -355,13 +366,13 @@ class SurveillanceBot(object): for nwst_id in self.station_list: self.write_html_figure(nwst_id) - def write_html_figure(self, nwst_id): + def write_html_figure(self, nwst_id, save_bytes=False): """ Write figure for html for specified station """ self.check_fig_dir() fig = plt.figure(figsize=(16, 9)) - fnout = self.get_fig_path_abs(nwst_id) - st = self.data.get(nwst_id) + fnames_out = [self.get_fig_path_abs(nwst_id), io.BytesIO()] + st = self.data.get(get_full_seed_id(nwst_id)) if st: # TODO: this section failed once, adding try-except block for analysis and to prevent program from crashing try: @@ -383,84 +394,111 @@ class SurveillanceBot(object): f'Refreshed hourly or on FAIL status.') for ax in fig.axes: ax.grid(True, alpha=0.1) - fig.savefig(fnout, dpi=150., bbox_inches='tight') + for fnout in fnames_out: + fig.savefig(fnout, dpi=150., bbox_inches='tight') + # if needed save figure as virtual object (e.g. for mailing) + if save_bytes: + fnames_out[-1].seek(0) + self.active_figures[nwst_id] = fnames_out[-1] plt.close(fig) - def write_html_table(self, default_color='#e6e6e6', default_header_color='#999', hide_keys_mobile=('other')): + def get_html_class(self, hide_keys_mobile=None, status=None, check_key=None): + """ helper function for html class if a certain condition is fulfilled """ + html_class = None + if status and status.is_active: + html_class = 'blink-bg' + if hide_keys_mobile and check_key in hide_keys_mobile: + html_class = 'hidden-mobile' + return html_class - def get_html_class(status=None, check_key=None): - """ helper function for html class if a certain condition is fulfilled """ - html_class = None - if status and status.is_active: - html_class = 'blink-bg' - if check_key in hide_keys_mobile: - html_class = 'hidden-mobile' - return html_class + def make_html_table_header(self, default_header_color, hide_keys_mobile=None, add_links=True): + # First write header items + header = self.keys.copy() + # add columns for additional links + if add_links: + for key in self.add_links: + header.insert(-1, key) + header_items = [dict(text='Station', color=default_header_color)] + for check_key in header: + html_class = self.get_html_class(hide_keys_mobile, check_key=check_key) + item = dict(text=check_key, color=default_header_color, html_class=html_class) + header_items.append(item) + + return header, header_items + + def get_html_row_items(self, status_dict, nwst_id, header, default_color, hide_keys_mobile=None, + hyperlinks=True): + ''' create a html table row for the different keys ''' + + fig_name = self.get_fig_path_rel(nwst_id) + nwst_id_str = nwst_id.rstrip('.') + col_items = [dict(text=nwst_id_str, color=default_color, hyperlink=fig_name if hyperlinks else None, + bold=True, tooltip=f'Show plot of {nwst_id_str}')] + + for check_key in header: + if check_key in self.keys: + status = status_dict.get(check_key) + message, detailed_message = status.get_status_str() + + # get background color + dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh] + bg_color = get_bg_color(check_key, status, dt_thresh, hex=True) + if not bg_color: + bg_color = default_color + + # add degree sign for temp + if check_key == 'temp': + if not type(message) in [str]: + message = str(message) + deg_str + + html_class = self.get_html_class(hide_keys_mobile, status=status, check_key=check_key) + item = dict(text=str(message), tooltip=str(detailed_message), color=bg_color, + html_class=html_class) + elif check_key in self.add_links: + value = self.add_links.get(check_key).get('URL') + link_text = self.add_links.get(check_key).get('text') + if not value: + continue + nw, st = nwst_id.split('.')[:2] + hyperlink_dict = dict(nw=nw, st=st, nwst_id=nwst_id) + link = value.format(**hyperlink_dict) + item = dict(text=link_text, tooltip=link, hyperlink=link if hyperlinks else None, color=default_color) + else: + item = dict(text='', tooltip='') + col_items.append(item) + + return col_items + + def write_html_table(self, default_color='#e6e6e6', default_header_color='#999999', hide_keys_mobile=('other',)): self.check_html_dir() fnout = pjoin(self.outpath_html, 'survBot_out.html') if not fnout: return try: with open(fnout, 'w') as outfile: - write_html_header(outfile, self.refresh_period) - # write_html_table_title(outfile, self.parameters) - init_html_table(outfile) + outfile.write(get_html_header(self.refresh_period)) - # First write header items - header = self.keys.copy() - # add columns for additional links - for key in self.add_links: - header.insert(-1, key) - header_items = [dict(text='Station', color=default_header_color)] - for check_key in header: - html_class = get_html_class(check_key=check_key) - item = dict(text=check_key, color=default_header_color, html_class=html_class) - header_items.append(item) - write_html_row(outfile, header_items, html_key='th') + # write_html_table_title(self.parameters) + outfile.write(init_html_table()) - # Write all cells + # write html header row + header, header_items = self.make_html_table_header(default_header_color, hide_keys_mobile) + html_row = get_html_row(header_items, html_key='th') + outfile.write(html_row) + + # Write all cells (row after row) for nwst_id in self.station_list: - fig_name = self.get_fig_path_rel(nwst_id) - nwst_id_str = nwst_id.rstrip('.') - col_items = [dict(text=nwst_id_str, color=default_color, hyperlink=fig_name, - bold=True, tooltip=f'Show plot of {nwst_id_str}')] - for check_key in header: - if check_key in self.keys: - status_dict = self.analysis_results.get(nwst_id) - status = status_dict.get(check_key) - message, detailed_message = status.get_status_str() + # get list with column-wise items to write as a html row + status_dict = self.analysis_results.get(nwst_id) + col_items = self.get_html_row_items(status_dict, nwst_id, header, default_color, hide_keys_mobile) + outfile.write(get_html_row(col_items)) - # get background color - dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh] - bg_color = get_bg_color(check_key, status, dt_thresh, hex=True) - if not bg_color: - bg_color = default_color + outfile.write(finish_html_table()) - # add degree sign for temp - if check_key == 'temp': - if not type(message) in [str]: - message = str(message) + deg_str + outfile.write(get_html_text(self.status_message)) + outfile.write(html_footer()) - html_class = get_html_class(status=status, check_key=check_key) - item = dict(text=str(message), tooltip=str(detailed_message), color=bg_color, - html_class=html_class) - elif check_key in self.add_links: - value = self.add_links.get(check_key).get('URL') - link_text = self.add_links.get(check_key).get('text') - if not value: - continue - nw, st = nwst_id.split('.')[:2] - hyperlink_dict = dict(nw=nw, st=st, nwst_id=nwst_id) - link = value.format(**hyperlink_dict) - item = dict(text=link_text, tooltip=link, hyperlink=link, color=default_color) - col_items.append(item) - - write_html_row(outfile, col_items) - - finish_html_table(outfile) - write_html_text(outfile, self.status_message) - write_html_footer(outfile) except Exception as e: print(f'Could not write HTML table to {fnout}:') print(traceback.format_exc()) @@ -566,6 +604,7 @@ class StationQC(object): # self.status_dict[key] = current_status_message + status_message def error(self, key, detailed_message, last_occurrence=None, count=1): + send_mail = False new_error = StatusError(count=count, show_count=self.parameters.get('warn_count')) current_status = self.status_dict.get(key) if current_status.is_error: @@ -574,20 +613,23 @@ class StationQC(object): current_status = new_error # if error is new and not on program-startup set active and refresh plot (using parent class) if self.search_previous_errors(key, n_errors=1) is True: - self.parent.write_html_figure(self.nwst_id) + self.parent.write_html_figure(self.nwst_id, save_bytes=True) if self.verbosity: self.print(f'{UTCDateTime()}: {detailed_message}', flush=False) # do not send error mail if this is the first run (e.g. program startup) or state was already error (unchanged) if self.search_previous_errors(key) is True: - self.send_mail(key, status_type='FAIL', additional_message=detailed_message) - # set status to "inactive" after sending info mail + send_mail = True + # set status to "inactive" when info mail is sent current_status.is_active = False elif self.search_previous_errors(key) == 'active': current_status.is_active = True + # first update status, then send mail self._update_status(key, current_status, detailed_message, last_occurrence) + if send_mail: + self.send_mail(key, status_type='FAIL', additional_message=detailed_message) def search_previous_errors(self, key, n_errors=None): """ @@ -604,6 +646,11 @@ class StationQC(object): # +1 to check whether n_errors +1 was no error (error is new) n_errors += 1 + # simulate an error specified in json file (dictionary: {nwst_id: key} ) + if self._simulated_error_check(key) is True: + print(f'Simulating Error on {self.nwst_id}, {key}') + return True + previous_errors = self.status_track.get(key) # only if error list is filled n_track times if previous_errors and len(previous_errors) == n_errors: @@ -615,6 +662,14 @@ class StationQC(object): return 'active' return False + def _simulated_error_check(self, key, fname='simulate_fail.json'): + if not os.path.isfile(fname): + return + with open(fname) as fid: + d = json.load(fid) + if d.get(self.nwst_id) == key: + return True + def send_mail(self, key, status_type, additional_message=''): """ Send info mail using parameters specified in parameters file """ if not mail_functionality: @@ -653,33 +708,81 @@ class StationQC(object): return dt = self.get_dt_for_action() text = f'{key}: Status {status_type} longer than {dt}: ' + additional_message - msg = MIMEText(text) + + msg = EmailMessage() + msg['Subject'] = f'new message on station {self.nwst_id}' msg['From'] = sender msg['To'] = ', '.join(addresses) + msg.set_content(text) + + # html mail version + html_str = self.add_html_mail_body(text) + msg.add_alternative(html_str, subtype='html') + # send message via SMTP server s = smtplib.SMTP(server) s.send_message(msg) s.quit() + def add_html_mail_body(self, text, default_color='#e6e6e6'): + parent = self.parent + + header, header_items = parent.make_html_table_header('#999999', add_links=False) + col_items = parent.get_html_row_items(self.status_dict, self.nwst_id, header, default_color, hyperlinks=False) + + # set general status text + html_str = get_html_text(text) + + # init html header and table + html_str += get_mail_html_header() + html_str += init_html_table() + + # add table header and row of current station + html_str += get_html_row(header_items, html_key='th') + html_str += get_html_row(col_items) + + html_str += finish_html_table() + + if self.nwst_id in self.parent.active_figures.keys(): + fid = self.parent.active_figures.pop(self.nwst_id) + html_str += add_html_image(img_data=fid.read()) + + html_str += html_footer() + + return html_str + def get_additional_mail_recipients(self, mail_params): """ return additional recipients from external mail list if this station (self.nwst_id) is specified """ eml_filename = mail_params.get('external_mail_list') - if not eml_filename: - return [] - try: - with open(eml_filename) as fid: - address_dict = yaml.safe_load(fid) - except FileNotFoundError as e: + if eml_filename: + # try to open file + try: + with open(eml_filename, 'r') as fid: + address_dict = yaml.safe_load(fid) + + for address, nwst_ids in address_dict.items(): + if self.nwst_id in nwst_ids: + yield address + # file not existing + except FileNotFoundError as e: + if self.verbosity: + print(e) + # no dictionary + except AttributeError as e: + if self.verbosity: + print(f'Could not read dictionary from file {eml_filename}: {e}') + # other exceptions + except Exception as e: + if self.verbosity: + print(f'Could not open file {eml_filename}: {e}') + # no file specified + else: if self.verbosity: - print(e) - if not isinstance(address_dict, dict): - if self.verbosity: - print(f'Could not read dictionary from file {eml_filename}') - for address, nwst_ids in address_dict.items(): - if self.nwst_id in nwst_ids: - yield address + print('No external mail list set.') + + return [] def get_dt_for_action(self): n_track = self.parameters.get('n_track') @@ -756,6 +859,13 @@ class StationQC(object): # activity check should be done last for useful status output (e.g. email) self.activity_check() + self._simulate_error() + + def _simulate_error(self): + for key in self.keys: + if self._simulated_error_check(key): + self.error(key, 'SIMULATED ERROR') + def return_print_analysis(self): items = [self.nwst_id] for key in self.keys: diff --git a/write_utils.py b/write_utils.py index 785805d..c1ae268 100644 --- a/write_utils.py +++ b/write_utils.py @@ -1,16 +1,21 @@ +from base64 import b64encode from datetime import timedelta -def write_html_table_title(fobj, parameters): +def _convert_to_textstring(lst): + return '\n'.join(lst) + + +def get_html_table_title(parameters): title = get_print_title_str(parameters) - fobj.write(f'

{title}

\n') + return f'

{title}

\n' -def write_html_text(fobj, text): - fobj.write(f'

{text}

\n') +def get_html_text(text): + return f'

{text}

\n' -def write_html_header(fobj, refresh_rate=10): +def get_html_header(refresh_rate=10): header = ['', '', '', @@ -21,28 +26,42 @@ def write_html_header(fobj, refresh_rate=10): '', '', ''] - for item in header: - fobj.write(item + '\n') + header = _convert_to_textstring(header) + return header -def init_html_table(fobj): - fobj.write('\n') +def get_mail_html_header(): + header = ['', + '', + '', + ''] + header = _convert_to_textstring(header) + return header -def finish_html_table(fobj): - fobj.write('
\n') +def init_html_table(): + return '\n' -def write_html_footer(fobj): +def finish_html_table(): + return '
\n' + + +def html_footer(): footer = ['', ''] - for item in footer: - fobj.write(item + '\n') + footer = _convert_to_textstring(footer) + return footer -def write_html_row(fobj, items, html_key='td'): +def add_html_image(img_data, img_format='png'): + return f"""
\n""" + + +def get_html_row(items, html_key='td'): + row_string = '' default_space = ' ' - fobj.write(default_space + '\n') + row_string += default_space + '\n' for item in items: text = item.get('text') if item.get('bold'): @@ -57,9 +76,10 @@ def write_html_row(fobj, items, html_key='td'): image_str = f'' if hyperlink else '' html_class = item.get('html_class') class_str = f' class="{html_class}"' if html_class else '' - fobj.write(2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"> {image_str}' - + text + f'\n') - fobj.write(default_space + '\n') + row_string += 2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"> {image_str}'\ + + text + f'\n' + row_string += default_space + '\n' + return row_string def get_print_title_str(parameters): From a6c792b27170745b91a81173feeb03ccec034d05 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 16 Feb 2023 15:49:07 +0100 Subject: [PATCH 22/40] [bugfix] some bugfixes/tweaks --- parameters.yaml | 4 ++-- simulate_fail.json | 2 -- survBot.py | 7 +++++-- utils.py | 9 +++++---- write_utils.py | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) delete mode 100644 simulate_fail.json diff --git a/parameters.yaml b/parameters.yaml index 4d12005..4bc4d72 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -7,7 +7,7 @@ stations_blacklist: ["TEST", "EREA"] # exclude these stations networks_blacklist: [] # exclude these networks interval: 60 # Perform checks every x seconds n_track: 300 # wait n_track * intervals before performing an action (i.e. send mail/end highlight status) -timespan: 1 # Check data of the recent x days +timespan: 3 # Check data of the recent x days verbosity: 0 # verbosity flag track_changes: True # tracks all changes since GUI startup by text highlighting (GUI only) warn_count: False # show number of warnings and errors in table @@ -125,7 +125,7 @@ add_links: # E-mail notifications EMAIL: mailserver: "localhost" - addresses: ["marcel.paffrath@rub.de",]# "kasper.fischer@rub.de"] # list of mail addresses for info mails + addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"] # list of mail addresses for info mails sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender stations_blacklist: ['GR33'] # do not send emails for specific stations networks_blacklist: [] # do not send emails for specific network diff --git a/simulate_fail.json b/simulate_fail.json deleted file mode 100644 index 33bd01e..0000000 --- a/simulate_fail.json +++ /dev/null @@ -1,2 +0,0 @@ -{"1Y.GR01": "230V", -"1Y.GR03": "charger"} \ No newline at end of file diff --git a/survBot.py b/survBot.py index 9cd5be1..44c2a8b 100755 --- a/survBot.py +++ b/survBot.py @@ -69,7 +69,7 @@ def get_nwst_id(trace): def fancy_timestr(dt, thresh=600, modif='+'): - if dt > timedelta(seconds=thresh): + if isinstance(dt, timedelta) and dt > timedelta(seconds=thresh): value = f'{modif} ' + str(dt) + f' {modif}' else: value = str(dt) @@ -643,7 +643,7 @@ class StationQC(object): if n_errors is None: n_errors = self.parameters.get('n_track') - # +1 to check whether n_errors +1 was no error (error is new) + # +1 to check whether n_errors + 1 was no error (error is new) n_errors += 1 # simulate an error specified in json file (dictionary: {nwst_id: key} ) @@ -657,6 +657,9 @@ class StationQC(object): # if first entry was no error but all others are, return True (-> new Fail n_track times) if not previous_errors[0] and all(previous_errors[1:]): return True + # special case: n_errors set to 1 (+1) to check for upcoming error (refresh plot etc.), but not on startup + if not previous_errors[0] and n_errors == 2: + return True # in case previous_errors exists, last item is error but not all items are error, error still active elif previous_errors and previous_errors[-1] and not all(previous_errors): return 'active' diff --git a/utils.py b/utils.py index f433d9c..cacbb5d 100644 --- a/utils.py +++ b/utils.py @@ -47,10 +47,11 @@ def get_color_mpl(key): def get_time_delay_color(dt, dt_thresh): """ Set color of time delay after thresholds specified in self.dt_thresh """ - if dt < dt_thresh[0]: - return get_color('OK') - elif dt_thresh[0] <= dt < dt_thresh[1]: - return get_color('WARN') + if isinstance(dt, type(dt_thresh[0])): + if dt < dt_thresh[0]: + return get_color('OK') + elif dt_thresh[0] <= dt < dt_thresh[1]: + return get_color('WARN') return get_color('FAIL') diff --git a/write_utils.py b/write_utils.py index c1ae268..d571c2a 100644 --- a/write_utils.py +++ b/write_utils.py @@ -55,7 +55,7 @@ def html_footer(): def add_html_image(img_data, img_format='png'): - return f"""
\n""" + return f"""
\n""" def get_html_row(items, html_key='td'): From 20635e3433c13ae6327e39b991c2ecbfce3b2c77 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 16 Feb 2023 15:57:13 +0100 Subject: [PATCH 23/40] [new] add file mailing_list.yaml --- mailing_list.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 mailing_list.yaml diff --git a/mailing_list.yaml b/mailing_list.yaml new file mode 100644 index 0000000..5bae70a --- /dev/null +++ b/mailing_list.yaml @@ -0,0 +1,9 @@ +# specify mail addresses and station network ids for which information mails shall be sent, e.g.: +# "mail.address@provider.com, mail.address2@provider2.com": +# - 1Y.GR01 +# - 1Y.GR02 +# "mail.address3@provder.com": +# - 1Y.GR03 + +#"kasper.fischer@rub.de": +# - 1Y.GR01 From b875e63f837f82f3e9c92ef9b4f757fb765822bd Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 16 Feb 2023 16:05:12 +0100 Subject: [PATCH 24/40] [bugfix] check for upcoming error --- survBot.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/survBot.py b/survBot.py index 44c2a8b..4929c7a 100755 --- a/survBot.py +++ b/survBot.py @@ -612,7 +612,7 @@ class StationQC(object): else: current_status = new_error # if error is new and not on program-startup set active and refresh plot (using parent class) - if self.search_previous_errors(key, n_errors=1) is True: + if self.status_track.get(key) and not self.status_track.get(key)[-1]: self.parent.write_html_figure(self.nwst_id, save_bytes=True) if self.verbosity: @@ -657,9 +657,6 @@ class StationQC(object): # if first entry was no error but all others are, return True (-> new Fail n_track times) if not previous_errors[0] and all(previous_errors[1:]): return True - # special case: n_errors set to 1 (+1) to check for upcoming error (refresh plot etc.), but not on startup - if not previous_errors[0] and n_errors == 2: - return True # in case previous_errors exists, last item is error but not all items are error, error still active elif previous_errors and previous_errors[-1] and not all(previous_errors): return 'active' From 305ab25ab20af00f74e22a08d9234cbc61cd4c12 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 17 Feb 2023 14:16:38 +0100 Subject: [PATCH 25/40] [bugfix] slightly wrong if condition --- .gitignore | 1 + survBot.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index d65f12b..54ff8ca 100644 --- a/.gitignore +++ b/.gitignore @@ -211,3 +211,4 @@ flycheck_*.el /network-security.data +/__simulate_fail.json diff --git a/survBot.py b/survBot.py index 4929c7a..c895723 100755 --- a/survBot.py +++ b/survBot.py @@ -657,8 +657,8 @@ class StationQC(object): # if first entry was no error but all others are, return True (-> new Fail n_track times) if not previous_errors[0] and all(previous_errors[1:]): return True - # in case previous_errors exists, last item is error but not all items are error, error still active - elif previous_errors and previous_errors[-1] and not all(previous_errors): + # in case previous_errors exist, last item is error but not all items are error, error still active + if previous_errors and previous_errors[-1] and not all(previous_errors): return 'active' return False From 9d9ced7c83f13c13369b31389d71360be89ac007 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 17 Feb 2023 16:14:59 +0100 Subject: [PATCH 26/40] [bugfix] no warning for volt_lvl = -1 --- survBot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/survBot.py b/survBot.py index c895723..088d3fb 100755 --- a/survBot.py +++ b/survBot.py @@ -1128,7 +1128,7 @@ class StationQC(object): if message == 'OK': self.status_ok(key) continue - if volt_lvl > 1: + if volt_lvl != 1: n_occurrences = self.calc_occurrences(ind_array) self.warn(key=key, detailed_message=f'Trace {trace.get_id()}: ' From 477727018fa711da7a0974adc52dfe5c1017926e Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 23 Feb 2023 11:50:38 +0100 Subject: [PATCH 27/40] [minor] added call argument "parameter path" --- survBot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/survBot.py b/survBot.py index 088d3fb..4bf587a 100755 --- a/survBot.py +++ b/survBot.py @@ -1364,7 +1364,9 @@ class StatusOther(Status): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Call survBot') parser.add_argument('-html', dest='html_path', default=None, help='filepath for HTML output') + parser.add_argument('-parfile', dest='parfile', default='parameters.yaml', + help='parameter file (default: parameters.yaml)') args = parser.parse_args() - survBot = SurveillanceBot(parameter_path='parameters.yaml', outpath_html=args.html_path) + survBot = SurveillanceBot(parameter_path=args.parfile, outpath_html=args.html_path) survBot.start() From 5a4733b031093a1bb295fefb3feeb6d89a6f6b74 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 28 Feb 2023 11:25:43 +0100 Subject: [PATCH 28/40] [minor] only send mail on inactive when no FAIL state is present --- survBot.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/survBot.py b/survBot.py index 4bf587a..e39a7d6 100755 --- a/survBot.py +++ b/survBot.py @@ -829,14 +829,24 @@ class StationQC(object): return max(endtimes) def check_for_inactive_message(self, key, dt_active): + """ send mail if station is inactive longer than dt_action and no FAIL status is present """ + + # check if any error is present in status_dict (in that case an email is sent already) + if self.check_for_any_error(): + return + dt_action = self.get_dt_for_action() interval = self.parameters.get('interval') + if dt_action <= dt_active < dt_action + timedelta(seconds=interval): detailed_message = f'\n{self.nwst_id}\n\n' for key, status in self.status_dict.items(): detailed_message += f'{key}: {status.message}\n' self.send_mail(key, status_type='Inactive', additional_message=detailed_message) + def check_for_any_error(self): + return any([status.is_error for status in self.status_dict.values()]) + def start(self): self.analyse_channels() From 174a6148bfafb15d606db4e909efa097ea5b0a30 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 12 Apr 2023 13:13:04 +0200 Subject: [PATCH 29/40] [update] disconnect status if PowBox voltage < 1V --- .gitignore | 1 + survBot.py | 29 +++++++++++++++++++++++------ utils.py | 8 ++++++-- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 54ff8ca..3c5d5a8 100644 --- a/.gitignore +++ b/.gitignore @@ -212,3 +212,4 @@ flycheck_*.el /__simulate_fail.json +/mailing_list.yaml diff --git a/survBot.py b/survBot.py index e39a7d6..c824d15 100755 --- a/survBot.py +++ b/survBot.py @@ -603,9 +603,11 @@ class StationQC(object): # current_status_message = '' if current_status_message in [None, 'OK', '-'] else current_status_message + ' | ' # self.status_dict[key] = current_status_message + status_message - def error(self, key, detailed_message, last_occurrence=None, count=1): + def error(self, key, detailed_message, last_occurrence=None, count=1, disc=False): send_mail = False new_error = StatusError(count=count, show_count=self.parameters.get('warn_count')) + if disc: + new_error.set_disconnected() current_status = self.status_dict.get(key) if current_status.is_error: current_status.count += count @@ -831,8 +833,8 @@ class StationQC(object): def check_for_inactive_message(self, key, dt_active): """ send mail if station is inactive longer than dt_action and no FAIL status is present """ - # check if any error is present in status_dict (in that case an email is sent already) - if self.check_for_any_error(): + # check if any error is present in status_dict and not disconnected (in that case an email is sent already) + if self.check_for_any_error_no_dcn(): return dt_action = self.get_dt_for_action() @@ -844,8 +846,8 @@ class StationQC(object): detailed_message += f'{key}: {status.message}\n' self.send_mail(key, status_type='Inactive', additional_message=detailed_message) - def check_for_any_error(self): - return any([status.is_error for status in self.status_dict.values()]) + def check_for_any_error_no_dcn(self): + return any([status.is_error and not status.connection_error for status in self.status_dict.values()]) def start(self): self.analyse_channels() @@ -1147,8 +1149,10 @@ class StationQC(object): count=n_occurrences, last_occurrence=self.get_last_occurrence(trace, ind_array)) # if last_val == current voltage (which is not 1) -> FAIL or last_val < 1: PBox no data - if volt_lvl == last_val or (volt_lvl == -1 and last_val < 1): + if volt_lvl == last_val: self.error(key, detailed_message=f'Last PowBox voltage state {last_val}V: {message}') + elif volt_lvl == -1 and last_val < 1: + self.error(key, detailed_message=f'PowBox under 1V - connection error', disc=True) def gaps_analysis(self, key='gaps'): """ return gaps of a given nwst_id """ @@ -1282,19 +1286,23 @@ class StationQC(object): class Status(object): + """ Basic Status class. All status classes are derived from this class.""" def __init__(self, message=None, detailed_messages=None, count: int = 0, last_occurrence=None, show_count=True): if message is None: message = '-' if detailed_messages is None: detailed_messages = [] + self.show_count = show_count self.message = message self.messages = [message] self.detailed_messages = detailed_messages self.count = count self.last_occurrence = last_occurrence + self.is_warn = None self.is_error = None + self.connection_error = None self.is_other = False self.is_active = False @@ -1341,6 +1349,15 @@ class StatusError(Status): super(StatusError, self).__init__(message=message, count=count, last_occurrence=last_occurence, detailed_messages=detailed_messages, show_count=show_count) self.set_error() + self.default_message = message + + def set_disconnected(self, message='DCN'): + self.connection_error = True + self.message = message + + def set_connected(self): + self.connection_error = False + self.message = self.default_message class StatusOther(Status): diff --git a/utils.py b/utils.py index cacbb5d..2168c01 100644 --- a/utils.py +++ b/utils.py @@ -19,7 +19,10 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False): if status.is_warn: bg_color = get_warn_color(status.count) elif status.is_error: - bg_color = get_color('FAIL') + if status.connection_error: + bg_color = get_color('disc') + else: + bg_color = get_color('FAIL') else: bg_color = get_color(message) if not bg_color: @@ -36,7 +39,8 @@ def get_color(key): 'NO DATA': (255, 255, 125, 255), 'WARN': (255, 255, 80, 255), 'OK': (125, 255, 125, 255), - 'undefined': (230, 230, 230, 255)} + 'undefined': (230, 230, 230, 255), + 'disc': (255, 160, 40, 255),} return colors_dict.get(key) From 1b010ecb611e8281babb4f78f569538910beed69 Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 17 Apr 2023 17:20:35 +0200 Subject: [PATCH 30/40] [minor] add missing at the end of html link --- write_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/write_utils.py b/write_utils.py index d571c2a..ecf131f 100644 --- a/write_utils.py +++ b/write_utils.py @@ -73,11 +73,11 @@ def get_html_row(items, html_key='td'): # check for black background of headers (shouldnt happen anymore) color = '#e6e6e6' if color == '#000000' else color hyperlink = item.get('hyperlink') - image_str = f'' if hyperlink else '' + text_str = f' {text} ' if hyperlink else text html_class = item.get('html_class') class_str = f' class="{html_class}"' if html_class else '' - row_string += 2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"> {image_str}'\ - + text + f'\n' + row_string += 2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"> {text_str}'\ + + f'\n' row_string += default_space + '\n' return row_string From acc8575d707cf77dc880787ea0607d9d528badeb Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 17 Apr 2023 17:59:09 +0200 Subject: [PATCH 31/40] [update] can add global (station-independent) links on html web page --- parameters.yaml | 20 +++++++++++++------- survBot.py | 13 ++++++++++++- write_utils.py | 8 ++++++-- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/parameters.yaml b/parameters.yaml index 4bc4d72..c59f05d 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -1,17 +1,17 @@ # Parameters file for Surveillance Bot datapath: "/data/SDS/" # SC3 Datapath -networks: ["1Y", "HA"] # select networks, list or str +networks: ["1Y", "HA", "MK"] # select networks, list or str stations: "*" # select stations, list or str locations: "*" # select locations, list or str -stations_blacklist: ["TEST", "EREA"] # exclude these stations +stations_blacklist: ["TEST", "EREA", "DOMV"] # exclude these stations networks_blacklist: [] # exclude these networks interval: 60 # Perform checks every x seconds -n_track: 300 # wait n_track * intervals before performing an action (i.e. send mail/end highlight status) +n_track: 360 # wait n_track * intervals before performing an action (i.e. send mail/end highlight status) timespan: 3 # Check data of the recent x days verbosity: 0 # verbosity flag track_changes: True # tracks all changes since GUI startup by text highlighting (GUI only) -warn_count: False # show number of warnings and errors in table -min_sample: 3 # minimum samples for raising Warn/FAIL +warn_count: False # show number of warnings and errors in table +min_sample: 5 # minimum samples for raising Warn/FAIL dt_thresh: [300, 1800] # threshold (s) for timing delay colourisation (yellow/red) html_figures: True # Create html figure directory and links reread_parameters: True # reread parameters file (change parameters on runtime, not for itself/GUI refresh/datapath) @@ -72,12 +72,12 @@ CHANNELS: EX2: unit: 1e-6 name: "PowBox 230V/12V (V)" - ticks: [1, 5, 1] + ticks: [0, 5, 1] warn: [2, 3, 4, 4.5, 5] EX3: unit: 1e-6 name: "PowBox Router/Charger (V)" - ticks: [1, 5, 1] + ticks: [0, 5, 1] warn: [2, 2.5, 3, 4, 5] VEI: unit: 1e-3 @@ -122,6 +122,12 @@ add_links: slmon: {"URL": "../slmon/{nw}_{st}.html", "text": "show"} 24h-plot: {"URL": "../scheli/{nw}/{st}.png", "text": "plot"} +# add station-independent links below html table (list items separated with -) +add_global_links: + # for example: - {"text": "our homepage", "URL": "https://www.rub.de"} + - {"text": "show recent events on map", + "URL": "https://fdsnws.geophysik.ruhr-uni-bochum.de/map/?lat=39.5&lon=21&zoom=7&baselayer=mapnik"} + # E-mail notifications EMAIL: mailserver: "localhost" diff --git a/survBot.py b/survBot.py index c824d15..55c5083 100755 --- a/survBot.py +++ b/survBot.py @@ -20,7 +20,7 @@ import matplotlib.pyplot as plt from obspy import read, UTCDateTime, Stream from obspy.clients.filesystem.sds import Client -from write_utils import get_html_text, get_html_row, html_footer, get_html_header, get_print_title_str, \ +from write_utils import get_html_text, get_html_link, get_html_row, html_footer, get_html_header, get_print_title_str, \ init_html_table, finish_html_table, get_mail_html_header, add_html_image from utils import get_bg_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds @@ -120,9 +120,16 @@ class SurveillanceBot(object): self.networks_blacklist = self.parameters.get('networks_blacklist') self.refresh_period = self.parameters.get('interval') self.transform_parameters() + add_links = self.parameters.get('add_links') self.add_links = add_links if add_links else {} + add_global_links = self.parameters.get('add_global_links') + # in case user forgets "-" in parameters file + if isinstance(add_global_links, dict): + add_global_links = [add_global_links] + self.add_global_links = add_global_links if add_global_links else [] + def transform_parameters(self): for key in ['networks', 'stations', 'locations', 'channels']: parameter = self.parameters.get(key) @@ -496,6 +503,10 @@ class SurveillanceBot(object): outfile.write(finish_html_table()) + for dct in self.add_global_links: + link_str = get_html_link(dct.get('text'), dct.get('URL')) + outfile.write(get_html_text(link_str)) + outfile.write(get_html_text(self.status_message)) outfile.write(html_footer()) diff --git a/write_utils.py b/write_utils.py index ecf131f..dd35dd5 100644 --- a/write_utils.py +++ b/write_utils.py @@ -49,7 +49,7 @@ def finish_html_table(): def html_footer(): footer = ['', - ''] + '\n'] footer = _convert_to_textstring(footer) return footer @@ -58,6 +58,10 @@ def add_html_image(img_data, img_format='png'): return f"""
\n""" +def get_html_link(text, link): + return f' {text} ' + + def get_html_row(items, html_key='td'): row_string = '' default_space = ' ' @@ -73,7 +77,7 @@ def get_html_row(items, html_key='td'): # check for black background of headers (shouldnt happen anymore) color = '#e6e6e6' if color == '#000000' else color hyperlink = item.get('hyperlink') - text_str = f' {text} ' if hyperlink else text + text_str = get_html_link(text, hyperlink) if hyperlink else text html_class = item.get('html_class') class_str = f' class="{html_class}"' if html_class else '' row_string += 2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"> {text_str}'\ From 1983cc3b1ea8c862af6b9241e2055eed2c0d6ea5 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 19 Apr 2023 14:59:45 +0200 Subject: [PATCH 32/40] [new] add logo on html webpage --- parameters.yaml | 3 +++ survBot.py | 2 +- write_utils.py | 12 +++++++++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/parameters.yaml b/parameters.yaml index c59f05d..68b9b9d 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -128,6 +128,9 @@ add_global_links: - {"text": "show recent events on map", "URL": "https://fdsnws.geophysik.ruhr-uni-bochum.de/map/?lat=39.5&lon=21&zoom=7&baselayer=mapnik"} +# html logo at page bottom +html_logo: "figures/Logo_RUB_BLAU_rgb.png" + # E-mail notifications EMAIL: mailserver: "localhost" diff --git a/survBot.py b/survBot.py index 55c5083..261965a 100755 --- a/survBot.py +++ b/survBot.py @@ -508,7 +508,7 @@ class SurveillanceBot(object): outfile.write(get_html_text(link_str)) outfile.write(get_html_text(self.status_message)) - outfile.write(html_footer()) + outfile.write(html_footer(footer_logo=self.parameters.get('html_logo'))) except Exception as e: print(f'Could not write HTML table to {fnout}:') diff --git a/write_utils.py b/write_utils.py index dd35dd5..58a6747 100644 --- a/write_utils.py +++ b/write_utils.py @@ -47,9 +47,15 @@ def finish_html_table(): return '\n' -def html_footer(): - footer = ['', - '\n'] +def html_footer(footer_logo=None): + footer = [''] + if footer_logo: + logo_items = [f'
', + f' ', + f'
'] + footer += logo_items + footer.append('\n') + footer = _convert_to_textstring(footer) return footer From fb4d5c29296bd0cf6a173f5d6cfb99be8282ea28 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 19 Apr 2023 15:44:31 +0200 Subject: [PATCH 33/40] [minor] optimized stylesheets --- stylesheets/desktop.css | 11 +++++++++++ stylesheets/mobile.css | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/stylesheets/desktop.css b/stylesheets/desktop.css index 24e8015..7d0371a 100644 --- a/stylesheets/desktop.css +++ b/stylesheets/desktop.css @@ -2,11 +2,13 @@ body { background-color: #ffffff; place-items: center; text-align: center; + padding-bottom: 30px; } td { border-radius: 4px; padding: 0px; + white-space: nowrap; } th { @@ -37,3 +39,12 @@ a:hover { 50% { background-color: #ff3200;} 100% { background-color: #ffcc00;} } + +.footer { + position: fixed; + left: 0; + bottom: 0; + width: 100%; + height: 50px; + text-align: center; +} diff --git a/stylesheets/mobile.css b/stylesheets/mobile.css index ac7b549..dde8d87 100644 --- a/stylesheets/mobile.css +++ b/stylesheets/mobile.css @@ -2,11 +2,13 @@ body { background-color: #ffffff; place-items: center; text-align: center; + padding-bottom: 30px; } td { border-radius: 4px; padding: 10px 2px; + white-space: nowrap; } th { @@ -41,3 +43,12 @@ a:hover { 50% { background-color: #ff3200;} 100% { background-color: #ffee00;} } + +.footer { + position: fixed; + left: 0; + bottom: 0; + width: 100%; + height: 50px; + text-align: center; +} From 72e3ede52fdc3514eabff3aa8c08eb9d09e04928 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 19 Apr 2023 16:15:03 +0200 Subject: [PATCH 34/40] [minor] small adjustments --- parameters.yaml | 2 +- survBot.py | 11 ++++++++++- utils.py | 6 +++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/parameters.yaml b/parameters.yaml index 68b9b9d..7821e3c 100644 --- a/parameters.yaml +++ b/parameters.yaml @@ -128,7 +128,7 @@ add_global_links: - {"text": "show recent events on map", "URL": "https://fdsnws.geophysik.ruhr-uni-bochum.de/map/?lat=39.5&lon=21&zoom=7&baselayer=mapnik"} -# html logo at page bottom +# html logo at page bottom (path relative to html directory) html_logo: "figures/Logo_RUB_BLAU_rgb.png" # E-mail notifications diff --git a/survBot.py b/survBot.py index 261965a..2a06ac2 100755 --- a/survBot.py +++ b/survBot.py @@ -503,12 +503,21 @@ class SurveillanceBot(object): outfile.write(finish_html_table()) + # add optional links below html table for dct in self.add_global_links: link_str = get_html_link(dct.get('text'), dct.get('URL')) outfile.write(get_html_text(link_str)) + # add status message outfile.write(get_html_text(self.status_message)) - outfile.write(html_footer(footer_logo=self.parameters.get('html_logo'))) + + # write footer with optional logo + logo_file = self.parameters.get('html_logo') + if not os.path.isfile(logo_file): + print(f'Specified file {logo_file} not found.') + logo_file = None + + outfile.write(html_footer(footer_logo=logo_file)) except Exception as e: print(f'Could not write HTML table to {fnout}:') diff --git a/utils.py b/utils.py index 2168c01..a34de20 100644 --- a/utils.py +++ b/utils.py @@ -35,10 +35,10 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False): def get_color(key): # some GUI default colors - colors_dict = {'FAIL': (255, 50, 0, 255), + colors_dict = {'FAIL': (255, 85, 50, 255), 'NO DATA': (255, 255, 125, 255), 'WARN': (255, 255, 80, 255), - 'OK': (125, 255, 125, 255), + 'OK': (173, 255, 133, 255), 'undefined': (230, 230, 230, 255), 'disc': (255, 160, 40, 255),} return colors_dict.get(key) @@ -60,7 +60,7 @@ def get_time_delay_color(dt, dt_thresh): def get_warn_color(count): - color = (min([255, 200 + count ** 2]), 255, 80, 255) + color = (min([255, 220 + count ** 2]), 255, 80, 255) return color From a6475f2c3bd24834f5c1eeba63984b03b353517b Mon Sep 17 00:00:00 2001 From: Anne Mohr Date: Fri, 21 Apr 2023 10:56:52 +0200 Subject: [PATCH 35/40] tweak website design in html and css --- stylesheets/desktop.css | 14 +++++++++++- stylesheets/mobile.css | 12 ++++++++++- survBot.py | 7 +++--- utils.py | 47 +++++++++++++++++++++++++++++++++++------ write_utils.py | 9 ++++---- 5 files changed, 74 insertions(+), 15 deletions(-) diff --git a/stylesheets/desktop.css b/stylesheets/desktop.css index 7d0371a..aa23f6e 100644 --- a/stylesheets/desktop.css +++ b/stylesheets/desktop.css @@ -3,6 +3,16 @@ body { place-items: center; text-align: center; padding-bottom: 30px; + font-family: "Helvetica", "sans-serif" +} + +table { + position: relative +} + +#managerTable { + max-height: 800px; + overflow: auto; } td { @@ -15,10 +25,12 @@ th { background-color: #999; border-radius: 4px; padding: 3px 1px; + position: sticky; + top: 0; } a:link, a:visited { - background-color: #ccc; + background-color: #e8e8e8; color: #000; text-decoration: none; display: block; diff --git a/stylesheets/mobile.css b/stylesheets/mobile.css index dde8d87..1691271 100644 --- a/stylesheets/mobile.css +++ b/stylesheets/mobile.css @@ -3,6 +3,16 @@ body { place-items: center; text-align: center; padding-bottom: 30px; + font-family: "Helvetica", "sans-serif" +} + +table { + position: relative +} + +#managerTable { + max-height: 800px; + overflow: auto; } td { @@ -18,7 +28,7 @@ th { } a:link { - background-color: #ccc; + background-color: #e8e8e8; color: #000; text-decoration: none; display: block; diff --git a/survBot.py b/survBot.py index 261965a..54d2160 100755 --- a/survBot.py +++ b/survBot.py @@ -22,7 +22,7 @@ from obspy.clients.filesystem.sds import Client from write_utils import get_html_text, get_html_link, get_html_row, html_footer, get_html_header, get_print_title_str, \ init_html_table, finish_html_table, get_mail_html_header, add_html_image -from utils import get_bg_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds +from utils import get_bg_color, get_font_color, modify_stream_for_plot, set_axis_yticks, set_axis_color, plot_axis_thresholds try: import smtplib @@ -441,7 +441,7 @@ class SurveillanceBot(object): fig_name = self.get_fig_path_rel(nwst_id) nwst_id_str = nwst_id.rstrip('.') col_items = [dict(text=nwst_id_str, color=default_color, hyperlink=fig_name if hyperlinks else None, - bold=True, tooltip=f'Show plot of {nwst_id_str}')] + bold=True, tooltip=f'Show plot of {nwst_id_str}', font_color='#000000')] for check_key in header: if check_key in self.keys: @@ -453,6 +453,7 @@ class SurveillanceBot(object): bg_color = get_bg_color(check_key, status, dt_thresh, hex=True) if not bg_color: bg_color = default_color + font_color = get_font_color(bg_color, hex=True) # add degree sign for temp if check_key == 'temp': @@ -461,7 +462,7 @@ class SurveillanceBot(object): html_class = self.get_html_class(hide_keys_mobile, status=status, check_key=check_key) item = dict(text=str(message), tooltip=str(detailed_message), color=bg_color, - html_class=html_class) + html_class=html_class, font_color=font_color) elif check_key in self.add_links: value = self.add_links.get(check_key).get('URL') link_text = self.add_links.get(check_key).get('text') diff --git a/utils.py b/utils.py index 2168c01..600889a 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from functools import partial + import matplotlib import numpy as np @@ -35,12 +37,18 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False): def get_color(key): # some GUI default colors - colors_dict = {'FAIL': (255, 50, 0, 255), + # colors_dict = {'FAIL': (255, 50, 0, 255), + # 'NO DATA': (255, 255, 125, 255), + # 'WARN': (255, 255, 80, 255), + # 'OK': (125, 255, 125, 255), + # 'undefined': (230, 230, 230, 255), + # 'disc': (255, 160, 40, 255),} + colors_dict = {'FAIL': (195, 29, 14, 255), 'NO DATA': (255, 255, 125, 255), - 'WARN': (255, 255, 80, 255), - 'OK': (125, 255, 125, 255), - 'undefined': (230, 230, 230, 255), - 'disc': (255, 160, 40, 255),} + 'WARN': (249, 238, 139, 255), + 'OK': (179, 219, 153, 255), + 'undefined': (240, 240, 240, 255), + 'disc': (126, 127, 131, 255), } return colors_dict.get(key) @@ -60,7 +68,16 @@ def get_time_delay_color(dt, dt_thresh): def get_warn_color(count): - color = (min([255, 200 + count ** 2]), 255, 80, 255) + n_colors = 20 + # r = np.linspace(158, 255, n_colors, dtype=int) + # g = np.linspace(255, 255, n_colors, dtype=int) + # b = np.linspace(114, 80, n_colors, dtype=int) + r = np.linspace(204, 249, n_colors, dtype=int) + g = np.linspace(226, 238, n_colors, dtype=int) + b = np.linspace(149, 139, n_colors, dtype=int) + pad = partial(np.pad, pad_width=(0, 100), mode='edge') + r, g, b = map(pad, [r, g, b]) + color = (r[count], g[count], b[count], 255) return color @@ -81,6 +98,24 @@ def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'): return rgba +def get_font_color(bg_color, hex=False): + if hex: + bg_color = matplotlib.colors.to_rgb(bg_color) + bg_color_hsv = matplotlib.colors.rgb_to_hsv(bg_color) + bg_color_hsl = hsv_to_hsl(bg_color_hsv) + font_color = (255, 255, 255, 255) if bg_color_hsl[2] < 0.6 else (0, 0, 0, 255) + if hex: + font_color = '#{:02x}{:02x}{:02x}'.format(*font_color[:3]) + return font_color + + +def hsv_to_hsl(hsv): + hue, saturation, value = hsv + lightness = value * (1 - saturation / 2) + saturation = 0 if lightness in (0, 1) else (value - lightness) / min(lightness, 1 - lightness) + return hue, saturation, lightness + + def modify_stream_for_plot(input_stream, parameters): """ copy (if necessary) and modify stream for plotting """ diff --git a/write_utils.py b/write_utils.py index 58a6747..d595c06 100644 --- a/write_utils.py +++ b/write_utils.py @@ -40,11 +40,11 @@ def get_mail_html_header(): def init_html_table(): - return '\n' + return '
\n' def finish_html_table(): - return '
\n' + return '\n' def html_footer(footer_logo=None): @@ -82,12 +82,13 @@ def get_html_row(items, html_key='td'): color = item.get('color') # check for black background of headers (shouldnt happen anymore) color = '#e6e6e6' if color == '#000000' else color + font_color = item.get('font_color') hyperlink = item.get('hyperlink') text_str = get_html_link(text, hyperlink) if hyperlink else text html_class = item.get('html_class') class_str = f' class="{html_class}"' if html_class else '' - row_string += 2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"> {text_str}'\ - + f'\n' + row_string += 2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"' \ + + f'style="color:{font_color}"> {text_str}\n' row_string += default_space + '\n' return row_string From 353f073d12c2fc2eadefe209e7c0179470888333 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 21 Apr 2023 16:33:02 +0200 Subject: [PATCH 36/40] [update] added color palette suggested by AM, some visual tweaks --- survBot.py | 2 +- utils.py | 25 +++++++++---------------- write_utils.py | 8 +++----- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/survBot.py b/survBot.py index e1c16a9..d117ed6 100755 --- a/survBot.py +++ b/survBot.py @@ -514,7 +514,7 @@ class SurveillanceBot(object): # write footer with optional logo logo_file = self.parameters.get('html_logo') - if not os.path.isfile(logo_file): + if not os.path.isfile(pjoin(self.outpath_html, logo_file)): print(f'Specified file {logo_file} not found.') logo_file = None diff --git a/utils.py b/utils.py index 07fa9c9..db2dbac 100644 --- a/utils.py +++ b/utils.py @@ -1,8 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from functools import partial - import matplotlib import numpy as np @@ -36,7 +34,7 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False): def get_color(key): - # some GUI default colors + # some old GUI default colors # colors_dict = {'FAIL': (255, 85, 50, 255), # 'NO DATA': (255, 255, 125, 255), # 'WARN': (255, 255, 80, 255), @@ -45,8 +43,8 @@ def get_color(key): # 'disc': (255, 160, 40, 255),} colors_dict = {'FAIL': (195, 29, 14, 255), 'NO DATA': (255, 255, 125, 255), - 'WARN': (249, 238, 139, 255), - 'OK': (179, 219, 153, 255), + 'WARN': (250, 192, 63, 255), + 'OK': (185, 245, 145, 255), 'undefined': (240, 240, 240, 255), 'disc': (126, 127, 131, 255), } return colors_dict.get(key) @@ -67,16 +65,11 @@ def get_time_delay_color(dt, dt_thresh): return get_color('FAIL') -def get_warn_color(count): - #color = (min([255, 220 + count ** 2]), 255, 80, 255) - n_colors = 20 - r = np.linspace(204, 249, n_colors, dtype=int) - g = np.linspace(226, 238, n_colors, dtype=int) - b = np.linspace(149, 139, n_colors, dtype=int) - pad = partial(np.pad, pad_width=(0, 100), mode='edge') - r, g, b = map(pad, [r, g, b]) - color = (r[count], g[count], b[count], 255) - return color +def get_warn_color(count, n_colors=20): + if count > n_colors: + count = -1 + gradient = np.linspace((240, 245, 110, 255), (250, 192, 63, 255), n_colors, dtype=int) + return tuple(gradient[count]) def get_mass_color(message): @@ -252,4 +245,4 @@ def plot_threshold_lines(fig, channel_threshold_list, parameters, **kwargs): if isinstance(warn_thresh, str): warn_thresh = parameters.get('THRESHOLDS').get(warn_thresh) if type(warn_thresh in (float, int)): - ax.axhline(warn_thresh, **kwargs) \ No newline at end of file + ax.axhline(warn_thresh, **kwargs) diff --git a/write_utils.py b/write_utils.py index d595c06..a042e89 100644 --- a/write_utils.py +++ b/write_utils.py @@ -40,11 +40,11 @@ def get_mail_html_header(): def init_html_table(): - return '
\n' + return '
\n' def finish_html_table(): - return '
\n' + return '\n' def html_footer(footer_logo=None): @@ -79,11 +79,9 @@ def get_html_row(items, html_key='td'): if item.get('italic'): text = '' + text + '' tooltip = item.get('tooltip') - color = item.get('color') - # check for black background of headers (shouldnt happen anymore) - color = '#e6e6e6' if color == '#000000' else color font_color = item.get('font_color') hyperlink = item.get('hyperlink') + color = 'transparent' if hyperlink else item.get('color') text_str = get_html_link(text, hyperlink) if hyperlink else text html_class = item.get('html_class') class_str = f' class="{html_class}"' if html_class else '' From a15aee1da6e85b72eaf600956a3755d38c6a8233 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 21 Apr 2023 16:40:58 +0200 Subject: [PATCH 37/40] [minor] update stylesheets --- stylesheets/desktop.css | 16 ++++++---------- stylesheets/mobile.css | 26 ++++++++++++-------------- 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/stylesheets/desktop.css b/stylesheets/desktop.css index aa23f6e..713feb1 100644 --- a/stylesheets/desktop.css +++ b/stylesheets/desktop.css @@ -3,27 +3,23 @@ body { place-items: center; text-align: center; padding-bottom: 30px; - font-family: "Helvetica", "sans-serif" + font-family: "Helvetica", "sans-serif"; } table { position: relative } -#managerTable { - max-height: 800px; - overflow: auto; -} - td { - border-radius: 4px; + border-radius: 2px; padding: 0px; white-space: nowrap; } th { - background-color: #999; - border-radius: 4px; + background-color: #17365c; + color: #fff; + border-radius: 2px; padding: 3px 1px; position: sticky; top: 0; @@ -35,7 +31,7 @@ a:link, a:visited { text-decoration: none; display: block; border-radius: 4px; - border: 1px solid #bbb; + border: 1px solid #ccc; } a:hover { diff --git a/stylesheets/mobile.css b/stylesheets/mobile.css index 1691271..1be0ce2 100644 --- a/stylesheets/mobile.css +++ b/stylesheets/mobile.css @@ -3,37 +3,35 @@ body { place-items: center; text-align: center; padding-bottom: 30px; - font-family: "Helvetica", "sans-serif" + font-family: "Helvetica", "sans-serif"; } table { position: relative } -#managerTable { - max-height: 800px; - overflow: auto; -} - td { - border-radius: 4px; + border-radius: 3px; padding: 10px 2px; white-space: nowrap; } th { - background-color: #999; - border-radius: 4px; + background-color: #17365c; + color: #fff; + border-radius: 3px; padding: 10px, 2px; + position: sticky; + top: 0; } -a:link { +a:link, a:visited { background-color: #e8e8e8; color: #000; text-decoration: none; display: block; - border-radius: 4px; - border: 1px solid #bbb; + border-radius: 6px; + border: 1px solid #ccc; } a:hover { @@ -49,9 +47,9 @@ a:hover { animation: blinkingBackground 2s infinite; } @keyframes blinkingBackground{ - 0% { background-color: #ffee00;} + 0% { background-color: #ffcc00;} 50% { background-color: #ff3200;} - 100% { background-color: #ffee00;} + 100% { background-color: #ffcc00;} } .footer { From f3ccaaefd854f2721f23f9feb295f7cd72c1800a Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 21 Apr 2023 17:18:16 +0200 Subject: [PATCH 38/40] [minor] changed colors in examples stylesheets back to default --- stylesheets/desktop.css | 2 +- stylesheets/mobile.css | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stylesheets/desktop.css b/stylesheets/desktop.css index 713feb1..f25fc28 100644 --- a/stylesheets/desktop.css +++ b/stylesheets/desktop.css @@ -17,7 +17,7 @@ td { } th { - background-color: #17365c; + background-color: #999; color: #fff; border-radius: 2px; padding: 3px 1px; diff --git a/stylesheets/mobile.css b/stylesheets/mobile.css index 1be0ce2..10701b6 100644 --- a/stylesheets/mobile.css +++ b/stylesheets/mobile.css @@ -17,7 +17,7 @@ td { } th { - background-color: #17365c; + background-color: #999; color: #fff; border-radius: 3px; padding: 10px, 2px; From 2af30f3a32ed6767520b3ea76ad5cd50ff5aad1f Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 25 May 2023 15:28:11 +0200 Subject: [PATCH 39/40] [bugfix] count index exceeded array length --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index db2dbac..91b425c 100644 --- a/utils.py +++ b/utils.py @@ -66,7 +66,7 @@ def get_time_delay_color(dt, dt_thresh): def get_warn_color(count, n_colors=20): - if count > n_colors: + if count >= n_colors: count = -1 gradient = np.linspace((240, 245, 110, 255), (250, 192, 63, 255), n_colors, dtype=int) return tuple(gradient[count]) From 9e1ebebeb26f0438a8b61bb0b13ae3f05a69121b Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 1 Jun 2023 10:08:26 +0200 Subject: [PATCH 40/40] [release] version 0.2 --- README.md | 12 ++++++++++-- __init__.py | 3 +++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c7a8509..29b7733 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # survBot -version: 0.1 +version: 0.2 survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output over SOH channels by analysing contents of a Seiscomp3 datapath. @@ -40,8 +40,16 @@ The GUI can be loaded via python survBotGui.py ``` +## Version Changes +- surveillance of mass, clock and gaps +- individual mailing lists for different stations +- html mail with recent status information +- updated web page design +- restructured parameter file +- recognize if PBox is disconnected + ## Staff Original author: M.Paffrath (marcel.paffrath@rub.de) -November 2022 +June 2023 diff --git a/__init__.py b/__init__.py index e69de29..90d7001 100644 --- a/__init__.py +++ b/__init__.py @@ -0,0 +1,3 @@ +# survBot is a small program used to track station quality channels of DSEBRA stations via PowBox output +# over SOH channels by analysing contents of a Seiscomp3 datapath. +__version__ = "0.2"