Merge remote-tracking branch 'origin/issue/1' into issue/1

Reviewed-on: #3

README.md

@@ -32,6 +32,8 @@ The main program with html output is executed by entering
 python survBot.py -html path_for_html_output
 ```
 
+There are example stylesheets in the folder *stylesheets* that can be copied into the path_for_html_output if desired.
+
 The GUI can be loaded via
 
 ```shell script

parameters.yaml

@@ -1,32 +1,27 @@
 # Parameters file for Surveillance Bot
-datapath: '/data/SDS/'      # SC3 Datapath
-networks: ['1Y', 'HA']
-stations: '*'
-locations: '*'
-channels: ['EX1', 'EX2', 'EX3', 'VEI']    # Specify SOH channels, currently supported EX[1-3] and VEI
-stations_blacklist: ['TEST', 'EREA']
-networks_blacklist: []
+datapath: "/data/SDS/"                    # SC3 Datapath
+networks: ["1Y", "HA"]                    # select networks, list or str
+stations: "*"                             # select stations, list or str
+locations: "*"                            # select locations, list or str
+channels: ["EX1", "EX2", "EX3", "VEI", "LCQ"]    # Specify SOH channels, currently supported EX[1-3], VEI and LCQ
+stations_blacklist: ["TEST", "EREA"]      # exclude these stations
+networks_blacklist: []                    # exclude these networks
 interval: 60               # Perform checks every x seconds
-n_track: 120               # wait number of intervals after FAIL before performing an action (i.e. send mail)
-timespan: 3                # Check data of the recent x days
-verbosity: 0
+n_track: 300               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
+timespan: 7                # Check data of the recent x days
+verbosity: 0               # verbosity flag
 track_changes: True        # tracks all changes since GUI startup by text highlighting (GUI only)
-warn_count: False          # show number of warnings and errors in table
+warn_count: False           # show number of warnings and errors in table
+min_sample: 3              # minimum samples for raising Warn/FAIL
 dt_thresh: [300, 1800]     # threshold (s) for timing delay colourisation (yellow/red)
 html_figures: True         # Create html figure directory and links
 reread_parameters: True    # reread parameters file (change parameters on runtime, not for itself/GUI refresh/datapath)
 
-# add links to html table with specified key as column and value as relative link, interpretable string parameters:
-# nw (e.g. 1Y), st (e.g. GR01A), nwst_id (e.g. 1Y.GR01A)
-# can also be empty!
-add_links:
-  slmon: {"URL": "{nw}_{st}.html", "text": "show"}   # for example: slmon: {"URL": "{nw}_{st}.html", "text": "link"}
-
 POWBOX:
   pb_ok: 1  # Voltage for PowBox OK
   pb_SOH2: # PowBox channel 2 voltage translations
     -1: {"230V": "PBox under 1V", "12V": "PBox under 1V"}
-    1: {"230V": 'OK', "12V": "OK"}
+    1: {"230V": "OK", "12V": "OK"}
     2: {"230V": "OFF", "12V": "OK"}
     3: {"230V": "OK", "12V": "overvoltage"}
     4: {"230V": "OK", "12V": "undervoltage"}
@@ -46,12 +41,47 @@ THRESHOLDS:
   pb_thresh: 0.2               # Threshold for PowBox Voltage check +/- (V)
   max_temp: 50                 # max temperature for temperature warning
   low_volt: 12                 # min voltage for low voltage warning
-  high_volt: 14.8               # max voltage for over voltage warning
+  high_volt: 14.8              # max voltage for over voltage warning
   unclassified: 5              # min voltage samples not classified for warning
+  clockquality_warn: 90        # clock quality ranges from 0 % to 100 % with 100 % being the best level
+  clockquality_fail: 70
+
+# ---------------------------------------- OPTIONAL PARAMETERS ---------------------------------------------------------
+
+# add links to html table with specified key as column and value as relative link, interpretable string parameters:
+# nw (e.g. 1Y), st (e.g. GR01A), nwst_id (e.g. 1Y.GR01A)
+add_links:
+  # for example: slmon: {"URL": "path/{nw}_{st}.html", "text": "link"}
+  slmon: {"URL": "../slmon/{nw}_{st}.html", "text": "show"}
+  24h-plot: {"URL": "../scheli/{nw}/{st}.png", "text": "plot"}
 
 # E-mail notifications
 EMAIL:
-  mailserver: 'localhost'
-  addresses: ['marcel.paffrath@rub.de', 'kasper.fischer@rub.de']  # list of mail addresses for info mails
-  sender: 'webmaster@geophysik.ruhr-uni-bochum.de' # mail sender
+  mailserver: "localhost"
+  addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"]  # list of mail addresses for info mails
+  sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender
+  stations_blacklist: ['GR33']                     # do not send emails for specific stations
+  networks_blacklist: []                           # do not send emails for specific network
 
+# names for plotting of the above defined parameter "channels" in the same order
+channel_names: ["Clock Quality (%)", "Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"] # names for plotting (optional)
+# specify y-ticks (and ylims) giving, (ymin, ymax, step) for each of the above channels (0: default)
+CHANNEL_TICKS:
+  - [0, 100, 20]
+  - [-10, 50, 10]
+  - [1, 5, 1]
+  - [1, 5, 1]
+  - [9, 15, 1]
+
+# Factor for channel to SI-units (for plotting)
+CHANNEL_UNITS:
+  EX1: 1e-6
+  EX2: 1e-6
+  EX3: 1e-6
+  VEI: 1e-3
+
+# Transform channel for plotting, perform arithmetic operations in given order, e.g.: PBox EX1 V to deg C: 20 * x -20
+CHANNEL_TRANSFORM:
+  EX1:
+    - ["*", 20]
+    - ["-", 20]

stylesheets/desktop.css

@@ -0,0 +1,39 @@
+body {
+    background-color: #ffffff;
+    place-items: center;
+    text-align: center;
+}
+
+td {
+    border-radius: 4px;
+    padding: 0px;
+}
+
+th {
+    background-color: #999;
+    border-radius: 4px;
+    padding: 3px 1px;
+}
+
+a:link, a:visited {
+    background-color: #ccc;
+    color: #000;
+    text-decoration: none;
+    display: block;
+    border-radius: 4px;
+    border: 1px solid #bbb;
+}
+
+a:hover {
+    background-color: #aaa;
+    display: block;			  
+}
+
+.blink-bg {
+    animation: blinkingBackground 2s infinite;
+}
+@keyframes blinkingBackground{
+    0%		{ background-color: #ffcc00;}
+    50%		{ background-color: #ff3200;}
+    100%	{ background-color: #ffcc00;}
+}

stylesheets/mobile.css

@@ -0,0 +1,43 @@
+body {
+    background-color: #ffffff;
+    place-items: center;
+    text-align: center;
+}
+
+td {
+    border-radius: 4px;
+    padding: 10px 2px;
+}
+
+th {
+    background-color: #999;
+    border-radius: 4px;
+    padding: 10px, 2px;
+}
+
+a:link {
+    background-color: #ccc;
+    color: #000;
+    text-decoration: none;
+    display: block;
+    border-radius: 4px;
+    border: 1px solid #bbb;
+}
+
+a:hover {
+    background-color: #aaa;
+    display: block;			  
+}
+
+.hidden-mobile {
+    display: none;
+}
+
+.blink-bg {
+    animation: blinkingBackground 2s infinite;
+}
+@keyframes blinkingBackground{
+    0%		{ background-color: #ffee00;}
+    50%		{ background-color: #ff3200;}
+    100%	{ background-color: #ffee00;}
+}

survBot.py

@@ -17,13 +17,14 @@ import matplotlib.pyplot as plt
 from obspy import read, UTCDateTime, Stream
 from obspy.clients.filesystem.sds import Client
 
-from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str,\
+from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str, \
     init_html_table, finish_html_table
-from utils import get_bg_color
+from utils import get_bg_color, modify_stream_for_plot, trace_ylabels, trace_yticks
 
 try:
     import smtplib
     from email.mime.text import MIMEText
+
     mail_functionality = True
 except ImportError:
     print('Could not import smtplib or mail. Disabled sending mails.')
@@ -60,7 +61,7 @@ def fancy_timestr(dt, thresh=600, modif='+'):
 
 class SurveillanceBot(object):
     def __init__(self, parameter_path, outpath_html=None):
-        self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'temp', 'other']
+        self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'clock', 'temp', 'other']
         self.parameter_path = parameter_path
         self.update_parameters()
         self.starttime = UTCDateTime()
@@ -156,7 +157,7 @@ class SurveillanceBot(object):
             if filename in self.filenames_read:
                 continue
             try:
-                st_new = read(filename)
+                st_new = read(filename, dtype=float)
                 # add file to read filenames to prevent re-reading in case it is not the current day (or end of
                 # previous day)
                 if not filename.endswith(f'{current_day:03}') and not (
@@ -166,7 +167,7 @@ class SurveillanceBot(object):
                 print(f'Could not read file {filename}:', e)
                 continue
             self.dataStream += st_new
-        self.dataStream.merge()
+        self.dataStream.merge(fill_value=np.nan)
 
         # organise data in dictionary with key for each station
         for trace in self.dataStream:
@@ -188,7 +189,7 @@ class SurveillanceBot(object):
             stream = self.data.get(nwst_id)
             if stream:
                 nsl = nsl_from_id(nwst_id)
-                station_qc = StationQC(stream, nsl, self.parameters, self.keys, qc_starttime,
+                station_qc = StationQC(self, stream, nsl, self.parameters, self.keys, qc_starttime,
                                        self.verbosity, print_func=self.print,
                                        status_track=self.status_track.get(nwst_id))
                 analysis_print_result = station_qc.return_print_analysis()
@@ -277,7 +278,7 @@ class SurveillanceBot(object):
             if self.outpath_html:
                 self.write_html_table()
                 if self.parameters.get('html_figures'):
-                    self.write_html_figures(check_plot_time=not(first_exec))
+                    self.write_html_figures(check_plot_time=not (first_exec))
             else:
                 self.print_analysis()
             time.sleep(self.refresh_period)
@@ -321,23 +322,50 @@ class SurveillanceBot(object):
             os.mkdir(self.outpath_html)
 
     def write_html_figures(self, check_plot_time=True):
-        """ Write figures for html, right now hardcoded hourly """
+        """ Write figures for html (e.g. hourly) """
         if check_plot_time and not self.check_plot_hour():
             return
-        self.check_fig_dir()
 
         for nwst_id in self.station_list:
-            fig = plt.figure(figsize=(16, 9))
-            fnout = self.get_fig_path_abs(nwst_id)
-            st = self.data.get(nwst_id)
-            if st:
-                st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full')
-                ax = fig.axes[0]
-                ax.set_title(f'Hourly refreshed plot at (UTC) {UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S")}')
-                fig.savefig(fnout, dpi=150., bbox_inches='tight')
-            plt.close(fig)
+            self.write_html_figure(nwst_id)
+
+    def write_html_figure(self, nwst_id):
+        """ Write figure for html for specified station """
+        self.check_fig_dir()
+
+        fig = plt.figure(figsize=(16, 9))
+        fnout = self.get_fig_path_abs(nwst_id)
+        st = self.data.get(nwst_id)
+        if st:
+            # TODO: this section might fail, adding try-except block for analysis and to prevent program from crashing
+            try:
+                st = modify_stream_for_plot(st, parameters=self.parameters)
+                st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full')
+                trace_ylabels(fig, self.parameters, self.verbosity)
+                trace_yticks(fig, self.parameters, self.verbosity)
+            except Exception as e:
+                print(f'Could not generate plot for {nwst_id}:')
+                print(traceback.format_exc())
+            if len(fig.axes) > 0:
+                ax = fig.axes[0]
+                ax.set_title(f'Plot refreshed at (UTC) {UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S")}. '
+                             f'Refreshed hourly or on FAIL status.')
+                for ax in fig.axes:
+                    ax.grid(True, alpha=0.1)
+                fig.savefig(fnout, dpi=150., bbox_inches='tight')
+        plt.close(fig)
+
+    def write_html_table(self, default_color='#e6e6e6', default_header_color='#999', hide_keys_mobile=('other')):
+
+        def get_html_class(status=None, check_key=None):
+            """ helper function for html class if a certain condition is fulfilled """
+            html_class = None
+            if status and status.is_active:
+                html_class = 'blink-bg'
+            if check_key in hide_keys_mobile:
+                html_class = 'hidden-mobile'
+            return html_class
 
-    def write_html_table(self, default_color='#e6e6e6'):
         self.check_html_dir()
         fnout = pjoin(self.outpath_html, 'survBot_out.html')
         if not fnout:
@@ -345,7 +373,7 @@ class SurveillanceBot(object):
         try:
             with open(fnout, 'w') as outfile:
                 write_html_header(outfile, self.refresh_period)
-                #write_html_table_title(outfile, self.parameters)
+                # write_html_table_title(outfile, self.parameters)
                 init_html_table(outfile)
 
                 # First write header items
@@ -353,16 +381,19 @@ class SurveillanceBot(object):
                 # add columns for additional links
                 for key in self.add_links:
                     header.insert(-1, key)
-                header_items = [dict(text='Station', color=default_color)]
+                header_items = [dict(text='Station', color=default_header_color)]
                 for check_key in header:
-                    item = dict(text=check_key, color=default_color)
+                    html_class = get_html_class(check_key=check_key)
+                    item = dict(text=check_key, color=default_header_color, html_class=html_class)
                     header_items.append(item)
                 write_html_row(outfile, header_items, html_key='th')
 
                 # Write all cells
                 for nwst_id in self.station_list:
                     fig_name = self.get_fig_path_rel(nwst_id)
-                    col_items = [dict(text=nwst_id.rstrip('.'), color=default_color, hyperlink=fig_name)]
+                    nwst_id_str = nwst_id.rstrip('.')
+                    col_items = [dict(text=nwst_id_str, color=default_color, hyperlink=fig_name,
+                                      bold=True, tooltip=f'Show plot of {nwst_id_str}')]
                     for check_key in header:
                         if check_key in self.keys:
                             status_dict = self.analysis_results.get(nwst_id)
@@ -380,7 +411,9 @@ class SurveillanceBot(object):
                                 if not type(message) in [str]:
                                     message = str(message) + deg_str
 
-                            item = dict(text=str(message), tooltip=str(detailed_message), color=bg_color)
+                            html_class = get_html_class(status=status, check_key=check_key)
+                            item = dict(text=str(message), tooltip=str(detailed_message), color=bg_color,
+                                        html_class=html_class)
                         elif check_key in self.add_links:
                             value = self.add_links.get(check_key).get('URL')
                             link_text = self.add_links.get(check_key).get('text')
@@ -405,7 +438,7 @@ class SurveillanceBot(object):
         timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
         self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \
                               f'Current time (UTC) {UTCDateTime().strftime("%Y-%m-%d %H:%M:%S")} | ' \
-                              f'Refresh period: {self.refresh_period}s | '\
+                              f'Refresh period: {self.refresh_period}s | ' \
                               f'Showing data of last {timespan}'
 
     def print(self, string, **kwargs):
@@ -422,12 +455,13 @@ class SurveillanceBot(object):
 
 
 class StationQC(object):
-    def __init__(self, stream, nsl, parameters, keys, starttime, verbosity, print_func, status_track={}):
+    def __init__(self, parent, stream, nsl, parameters, keys, starttime, verbosity, print_func, status_track={}):
         """
         Station Quality Check class.
         :param nsl: dictionary containing network, station and location (key: str)
         :param parameters: parameters dictionary from parameters.yaml file
         """
+        self.parent = parent
         self.stream = stream
         self.nsl = nsl
         self.network = nsl.get('network')
@@ -448,6 +482,10 @@ class StationQC(object):
 
         self.start()
 
+    @property
+    def nwst_id(self):
+        return f'{self.network}.{self.station}'
+
     def status_ok(self, key, detailed_message="Everything OK", status_message='OK', overwrite=False):
         current_status = self.status_dict.get(key)
         # do not overwrite existing warnings or errors
@@ -497,44 +535,71 @@ class StationQC(object):
             current_status.count += count
         else:
             current_status = new_error
-
-        self._update_status(key, current_status, detailed_message, last_occurrence)
+            # if error is new and not on program-startup set active and refresh plot (using parent class)
+            if self.search_previous_errors(key, n_errors=1) is True:
+                self.parent.write_html_figure(self.nwst_id)
 
         if self.verbosity:
             self.print(f'{UTCDateTime()}: {detailed_message}', flush=False)
 
         # do not send error mail if this is the first run (e.g. program startup) or state was already error (unchanged)
-        if self.search_previous_errors(key):
-            self.send_mail(key, detailed_message)
+        if self.search_previous_errors(key) is True:
+            self.send_mail(key, status_type='FAIL', additional_message=detailed_message)
+            # set status to "inactive" after sending info mail
+            current_status.is_active = False
+        elif self.search_previous_errors(key) == 'active':
+            current_status.is_active = True
 
-    def search_previous_errors(self, key):
+        self._update_status(key, current_status, detailed_message, last_occurrence)
+
+    def search_previous_errors(self, key, n_errors=None):
         """
         Check n_track + 1 previous statuses for errors.
-        If first item in list is no error but all others are return True (first time n_track errors appeared --
-        if ALL n_track + 1 are error: error is old)
-        In all other cases return True.
+        If first item in list is no error but all others are return True
+        (first time n_track errors appeared if ALL n_track + 1 are error: error is old)
+        If last item is error but not all items are error yet return keyword 'active' -> error active, no message sent
+        In all other cases return False.
         This also prevents sending status (e.g. mail) in case of program startup
         """
+        if n_errors is not None:
+            n_errors = self.parameters.get('n_track') + 1
+
         previous_errors = self.status_track.get(key)
         # only if error list is filled n_track times
-        if previous_errors and len(previous_errors) == self.parameters.get('n_track') + 1:
+        if previous_errors and len(previous_errors) == n_errors:
             # if first entry was no error but all others are, return True (-> new Fail n_track times)
             if not previous_errors[0] and all(previous_errors[1:]):
                 return True
-        else:
-            return False
+        # in case previous_errors exists, last item is error but not all items are error, error still active
+        elif previous_errors and previous_errors[-1] and not all(previous_errors):
+            return 'active'
+        return False
 
-    def send_mail(self, key, message):
+    def send_mail(self, key, status_type, additional_message=''):
         """ Send info mail using parameters specified in parameters file """
         if not mail_functionality:
             if self.verbosity:
                 print('Mail functionality disabled. Return')
             return
+
         mail_params = self.parameters.get('EMAIL')
         if not mail_params:
             if self.verbosity:
                 print('parameter "EMAIL" not set in parameter file. Return')
             return
+
+        stations_blacklist = mail_params.get('stations_blacklist')
+        if stations_blacklist and self.station in stations_blacklist:
+            if self.verbosity:
+                print(f'Station {self.station} listed in blacklist. Return')
+            return
+
+        networks_blacklist = mail_params.get('networks_blacklist')
+        if networks_blacklist and self.network in networks_blacklist:
+            if self.verbosity:
+                print(f'Station {self.station} of network {self.network} listed in blacklist. Return')
+            return
+
         sender = mail_params.get('sender')
         addresses = mail_params.get('addresses')
         server = mail_params.get('mailserver')
@@ -542,12 +607,10 @@ class StationQC(object):
             if self.verbosity:
                 print('Mail sender or addresses not correctly defined. Return')
             return
-        n_track = self.parameters.get('n_track')
-        interval = self.parameters.get('interval')
-        dt = timedelta(seconds=n_track * interval)
-        text = f'{key} FAIL status longer than {dt}: ' + message
+        dt = self.get_dt_for_action()
+        text = f'{key}: Status {status_type} longer than {dt}: ' + additional_message
         msg = MIMEText(text)
-        msg['Subject'] = f'new FAIL status on station {self.network}.{self.station}'
+        msg['Subject'] = f'new message on station {self.nwst_id}'
         msg['From'] = sender
         msg['To'] = ', '.join(addresses)
 
@@ -556,7 +619,12 @@ class StationQC(object):
         s.sendmail(sender, addresses, msg.as_string())
         s.quit()
 
-        
+    def get_dt_for_action(self):
+        n_track = self.parameters.get('n_track')
+        interval = self.parameters.get('interval')
+        dt = timedelta(seconds=n_track * interval)
+        return dt
+
     def status_other(self, detailed_message, status_message, last_occurrence=None, count=1):
         key = 'other'
         new_status = StatusOther(count=count, messages=[status_message])
@@ -575,13 +643,15 @@ class StationQC(object):
 
         self.status_dict[key] = current_status
 
-    def activity_check(self):
+    def activity_check(self, key='last_active'):
         self.last_active = self.last_activity()
         if not self.last_active:
             status = StatusError()
         else:
-            message = timedelta(seconds=int(self.program_starttime - self.last_active))
-            status = Status(message=message)
+            dt_active = timedelta(seconds=int(self.program_starttime - self.last_active))
+            status = Status(message=dt_active)
+            self.check_for_inactive_message(key, dt_active)
+
         self.status_dict['last active'] = status
 
     def last_activity(self):
@@ -593,6 +663,12 @@ class StationQC(object):
         if len(endtimes) > 0:
             return max(endtimes)
 
+    def check_for_inactive_message(self, key, dt_active):
+        dt_action = self.get_dt_for_action()
+        interval = self.parameters.get('interval')
+        if dt_action <= dt_active < dt_action + timedelta(seconds=interval):
+            self.send_mail(key, status_type='Inactive')
+
     def start(self):
         self.analyse_channels()
 
@@ -609,9 +685,10 @@ class StationQC(object):
         self.pb_temp_analysis()
         self.pb_power_analysis()
         self.pb_rout_charge_analysis()
+        self.clock_quality_analysis()
 
     def return_print_analysis(self):
-        items = [f'{self.network}.{self.station}']
+        items = [self.nwst_id]
         for key in self.keys:
             status = self.status_dict[key]
             message = status.message
@@ -637,6 +714,44 @@ class StationQC(object):
     def get_last_occurrence(self, trace, indices):
         return self.get_time(trace, indices[-1])
 
+    def clock_quality_analysis(self, channel='LCQ'):
+        """ Analyse clock quality """
+        key = 'clock'
+        st = self.stream.select(channel=channel)
+        trace = self.get_trace(st, key)
+        if not trace: return
+        clockQuality = trace.data
+        clockQuality_warn_level = self.parameters.get('THRESHOLDS').get('clockquality_warn')
+        clockQuality_fail_level = self.parameters.get('THRESHOLDS').get('clockquality_fail')
+
+        if self.verbosity > 1:
+            self.print(40 * '-')
+            self.print('Performing Clock Quality check', flush=False)
+        
+        clockQuality_warn = np.where(clockQuality < clockQuality_warn_level)[0]
+        clockQuality_fail = np.where(clockQuality < clockQuality_fail_level)[0]
+
+        if len(clockQuality_warn) == 0 and len(clockQuality_fail) == 0:
+            self.status_ok(key, detailed_message=f'ClockQuality={(clockQuality[-1])}')
+            return
+ 
+        warn_message = f'Trace {trace.get_id()}:'
+        if len(clockQuality_warn) > 0:
+            # try calculate number of warn peaks from gaps between indices
+            n_qc_warn = self.calc_occurrences(clockQuality_warn)
+            detailed_message = warn_message + f' {n_qc_warn}x Qlock Quality less then {clockQuality_warn_level}' \
+                               + self.get_last_occurrence_timestring(trace, clockQuality_warn)
+            self.warn(key, detailed_message=detailed_message, count=n_qc_warn,
+                      last_occurrence=self.get_last_occurrence(trace, clockQuality_warn))
+
+        if len(clockQuality_fail) > 0:
+            # try calculate number of fail peaks from gaps between indices
+            n_qc_fail = self.calc_occurrences(clockQuality_fail)
+            detailed_message = warn_message + f' {n_qc_fail}x Qlock Quality less then {clockQuality_fail_level}V ' \
+                               + self.get_last_occurrence_timestring(trace, clockQuality_fail)
+            self.error(key, detailed_message=detailed_message, count=n_qc_fail,
+                      last_occurrence=self.get_last_occurrence(trace, clockQuality_fail))
+
     def voltage_analysis(self, channel='VEI'):
         """ Analyse voltage channel for over/undervoltage """
         key = 'voltage'
@@ -658,9 +773,6 @@ class StationQC(object):
             self.status_ok(key, detailed_message=f'U={(voltage[-1])}V')
             return
 
-        n_overvolt = 0
-        n_undervolt = 0
-
         warn_message = f'Trace {trace.get_id()}:'
         if len(overvolt) > 0:
             # try calculate number of voltage peaks from gaps between indices
@@ -670,11 +782,10 @@ class StationQC(object):
             self.warn(key, detailed_message=detailed_message, count=n_overvolt,
                       last_occurrence=self.get_last_occurrence(trace, overvolt))
 
-
         if len(undervolt) > 0:
             # try calculate number of voltage peaks from gaps between indices
             n_undervolt = len(np.where(np.diff(undervolt) > 1)[0]) + 1
-            detailed_message = warn_message + f' {n_undervolt}x Voltage under {low_volt}V '\
+            detailed_message = warn_message + f' {n_undervolt}x Voltage under {low_volt}V ' \
                                + self.get_last_occurrence_timestring(trace, undervolt)
             self.warn(key, detailed_message=detailed_message, count=n_undervolt,
                       last_occurrence=self.get_last_occurrence(trace, undervolt))
@@ -693,7 +804,7 @@ class StationQC(object):
         nsamp_av = int(trace.stats.sampling_rate) * timespan
         av_temp_str = str(round(np.mean(temp[-nsamp_av:]), 1)) + deg_str
         # dt of average
-        dt_t_str = str(timedelta(seconds=int(timespan)))
+        dt_t_str = str(timedelta(seconds=int(timespan))).replace(', 0:00:00', '')
         # current temp
         cur_temp = round(temp[-1], 1)
         if self.verbosity > 1:
@@ -774,11 +885,10 @@ class StationQC(object):
                         self.status_ok(key)
                         continue
                     if volt_lvl > 1:
-                        # try calculate number of voltage peaks from gaps between indices
-                        n_occurrences = len(np.where(np.diff(ind_array) > 1)[0]) + 1
+                        n_occurrences = self.calc_occurrences(ind_array)
                         self.warn(key=key,
                                   detailed_message=f'Trace {trace.get_id()}: '
-                                          f'Found {n_occurrences} occurrence(s) of {volt_lvl}V: {key}: {message}'
+                                                   f'Found {n_occurrences} occurrence(s) of {volt_lvl}V: {key}: {message}'
                                                    + self.get_last_occurrence_timestring(trace, ind_array),
                                   count=n_occurrences,
                                   last_occurrence=self.get_last_occurrence(trace, ind_array))
@@ -786,6 +896,32 @@ class StationQC(object):
                     if volt_lvl == last_val or (volt_lvl == -1 and last_val < 1):
                         self.error(key, detailed_message=f'Last PowBox voltage state {last_val}V: {message}')
 
+    def calc_occurrences(self, ind_array):
+        # try calculate number of voltage peaks/plateaus from gaps between indices
+        if len(ind_array) == 0:
+            return 0
+        else:
+            # start index at 1 if there are gaps (n_peaks = n_gaps + 1)
+            n_occurrences = 1
+
+        min_samples = self.parameters.get('min_sample')
+        if not min_samples:
+            min_samples = 1
+
+        # calculated differences in index array, diff > 1: gap, diff == 1: within peak/plateau
+        diffs = np.diff(ind_array)
+        gap_start_inds = np.where(np.diff(ind_array) > 1)[0]
+        # iterate over all gaps and check "min_samples" before the gap
+        for gsi in gap_start_inds:
+            # right boundary index of peak (gap index - 1)
+            peak_rb_ind = gsi - 1
+            # left boundary index of peak
+            peak_lb_ind = max([0, peak_rb_ind - min_samples])
+            if all(diffs[peak_lb_ind: peak_rb_ind] == 1):
+                n_occurrences += 1
+
+        return n_occurrences
+
     def get_trace(self, stream, keys):
         if not type(keys) == list:
             keys = [keys]
@@ -839,9 +975,9 @@ class StationQC(object):
             n_occurrences = len(np.where(np.diff(under) > 1)[0]) + 1
             voltage_dict[-1] = under
             self.status_other(detailed_message=f'Trace {trace.get_id()}: '
-                              f'Voltage below {pb_ok}V in {len(under)} samples, {n_occurrences} time(s). '
-                              f'Mean voltage: {np.mean(voltage):.2}'
-                                       + self.get_last_occurrence_timestring(trace, under),
+                                               f'Voltage below {pb_ok}V in {len(under)} samples, {n_occurrences} time(s). '
+                                               f'Mean voltage: {np.mean(voltage):.2}'
+                                               + self.get_last_occurrence_timestring(trace, under),
                               status_message='under 1V ({})'.format(n_occurrences))
 
         # classify last voltage values
@@ -860,8 +996,8 @@ class StationQC(object):
             max_uncl = self.parameters.get('THRESHOLDS').get('unclassified')
             if max_uncl and n_unclassified > max_uncl:
                 self.status_other(detailed_message=f'Trace {trace.get_id()}: '
-                                               f'{n_unclassified}/{len(all_indices)} '
-                                               f'unclassified voltage values in channel {trace.get_id()}',
+                                                   f'{n_unclassified}/{len(all_indices)} '
+                                                   f'unclassified voltage values in channel {trace.get_id()}',
                                   status_message=f'{channel}: {n_unclassified} uncl.')
 
         return False, voltage_dict, last_val
@@ -872,7 +1008,9 @@ class StationQC(object):
 
 
 class Status(object):
-    def __init__(self, message='-', detailed_messages=None, count: int = 0, last_occurrence=None, show_count=True):
+    def __init__(self, message=None, detailed_messages=None, count: int = 0, last_occurrence=None, show_count=True):
+        if message is None:
+            message = '-'
         if detailed_messages is None:
             detailed_messages = []
         self.show_count = show_count
@@ -884,6 +1022,7 @@ class Status(object):
         self.is_warn = None
         self.is_error = None
         self.is_other = False
+        self.is_active = False
 
     def set_warn(self):
         self.is_warn = True
@@ -929,7 +1068,7 @@ class StatusError(Status):
                                           detailed_messages=detailed_messages, show_count=show_count)
         self.set_error()
 
-    
+
 class StatusOther(Status):
     def __init__(self, messages=None, count=1, last_occurence=None, detailed_messages=None):
         super(StatusOther, self).__init__(count=count, last_occurrence=last_occurence,
@@ -938,11 +1077,11 @@ class StatusOther(Status):
             messages = []
         self.messages = messages
         self.is_other = True
-    
+
     def get_status_str(self):
         if self.messages == []:
             return '-'
-    
+
         message = ''
         for index, mes in enumerate(self.messages):
             if index > 0:

survBotGUI.py

@@ -22,7 +22,6 @@ except ImportError:
         except ImportError:
             raise ImportError('Could import neither of PySide2, PySide6 or PyQt5')
 
-import matplotlib
 from matplotlib.figure import Figure
 
 if QtGui.__package__ in ['PySide2', 'PyQt5', 'PySide6']:
@@ -35,7 +34,7 @@ from obspy import UTCDateTime
 
 from survBot import SurveillanceBot
 from write_utils import *
-from utils import get_bg_color
+from utils import get_bg_color, modify_stream_for_plot, trace_ylabels, trace_yticks
 
 try:
     from rest_api.utils import get_station_iccid
@@ -315,7 +314,10 @@ class MainWindow(QtWidgets.QMainWindow):
         if st:
             self.plot_widget = PlotWidget(self)
             self.plot_widget.setWindowTitle(nwst_id)
+            st = modify_stream_for_plot(st, parameters=self.parameters)
             st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig)
+            trace_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
+            trace_yticks(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
             self.plot_widget.show()
 
     def notification(self, text):

utils.py

@@ -3,6 +3,7 @@
 
 import matplotlib
 
+
 def get_bg_color(check_key, status, dt_thresh=None, hex=False):
     message = status.message
     if check_key == 'last active':
@@ -23,6 +24,7 @@ def get_bg_color(check_key, status, dt_thresh=None, hex=False):
         bg_color = '#{:02x}{:02x}{:02x}'.format(*bg_color[:3])
     return bg_color
 
+
 def get_color(key):
     # some GUI default colors
     colors_dict = {'FAIL': (255, 50, 0, 255),
@@ -33,6 +35,7 @@ def get_color(key):
                    'undefined': (230, 230, 230, 255)}
     return colors_dict.get(key)
 
+
 def get_time_delay_color(dt, dt_thresh):
     """ Set color of time delay after thresholds specified in self.dt_thresh """
     if dt < dt_thresh[0]:
@@ -41,6 +44,7 @@ def get_time_delay_color(dt, dt_thresh):
         return get_color('WARN')
     return get_color('FAIL')
 
+
 def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
     """ Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """
     if type(temp) in [str]:
@@ -50,3 +54,92 @@ def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
     rgba = [int(255 * c) for c in cmap(val)]
     return rgba
 
+
+def modify_stream_for_plot(st, parameters):
+    """ copy (if necessary) and modify stream for plotting """
+    ch_units = parameters.get('CHANNEL_UNITS')
+    ch_transf = parameters.get('CHANNEL_TRANSFORM')
+
+    # if either of both are defined make copy
+    if ch_units or ch_transf:
+        st = st.copy()
+
+    # modify trace for plotting by multiplying unit factor (e.g. 1e-3 mV to V)
+    if ch_units:
+        for tr in st:
+            channel = tr.stats.channel
+            unit_factor = ch_units.get(channel)
+            if unit_factor:
+                tr.data = tr.data * float(unit_factor)
+    # modify trace for plotting by other arithmetic expressions
+    if ch_transf:
+        for tr in st:
+            channel = tr.stats.channel
+            transf = ch_transf.get(channel)
+            if transf:
+                tr.data = transform_trace(tr.data, transf)
+
+    return st
+
+
+def transform_trace(data, transf):
+    """
+    Transform trace with arithmetic operations in order, specified in transf
+    @param data: numpy array
+    @param transf: list of lists with arithmetic operations (e.g. [['*', '20'], ] -> multiply data by 20
+    """
+    # This looks a little bit hardcoded, however it is safer than using e.g. "eval"
+    for operator_str, val in transf:
+        if operator_str == '+':
+            data = data + val
+        elif operator_str == '-':
+            data = data - val
+        elif operator_str == '*':
+            data = data * val
+        elif operator_str == '/':
+            data = data / val
+        else:
+            raise IOError(f'Unknown arithmethic operator string: {operator_str}')
+
+    return data
+
+
+def trace_ylabels(fig, parameters, verbosity=0):
+    """
+    Adds channel names to y-axis if defined in parameters.
+    Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is
+    difficult to assess the correct order from Obspy plotting routing.
+    """
+    names = parameters.get('channel_names')
+    if not names: # or not len(st.traces):
+        return
+    if not len(names) == len(fig.axes):
+        if verbosity:
+            print('Mismatch in axis and label lengths. Not adding plot labels')
+        return
+    for channel_name, ax in zip(names, fig.axes):
+        if channel_name:
+            ax.set_ylabel(channel_name)
+
+
+def trace_yticks(fig, parameters, verbosity=0):
+    """
+    Adds channel names to y-axis if defined in parameters.
+    Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is
+    difficult to assess the correct order from Obspy plotting routing.
+    """
+    ticks = parameters.get('CHANNEL_TICKS')
+    if not ticks:
+        return
+    if not len(ticks) == len(fig.axes):
+        if verbosity:
+            print('Mismatch in axis tick and label lengths. Not changing plot ticks.')
+        return
+    for ytick_tripple, ax in zip(ticks, fig.axes):
+        if not ytick_tripple:
+            continue
+        ymin, ymax, step = ytick_tripple
+
+        yticks = list(range(ymin, ymax + step, step))
+        ax.set_yticks(yticks)
+        ax.set_ylim(ymin - step, ymax + step)

write_utils.py

@@ -1,58 +1,68 @@
 from datetime import timedelta
 
+
 def write_html_table_title(fobj, parameters):
     title = get_print_title_str(parameters)
     fobj.write(f'<h3>{title}</h3>\n')
 
+
 def write_html_text(fobj, text):
     fobj.write(f'<p>{text}</p>\n')
 
+
 def write_html_header(fobj, refresh_rate=10):
     header = ['<!DOCTYPE html>',
               '<html>',
               '<head>',
-              '<link rel="stylesheet" href="stylesheet.css">',
+              '  <link rel="stylesheet" media="only screen and (max-width: 400px)" href="mobile.css" />',
+              '  <link rel="stylesheet" media="only screen and (min-width: 401px)" href="desktop.css" />',
               '</head>',
               f'<meta http-equiv="refresh" content="{refresh_rate}" >',
               '<meta charset="utf-8">',
+              '<meta name="viewport" content="width=device-width, initial-scale=1">',
               '<body>']
-    # style = ['<style>',
-    #           'table, th, td {',
-    #           'border:1px solid black;',
-    #           '}',
-    #           '</style>',]
     for item in header:
         fobj.write(item + '\n')
 
+
 def init_html_table(fobj):
     fobj.write('<table style="width:100%">\n')
 
+
 def finish_html_table(fobj):
     fobj.write('</table>\n')
 
+
 def write_html_footer(fobj):
     footer = ['</body>',
               '</html>']
     for item in footer:
         fobj.write(item + '\n')
 
+
 def write_html_row(fobj, items, html_key='td'):
     default_space = '  '
     fobj.write(default_space + '<tr>\n')
     for item in items:
         text = item.get('text')
+        if item.get('bold'):
+            text = '<b>' + text + '</b>'
+        if item.get('italic'):
+            text = '<i>' + text + '</i>'
         tooltip = item.get('tooltip')
         color = item.get('color')
         # check for black background of headers (shouldnt happen anymore)
         color = '#e6e6e6' if color == '#000000' else color
         hyperlink = item.get('hyperlink')
         image_str = f'<a href="{hyperlink}">' if hyperlink else ''
-        fobj.write(2 * default_space + f'<{html_key} bgcolor="{color}" title="{tooltip}"> {image_str}'
+        html_class = item.get('html_class')
+        class_str = f' class="{html_class}"' if html_class else ''
+        fobj.write(2 * default_space + f'<{html_key}{class_str} bgcolor="{color}" title="{tooltip}"> {image_str}'
                    + text + f'</{html_key}>\n')
     fobj.write(default_space + '</tr>\n')
 
+
 def get_print_title_str(parameters):
     timespan = parameters.get('timespan') * 24 * 3600
-    tdelta_str = str(timedelta(seconds=int(timespan)))
+    tdelta_str = str(timedelta(seconds=int(timespan))).replace(', 0:00:00', '')
     return f'Analysis table of router quality within the last {tdelta_str}'
-