fixed paramter file

README.md

@@ -32,8 +32,6 @@ The main program with html output is executed by entering
 python survBot.py -html path_for_html_output
 ```
 
-There are example stylesheets in the folder *stylesheets* that can be copied into the path_for_html_output if desired.
-
 The GUI can be loaded via
 
 ```shell script

parameters.yaml

@@ -3,9 +3,9 @@ datapath: "/data/SDS/"                    # SC3 Datapath
 networks: ["1Y", "HA"]                    # select networks, list or str
 stations: "*"                             # select stations, list or str
 locations: "*"                            # select locations, list or str
-channels: ["EX1", "EX2", "EX3", "VEI"]    # Specify SOH channels, currently supported EX[1-3] and VEI
+channels: ["EX1", "EX2", "EX3", "VEI", "LCQ"]    # Specify SOH channels, currently supported EX[1-3], VEI, LCQ
-stations_blacklist: ["TEST", "EREA"]      # exclude these stations
+stations_blacklist: ["TEST", "EREA"]
-networks_blacklist: []                    # exclude these networks
+networks_blacklist: []
 interval: 60               # Perform checks every x seconds
 n_track: 300               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
 timespan: 7                # Check data of the recent x days
@@ -41,8 +41,10 @@ THRESHOLDS:
   pb_thresh: 0.2               # Threshold for PowBox Voltage check +/- (V)
   max_temp: 50                 # max temperature for temperature warning
   low_volt: 12                 # min voltage for low voltage warning
-  high_volt: 14.8               # max voltage for over voltage warning
+  high_volt: 14.8              # max voltage for over voltage warning
   unclassified: 5              # min voltage samples not classified for warning
+  clockquality_warn: 90        # clock quality ranges from 0 % to 100 % with 100 % being the best level
+  clockquality_fail: 70
 
 # ---------------------------------------- OPTIONAL PARAMETERS ---------------------------------------------------------
 
@@ -58,13 +60,12 @@ EMAIL:
   mailserver: "localhost"
   addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"]  # list of mail addresses for info mails
   sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender
-  stations_blacklist: ['GR33']                     # do not send emails for specific stations
-  networks_blacklist: []                           # do not send emails for specific network
 
 # names for plotting of the above defined parameter "channels" in the same order
-channel_names: ["Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"]
+channel_names: ["Clock Quality (%)", "Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"] # names for plotting (optional)
 # specify y-ticks (and ylims) giving, (ymin, ymax, step) for each of the above channels (0: default)
 CHANNEL_TICKS:
+  - [0, 100, 20]
   - [-10, 50, 10]
   - [1, 5, 1]
   - [1, 5, 1]

stylesheets/desktop.css

@@ -1,39 +0,0 @@
-body {
-    background-color: #ffffff;
-    place-items: center;
-    text-align: center;
-}
-
-td {
-    border-radius: 4px;
-    padding: 0px;
-}
-
-th {
-    background-color: #999;
-    border-radius: 4px;
-    padding: 3px 1px;
-}
-
-a:link, a:visited {
-    background-color: #ccc;
-    color: #000;
-    text-decoration: none;
-    display: block;
-    border-radius: 4px;
-    border: 1px solid #bbb;
-}
-
-a:hover {
-    background-color: #aaa;
-    display: block;			  
-}
-
-.blink-bg {
-    animation: blinkingBackground 2s infinite;
-}
-@keyframes blinkingBackground{
-    0%		{ background-color: #ffcc00;}
-    50%		{ background-color: #ff3200;}
-    100%	{ background-color: #ffcc00;}
-}

stylesheets/mobile.css

@@ -1,43 +0,0 @@
-body {
-    background-color: #ffffff;
-    place-items: center;
-    text-align: center;
-}
-
-td {
-    border-radius: 4px;
-    padding: 10px 2px;
-}
-
-th {
-    background-color: #999;
-    border-radius: 4px;
-    padding: 10px, 2px;
-}
-
-a:link {
-    background-color: #ccc;
-    color: #000;
-    text-decoration: none;
-    display: block;
-    border-radius: 4px;
-    border: 1px solid #bbb;
-}
-
-a:hover {
-    background-color: #aaa;
-    display: block;			  
-}
-
-.hidden-mobile {
-    display: none;
-}
-
-.blink-bg {
-    animation: blinkingBackground 2s infinite;
-}
-@keyframes blinkingBackground{
-    0%		{ background-color: #ffee00;}
-    50%		{ background-color: #ff3200;}
-    100%	{ background-color: #ffee00;}
-}

submit_bot.sh

@@ -6,7 +6,7 @@ ulimit -s 8192
 #$ -cwd
 #$ -pe smp 1
 #$ -N survBot_bg
-#$ -l os=*stretch
+##$ -l os=*stretch
 
 source /opt/anaconda3/etc/profile.d/conda.sh
 conda activate py37

survBot.py

@@ -61,7 +61,7 @@ def fancy_timestr(dt, thresh=600, modif='+'):
 
 class SurveillanceBot(object):
     def __init__(self, parameter_path, outpath_html=None):
-        self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'temp', 'other']
+        self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'clock', 'temp', 'other']
         self.parameter_path = parameter_path
         self.update_parameters()
         self.starttime = UTCDateTime()
@@ -581,25 +581,11 @@ class StationQC(object):
             if self.verbosity:
                 print('Mail functionality disabled. Return')
             return
-
         mail_params = self.parameters.get('EMAIL')
         if not mail_params:
             if self.verbosity:
                 print('parameter "EMAIL" not set in parameter file. Return')
             return
-
-        stations_blacklist = mail_params.get('stations_blacklist')
-        if stations_blacklist and self.station in stations_blacklist:
-            if self.verbosity:
-                print(f'Station {self.station} listed in blacklist. Return')
-            return
-
-        networks_blacklist = mail_params.get('networks_blacklist')
-        if networks_blacklist and self.network in networks_blacklist:
-            if self.verbosity:
-                print(f'Station {self.station} of network {self.network} listed in blacklist. Return')
-            return
-
         sender = mail_params.get('sender')
         addresses = mail_params.get('addresses')
         server = mail_params.get('mailserver')
@@ -685,6 +671,7 @@ class StationQC(object):
         self.pb_temp_analysis()
         self.pb_power_analysis()
         self.pb_rout_charge_analysis()
+        self.clock_quality_analysis()
 
     def return_print_analysis(self):
         items = [self.nwst_id]
@@ -713,6 +700,44 @@ class StationQC(object):
     def get_last_occurrence(self, trace, indices):
         return self.get_time(trace, indices[-1])
 
+    def clock_quality_analysis(self, channel='LCQ'):
+        """ Analyse clock quality """
+        key = 'clock'
+        st = self.stream.select(channel=channel)
+        trace = self.get_trace(st, key)
+        if not trace: return
+        clockQuality = trace.data
+        clockQuality_warn_level = self.parameters.get('THRESHOLDS').get('clockquality_warn')
+        clockQuality_fail_level = self.parameters.get('THRESHOLDS').get('clockquality_fail')
+
+        if self.verbosity > 1:
+            self.print(40 * '-')
+            self.print('Performing Clock Quality check', flush=False)
+        
+        clockQuality_warn = np.where(clockQuality < clockQuality_warn_level)[0]
+        clockQuality_fail = np.where(clockQuality < clockQuality_fail_level)[0]
+
+        if len(clockQuality_warn) == 0 and len(clockQuality_fail) == 0:
+            self.status_ok(key, detailed_message=f'ClockQuality={(clockQuality[-1])}')
+            return
+ 
+        warn_message = f'Trace {trace.get_id()}:'
+        if len(clockQuality_warn) > 0:
+            # try calculate number of warn peaks from gaps between indices
+            n_qc_warn = self.calc_occurrences(clockQuality_warn)
+            detailed_message = warn_message + f' {n_qc_warn}x Qlock Quality less then {clockQuality_warn_level}' \
+                               + self.get_last_occurrence_timestring(trace, clockQuality_warn)
+            self.warn(key, detailed_message=detailed_message, count=n_qc_warn,
+                      last_occurrence=self.get_last_occurrence(trace, clockQuality_warn))
+
+        if len(clockQuality_fail) > 0:
+            # try calculate number of fail peaks from gaps between indices
+            n_qc_fail = self.calc_occurrences(clockQuality_fail)
+            detailed_message = warn_message + f' {n_qc_fail}x Qlock Quality less then {clockQuality_fail_level}V ' \
+                               + self.get_last_occurrence_timestring(trace, clockQuality_fail)
+            self.error(key, detailed_message=detailed_message, count=n_qc_fail,
+                      last_occurrence=self.get_last_occurrence(trace, clockQuality_fail))
+
     def voltage_analysis(self, channel='VEI'):
         """ Analyse voltage channel for over/undervoltage """
         key = 'voltage'

write_utils.py

@@ -14,13 +14,19 @@ def write_html_header(fobj, refresh_rate=10):
     header = ['<!DOCTYPE html>',
               '<html>',
               '<head>',
-              '  <link rel="stylesheet" media="only screen and (max-width: 400px)" href="mobile.css" />',
+              '<link rel="stylesheet" href="stylesheet.css">',
-              '  <link rel="stylesheet" media="only screen and (min-width: 401px)" href="desktop.css" />',
               '</head>',
               f'<meta http-equiv="refresh" content="{refresh_rate}" >',
               '<meta charset="utf-8">',
               '<meta name="viewport" content="width=device-width, initial-scale=1">',
+              '<link rel="stylesheet" media="only screen and (max-width: 400px)" href="mobile.css" />',
+              '<link rel="stylesheet" media="only screen and (min-width: 401px)" href="desktop.css" />',
               '<body>']
+    # style = ['<style>',
+    #           'table, th, td {',
+    #           'border:1px solid black;',
+    #           '}',
+    #           '</style>',]
     for item in header:
         fobj.write(item + '\n')