Merge pull request 'release/1.0' (#3) from release/1.0 into main

Reviewed-on: #3

README.md

@@ -32,6 +32,8 @@ The main program with html output is executed by entering
 python survBot.py -html path_for_html_output
 ```
 
+There are example stylesheets in the folder *stylesheets* that can be copied into the path_for_html_output if desired.
+
 The GUI can be loaded via
 
 ```shell script

parameters.yaml

@@ -3,10 +3,9 @@ datapath: "/data/SDS/"                    # SC3 Datapath
 networks: ["1Y", "HA"]                    # select networks, list or str
 stations: "*"                             # select stations, list or str
 locations: "*"                            # select locations, list or str
-channels: ["EX1", "EX2", "EX3", "VEI", "LCQ"]    # Specify SOH channels, currently supported EX[1-3], VEI, LCQ
-channel_names: ["Clock Quality (%)", "Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"] # names for plotting (optional)
-stations_blacklist: ["TEST", "EREA"]
-networks_blacklist: []
+channels: ["EX1", "EX2", "EX3", "VEI"]    # Specify SOH channels, currently supported EX[1-3] and VEI
+stations_blacklist: ["TEST", "EREA"]      # exclude these stations
+networks_blacklist: []                    # exclude these networks
 interval: 60               # Perform checks every x seconds
 n_track: 300               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
 timespan: 7                # Check data of the recent x days
@@ -44,8 +43,6 @@ THRESHOLDS:
   low_volt: 12                 # min voltage for low voltage warning
   high_volt: 14.8               # max voltage for over voltage warning
   unclassified: 5              # min voltage samples not classified for warning
-  clockquality_warn: 90        # clock quality ranges from 0 % to 100 % with 100 % being the best level
-  clockquality_fail: 70
 
 # ---------------------------------------- OPTIONAL PARAMETERS ---------------------------------------------------------
 
@@ -61,6 +58,8 @@ EMAIL:
   mailserver: "localhost"
   addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"]  # list of mail addresses for info mails
   sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender
+  stations_blacklist: ['GR33']                     # do not send emails for specific stations
+  networks_blacklist: []                           # do not send emails for specific network
 
 # names for plotting of the above defined parameter "channels" in the same order
 channel_names: ["Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"]

stylesheets/desktop.css

@@ -0,0 +1,39 @@
+body {
+    background-color: #ffffff;
+    place-items: center;
+    text-align: center;
+}
+
+td {
+    border-radius: 4px;
+    padding: 0px;
+}
+
+th {
+    background-color: #999;
+    border-radius: 4px;
+    padding: 3px 1px;
+}
+
+a:link, a:visited {
+    background-color: #ccc;
+    color: #000;
+    text-decoration: none;
+    display: block;
+    border-radius: 4px;
+    border: 1px solid #bbb;
+}
+
+a:hover {
+    background-color: #aaa;
+    display: block;			  
+}
+
+.blink-bg {
+    animation: blinkingBackground 2s infinite;
+}
+@keyframes blinkingBackground{
+    0%		{ background-color: #ffcc00;}
+    50%		{ background-color: #ff3200;}
+    100%	{ background-color: #ffcc00;}
+}

stylesheets/mobile.css

@@ -0,0 +1,43 @@
+body {
+    background-color: #ffffff;
+    place-items: center;
+    text-align: center;
+}
+
+td {
+    border-radius: 4px;
+    padding: 10px 2px;
+}
+
+th {
+    background-color: #999;
+    border-radius: 4px;
+    padding: 10px, 2px;
+}
+
+a:link {
+    background-color: #ccc;
+    color: #000;
+    text-decoration: none;
+    display: block;
+    border-radius: 4px;
+    border: 1px solid #bbb;
+}
+
+a:hover {
+    background-color: #aaa;
+    display: block;			  
+}
+
+.hidden-mobile {
+    display: none;
+}
+
+.blink-bg {
+    animation: blinkingBackground 2s infinite;
+}
+@keyframes blinkingBackground{
+    0%		{ background-color: #ffee00;}
+    50%		{ background-color: #ff3200;}
+    100%	{ background-color: #ffee00;}
+}

submit_bot.sh

@@ -6,7 +6,7 @@ ulimit -s 8192
 #$ -cwd
 #$ -pe smp 1
 #$ -N survBot_bg
-##$ -l os=*stretch
+#$ -l os=*stretch
 
 source /opt/anaconda3/etc/profile.d/conda.sh
 conda activate py37

survBot.py

@@ -61,7 +61,7 @@ def fancy_timestr(dt, thresh=600, modif='+'):
 
 class SurveillanceBot(object):
     def __init__(self, parameter_path, outpath_html=None):
-        self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'clock', 'temp', 'other']
+        self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'temp', 'other']
         self.parameter_path = parameter_path
         self.update_parameters()
         self.starttime = UTCDateTime()
@@ -157,7 +157,7 @@ class SurveillanceBot(object):
             if filename in self.filenames_read:
                 continue
             try:
-                st_new = read(filename)
+                st_new = read(filename, dtype=float)
                 # add file to read filenames to prevent re-reading in case it is not the current day (or end of
                 # previous day)
                 if not filename.endswith(f'{current_day:03}') and not (
@@ -167,7 +167,7 @@ class SurveillanceBot(object):
                 print(f'Could not read file {filename}:', e)
                 continue
             self.dataStream += st_new
-        self.dataStream.merge()
+        self.dataStream.merge(fill_value=np.nan)
 
         # organise data in dictionary with key for each station
         for trace in self.dataStream:
@@ -581,11 +581,25 @@ class StationQC(object):
             if self.verbosity:
                 print('Mail functionality disabled. Return')
             return
+
         mail_params = self.parameters.get('EMAIL')
         if not mail_params:
             if self.verbosity:
                 print('parameter "EMAIL" not set in parameter file. Return')
             return
+
+        stations_blacklist = mail_params.get('stations_blacklist')
+        if stations_blacklist and self.station in stations_blacklist:
+            if self.verbosity:
+                print(f'Station {self.station} listed in blacklist. Return')
+            return
+
+        networks_blacklist = mail_params.get('networks_blacklist')
+        if networks_blacklist and self.network in networks_blacklist:
+            if self.verbosity:
+                print(f'Station {self.station} of network {self.network} listed in blacklist. Return')
+            return
+
         sender = mail_params.get('sender')
         addresses = mail_params.get('addresses')
         server = mail_params.get('mailserver')
@@ -671,7 +685,6 @@ class StationQC(object):
         self.pb_temp_analysis()
         self.pb_power_analysis()
         self.pb_rout_charge_analysis()
-        self.clock_quality_analysis()
 
     def return_print_analysis(self):
         items = [self.nwst_id]
@@ -700,44 +713,6 @@ class StationQC(object):
     def get_last_occurrence(self, trace, indices):
         return self.get_time(trace, indices[-1])
 
-    def clock_quality_analysis(self, channel='LCQ'):
-        """ Analyse clock quality """
-        key = 'clock'
-        st = self.stream.select(channel=channel)
-        trace = self.get_trace(st, key)
-        if not trace: return
-        clockQuality = trace.data
-        clockQuality_warn_level = self.parameters.get('THRESHOLDS').get('clockquality_warn')
-        clockQuality_fail_level = self.parameters.get('THRESHOLDS').get('clockquality_fail')
-
-        if self.verbosity > 1:
-            self.print(40 * '-')
-            self.print('Performing Clock Quality check', flush=False)
-        
-        clockQuality_warn = np.where(clockQuality < clockQuality_warn_level)[0]
-        clockQuality_fail = np.where(clockQuality < clockQuality_fail_level)[0]
-
-        if len(clockQuality_warn) == 0 and len(clockQuality_fail) == 0:
-            self.status_ok(key, detailed_message=f'ClockQuality={(clockQuality[-1])}')
-            return
- 
-        warn_message = f'Trace {trace.get_id()}:'
-        if len(clockQuality_warn) > 0:
-            # try calculate number of warn peaks from gaps between indices
-            n_qc_warn = self.calc_occurrences(clockQuality_warn)
-            detailed_message = warn_message + f' {n_qc_warn}x Qlock Quality less then {clockQuality_warn_level}' \
-                               + self.get_last_occurrence_timestring(trace, clockQuality_warn)
-            self.warn(key, detailed_message=detailed_message, count=n_qc_warn,
-                      last_occurrence=self.get_last_occurrence(trace, clockQuality_warn))
-
-        if len(clockQuality_fail) > 0:
-            # try calculate number of fail peaks from gaps between indices
-            n_qc_fail = self.calc_occurrences(clockQuality_fail)
-            detailed_message = warn_message + f' {n_qc_fail}x Qlock Quality less then {clockQuality_fail_level}V ' \
-                               + self.get_last_occurrence_timestring(trace, clockQuality_fail)
-            self.error(key, detailed_message=detailed_message, count=n_qc_fail,
-                      last_occurrence=self.get_last_occurrence(trace, clockQuality_fail))
-
     def voltage_analysis(self, channel='VEI'):
         """ Analyse voltage channel for over/undervoltage """
         key = 'voltage'

write_utils.py

@@ -14,19 +14,13 @@ def write_html_header(fobj, refresh_rate=10):
     header = ['<!DOCTYPE html>',
               '<html>',
               '<head>',
-              '<link rel="stylesheet" href="stylesheet.css">',
+              '  <link rel="stylesheet" media="only screen and (max-width: 400px)" href="mobile.css" />',
+              '  <link rel="stylesheet" media="only screen and (min-width: 401px)" href="desktop.css" />',
               '</head>',
               f'<meta http-equiv="refresh" content="{refresh_rate}" >',
               '<meta charset="utf-8">',
               '<meta name="viewport" content="width=device-width, initial-scale=1">',
-              '<link rel="stylesheet" media="only screen and (max-width: 400px)" href="mobile.css" />',
-              '<link rel="stylesheet" media="only screen and (min-width: 401px)" href="desktop.css" />',
               '<body>']
-    # style = ['<style>',
-    #           'table, th, td {',
-    #           'border:1px solid black;',
-    #           '}',
-    #           '</style>',]
     for item in header:
         fobj.write(item + '\n')