Merge branch 'develop' into issue/1

2022-12-06 15:47:56 +01:00 · 2022-12-06 15:47:56 +01:00 · ac1ce5f6fa
commit ac1ce5f6fa
parent cb3623e4a9 f0ae7da2be
5 changed files with 78 additions and 20 deletions
--- a/parameters.yaml
+++ b/parameters.yaml
@ -8,11 +8,12 @@ channel_names: ["Clock Quality (%)", "Temperature (°C)", "230V/12V Status (V)",
 stations_blacklist: ["TEST", "EREA"]
 networks_blacklist: []
 interval: 60               # Perform checks every x seconds
-n_track: 120               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
+n_track: 300               # wait n_track * intervals before performing an action (i.e. send mail/end highlight status)
 timespan: 7                # Check data of the recent x days
-verbosity: 0
+verbosity: 0               # verbosity flag
 track_changes: True        # tracks all changes since GUI startup by text highlighting (GUI only)
-warn_count: False          # show number of warnings and errors in table
+warn_count: False           # show number of warnings and errors in table
+min_sample: 3              # minimum samples for raising Warn/FAIL
 dt_thresh: [300, 1800]     # threshold (s) for timing delay colourisation (yellow/red)
 html_figures: True         # Create html figure directory and links
 reread_parameters: True    # reread parameters file (change parameters on runtime, not for itself/GUI refresh/datapath)
@ -46,20 +47,31 @@ THRESHOLDS:
  clockquality_warn: 90        # clock quality ranges from 0 % to 100 % with 100 % being the best level
  clockquality_fail: 70

+# ---------------------------------------- OPTIONAL PARAMETERS ---------------------------------------------------------
+
 # add links to html table with specified key as column and value as relative link, interpretable string parameters:
 # nw (e.g. 1Y), st (e.g. GR01A), nwst_id (e.g. 1Y.GR01A)
-# optional!
 add_links:
-  slmon: {"URL": "../slmon/{nw}_{st}.html", "text": "show"}  # for example: slmon: {"URL": "path/{nw}_{st}.html", "text": "link"}
+  # for example: slmon: {"URL": "path/{nw}_{st}.html", "text": "link"}
+  slmon: {"URL": "../slmon/{nw}_{st}.html", "text": "show"}
  24h-plot: {"URL": "../scheli/{nw}/{st}.png", "text": "plot"}

-# E-mail notifications (optional)
+# E-mail notifications
 EMAIL:
  mailserver: "localhost"
  addresses: ["marcel.paffrath@rub.de", "kasper.fischer@rub.de"]  # list of mail addresses for info mails
  sender: "webmaster@geophysik.ruhr-uni-bochum.de" # mail sender

-# Factor for channel to SI-units (for plotting, optional)
+# names for plotting of the above defined parameter "channels" in the same order
+channel_names: ["Temperature (°C)", "230V/12V Status (V)", "Router/Charger State (V)", "Logger Voltage (V)"]
+# specify y-ticks (and ylims) giving, (ymin, ymax, step) for each of the above channels (0: default)
+CHANNEL_TICKS:
+  - [-10, 50, 10]
+  - [1, 5, 1]
+  - [1, 5, 1]
+  - [9, 15, 1]
+
+# Factor for channel to SI-units (for plotting)
 CHANNEL_UNITS:
  EX1: 1e-6
  EX2: 1e-6
@ -67,8 +79,7 @@ CHANNEL_UNITS:
  VEI: 1e-3

 # Transform channel for plotting, perform arithmetic operations in given order, e.g.: PBox EX1 V to deg C: 20 * x -20
-# optional!
 CHANNEL_TRANSFORM:
  EX1:
    - ["*", 20]
-    - ["-", 20]
+    - ["-", 20]
--- a/submit_bot.sh
+++ b/submit_bot.sh
@ -6,7 +6,7 @@ ulimit -s 8192
 #$ -cwd
 #$ -pe smp 1
 #$ -N survBot_bg
-#$ -l os=*stretch
+##$ -l os=*stretch

 source /opt/anaconda3/etc/profile.d/conda.sh
 conda activate py37
--- a/survBot.py
+++ b/survBot.py
@ -19,7 +19,7 @@ from obspy.clients.filesystem.sds import Client

 from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str, \
    init_html_table, finish_html_table
-from utils import get_bg_color, modify_stream_for_plot, annotate_trace_axes
+from utils import get_bg_color, modify_stream_for_plot, trace_ylabels, trace_yticks

 try:
    import smtplib
@ -341,7 +341,8 @@ class SurveillanceBot(object):
            try:
                st = modify_stream_for_plot(st, parameters=self.parameters)
                st.plot(fig=fig, show=False, draw=False, block=False, equal_scale=False, method='full')
-                annotate_trace_axes(fig, self.parameters, self.verbosity)
+                trace_ylabels(fig, self.parameters, self.verbosity)
+                trace_yticks(fig, self.parameters, self.verbosity)
            except Exception as e:
                print(f'Could not generate plot for {nwst_id}:')
                print(traceback.format_exc())
@ -349,6 +350,8 @@ class SurveillanceBot(object):
                ax = fig.axes[0]
                ax.set_title(f'Plot refreshed at (UTC) {UTCDateTime.now().strftime("%Y-%m-%d %H:%M:%S")}. '
                             f'Refreshed hourly or on FAIL status.')
+                for ax in fig.axes:
+                    ax.grid(True, alpha=0.1)
                fig.savefig(fnout, dpi=150., bbox_inches='tight')
        plt.close(fig)

@ -759,9 +762,6 @@ class StationQC(object):
            self.status_ok(key, detailed_message=f'U={(voltage[-1])}V')
            return

-        n_overvolt = 0
-        n_undervolt = 0
-
        warn_message = f'Trace {trace.get_id()}:'
        if len(overvolt) > 0:
            # try calculate number of voltage peaks from gaps between indices
@ -874,8 +874,7 @@ class StationQC(object):
                        self.status_ok(key)
                        continue
                    if volt_lvl > 1:
-                        # try calculate number of voltage peaks from gaps between indices
-                        n_occurrences = len(np.where(np.diff(ind_array) > 1)[0]) + 1
+                        n_occurrences = self.calc_occurrences(ind_array)
                        self.warn(key=key,
                                  detailed_message=f'Trace {trace.get_id()}: '
                                                   f'Found {n_occurrences} occurrence(s) of {volt_lvl}V: {key}: {message}'
@ -886,6 +885,32 @@ class StationQC(object):
                    if volt_lvl == last_val or (volt_lvl == -1 and last_val < 1):
                        self.error(key, detailed_message=f'Last PowBox voltage state {last_val}V: {message}')

+    def calc_occurrences(self, ind_array):
+        # try calculate number of voltage peaks/plateaus from gaps between indices
+        if len(ind_array) == 0:
+            return 0
+        else:
+            # start index at 1 if there are gaps (n_peaks = n_gaps + 1)
+            n_occurrences = 1
+
+        min_samples = self.parameters.get('min_sample')
+        if not min_samples:
+            min_samples = 1
+
+        # calculated differences in index array, diff > 1: gap, diff == 1: within peak/plateau
+        diffs = np.diff(ind_array)
+        gap_start_inds = np.where(np.diff(ind_array) > 1)[0]
+        # iterate over all gaps and check "min_samples" before the gap
+        for gsi in gap_start_inds:
+            # right boundary index of peak (gap index - 1)
+            peak_rb_ind = gsi - 1
+            # left boundary index of peak
+            peak_lb_ind = max([0, peak_rb_ind - min_samples])
+            if all(diffs[peak_lb_ind: peak_rb_ind] == 1):
+                n_occurrences += 1
+
+        return n_occurrences
+
    def get_trace(self, stream, keys):
        if not type(keys) == list:
            keys = [keys]
--- a/survBotGUI.py
+++ b/survBotGUI.py
@ -34,7 +34,7 @@ from obspy import UTCDateTime

 from survBot import SurveillanceBot
 from write_utils import *
-from utils import get_bg_color, modify_stream_for_plot, annotate_trace_axes
+from utils import get_bg_color, modify_stream_for_plot, trace_ylabels, trace_yticks

 try:
    from rest_api.utils import get_station_iccid
@ -316,7 +316,8 @@ class MainWindow(QtWidgets.QMainWindow):
            self.plot_widget.setWindowTitle(nwst_id)
            st = modify_stream_for_plot(st, parameters=self.parameters)
            st.plot(equal_scale=False, method='full', block=False, fig=self.plot_widget.canvas.fig)
-            annotate_trace_axes(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
+            trace_ylabels(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
+            trace_yticks(fig=self.plot_widget.canvas.fig, parameters=self.parameters)
            self.plot_widget.show()

    def notification(self, text):
--- a/utils.py
+++ b/utils.py
@ -104,7 +104,7 @@ def transform_trace(data, transf):
    return data


-def annotate_trace_axes(fig, parameters, verbosity=0):
+def trace_ylabels(fig, parameters, verbosity=0):
    """
    Adds channel names to y-axis if defined in parameters.
    Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is
@ -122,3 +122,24 @@ def annotate_trace_axes(fig, parameters, verbosity=0):
            ax.set_ylabel(channel_name)


+def trace_yticks(fig, parameters, verbosity=0):
+    """
+    Adds channel names to y-axis if defined in parameters.
+    Can get mixed up if channel order in stream and channel names defined in parameters.yaml differ, but it is
+    difficult to assess the correct order from Obspy plotting routing.
+    """
+    ticks = parameters.get('CHANNEL_TICKS')
+    if not ticks:
+        return
+    if not len(ticks) == len(fig.axes):
+        if verbosity:
+            print('Mismatch in axis tick and label lengths. Not changing plot ticks.')
+        return
+    for ytick_tripple, ax in zip(ticks, fig.axes):
+        if not ytick_tripple:
+            continue
+        ymin, ymax, step = ytick_tripple
+
+        yticks = list(range(ymin, ymax + step, step))
+        ax.set_yticks(yticks)
+        ax.set_ylim(ymin - step, ymax + step)