[update] moved html writing to survBot.py so that no GUI is needed

parameters.yaml

@@ -6,10 +6,12 @@ locations: '*'
 channels: ['EX1', 'EX2', 'EX3', 'VEI']    # Specify SOH channels, currently supported EX[1-3] and VEI
 stations_blacklist: ['TEST', 'EREA']
 networks_blacklist: []
-interval: 20               # Perform checks every x seconds
+interval: 60               # Perform checks every x seconds
 timespan: 7                # Check data of the recent x days
 verbosity: 0
+reread_parameters: True    # reread parameters file (change parameters on runtime, not for itself/GUI refresh/datapath)
 track_changes: True        # tracks all changes since GUI startup by text highlighting (GUI only)
+dt_thresh: [300, 1800]     # threshold (s) for timing delay colourisation (yellow/red)
 
 POWBOX:
   pb_ok: 1  # Voltage for PowBox OK

submit_bot.sh

@@ -17,5 +17,4 @@ export MKL_NUM_THREADS=1
 export NUMEXPR_NUM_THREADS=1
 export OMP_NUM_THREADS=1
 
-
+python survBot.py -html '/home/marcel/public_html/survBot_out.html'
-python survBotGUI.py -html '/home/marcel/public_html/survBot_out.html' --background

survBot.py

@@ -5,7 +5,9 @@ __version__ = '0.1'
 __author__ = 'Marcel Paffrath'
 
 import os
+import traceback
 import yaml
+import argparse
 
 import time
 from datetime import timedelta
@@ -14,7 +16,9 @@ import numpy as np
 from obspy import read, UTCDateTime, Stream
 from obspy.clients.filesystem.sds import Client
 
-from write_utils import get_print_title_str
+from write_utils import write_html_text, write_html_row, write_html_footer, write_html_header, get_print_title_str,\
+    init_html_table, finish_html_table
+from utils import get_bg_color
 
 pjoin = os.path.join
 UP = "\x1B[{length}A"
@@ -46,27 +50,35 @@ def fancy_timestr(dt, thresh=600, modif='+'):
 
 
 class SurveillanceBot(object):
-    def __init__(self, parameter_path):
+    def __init__(self, parameter_path, outpath_html=None):
         self.keys = ['last active', '230V', '12V', 'router', 'charger', 'voltage', 'temp', 'other']
-        self.parameters = read_yaml(parameter_path)
+        self.parameter_path = parameter_path
-        self.transform_parameters()
+        self.update_parameters()
         self.starttime = UTCDateTime()
-        self.verbosity = self.parameters.get('verbosity')
+        self.outpath_html = outpath_html
         self.filenames = []
         self.filenames_read = []
         self.station_list = []
         self.analysis_print_list = []
         self.analysis_results = {}
-        self.stations_blacklist = self.parameters.get('stations_blacklist')
-        self.networks_blacklist = self.parameters.get('networks_blacklist')
         self.dataStream = Stream()
         self.data = {}
         self.print_count = 0
-        self.refresh_period = 0
+        self.status_message = ''
 
         self.cl = Client(self.parameters.get('datapath'))  # TODO: Check if this has to be loaded again on update
         self.get_stations()
 
+    def update_parameters(self):
+        self.parameters = read_yaml(self.parameter_path)
+        self.reread_parameters = self.parameters.get('reread_parameters')
+        self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')]
+        self.verbosity = self.parameters.get('verbosity')
+        self.stations_blacklist = self.parameters.get('stations_blacklist')
+        self.networks_blacklist = self.parameters.get('networks_blacklist')
+        self.refresh_period = self.parameters.get('interval')
+        self.transform_parameters()
+
     def transform_parameters(self):
         for key in ['networks', 'stations', 'locations', 'channels']:
             parameter = self.parameters.get(key)
@@ -132,9 +144,11 @@ class SurveillanceBot(object):
             self.data[st_id].append(trace)
 
     def execute_qc(self):
-        self.starttime = UTCDateTime()
+        if self.reread_parameters:
+            self.update_parameters()
         self.get_filenames()
         self.read_data()
+        qc_starttime = UTCDateTime()
 
         self.analysis_print_list = []
         self.analysis_results = {}
@@ -142,7 +156,7 @@ class SurveillanceBot(object):
             stream = self.data.get(st_id)
             if stream:
                 nsl = nsl_from_id(st_id)
-                station_qc = StationQC(stream, nsl, self.parameters, self.keys, self.starttime, self.verbosity,
+                station_qc = StationQC(stream, nsl, self.parameters, self.keys, qc_starttime, self.verbosity,
                                        print_func=self.print)
                 analysis_print_result = station_qc.return_print_analysis()
                 station_dict, warn_dict = station_qc.return_analysis()
@@ -151,6 +165,8 @@ class SurveillanceBot(object):
                 station_dict, warn_dict = self.get_no_data_station(st_id)
             self.analysis_print_list.append(analysis_print_result)
             self.analysis_results[st_id] = (station_dict, warn_dict)
+
+        self.update_status_message()
         return 'ok'
 
     def get_no_data_station(self, st_id, no_data='-', to_print=False):
@@ -187,10 +203,6 @@ class SurveillanceBot(object):
         if len(times) > 0:
             return min(times)
 
-    def print_analysis_html(self, filename):
-        with open(filename, 'w') as outfile:
-            pass
-
     def print_analysis(self):
         self.print(200 * '+')
         title_str = get_print_title_str(self.parameters)
@@ -202,19 +214,22 @@ class SurveillanceBot(object):
         for items in self.analysis_print_list:
             self.console_print(items)
 
-    def start(self, refresh_period=30):
+    def start(self):
         '''
         Perform qc periodically.
         :param refresh_period: Update every x seconds
         :return:
         '''
-        self.refresh_period = refresh_period
         status = 'ok'
         while status == 'ok' and self.refresh_period > 0:
             status = self.execute_qc()
-            self.print_analysis()
+            if self.outpath_html:
+                self.write_html_table()
+            else:
+                self.print_analysis()
             time.sleep(self.refresh_period)
-            self.clear_prints()
+            if not self.outpath_html:
+                self.clear_prints()
 
     def console_print(self, itemlist, str_len=21, sep='|', seplen=3):
         assert len(sep) <= seplen, f'Make sure seperator has less than {seplen} characters'
@@ -225,6 +240,60 @@ class SurveillanceBot(object):
             string += item.center(str_len) + sr
         self.print(string, flush=False)
 
+    def write_html_table(self, default_color='#e6e6e6'):
+        fnout = self.outpath_html
+        if not fnout:
+            return
+        try:
+            with open(fnout, 'w') as outfile:
+                write_html_header(outfile, self.refresh_period)
+                #write_html_table_title(outfile, self.parameters)
+                init_html_table(outfile)
+
+                # First write header items
+                header_items = [dict(text='Station', color=default_color)]
+                for check_key in self.keys:
+                    item = dict(text=check_key, color=default_color)
+                    header_items.append(item)
+                write_html_row(outfile, header_items, html_key='th')
+
+                # Write all cells
+                for st_id in self.station_list:
+                    col_items = [dict(text=st_id.rstrip('.'), color=default_color)]
+                    for check_key in self.keys:
+                        status_dict, detailed_dict = self.analysis_results.get(st_id)
+                        status = status_dict.get(check_key)
+
+                        # get background color
+                        dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh]
+                        bg_color = get_bg_color(check_key, status, dt_thresh, hex=True)
+                        if not bg_color:
+                            bg_color = default_color
+
+                        # add degree sign for temp
+                        if check_key == 'temp':
+                            if not type(status) in [str]:
+                                status = str(status) + deg_str
+
+                        item = dict(text=str(status), tooltip=str(detailed_dict.get(check_key)),
+                                    color=bg_color)
+                        col_items.append(item)
+                    write_html_row(outfile, col_items)
+
+                finish_html_table(outfile)
+                write_html_text(outfile, self.status_message)
+                write_html_footer(outfile)
+        except Exception as e:
+            print(f'Could not write HTML table to {fnout}:')
+            print(traceback.format_exc())
+
+    def update_status_message(self):
+        timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
+        self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \
+                              f'Current time (UTC) {UTCDateTime().strftime("%Y-%m-%d %H:%M:%S")} | ' \
+                              f'Refresh period: {self.refresh_period}s | '\
+                              f'Showing data of last {timespan}'
+
     def print(self, string, **kwargs):
         clear_end = CLR + '\n'
         n_nl = string.count('\n')
@@ -553,7 +622,7 @@ class StationQC(object):
                 self.warn(key='other', detailed_message=f'Trace {trace.get_id()}: '
                                                f'{n_unclassified}/{len(all_indices)} '
                                                f'unclassified voltage values in channel {trace.get_id()}',
-                          status_message=f'{channel}: {n_unclassified} u')
+                          status_message=f'{channel}: {n_unclassified} uncl.')
 
         return False, voltage_dict, last_val
 
@@ -563,5 +632,9 @@ class StationQC(object):
 
 
 if __name__ == '__main__':
-    survBot = SurveillanceBot(parameter_path='parameters.yaml')
+    parser = argparse.ArgumentParser(description='Call survBot')
-    survBot.start(refresh_period=30)
+    parser.add_argument('-html', dest='html_filename', default=None, help='filename for HTML output')
+    args = parser.parse_args()
+
+    survBot = SurveillanceBot(parameter_path='parameters.yaml', outpath_html=args.html_filename)
+    survBot.start()

survBotGUI.py

@@ -10,7 +10,6 @@ __author__ = 'Marcel Paffrath'
 import os
 import sys
 import traceback
-import argparse
 
 try:
     from PySide2 import QtGui, QtCore, QtWidgets
@@ -36,6 +35,7 @@ from obspy import UTCDateTime
 
 from survBot import SurveillanceBot
 from write_utils import *
+from utils import get_bg_color
 
 try:
     from rest_api.utils import get_station_iccid
@@ -78,28 +78,17 @@ class Thread(QtCore.QThread):
 
 
 class MainWindow(QtWidgets.QMainWindow):
-    def __init__(self, parameters='parameters.yaml', outpath_html=None, dt_thresh=(300, 1800)):
+    def __init__(self, parameters='parameters.yaml'):
         """
         Main window of survBot GUI.
         :param parameters: Parameters dictionary file (yaml format)
-        :param dt_thresh: threshold for timing delay colourisation (yellow/red)
         """
         super(MainWindow, self).__init__()
 
-        # some GUI default colors
-        self.colors_dict = {'FAIL': (255, 50, 0, 255),
-                            'NO DATA': (255, 255, 125, 255),
-                            'WARN': (255, 255, 125, 255),
-                            'WARNX': lambda x: (min([255, 200 + x**2]), 255, 125, 255),
-                            'OK': (125, 255, 125, 255),
-                            'undefined': (230, 230, 230, 255)}
-
         # init some attributes
-        self.dt_thresh = dt_thresh
         self.last_mouse_loc = None
         self.status_message = ''
         self.starttime = UTCDateTime()
-        self.outpath_html = outpath_html
 
         # setup main layout of the GUI
         self.main_layout = QtWidgets.QVBoxLayout()
@@ -111,6 +100,7 @@ class MainWindow(QtWidgets.QMainWindow):
         self.survBot = SurveillanceBot(parameter_path=parameters)
         self.parameters = self.survBot.parameters
         self.refresh_period = self.parameters.get('interval')
+        self.dt_thresh = [int(val) for val in self.parameters.get('dt_thresh')]
 
         # create thread that is used to update
         self.thread = Thread(parent=self, runnable=self.survBot.execute_qc)
@@ -185,41 +175,6 @@ class MainWindow(QtWidgets.QMainWindow):
             self.last_mouse_loc = event.pos()
         return super(QtWidgets.QMainWindow, self).eventFilter(object, event)
 
-    def write_html_table(self):
-        fnout = self.outpath_html
-        if not fnout:
-            return
-        try:
-            with open(fnout, 'w') as outfile:
-                write_html_header(outfile, self.refresh_period)
-                #write_html_table_title(outfile, self.parameters)
-                init_html_table(outfile)
-                nrows = self.table.rowCount()
-                ncolumns = self.table.columnCount()
-
-                # add header item 0 fix default black bg color for headers
-                station_header = QtWidgets.QTableWidgetItem(text='Station')
-                station_header.setText('Station')
-                header_items = [station_header]
-                for column in range(ncolumns):
-                    hheader = self.table.horizontalHeaderItem(column)
-                    header_items.append(hheader)
-                write_html_row(outfile, header_items, html_key='th')
-
-                for row in range(nrows):
-                    vheader = self.table.verticalHeaderItem(row)
-                    col_items = [vheader]
-                    for column in range(ncolumns):
-                        col_items.append(self.table.item(row, column))
-                    write_html_row(outfile, col_items)
-
-                finish_html_table(outfile)
-                write_html_text(outfile, self.status_message)
-                write_html_footer(outfile)
-        except Exception as e:
-            print(f'Could not write HTML table to {fnout}:')
-            print(e)
-
     def sms_context_menu(self, row_ind):
         """ Open a context menu when left-clicking vertical header item """
         header_item = self.table.verticalHeaderItem(row_ind)
@@ -264,11 +219,7 @@ class MainWindow(QtWidgets.QMainWindow):
 
     def fill_status_bar(self):
         """ Set status bar text """
-        timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
+        self.status_message = self.survBot.status_message
-        self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \
-                              f'Current time (UTC) {UTCDateTime().strftime("%Y-%m-%d %H:%M:%S")} | ' \
-                              f'Refresh period: {self.refresh_period}s | '\
-                              f'Showing data of last {timespan}'
         status_bar = self.statusBar()
         status_bar.showMessage(self.status_message)
 
@@ -283,21 +234,12 @@ class MainWindow(QtWidgets.QMainWindow):
                 status_dict, detailed_dict = self.survBot.analysis_results.get(st_id)
                 status = status_dict.get(check_key)
                 detailed_message = detailed_dict.get(check_key)
-                if check_key == 'last active':
+
-                    bg_color = self.get_time_delay_color(status)
+                dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh]
-                elif check_key == 'temp':
+                bg_color = get_bg_color(check_key, status, dt_thresh)
-                    bg_color = self.get_temp_color(status)
+                if check_key == 'temp':
                     if not type(status) in [str]:
                         status = str(status) + deg_str
-                else:
-                    statussplit = status.split(' ')
-                    if len(statussplit) > 1 and statussplit[0] == 'WARN':
-                        x = int(status.split(' ')[-1].lstrip('(').rstrip(')'))
-                        bg_color = self.colors_dict.get('WARNX')(x)
-                    else:
-                        bg_color = self.colors_dict.get(status)
-                if not bg_color:
-                    bg_color = self.colors_dict.get('undefined')
 
                 # Continue if nothing changed
                 text = str(status)
@@ -336,26 +278,6 @@ class MainWindow(QtWidgets.QMainWindow):
 
         # table filling/refreshing done, set clear_on_refresh to False
         self.clear_on_refresh = False
-        # write html output if parameter is set
-        self.write_html_table()
-
-    def get_time_delay_color(self, dt):
-        """ Set color of time delay after thresholds specified in self.dt_thresh """
-        dt_thresh = [timedelta(seconds=sec) for sec in self.dt_thresh]
-        if dt < dt_thresh[0]:
-            return self.colors_dict.get('OK')
-        elif dt_thresh[0] <= dt < dt_thresh[1]:
-            return self.colors_dict.get('WARN')
-        return self.colors_dict.get('FAIL')
-
-    def get_temp_color(self, temp, vmin=-10, vmax=60, cmap='coolwarm'):
-        """ Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """
-        if type(temp) in [str]:
-            return self.colors_dict.get('undefined')
-        cmap = matplotlib.cm.get_cmap(cmap)
-        val = (temp - vmin) / (vmax - vmin)
-        rgba = [int(255 * c) for c in cmap(val)]
-        return rgba
 
     def set_font_bold(self, item):
         """ Set item font bold """
@@ -507,15 +429,9 @@ class SendSMSWidget(QtWidgets.QDialog):
 
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Call survBot GUI')
-    parser.add_argument('-html', dest='html_filename', default=None, help='filename for HTML output')
-    parser.add_argument('--background', dest='background', default=False, action='store_true', help='run in background')
-    args = parser.parse_args()
-
     program_path = sys.path[0]
     parameters = os.path.join(program_path, 'parameters.yaml')
     app = QtWidgets.QApplication([])
-    window = MainWindow(parameters=parameters, outpath_html=args.html_filename)
+    window = MainWindow(parameters=parameters)
-    if not args.background:
+    window.showMaximized()
-        window.showMaximized()
     sys.exit(app.exec_())

survBotGUI.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-ulimit -s 8192
-
-#$ -l os=*stretch
-##$ -cwd
-#$ -pe smp 1
-##$ -q "*@minos15"
-
-export PYTHONPATH="$PYTHONPATH:/home/marcel/git/"
-
-source /opt/anaconda3/etc/profile.d/conda.sh
-conda activate py37
-
-python /home/marcel/git/survBot/survBotGUI.py

utils.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import matplotlib
+
+def get_bg_color(check_key, status, dt_thresh=None, hex=False):
+    if check_key == 'last active':
+        bg_color = get_time_delay_color(status, dt_thresh)
+    elif check_key == 'temp':
+        bg_color = get_temp_color(status)
+    else:
+        statussplit = status.split(' ')
+        if len(statussplit) > 1 and statussplit[0] == 'WARN':
+            x = int(status.split(' ')[-1].lstrip('(').rstrip(')'))
+            bg_color = get_color('WARNX')(x)
+        else:
+            bg_color = get_color(status)
+    if not bg_color:
+        bg_color = get_color('undefined')
+
+    if hex:
+        bg_color = '#{:02x}{:02x}{:02x}'.format(*bg_color[:3])
+    return bg_color
+
+def get_color(key):
+    # some GUI default colors
+    colors_dict = {'FAIL': (255, 50, 0, 255),
+                   'NO DATA': (255, 255, 125, 255),
+                   'WARN': (255, 255, 125, 255),
+                   'WARNX': lambda x: (min([255, 200 + x ** 2]), 255, 125, 255),
+                   'OK': (125, 255, 125, 255),
+                   'undefined': (230, 230, 230, 255)}
+    return colors_dict.get(key)
+
+def get_time_delay_color(dt, dt_thresh):
+    """ Set color of time delay after thresholds specified in self.dt_thresh """
+    if dt < dt_thresh[0]:
+        return get_color('OK')
+    elif dt_thresh[0] <= dt < dt_thresh[1]:
+        return get_color('WARN')
+    return get_color('FAIL')
+
+def get_temp_color(temp, vmin=-10, vmax=60, cmap='coolwarm'):
+    """ Get an rgba temperature value back from specified cmap, linearly interpolated between vmin and vmax. """
+    if type(temp) in [str]:
+        return get_color('undefined')
+    cmap = matplotlib.cm.get_cmap(cmap)
+    val = (temp - vmin) / (vmax - vmin)
+    rgba = [int(255 * c) for c in cmap(val)]
+    return rgba
+

write_utils.py

@@ -37,10 +37,10 @@ def write_html_row(fobj, items, html_key='td'):
     default_space = '  '
     fobj.write(default_space + '<tr>\n')
     for item in items:
-        text = item.text()
+        text = item.get('text')
-        tooltip = item.toolTip()
+        tooltip = item.get('tooltip')
-        color = item.backgroundColor().name()
+        color = item.get('color')
-        # fix for black background of headers
+        # check for black background of headers (shouldnt happen anymore)
         color = '#e6e6e6' if color == '#000000' else color
         fobj.write(2 * default_space + f'<{html_key} bgcolor="{color}" title="{tooltip}">' + text + f'</{html_key}>\n')
     fobj.write(default_space + '</tr>\n')