Merge remote-tracking branch 'origin/develop' into develop

parameters.yaml

@@ -1,5 +1,6 @@
 # Parameters file for Surveillance Bot
 datapath: '/data/SDS/'      # SC3 Datapath
+outpath_html: '/home/marcel/tmp/survBot_out.html' # output of HTML table
 networks: ['1Y', 'HA']
 stations: '*'
 locations: '*'

submit_bot.sh

@@ -8,6 +8,7 @@ ulimit -s 8192
 ##$ -q "*@minos15"
 
 export PYTHONPATH="$PYTHONPATH:/home/marcel/git/"
+export PYTHONPATH="$PYTHONPATH:/home/marcel/git/code_base/"
 
 source /opt/anaconda3/etc/profile.d/conda.sh
 conda activate py37

survBot.py

@@ -13,11 +13,14 @@ import numpy as np
 from obspy import read, UTCDateTime, Stream
 from obspy.clients.filesystem.sds import Client
 
+from write_utils import get_print_title_str
+
 pjoin = os.path.join
 UP = "\x1B[{length}A"
 CLR = "\x1B[0K"
 deg_str = '\N{DEGREE SIGN}C'
 
+
 def read_yaml(file_path):
     with open(file_path, "r") as f:
         return yaml.safe_load(f)
@@ -30,7 +33,7 @@ def nsl_from_id(st_id):
 
 def get_st_id(trace):
     stats = trace.stats
-    return f'{stats.network}.{stats.station}.'#{stats.location}'
+    return f'{stats.network}.{stats.station}.'  # {stats.location}'
 
 
 def fancy_timestr(dt, thresh=600, modif='+'):
@@ -60,7 +63,7 @@ class SurveillanceBot(object):
         self.print_count = 0
         self.refresh_period = 0
 
-        self.cl = Client(self.parameters.get('datapath')) #TODO: Check if this has to be loaded again on update
+        self.cl = Client(self.parameters.get('datapath'))  # TODO: Check if this has to be loaded again on update
         self.get_stations()
 
     def transform_parameters(self):
@@ -99,7 +102,7 @@ class SurveillanceBot(object):
                 for location in locations:
                     for channel in channels:
                         self.filenames += list(self.cl._get_filenames(network, station, location, channel,
-                                                                 starttime=t1, endtime=time_now))
+                                                                      starttime=t1, endtime=time_now))
 
     def read_data(self):
         self.data = {}
@@ -183,11 +186,13 @@ class SurveillanceBot(object):
         if len(times) > 0:
             return min(times)
 
+    def print_analysis_html(self, filename):
+        with open(filename, 'w') as outfile:
+            pass
+
     def print_analysis(self):
-        timespan = self.parameters.get('timespan') * 24 * 3600
-        self.print(200*'+')
-        tdelta_str = str(timedelta(seconds=int(timespan)))
-        title_str = f'Analysis table of router quality within the last {tdelta_str}'
+        self.print(200 * '+')
+        title_str = get_print_title_str(self.parameters)
         self.print(title_str)
         if self.refresh_period > 0:
             self.print(f'Refreshing every {self.refresh_period}s.')
@@ -224,8 +229,8 @@ class SurveillanceBot(object):
         n_nl = string.count('\n')
         string.replace('\n', clear_end)
         print(string, end=clear_end, **kwargs)
-        self.print_count += n_nl + 1 #number of newlines + actual print with end='\n' (no check for kwargs end!)
-        #print('pc:', self.print_count)
+        self.print_count += n_nl + 1  # number of newlines + actual print with end='\n' (no check for kwargs end!)
+        # print('pc:', self.print_count)
 
     def clear_prints(self):
         print(UP.format(length=self.print_count), end='')
@@ -300,7 +305,7 @@ class StationQC(object):
 
     def analyse_channels(self):
         if self.verbosity > 0:
-            self.print(150*'#')
+            self.print(150 * '#')
             self.print('This is StationQT. Calculating quality for station'
                        ' {network}.{station}.{location}'.format(**self.nsl))
         self.voltage_analysis()
@@ -333,7 +338,7 @@ class StationQC(object):
 
     def voltage_analysis(self, channel='VEI'):
         """ Analyse voltage channel for over/undervoltage """
-        key='voltage'
+        key = 'voltage'
         st = self.stream.select(channel=channel)
         trace = self.get_trace(st, key)
         if not trace: return
@@ -367,7 +372,7 @@ class StationQC(object):
 
     def pb_temp_analysis(self, channel='EX1'):
         """ Analyse PowBox temperature output. """
-        key='temp'
+        key = 'temp'
         st = self.stream.select(channel=channel)
         trace = self.get_trace(st, key)
         if not trace: return
@@ -375,7 +380,7 @@ class StationQC(object):
         thresholds = self.parameters.get('THRESHOLDS')
         temp = 20. * voltage - 20
         # average temp
-        timespan = min([self.parameters.get('timespan') * 24 * 3600, int(len(temp)/trace.stats.sampling_rate)])
+        timespan = min([self.parameters.get('timespan') * 24 * 3600, int(len(temp) / trace.stats.sampling_rate)])
         nsamp_av = int(trace.stats.sampling_rate) * timespan
         av_temp_str = str(round(np.mean(temp[-nsamp_av:]), 1)) + deg_str
         # dt of average

survBotGUI.py

@@ -9,7 +9,7 @@ __author__ = 'Marcel Paffrath'
 import os
 import sys
 import traceback
-from datetime import timedelta
+import argparse
 
 try:
     from PySide2 import QtGui, QtCore, QtWidgets
@@ -34,6 +34,7 @@ else:
 from obspy import UTCDateTime
 
 from survBot import SurveillanceBot
+from write_utils import *
 
 try:
     from rest_api.utils import get_station_iccid
@@ -95,6 +96,7 @@ class MainWindow(QtWidgets.QMainWindow):
         # init some attributes
         self.dt_thresh = dt_thresh
         self.last_mouse_loc = None
+        self.status_message = ''
         self.starttime = UTCDateTime()
 
         # setup main layout of the GUI
@@ -181,6 +183,41 @@ class MainWindow(QtWidgets.QMainWindow):
             self.last_mouse_loc = event.pos()
         return super(QtWidgets.QMainWindow, self).eventFilter(object, event)
 
+    def write_html_table(self):
+        fnout = self.parameters.get('outpath_html')
+        if not fnout:
+            return
+        try:
+            with open(fnout, 'w') as outfile:
+                write_html_header(outfile)
+                #write_html_table_title(outfile, self.parameters)
+                init_html_table(outfile)
+                nrows = self.table.rowCount()
+                ncolumns = self.table.columnCount()
+
+                # add header item 0 fix default black bg color for headers
+                station_header = QtWidgets.QTableWidgetItem(text='Station')
+                station_header.setText('Station')
+                header_items = [station_header]
+                for column in range(ncolumns):
+                    hheader = self.table.horizontalHeaderItem(column)
+                    header_items.append(hheader)
+                write_html_row(outfile, header_items, html_key='th')
+
+                for row in range(nrows):
+                    vheader = self.table.verticalHeaderItem(row)
+                    col_items = [vheader]
+                    for column in range(ncolumns):
+                        col_items.append(self.table.item(row, column))
+                    write_html_row(outfile, col_items)
+
+                finish_html_table(outfile)
+                write_html_text(outfile, self.status_message)
+                write_html_footer(outfile)
+        except Exception as e:
+            print(f'Could not write HTML table to {fnout}:')
+            print(e)
+
     def sms_context_menu(self, row_ind):
         """ Open a context menu when left-clicking vertical header item """
         header_item = self.table.verticalHeaderItem(row_ind)
@@ -225,11 +262,13 @@ class MainWindow(QtWidgets.QMainWindow):
 
     def fill_status_bar(self):
         """ Set status bar text """
-        status_bar = self.statusBar()
         timespan = timedelta(seconds=int(self.parameters.get('timespan') * 24 * 3600))
-        status_bar.showMessage(f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | '
-                               f'Refresh period: {self.refresh_period}s | '
-                               f'Showing data of last {timespan}')
+        self.status_message = f'Program starttime (UTC) {self.starttime.strftime("%Y-%m-%d %H:%M:%S")} | ' \
+                              f'Current time (UTC) {UTCDateTime().strftime("%Y-%m-%d %H:%M:%S")} | ' \
+                              f'Refresh period: {self.refresh_period}s | '\
+                              f'Showing data of last {timespan}'
+        status_bar = self.statusBar()
+        status_bar.showMessage(self.status_message)
 
     def fill_table(self):
         """ Fills the table with most recent information. Executed after execute_qc thread is done or on refresh. """
@@ -291,6 +330,8 @@ class MainWindow(QtWidgets.QMainWindow):
 
         # table filling/refreshing done, set clear_on_refresh to False
         self.clear_on_refresh = False
+        # write html output if parameter is set
+        self.write_html_table()
 
     def get_time_delay_color(self, dt):
         """ Set color of time delay after thresholds specified in self.dt_thresh """

write_utils.py

@@ -0,0 +1,48 @@
+from datetime import timedelta
+
+def write_html_table_title(fobj, parameters):
+    title = get_print_title_str(parameters)
+    fobj.write(f'<h3>{title}</h3>\n')
+
+def write_html_text(fobj, text):
+    fobj.write(f'<p>{text}</p>\n')
+
+def write_html_header(fobj):
+    header = ['<!DOCTYPE html>',
+              '<html>',
+              '<style>',
+              'table, th, td {',
+              'border:1px solid black;',
+              '}',
+              '</style>',
+              '<body>']
+    for item in header:
+        fobj.write(item + '\n')
+
+def init_html_table(fobj):
+    fobj.write('<table style="width:100%">\n')
+
+def finish_html_table(fobj):
+    fobj.write('</table>\n')
+
+def write_html_footer(fobj):
+    footer = ['</body>',
+              '</html>']
+    for item in footer:
+        fobj.write(item + '\n')
+
+def write_html_row(fobj, items, html_key='td'):
+    fobj.write('<tr>\n')
+    for item in items:
+        text = item.text()
+        color = item.backgroundColor().name()
+        # fix for black background of headers
+        color = '#e6e6e6' if color == '#000000' else color
+        fobj.write(f'<{html_key} bgcolor="{color}">' + text + f'</{html_key}>\n')
+    fobj.write('</tr>\n')
+
+def get_print_title_str(parameters):
+    timespan = parameters.get('timespan') * 24 * 3600
+    tdelta_str = str(timedelta(seconds=int(timespan)))
+    return f'Analysis table of router quality within the last {tdelta_str}'
+