[update] WIP: adding tests for autopylot (global)

[refactor] minor

PyLoT.py

@@ -25,6 +25,7 @@ https://www.iconfinder.com/iconsets/flavour
 
 import argparse
 import json
+import logging
 import os
 import platform
 import shutil
@@ -60,7 +61,7 @@ except ImportError:
     from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
 from matplotlib.figure import Figure
 
-from pylot.core.analysis.magnitude import LocalMagnitude, MomentMagnitude, calcsourcespec
+from pylot.core.analysis.magnitude import LocalMagnitude, MomentMagnitude
 from pylot.core.io.data import Data
 from pylot.core.io.inputs import FilterOptions, PylotParameter
 from autoPyLoT import autoPyLoT
@@ -72,11 +73,11 @@ from pylot.core.util.errors import DatastructureError, \
     OverwriteError
 from pylot.core.util.connection import checkurl
 from pylot.core.util.dataprocessing import Metadata, restitute_data
-from pylot.core.util.utils import fnConstructor, getLogin, \
+from pylot.core.util.utils import fnConstructor, get_login, \
     full_range, readFilterInformation, pick_color_plt, \
     pick_linestyle_plt, identifyPhaseID, excludeQualityClasses, \
     transform_colors_mpl, transform_colors_mpl_str, getAutoFilteroptions, check_all_obspy, \
-    check_all_pylot, get_bool, get_None
+    check_all_pylot, get_bool, get_none
 from pylot.core.util.gui import make_pen
 from pylot.core.util.event import Event
 from pylot.core.io.location import create_creation_info, create_event
@@ -84,7 +85,7 @@ from pylot.core.util.widgets import FilterOptionsDialog, NewEventDlg, \
     PylotCanvas, WaveformWidgetPG, PropertiesDlg, HelpForm, createAction, PickDlg, \
     ComparisonWidget, TuneAutopicker, PylotParaBox, AutoPickDlg, CanvasWidget, AutoPickWidget, \
     CompareEventsWidget, ProgressBarWidget, AddMetadataWidget, SingleTextLineDialog, LogWidget, PickQualitiesFromXml, \
-    SourceSpecWindow, ChooseWaveFormWindow, SpectrogramTab
+    SpectrogramTab, SearchFileByExtensionDialog
 from pylot.core.util.array_map import Array_map
 from pylot.core.util.structure import DATASTRUCTURE
 from pylot.core.util.thread import Thread, Worker
@@ -113,11 +114,7 @@ class MainWindow(QMainWindow):
     def __init__(self, parent=None, infile=None, reset_qsettings=False):
         super(MainWindow, self).__init__(parent)
 
-        # check for default pylot.in-file
-        if not infile:
-            infile = os.path.join(os.path.expanduser('~'), '.pylot', 'pylot.in')
-            print('Using default input file {}'.format(infile))
-        if os.path.isfile(infile) is False:
+        if infile and os.path.isfile(infile) is False:
             infile = QFileDialog().getOpenFileName(caption='Choose PyLoT-input file')[0]
 
             if not os.path.exists(infile):
@@ -197,7 +194,7 @@ class MainWindow(QMainWindow):
                 if settings.value("user/FullName", None) is None:
                     fulluser = QInputDialog.getText(self, "Enter Name:", "Full name")
                     settings.setValue("user/FullName", fulluser)
-                    settings.setValue("user/Login", getLogin())
+                    settings.setValue("user/Login", get_login())
                 if settings.value("agency_id", None) is None:
                     agency = QInputDialog.getText(self,
                                                   "Enter authority/institution name:",
@@ -254,7 +251,7 @@ class MainWindow(QMainWindow):
             self._inputs.reset_defaults()
             # check for default pylot.in-file
             infile = os.path.join(pylot_config_dir, '.pylot.in')
-            print('Using default input file {}'.format(infile))
+            logging.warning('Using default input file {}'.format(infile))
             self._inputs.export2File(infile)
         self.infile = infile
 
@@ -1007,18 +1004,17 @@ class MainWindow(QMainWindow):
             return
         refresh = False
         events = self.project.eventlist
-        sld = SingleTextLineDialog(label='Specify file extension: ', default_text='.xml')
+        sld = SearchFileByExtensionDialog(label='Specify file extension: ', default_text='.xml',
+                                          events=events)
         if not sld.exec_():
             return
-        fext = sld.lineEdit.text()
-        # fext = '.xml'
+
+        filenames = sld.getChecked()
         for event in events:
-            path = event.path
-            eventname = path.split('/')[-1]  # or event.pylot_id
-            filename = os.path.join(path, 'PyLoT_' + eventname + fext)
-            if os.path.isfile(filename):
-                self.load_data(filename, draw=False, event=event, overwrite=True)
-                refresh = True
+            for filename in filenames:
+                if os.path.isfile(filename) and event.pylot_id in filename:
+                    self.load_data(filename, draw=False, event=event, ask_user=True, merge_strategy=sld.merge_strategy)
+                    refresh = True
         if not refresh:
             return
         if self.get_current_event().pylot_picks:
@@ -1026,8 +1022,8 @@ class MainWindow(QMainWindow):
         self.fill_eventbox()
         self.setDirty(True)
 
-    def load_data(self, fname=None, loc=False, draw=True, event=None, overwrite=False):
-        if not overwrite:
+    def load_data(self, fname=None, loc=False, draw=True, event=None, ask_user=False, merge_strategy='Overwrite'):
+        if not ask_user:
             if not self.okToContinue():
                 return
         if fname is None:
@@ -1041,9 +1037,12 @@ class MainWindow(QMainWindow):
         data = Data(self, event)
         try:
             data_new = Data(self, evtdata=str(fname))
-            # MP MP commented because adding several picks might cause inconsistencies
-            data = data_new
-            # data += data_new
+            if merge_strategy == 'Overwrite':
+                data = data_new
+            elif merge_strategy == 'Merge':
+                data += data_new
+            else:
+                raise NotImplementedError(f'Unknown merge strategy: {merge_strategy}')
         except ValueError:
             qmb = QMessageBox(self, icon=QMessageBox.Question,
                               text='Warning: Missmatch in event identifiers {} and {}. Continue?'.format(
@@ -1408,25 +1407,28 @@ class MainWindow(QMainWindow):
 
         for id, event in enumerate(self.project.eventlist):
             event_path = event.path
-            phaseErrors = {'P': self._inputs['timeerrorsP'],
-                           'S': self._inputs['timeerrorsS']}
+            #phaseErrors = {'P': self._inputs['timeerrorsP'],
+            #               'S': self._inputs['timeerrorsS']}
 
-            ma_props = {'manual': event.pylot_picks,
-                        'auto': event.pylot_autopicks}
-            ma_count = {'manual': 0,
-                        'auto': 0}
-            ma_count_total = {'manual': 0,
-                              'auto': 0}
+            man_au_picks = {'manual': event.pylot_picks,
+                            'auto': event.pylot_autopicks}
+            npicks = {'manual': {'P': 0, 'S': 0},
+                      'auto': {'P': 0, 'S': 0}}
+            npicks_total = {'manual': {'P': 0, 'S': 0},
+                            'auto': {'P': 0, 'S': 0}}
 
-            for ma in ma_props.keys():
-                if ma_props[ma]:
-                    for picks in ma_props[ma].values():
+            for ma in man_au_picks.keys():
+                if man_au_picks[ma]:
+                    for picks in man_au_picks[ma].values():
                         for phasename, pick in picks.items():
                             if not type(pick) in [dict, AttribDict]:
                                 continue
+                            phase_ID = identifyPhaseID(phasename)
+                            if not phase_ID in npicks[ma].keys():
+                                continue
                             if pick.get('spe'):
-                                ma_count[ma] += 1
-                            ma_count_total[ma] += 1
+                                npicks[ma][phase_ID] += 1
+                            npicks_total[ma][phase_ID] += 1
 
             event_ref = event.isRefEvent()
             event_test = event.isTestEvent()
@@ -1461,16 +1463,23 @@ class MainWindow(QMainWindow):
             if event.dirty:
                 event_str += '*'
             item_path = QStandardItem(event_str)
-            item_time = QStandardItem('{}'.format(time))
+            item_time = QStandardItem('{}'.format(time.strftime("%Y-%m-%d %H:%M:%S") if time else ''))
             item_lat = QStandardItem('{}'.format(lat))
             item_lon = QStandardItem('{}'.format(lon))
             item_depth = QStandardItem('{}'.format(depth))
             item_localmag = QStandardItem('{}'.format(localmag))
             item_momentmag = QStandardItem('{}'.format(momentmag))
-            item_nmp = QStandardItem('{}({})'.format(ma_count['manual'], ma_count_total['manual']))
+
+            item_nmp = QStandardItem()
+            item_nap = QStandardItem()
             item_nmp.setIcon(self.manupicksicon_small)
-            item_nap = QStandardItem('{}({})'.format(ma_count['auto'], ma_count_total['auto']))
             item_nap.setIcon(self.autopicksicon_small)
+
+            for picktype, item_np in [('manual', item_nmp), ('auto', item_nap)]:
+                npicks_str = f"{npicks[picktype]['P']}|{npicks[picktype]['S']}"
+                #npicks_str += f"({npicks_total[picktype]['P']}/{npicks_total[picktype]['S']})"
+                item_np.setText(npicks_str)
+
             item_ref = QStandardItem()  # str(event_ref))
             item_test = QStandardItem()  # str(event_test))
             if event_ref:
@@ -1891,6 +1900,7 @@ class MainWindow(QMainWindow):
         # which will read in data input twice. Therefore current tab is changed to 0
         # in loadProject before calling this function.
         self.fill_eventbox()
+        #print(f'{self.get_current_event()=}')
         plotted = False
         if self.tabs.currentIndex() == 2:
             self.init_event_table()
@@ -1925,7 +1935,6 @@ class MainWindow(QMainWindow):
             self.spectro_layout.addWidget(newSpectroWidget)
             self.spectroWidget = newSpectroWidget
 
-
     def newWF(self, event=None, plot=True):
         '''
         Load new data and plot if necessary.
@@ -1983,6 +1992,8 @@ class MainWindow(QMainWindow):
         #     ans = False
 
         settings = QSettings()
+        # process application events to wait for event items to appear in event box
+        QApplication.processEvents()
         curr_event = self.get_current_event()
         if not curr_event:
             print('Could not find current event. Try reload?')
@@ -1990,8 +2001,8 @@ class MainWindow(QMainWindow):
 
         if len(curr_event.origins) > 0:
             origin_time = curr_event.origins[0].time
-            tstart = settings.value('tstart') if get_None(settings.value('tstart')) else 0
-            tstop = settings.value('tstop') if get_None(settings.value('tstop')) else 0
+            tstart = settings.value('tstart') if get_none(settings.value('tstart')) else 0
+            tstop = settings.value('tstop') if get_none(settings.value('tstop')) else 0
             tstart = origin_time + float(tstart)
             tstop = origin_time + float(tstop)
         else:
@@ -2138,7 +2149,7 @@ class MainWindow(QMainWindow):
 
     def finish_pg_plot(self):
         self.getPlotWidget().updateWidget()
-        plots, gaps = self.wfp_thread.data
+        plots = self.wfp_thread.data
         # do not show plot if no data are given
         self.wf_scroll_area.setVisible(len(plots) > 0)
         self.no_data_label.setVisible(not len(plots) > 0)
@@ -2186,7 +2197,8 @@ class MainWindow(QMainWindow):
         if event.pylot_autopicks:
             self.drawPicks(picktype='auto')
         if event.pylot_picks or event.pylot_autopicks:
-            self.locateEventAction.setEnabled(True)
+            if not self._inputs.get('extent') == 'global':
+                self.locateEventAction.setEnabled(True)
             self.qualities_action.setEnabled(True)
             self.eventlist_xml_action.setEnabled(True)
 
@@ -2306,14 +2318,14 @@ class MainWindow(QMainWindow):
         # wfst += self.get_data().getWFData().select(component=alter_comp)
         plotWidget = self.getPlotWidget()
         self.adjustPlotHeight()
-        if get_bool(settings.value('large_dataset')):
+        if get_bool(settings.value('large_dataset')) == True:
             self.plot_method = 'fast'
         else:
             self.plot_method = 'normal'
         rval = plotWidget.plotWFData(wfdata=wfst, wfsyn=wfsyn, title=title, mapping=False, component=comp,
                                      nth_sample=int(nth_sample), method=self.plot_method, gain=self.gain)
-        plots, gaps = rval if rval else ([], [])
-        return plots, gaps
+        plots = rval if rval else []
+        return plots
 
     def adjustPlotHeight(self):
         if self.pg:
@@ -2621,7 +2633,6 @@ class MainWindow(QMainWindow):
                           picks=self.getPicksOnStation(station, 'manual'),
                           autopicks=self.getPicksOnStation(station, 'auto'),
                           metadata=self.metadata, event=event,
-                          model=self.inputs.get('taup_model'),
                           filteroptions=self.filteroptions, wftype=wftype,
                           show_comp_data=self.dataPlot.comp_checkbox.isChecked())
         if self.filterActionP.isChecked():
@@ -3002,10 +3013,16 @@ class MainWindow(QMainWindow):
             event = self.get_current_event()
         event.pylot_picks = {}
         event.pylot_autopicks = {}
-        picksdict = picksdict_from_picks(evt=self.get_data().get_evt_data())
+        picksdict = picksdict_from_picks(evt=self.get_data().get_evt_data(), parameter=self.getParameter())
         event.addPicks(picksdict['manual'])
         event.addAutopicks(picksdict['auto'])
 
+    def getParameter(self):
+        if hasattr(self.project, 'parameter') and isinstance(self.project.parameter, PylotParameter):
+            return self.project.parameter
+        else:
+            return self._inputs
+
     def drawPicks(self, station=None, picktype=None, stime=None):
         # if picktype not specified, draw both
         if not stime:
@@ -3647,7 +3664,7 @@ class MainWindow(QMainWindow):
             return True
         return False
 
-    def update_status(self, message, duration=5000):
+    def update_status(self, message, duration=10000):
         self.statusBar().showMessage(message, duration)
         if self.get_data() is not None:
             if not self.get_current_event() or not self.project.location:
@@ -3700,10 +3717,13 @@ class MainWindow(QMainWindow):
         if not self.okToContinue():
             return
         if not fnm:
-            dlg = QFileDialog(parent=self)
+            settings = QSettings()
+            dir = settings.value('current_project_path')
+            dlg = QFileDialog(parent=self, directory=dir)
             fnm = dlg.getOpenFileName(self, 'Open project file...', filter='Pylot project (*.plp)')[0]
             if not fnm:
                 return
+            settings.setValue('current_project_path', os.path.split(fnm)[0])
         if not os.path.exists(fnm):
             QMessageBox.warning(self, 'Could not open file',
                                 'Could not open project file {}. File does not exist.'.format(fnm))
@@ -3821,7 +3841,8 @@ class MainWindow(QMainWindow):
 
     def closeEvent(self, event):
         if self.okToContinue():
-            self.logwidget.close()
+            if hasattr(self, 'logwidget'):
+                self.logwidget.close()
             event.accept()
         else:
             event.ignore()

README.md

@@ -11,7 +11,7 @@ PILOT has originally been developed in Mathworks' MatLab. In order to distribute
 problems, it has been decided to redevelop the software package in Python. The great work of the ObsPy group allows easy
 handling of a bunch of seismic data and PyLoT will benefit a lot compared to the former MatLab version.
 
-The development of PyLoT is part of the joint research project MAGS2 and AlpArray.
+The development of PyLoT is part of the joint research project MAGS2, AlpArray and AdriaArray.
 
 ## Installation
 
@@ -27,28 +27,30 @@ Afterwards run (from the PyLoT main directory where the files *requirements.txt*
     conda env create -f pylot.yml
 or
     
-    conda create --name pylot_38 --file requirements.txt
+    conda create -c conda-forge --name pylot_311 python=3.11 --file requirements.txt
 
-to create a new Anaconda environment called "pylot_38".
+to create a new Anaconda environment called *pylot_311*.
 
 Afterwards activate the environment by typing
 
-    conda activate pylot_38
+    conda activate pylot_311
 
 #### Prerequisites:
 
 In order to run PyLoT you need to install:
 
 - Python 3
-- obspy
-- pyside2
-- pyqtgraph
 - cartopy
+- joblib
+- obspy
+- pyaml
+- pyqtgraph
+- pyside2
 
 (the following are already dependencies of the above packages):
 - scipy
 - numpy
-- matplotlib <= 3.3.x
+- matplotlib
 
 #### Some handwork:
 
@@ -108,4 +110,4 @@ Others: A. Bruestle, T. Meier, W. Friederich
 
 [ObsPy]: http://github.com/obspy/obspy/wiki
 
-April 2022
+August 2024

autoPyLoT.py

@@ -28,7 +28,7 @@ from pylot.core.util.dataprocessing import restitute_data, Metadata
 from pylot.core.util.defaults import SEPARATOR
 from pylot.core.util.event import Event
 from pylot.core.util.structure import DATASTRUCTURE
-from pylot.core.util.utils import get_None, trim_station_components, check4gapsAndRemove, check4doubled, \
+from pylot.core.util.utils import get_none, trim_station_components, check4gapsAndRemove, check4doubled, \
     check4rotated
 from pylot.core.util.version import get_git_version as _getVersionString
 
@@ -91,9 +91,9 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even
                                                               sp=sp_info)
     print(splash)
 
-    parameter = get_None(parameter)
-    inputfile = get_None(inputfile)
-    eventid = get_None(eventid)
+    parameter = get_none(parameter)
+    inputfile = get_none(inputfile)
+    eventid = get_none(eventid)
 
     fig_dict = None
     fig_dict_wadatijack = None
@@ -119,13 +119,9 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even
             obspyDMT_wfpath = input_dict['obspyDMT_wfpath']
 
     if not parameter:
-        if inputfile:
-            parameter = PylotParameter(inputfile)
-            # iplot = parameter['iplot']
-        else:
-            infile = os.path.join(os.path.expanduser('~'), '.pylot', 'pylot.in')
-            print('Using default input file {}'.format(infile))
-            parameter = PylotParameter(infile)
+        if not inputfile:
+            print('Using default input parameter')
+        parameter = PylotParameter(inputfile)
     else:
         if not type(parameter) == PylotParameter:
             print('Wrong input type for parameter: {}'.format(type(parameter)))
@@ -154,7 +150,7 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even
         datastructure.setExpandFields(exf)
 
         # check if default location routine NLLoc is available and all stations are used
-        if get_None(parameter['nllocbin']) and station == 'all':
+        if get_none(parameter['nllocbin']) and station == 'all':
             locflag = 1
             # get NLLoc-root path
             nllocroot = parameter.get('nllocroot')

autopylot.sh

@@ -3,10 +3,10 @@
 #$ -l low
 #$ -cwd
 #$ -pe smp 40
-#$ -l mem=2G
-#$ -l h_vmem=2G
+##$ -l mem=3G
+#$ -l h_vmem=6G
 #$ -l os=*stretch
 
-conda activate pylot_38
+conda activate pylot_311
 
-python ./autoPyLoT.py -i /home/marcel/.pylot/pylot_janis_noisy.in -c $NSLOTS
+python ./autoPyLoT.py -i /home/marcel/.pylot/pylot_adriaarray.in -c 20 -dmt processed

pylot.yml

@@ -1,14 +1,12 @@
-name: pylot_38
+name: pylot_311
 channels:
   - conda-forge
   - defaults
 dependencies:
-  - cartopy=0.20.2
-  - matplotlib-base=3.3.4
-  - numpy=1.22.3
-  - obspy=1.3.0
-  - pyqtgraph=0.12.4
-  - pyside2>=5.13.2
-  - python=3.8.12
-  - qt>=5.12.9
-  - scipy=1.8.0
+  - cartopy=0.23.0=py311hcf9f919_1
+  - joblib=1.4.2=pyhd8ed1ab_0
+  - obspy=1.4.1=py311he736701_3
+  - pyaml=24.7.0=pyhd8ed1ab_0
+  - pyqtgraph=0.13.7=pyhd8ed1ab_0
+  - pyside2=5.15.8=py311h3d699ce_4
+  - pytest=8.3.2=pyhd8ed1ab_0

pylot/__init__.py

@@ -9,7 +9,7 @@ PyLoT - the Python picking and Localization Tool
 
 This python library contains a graphical user interfaces for picking
 seismic phases. This software needs ObsPy (http://github.com/obspy/obspy/wiki)
-and the Qt4 libraries to be installed first.
+and the Qt libraries to be installed first.
 
 PILOT has been developed in Mathworks' MatLab. In order to distribute
 PILOT without facing portability problems, it has been decided to re-

pylot/core/io/data.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 import copy
+import logging
 import os
 
 from PySide2.QtWidgets import QMessageBox
@@ -19,7 +20,7 @@ from pylot.core.util.errors import FormatError, OverwriteError
 from pylot.core.util.event import Event
 from pylot.core.util.obspyDMT_interface import qml_from_obspyDMT
 from pylot.core.util.utils import fnConstructor, full_range, check4rotated, \
-    check4gapsAndMerge, trim_station_components
+    check_for_gaps_and_merge, trim_station_components, check_for_nan
 
 
 class Data(object):
@@ -64,7 +65,7 @@ class Data(object):
                     elif 'LOC' in evtdata:
                         raise NotImplementedError('PILOT location information '
                                                   'read support not yet '
-                                                  'implemeted.')
+                                                  'implemented.')
                     elif 'event.pkl' in evtdata:
                         evtdata = qml_from_obspyDMT(evtdata)
                     else:
@@ -408,18 +409,16 @@ class Data(object):
                                      not implemented: {1}'''.format(evtformat, e))
             if fnext == '_focmec.in':
                 try:
-                    infile = os.path.join(os.path.expanduser('~'), '.pylot', 'pylot.in')
-                    print('Using default input file {}'.format(infile))
-                    parameter = PylotParameter(infile)
+                    parameter = PylotParameter()
+                    logging.warning('Using default input parameter')
                     focmec.export(picks_copy, fnout + fnext, parameter, eventinfo=self.get_evt_data())
                 except KeyError as e:
                     raise KeyError('''{0} export format
                                      not implemented: {1}'''.format(evtformat, e))
             if fnext == '.pha':
                 try:
-                    infile = os.path.join(os.path.expanduser('~'), '.pylot', 'pylot.in')
-                    print('Using default input file {}'.format(infile))
-                    parameter = PylotParameter(infile)
+                    parameter = PylotParameter()
+                    logging.warning('Using default input parameter')
                     hypodd.export(picks_copy, fnout + fnext, parameter, eventinfo=self.get_evt_data())
                 except KeyError as e:
                     raise KeyError('''{0} export format
@@ -458,6 +457,11 @@ class Data(object):
         :param fnames_alt: alternative data to show (e.g. synthetic/processed)
         :type fnames: list
         """
+        def check_fname_exists(filenames: list) -> list:
+            if filenames:
+                filenames = [fn for fn in filenames if os.path.isfile(fn)]
+            return filenames
+
         self.wfdata = Stream()
         self.wforiginal = None
         self.wf_alt = Stream()
@@ -467,8 +471,8 @@ class Data(object):
         self.tstop = tstop
 
         # remove directories
-        fnames = [fname for fname in fnames if not os.path.isdir(fname)]
-        fnames_alt = [fname for fname in fnames_alt if not os.path.isdir(fname)]
+        fnames = check_fname_exists(fnames)
+        fnames_alt = check_fname_exists(fnames_alt)
 
         # if obspy_dmt:
         #     wfdir = 'raw'
@@ -496,7 +500,9 @@ class Data(object):
         # remove possible underscores in station names
         # self.wfdata = remove_underscores(self.wfdata)
         # check for gaps and merge
-        self.wfdata = check4gapsAndMerge(self.wfdata)
+        self.wfdata, _ = check_for_gaps_and_merge(self.wfdata)
+        # check for nans
+        check_for_nan(self.wfdata)
         # check for stations with rotated components
         if checkRotated and metadata is not None:
             self.wfdata = check4rotated(self.wfdata, metadata, verbosity=0)

pylot/core/io/default_parameters.py

@@ -511,7 +511,7 @@ defaults = {'rootpath': {'type': str,
 
             'taup_model': {'type': str,
                            'tooltip': 'Define TauPy model for traveltime estimation. Possible values: 1066a, 1066b, ak135, ak135f, herrin, iasp91, jb, prem, pwdk, sp6',
-                           'value': None,
+                           'value': 'iasp91',
                            'namestring': 'TauPy model'},
 
             'taup_phases': {'type': str,

pylot/core/io/location.py

@@ -1,7 +1,7 @@
 from obspy import UTCDateTime
 from obspy.core import event as ope
 
-from pylot.core.util.utils import getLogin, getHash
+from pylot.core.util.utils import get_login, get_hash
 
 
 def create_amplitude(pickID, amp, unit, category, cinfo):
@@ -61,7 +61,7 @@ def create_creation_info(agency_id=None, creation_time=None, author=None):
     :return:
     '''
     if author is None:
-        author = getLogin()
+        author = get_login()
     if creation_time is None:
         creation_time = UTCDateTime()
     return ope.CreationInfo(agency_id=agency_id, author=author,
@@ -210,7 +210,7 @@ def create_resourceID(timetohash, restype, authority_id=None, hrstr=None):
     '''
     assert isinstance(timetohash, UTCDateTime), "'timetohash' is not an ObsPy" \
                                                 "UTCDateTime object"
-    hid = getHash(timetohash)
+    hid = get_hash(timetohash)
     if hrstr is None:
         resID = ope.ResourceIdentifier(restype + '/' + hid[0:6])
     else:

pylot/core/io/phases.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import glob
+import logging
 import os
 import warnings
 
@@ -16,7 +17,7 @@ from pylot.core.io.inputs import PylotParameter
 from pylot.core.io.location import create_event, \
     create_magnitude
 from pylot.core.pick.utils import select_for_phase, get_quality_class
-from pylot.core.util.utils import getOwner, full_range, four_digits, transformFilterString4Export, \
+from pylot.core.util.utils import get_owner, full_range, four_digits, transformFilterString4Export, \
     backtransformFilterString, loopIdentifyPhase, identifyPhase
 
 
@@ -58,7 +59,7 @@ def readPILOTEvent(phasfn=None, locfn=None, authority_id='RUB', **kwargs):
     if phasfn is not None and os.path.isfile(phasfn):
         phases = sio.loadmat(phasfn)
         phasctime = UTCDateTime(os.path.getmtime(phasfn))
-        phasauthor = getOwner(phasfn)
+        phasauthor = get_owner(phasfn)
     else:
         phases = None
         phasctime = None
@@ -66,7 +67,7 @@ def readPILOTEvent(phasfn=None, locfn=None, authority_id='RUB', **kwargs):
     if locfn is not None and os.path.isfile(locfn):
         loc = sio.loadmat(locfn)
         locctime = UTCDateTime(os.path.getmtime(locfn))
-        locauthor = getOwner(locfn)
+        locauthor = get_owner(locfn)
     else:
         loc = None
         locctime = None
@@ -217,7 +218,7 @@ def picksdict_from_obs(fn):
     return picks
 
 
-def picksdict_from_picks(evt):
+def picksdict_from_picks(evt, parameter=None):
     """
     Takes an Event object and return the pick dictionary commonly used within
     PyLoT
@@ -230,6 +231,7 @@ def picksdict_from_picks(evt):
         'auto': {}
     }
     for pick in evt.picks:
+        errors = None
         phase = {}
         station = pick.waveform_id.station_code
         if pick.waveform_id.channel_code is None:
@@ -273,32 +275,28 @@ def picksdict_from_picks(evt):
         phase['epp'] = epp
         phase['lpp'] = lpp
         phase['spe'] = spe
-        try:
-            phase['weight'] = weight
-        except:
-            # get onset weight from uncertainty
-            infile = os.path.join(os.path.expanduser('~'), '.pylot', 'pylot.in')
-            print('Using default input file {}'.format(infile))
-            parameter = PylotParameter(infile)
+        weight = phase.get('weight')
+        if not weight:
+            if not parameter:
+                logging.warning('Using ')
+                logging.warning('Using default input parameter')
+                parameter = PylotParameter()
             pick.phase_hint = identifyPhase(pick.phase_hint)
             if pick.phase_hint == 'P':
                 errors = parameter['timeerrorsP']
             elif pick.phase_hint == 'S':
                 errors = parameter['timeerrorsS']
-            weight = get_quality_class(spe, errors)
-            phase['weight'] = weight
+            if errors:
+                weight = get_quality_class(spe, errors)
+                phase['weight'] = weight
         phase['channel'] = channel
         phase['network'] = network
         phase['picker'] = pick_method
-        try:
-            if pick.polarity == 'positive':
-                phase['fm'] = 'U'
-            elif pick.polarity == 'negative':
-                phase['fm'] = 'D'
-            else:
-                phase['fm'] = 'N'
-        except:
-            print("No FM info available!")
+        if pick.polarity == 'positive':
+            phase['fm'] = 'U'
+        elif pick.polarity == 'negative':
+            phase['fm'] = 'D'
+        else:
             phase['fm'] = 'N'
         phase['filter_id'] = filter_id if filter_id is not None else ''
 

pylot/core/pick/autopick.py

@@ -20,9 +20,9 @@ from pylot.core.pick.charfuns import CharacteristicFunction
 from pylot.core.pick.charfuns import HOScf, AICcf, ARZcf, ARHcf, AR3Ccf
 from pylot.core.pick.picker import AICPicker, PragPicker
 from pylot.core.pick.utils import checksignallength, checkZ4S, earllatepicker, \
-    getSNR, fmpicker, checkPonsets, wadaticheck, get_quality_class
+    getSNR, fmpicker, checkPonsets, wadaticheck, get_quality_class, PickingFailedException, MissingTraceException
 from pylot.core.util.utils import getPatternLine, gen_Pool, \
-    get_bool, identifyPhaseID, get_None, correct_iplot
+    get_bool, identifyPhaseID, get_none, correct_iplot
 
 
 def autopickevent(data, param, iplot=0, fig_dict=None, fig_dict_wadatijack=None, ncores=0, metadata=None, origin=None):
@@ -232,20 +232,6 @@ class PickingContainer:
         self.Sflag = 0
 
 
-class MissingTraceException(ValueError):
-    """
-    Used to indicate missing traces in a obspy.core.stream.Stream object
-    """
-    pass
-
-
-class PickingFailedException(Exception):
-    """
-    Raised when picking fails due to missing values etc.
-    """
-    pass
-
-
 class AutopickStation(object):
 
     def __init__(self, wfstream, pickparam, verbose, iplot=0, fig_dict=None, metadata=None, origin=None):
@@ -272,7 +258,7 @@ class AutopickStation(object):
         self.pickparams = copy.deepcopy(pickparam)
         self.verbose = verbose
         self.iplot = correct_iplot(iplot)
-        self.fig_dict = get_None(fig_dict)
+        self.fig_dict = get_none(fig_dict)
         self.metadata = metadata
         self.origin = origin
 
@@ -660,7 +646,7 @@ class AutopickStation(object):
             ax1.set_ylim([-1.5, 1.5])
             ax1.set_ylabel('Normalized Counts')
 
-            if self.horizontal_traces_exist() and self.s_data.Sflag == 1:
+            if self.horizontal_traces_exist():# and self.s_data.Sflag == 1:
                 # plot E trace
                 ax2 = fig.add_subplot(3, 1, 2, sharex=ax1)
                 th1data = np.linspace(0, self.etrace.stats.endtime - self.etrace.stats.starttime,

pylot/core/pick/charfuns.py

@@ -20,6 +20,8 @@ import numpy as np
 from scipy import signal
 from obspy.core import Stream
 
+from pylot.core.pick.utils import PickingFailedException
+
 
 class CharacteristicFunction(object):
     """
@@ -293,7 +295,7 @@ class HOScf(CharacteristicFunction):
             if j < 4:
                 LTA[j] = 0
                 STA[j] = 0
-            elif j <= ista:
+            elif j <= ista and self.getOrder() == 2:
                 lta = (y[j] + lta * (j - 1)) / j
                 if self.getOrder() == 2:
                     sta = (y[j] + sta * (j - 1)) / j
@@ -488,6 +490,9 @@ class ARHcf(CharacteristicFunction):
         print('Calculating AR-prediction error from both horizontal traces ...')
 
         xnp = self.getDataArray(self.getCut())
+        if len(xnp[0]) == 0:
+            raise PickingFailedException('calcCF: Found empty data trace for cut times. Return')
+
         n0 = np.isnan(xnp[0].data)
         if len(n0) > 1:
             xnp[0].data[n0] = 0

pylot/core/pick/picker.py

@@ -178,7 +178,9 @@ class AICPicker(AutoPicker):
         aic = tap * self.cf + max(abs(self.cf))
         # smooth AIC-CF
         ismooth = int(round(self.Tsmooth / self.dt))
-        aicsmooth = np.zeros(len(aic))
+        # MP MP better start with original data than zeros if array shall be smoothed, created artificial value before
+        # when starting with i in range(1...) loop below and subtracting offset afterwards
+        aicsmooth = np.copy(aic)
         if len(aic) < ismooth:
             print('AICPicker: Tsmooth larger than CF!')
             return
@@ -188,7 +190,7 @@ class AICPicker(AutoPicker):
                     ii1 = i - ismooth
                     aicsmooth[i] = aicsmooth[i - 1] + (aic[i] - aic[ii1]) / ismooth
                 else:
-                    aicsmooth[i] = np.mean(aic[1: i])
+                    aicsmooth[i] = np.mean(aic[0: i]) # MP MP created np.nan for i=1
         # remove offset in AIC function
         offset = abs(min(aic) - min(aicsmooth))
         aicsmooth = aicsmooth - offset
@@ -197,7 +199,7 @@ class AICPicker(AutoPicker):
         # minimum in AIC function 
         icfmax = np.argmax(cf)
 
-        # MP MP testing threshold
+        # TODO: If this shall be kept, maybe add thresh_factor to pylot parameters
         thresh_hit = False
         thresh_factor = 0.7
         thresh = thresh_factor * cf[icfmax]
@@ -209,7 +211,6 @@ class AICPicker(AutoPicker):
                 if sample <= cf[index - 1]:
                     icfmax = index - 1
                     break
-        # MP MP ---
 
         # find minimum in AIC-CF front of maximum of HOS/AR-CF
         lpickwindow = int(round(self.PickWindow / self.dt))

pylot/core/pick/utils.py

@@ -15,7 +15,7 @@ import numpy as np
 from obspy.core import Stream, UTCDateTime
 from scipy.signal import argrelmax
 
-from pylot.core.util.utils import get_bool, get_None, SetChannelComponents
+from pylot.core.util.utils import get_bool, get_none, SetChannelComponents
 
 
 def earllatepicker(X, nfac, TSNR, Pick1, iplot=0, verbosity=1, fig=None, linecolor='k'):
@@ -136,7 +136,7 @@ def earllatepicker(X, nfac, TSNR, Pick1, iplot=0, verbosity=1, fig=None, linecol
     PickError = symmetrize_error(diffti_te, diffti_tl)
 
     if iplot > 1:
-        if get_None(fig) is None:
+        if get_none(fig) is None:
             fig = plt.figure()  # iplot)
             plt_flag = 1
         fig._tight = True
@@ -275,7 +275,7 @@ def fmpicker(Xraw, Xfilt, pickwin, Pick, iplot=0, fig=None, linecolor='k'):
             try:
                 P1 = np.polyfit(xslope1, xraw[islope1], 1)
                 datafit1 = np.polyval(P1, xslope1)
-            except Exception as e:
+            except ValueError as e:
                 print("fmpicker: Problems with data fit! {}".format(e))
                 print("Skip first motion determination!")
                 return FM
@@ -321,7 +321,7 @@ def fmpicker(Xraw, Xfilt, pickwin, Pick, iplot=0, fig=None, linecolor='k'):
             try:
                 P2 = np.polyfit(xslope2, xfilt[islope2], 1)
                 datafit2 = np.polyval(P2, xslope2)
-            except Exception as e:
+            except ValueError as e:
                 emsg = 'fmpicker: polyfit failed: {}'.format(e)
                 print(emsg)
                 return FM
@@ -344,7 +344,7 @@ def fmpicker(Xraw, Xfilt, pickwin, Pick, iplot=0, fig=None, linecolor='k'):
         print("fmpicker: Found polarity %s" % FM)
 
     if iplot > 1:
-        if get_None(fig) is None:
+        if get_none(fig) is None:
             fig = plt.figure()  # iplot)
             plt_flag = 1
         fig._tight = True
@@ -868,7 +868,7 @@ def checksignallength(X, pick, minsiglength, pickparams, iplot=0, fig=None, line
             returnflag = 0
 
     if iplot > 1:
-        if get_None(fig) is None:
+        if get_none(fig) is None:
             fig = plt.figure()  # iplot)
             plt_flag = 1
         fig._tight = True
@@ -890,6 +890,8 @@ def checksignallength(X, pick, minsiglength, pickparams, iplot=0, fig=None, line
                 input()
             except SyntaxError:
                 pass
+            except EOFError:
+                pass
             plt.close(fig)
 
     return returnflag
@@ -1211,14 +1213,14 @@ def checkZ4S(X, pick, pickparams, iplot, fig=None, linecolor='k'):
             t = np.linspace(diff_dict[key], trace.stats.endtime - trace.stats.starttime + diff_dict[key],
                             trace.stats.npts)
             if i == 0:
-                if get_None(fig) is None:
+                if get_none(fig) is None:
                     fig = plt.figure()  # self.iplot) ### WHY? MP MP
                     plt_flag = 1
                 ax1 = fig.add_subplot(3, 1, i + 1)
                 ax = ax1
                 ax.set_title('CheckZ4S, Station %s' % zdat[0].stats.station)
             else:
-                if get_None(fig) is None:
+                if get_none(fig) is None:
                     fig = plt.figure()  # self.iplot) ### WHY? MP MP
                     plt_flag = 1
                 ax = fig.add_subplot(3, 1, i + 1, sharex=ax1)
@@ -1332,22 +1334,6 @@ def get_quality_class(uncertainty, weight_classes):
     return quality
 
 
-def set_NaNs_to(data, nan_value):
-    """
-    Replace all NaNs in data with nan_value
-    :param data: array holding data
-    :type data: `~numpy.ndarray`
-    :param nan_value: value which all NaNs are set to
-    :type nan_value: float, int
-    :return: data array with all NaNs replaced with nan_value
-    :rtype: `~numpy.ndarray`
-    """
-    nn = np.isnan(data)
-    if np.any(nn):
-        data[nn] = nan_value
-    return data
-
-
 def taper_cf(cf):
     """
     Taper cf data to get rid off of side maximas
@@ -1508,7 +1494,7 @@ def getQualityFromUncertainty(uncertainty, Errors):
     # set initial quality to 4 (worst) and change only if one condition is hit
     quality = 4
 
-    if get_None(uncertainty) is None:
+    if get_none(uncertainty) is None:
         return quality
 
     if uncertainty <= Errors[0]:
@@ -1532,3 +1518,17 @@ if __name__ == '__main__':
     import doctest
 
     doctest.testmod()
+
+
+class PickingFailedException(Exception):
+    """
+    Raised when picking fails due to missing values etc.
+    """
+    pass
+
+
+class MissingTraceException(ValueError):
+    """
+    Used to indicate missing traces in a obspy.core.stream.Stream object
+    """
+    pass

pylot/core/util/array_map.py

@@ -13,6 +13,7 @@ import obspy
 from PySide2 import QtWidgets
 from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
 from mpl_toolkits.axes_grid1.inset_locator import inset_axes
+from pylot.core.util.utils import identifyPhaseID
 from scipy.interpolate import griddata
 
 from pylot.core.pick.utils import get_quality_class
@@ -123,8 +124,8 @@ class Array_map(QtWidgets.QWidget):
         self.cmaps_box = QtWidgets.QComboBox()
         self.cmaps_box.setMaxVisibleItems(20)
         [self.cmaps_box.addItem(map_name) for map_name in sorted(plt.colormaps())]
-        # try to set to hsv as default
-        self.cmaps_box.setCurrentIndex(self.cmaps_box.findText('hsv'))
+        # try to set to viridis as default
+        self.cmaps_box.setCurrentIndex(self.cmaps_box.findText('viridis'))
 
         self.top_row.addWidget(QtWidgets.QLabel('Select a phase: '))
         self.top_row.addWidget(self.comboBox_phase)
@@ -279,9 +280,12 @@ class Array_map(QtWidgets.QWidget):
         self.canvas.axes.figure.canvas.draw_idle()
 
     def onpick(self, event):
+        btn_msg = {1: ' in selection. Aborted', 2: ' to delete a pick on. Aborted', 3: ' to display info.'}
         ind = event.ind
         button = event.mouseevent.button
-        if ind == []:
+        msg_reason = None
+        if len(ind) > 1:
+            self._parent.update_status(f'Found more than one station {btn_msg.get(button)}')
             return
         if button == 1:
             self.openPickDlg(ind)
@@ -384,7 +388,14 @@ class Array_map(QtWidgets.QWidget):
                 try:
                     station_name = st_id.split('.')[-1]
                     # current_picks_dict: auto or manual
-                    pick = self.current_picks_dict()[station_name][phase]
+                    station_picks = self.current_picks_dict().get(station_name)
+                    if not station_picks:
+                        continue
+                    for phase_hint, pick in station_picks.items():
+                        if identifyPhaseID(phase_hint) == phase:
+                            break
+                    else:
+                        continue
                     if pick['picker'] == 'auto':
                         if not pick['spe']:
                             continue
@@ -464,7 +475,7 @@ class Array_map(QtWidgets.QWidget):
 
     def openPickDlg(self, ind):
         wfdata = self._parent.get_data().getWFData()
-        wfdata_comp = self._parent.get_data().getWFDataComp()
+        wfdata_comp = self._parent.get_data().getAltWFdata()
         for index in ind:
             network, station = self._station_onpick_ids[index].split('.')[:2]
             pyl_mw = self._parent
@@ -479,7 +490,6 @@ class Array_map(QtWidgets.QWidget):
                                   picks=self._parent.get_current_event().getPick(station),
                                   autopicks=self._parent.get_current_event().getAutopick(station),
                                   filteroptions=self._parent.filteroptions, metadata=self.metadata,
-                                  model=self.parameter.get('taup_model'),
                                   event=pyl_mw.get_current_event())
             except Exception as e:
                 message = 'Could not generate Plot for station {st}.\n {er}'.format(st=station, er=e)

pylot/core/util/dataprocessing.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 import glob
+import logging
 import os
 import sys
 
@@ -189,7 +190,11 @@ class Metadata(object):
         metadata = self.get_metadata(seed_id, time)
         if not metadata:
             return
-        return metadata['data'].get_coordinates(seed_id, time)
+        try:
+            return metadata['data'].get_coordinates(seed_id, time)
+        # no specific exception defined in obspy inventory
+        except Exception as e:
+            logging.warning(f'Could not get metadata for {seed_id}')
 
     def get_all_coordinates(self):
         def stat_info_from_parser(parser):

pylot/core/util/generate_array_maps.py

@@ -42,7 +42,7 @@ def main(project_file_path, manual=False, auto=True, file_format='png', f_ext=''
         for item in input_list:
             array_map_worker(item)
     else:
-        pool = multiprocessing.Pool(ncores)
+        pool = multiprocessing.Pool(ncores, maxtasksperchild=1000)
         pool.map(array_map_worker, input_list)
         pool.close()
         pool.join()

pylot/core/util/thread.py

@@ -160,7 +160,7 @@ class MultiThread(QThread):
         try:
             if not self.ncores:
                 self.ncores = multiprocessing.cpu_count()
-            pool = multiprocessing.Pool(self.ncores)
+            pool = multiprocessing.Pool(self.ncores, maxtasksperchild=1000)
             self.data = pool.map_async(self.func, self.args, callback=self.emitDone)
             # self.data = pool.apply_async(self.func, self.shotlist, callback=self.emitDone) #emit each time returned
             pool.close()

pylot/core/util/utils.py

@@ -1,12 +1,15 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-
+import glob
 import hashlib
+import logging
 import os
 import platform
 import re
 import subprocess
 import warnings
+from typing import Literal, Tuple, Type
+from functools import lru_cache
 
 import numpy as np
 from obspy import UTCDateTime, read
@@ -18,6 +21,10 @@ from pylot.core.io.inputs import PylotParameter, FilterOptions
 from pylot.core.util.obspyDMT_interface import check_obspydmt_eventfolder
 from pylot.styles import style_settings
 
+Rgba: Type[tuple] = Tuple[int, int, int, int]
+Mplrgba: Type[tuple] = Tuple[float, float, float, float]
+Mplrgbastr: Type[tuple] = Tuple[str, str, str, str]
+
 
 def _pickle_method(m):
     if m.im_self is None:
@@ -81,25 +88,6 @@ def fit_curve(x, y):
     return splev, splrep(x, y)
 
 
-def getindexbounds(f, eta):
-    """
-    Get indices of values closest below and above maximum value in an array
-    :param f: array
-    :type f: `~numpy.ndarray`
-    :param eta: look for value in array that is closes to max_value * eta
-    :type eta: float
-    :return: tuple containing index of max value, index of value closest below max value,
-     index of value closest above max value
-    :rtype: (int, int, int)
-    """
-    mi = f.argmax()  # get indices of max values
-    m = max(f)  # get maximum value
-    b = m * eta  #
-    l = find_nearest(f[:mi], b)  # find closest value below max value
-    u = find_nearest(f[mi:], b) + mi  # find closest value above max value
-    return mi, l, u
-
-
 def gen_Pool(ncores=0):
     """
     Generate mulitprocessing pool object utilizing ncores amount of cores
@@ -119,7 +107,7 @@ def gen_Pool(ncores=0):
 
     print('gen_Pool: Generated multiprocessing Pool with {} cores\n'.format(ncores))
 
-    pool = multiprocessing.Pool(ncores)
+    pool = multiprocessing.Pool(ncores, maxtasksperchild=100)
     return pool
 
 
@@ -165,11 +153,11 @@ def clims(lim1, lim2):
     """
     takes two pairs of limits and returns one pair of common limts
     :param lim1: limit 1
-    :type lim1: int
+    :type lim1: List[int]
     :param lim2: limit 2
-    :type lim2: int
+    :type lim2: List[int]
     :return: new upper and lower limit common to both given limits
-    :rtype: [int, int]
+    :rtype: List[int]
 
     >>> clims([0, 4], [1, 3])
     [0, 4]
@@ -301,7 +289,7 @@ def fnConstructor(s):
     if type(s) is str:
         s = s.split(':')[-1]
     else:
-        s = getHash(UTCDateTime())
+        s = get_hash(UTCDateTime())
 
     badchars = re.compile(r'[^A-Za-z0-9_. ]+|^\.|\.$|^ | $|^$')
     badsuffix = re.compile(r'(aux|com[1-9]|con|lpt[1-9]|prn)(\.|$)')
@@ -313,15 +301,32 @@ def fnConstructor(s):
     return fn
 
 
-def get_None(value):
+def get_none(value):
     """
     Convert "None" to None
     :param value:
-    :type value: str, bool
+    :type value: str, NoneType
     :return:
-    :rtype: bool
+    :rtype: type(value) or NoneType
+
+    >>> st = read()
+    >>> print(get_none(st))
+    3 Trace(s) in Stream:
+    BW.RJOB..EHZ | 2009-08-24T00:20:03.000000Z - 2009-08-24T00:20:32.990000Z | 100.0 Hz, 3000 samples
+    BW.RJOB..EHN | 2009-08-24T00:20:03.000000Z - 2009-08-24T00:20:32.990000Z | 100.0 Hz, 3000 samples
+    BW.RJOB..EHE | 2009-08-24T00:20:03.000000Z - 2009-08-24T00:20:32.990000Z | 100.0 Hz, 3000 samples
+    >>> get_none('Stream')
+    'Stream'
+    >>> get_none(0)
+    0
+    >>> get_none(0.)
+    0.0
+    >>> print(get_none('None'))
+    None
+    >>> print(get_none(None))
+    None
     """
-    if value == 'None':
+    if value is None or (type(value) is str and value == 'None'):
         return None
     else:
         return value
@@ -329,11 +334,30 @@ def get_None(value):
 
 def get_bool(value):
     """
-    Convert string representations of bools to their true boolean value
+    Convert string representations of bools to their true boolean value. Return value if it cannot be identified as bool.
     :param value:
-    :type value: str, bool
+    :type value: str, bool, int, float
     :return: true boolean value
     :rtype: bool
+
+    >>> get_bool(True)
+    True
+    >>> get_bool(False)
+    False
+    >>> get_bool(0)
+    False
+    >>> get_bool(0.)
+    False
+    >>> get_bool(0.1)
+    True
+    >>> get_bool(2)
+    True
+    >>> get_bool(-1)
+    False
+    >>> get_bool(-0.3)
+    False
+    >>> get_bool(None)
+    None
     """
     if type(value) is bool:
         return value
@@ -341,8 +365,14 @@ def get_bool(value):
         return True
     elif value in ['False', 'false']:
         return False
+    elif isinstance(value, float) or isinstance(value, int):
+        if value > 0. or value > 0:
+            return True
+        else:
+            return False
     else:
-        return bool(value)
+        return value
+
 
 def four_digits(year):
     """
@@ -353,8 +383,8 @@ def four_digits(year):
     :return: four digit year correspondent
     :rtype: int
 
-    >>> four_digits(20)
-    1920
+    >>> four_digits(75)
+    1975
     >>> four_digits(16)
     2016
     >>> four_digits(00)
@@ -436,36 +466,53 @@ def backtransformFilterString(st):
     return st
 
 
-def getHash(time):
+def get_hash(time):
     """
     takes a time object and returns the corresponding SHA1 hash of the formatted date string
     :param time: time object for which a hash should be calculated
     :type time: `~obspy.core.utcdatetime.UTCDateTime`
     :return: SHA1 hash
     :rtype: str
+
+    >>> time = UTCDateTime(0)
+    >>> get_hash(time)
+    '7627cce3b1b58dd21b005dac008b34d18317dd15'
+    >>> get_hash(0)
+    Traceback (most recent call last):
+    ...
+    AssertionError: 'time' is not an ObsPy UTCDateTime object
     """
+    assert isinstance(time, UTCDateTime), '\'time\' is not an ObsPy UTCDateTime object'
     hg = hashlib.sha1()
-    hg.update(time.strftime('%Y-%m-%d %H:%M:%S.%f'))
+    hg.update(time.strftime('%Y-%m-%d %H:%M:%S.%f').encode('utf-8'))
     return hg.hexdigest()
 
 
-def getLogin():
+def get_login():
     """
-    returns the actual user's login ID
-    :return: login ID
+    returns the actual user's name
+    :return: login name
     :rtype: str
     """
     import getpass
     return getpass.getuser()
 
 
-def getOwner(fn):
+def get_owner(fn):
     """
     takes a filename and return the login ID of the actual owner of the file
     :param fn: filename of the file tested
     :type fn: str
     :return: login ID of the file's owner
     :rtype: str
+
+    >>> import tempfile
+    >>> with tempfile.NamedTemporaryFile() as tmpfile:
+    ...     tmpfile.write(b'') and True
+    ...     tmpfile.flush()
+    ...     get_owner(tmpfile.name) == os.path.expanduser('~').split('/')[-1]
+    0
+    True
     """
     system_name = platform.system()
     if system_name in ["Linux", "Darwin"]:
@@ -511,6 +558,11 @@ def is_executable(fn):
     :param fn: path to the file to be tested
     :return: True or False
     :rtype: bool
+
+    >>> is_executable('/bin/ls')
+    True
+    >>> is_executable('/var/log/system.log')
+    False
     """
     return os.path.isfile(fn) and os.access(fn, os.X_OK)
 
@@ -537,24 +589,36 @@ def isSorted(iterable):
     >>> isSorted([2,3,1,4])
     False
     """
-    assert isIterable(iterable), 'object is not iterable; object: {' \
-                                 '}'.format(iterable)
+    assert is_iterable(iterable), "object is not iterable; object: {}".format(iterable)
     if type(iterable) is str:
         iterable = [s for s in iterable]
     return sorted(iterable) == iterable
 
 
-def isIterable(obj):
+def is_iterable(obj):
     """
     takes a python object and returns True is the object is iterable and
     False otherwise
     :param obj: a python object
-    :type obj: object
+    :type obj: obj
     :return: True of False
     :rtype: bool
+
+    >>> is_iterable(1)
+    False
+    >>> is_iterable(True)
+    False
+    >>> is_iterable(0.)
+    False
+    >>> is_iterable((0,1,3,4))
+    True
+    >>> is_iterable([1])
+    True
+    >>> is_iterable('a')
+    True
     """
     try:
-        iterator = iter(obj)
+        iter(obj)
     except TypeError as te:
         return False
     return True
@@ -563,13 +627,19 @@ def isIterable(obj):
 def key_for_set_value(d):
     """
     takes a dictionary and returns the first key for which's value the
-    boolean is True
+    boolean representation is True
     :param d: dictionary containing values
     :type d: dict
     :return: key to the first non-False value found; None if no value's
     boolean equals True
-    :rtype:
+    :rtype: bool or NoneType
+
+    >>> key_for_set_value({'one': 0, 'two': 1})
+    'two'
+    >>> print(key_for_set_value({1: 0, 2: False}))
+    None
     """
+    assert type(d) is dict, "Function only defined for inputs of type 'dict'."
     r = None
     for k, v in d.items():
         if v:
@@ -577,32 +647,53 @@ def key_for_set_value(d):
     return r
 
 
-def prepTimeAxis(stime, trace, verbosity=0):
+def prep_time_axis(offset, trace, verbosity=0):
     """
-    takes a starttime and a trace object and returns a valid time axis for
+    takes an offset and a trace object and returns a valid time axis for
     plotting
-    :param stime: start time of the actual seismogram as UTCDateTime
-    :type stime: `~obspy.core.utcdatetime.UTCDateTime`
+    :param offset: offset of the actual seismogram on plotting axis
+    :type offset: float or int
     :param trace: seismic trace object
     :type trace: `~obspy.core.trace.Trace`
     :param verbosity: if != 0, debug output will be written to console
     :type verbosity: int
     :return: valid numpy array with time stamps for plotting
     :rtype: `~numpy.ndarray`
+
+    >>> tr = read()[0]
+    >>> prep_time_axis(0., tr)
+    array([0.00000000e+00, 1.00033344e-02, 2.00066689e-02, ...,
+           2.99799933e+01, 2.99899967e+01, 3.00000000e+01])
+    >>> prep_time_axis(22.5, tr)
+    array([22.5       , 22.51000333, 22.52000667, ..., 52.47999333,
+           52.48999667, 52.5       ])
+    >>> prep_time_axis(tr.stats.starttime, tr)
+    Traceback (most recent call last):
+    ...
+    AssertionError: 'offset' is not of type 'float' or 'int'; type: <class 'obspy.core.utcdatetime.UTCDateTime'>
+    >>> tr.stats.npts -= 1
+    >>> prep_time_axis(0, tr)
+    array([0.00000000e+00, 1.00033356e-02, 2.00066711e-02, ...,
+           2.99699933e+01, 2.99799967e+01, 2.99900000e+01])
+    >>> tr.stats.npts += 2
+    >>> prep_time_axis(0, tr)
+    array([0.00000000e+00, 1.00033333e-02, 2.00066667e-02, ...,
+           2.99899933e+01, 2.99999967e+01, 3.00100000e+01])
     """
+    assert isinstance(offset, (float, int)), "'offset' is not of type 'float' or 'int'; type: {}".format(type(offset))
     nsamp = trace.stats.npts
     srate = trace.stats.sampling_rate
     tincr = trace.stats.delta
-    etime = stime + nsamp / srate
-    time_ax = np.linspace(stime, etime, nsamp)
+    etime = offset + nsamp / srate
+    time_ax = np.linspace(offset, etime, nsamp)
     if len(time_ax) < nsamp:
         if verbosity:
             print('elongate time axes by one datum')
-        time_ax = np.arange(stime, etime + tincr, tincr)
+        time_ax = np.arange(offset, etime + tincr, tincr)
     elif len(time_ax) > nsamp:
         if verbosity:
             print('shorten time axes by one datum')
-        time_ax = np.arange(stime, etime - tincr, tincr)
+        time_ax = np.arange(offset, etime - tincr, tincr)
     if len(time_ax) != nsamp:
         print('Station {0}, {1} samples of data \n '
               '{2} length of time vector \n'
@@ -618,13 +709,13 @@ def find_horizontals(data):
     :param data: waveform data
     :type data: `obspy.core.stream.Stream`
     :return: components list
-    :rtype: list
+    :rtype: List(str)
 
     ..example::
 
     >>> st = read()
     >>> find_horizontals(st)
-    [u'N', u'E']
+    ['N', 'E']
     """
     rval = []
     for tr in data:
@@ -635,7 +726,7 @@ def find_horizontals(data):
     return rval
 
 
-def pick_color(picktype, phase, quality=0):
+def pick_color(picktype: Literal['manual', 'automatic'], phase: Literal['P', 'S'], quality: int = 0) -> Rgba:
     """
     Create pick color by modifying the base color by the quality.
 
@@ -648,7 +739,7 @@ def pick_color(picktype, phase, quality=0):
     :param quality: quality of pick. Decides the new intensity of the modifier color
     :type quality: int
     :return: tuple containing modified rgba color values
-    :rtype: (int, int, int, int)
+    :rtype: Rgba
     """
     min_quality = 3
     bpc = base_phase_colors(picktype, phase)  # returns dict like {'modifier': 'g', 'rgba': (0, 0, 255, 255)}
@@ -704,17 +795,17 @@ def pick_linestyle_plt(picktype, key):
     return linestyles[picktype][key]
 
 
-def modify_rgba(rgba, modifier, intensity):
+def modify_rgba(rgba: Rgba, modifier: Literal['r', 'g', 'b'], intensity: float) -> Rgba:
     """
     Modify rgba color by adding the given intensity to the modifier color
     :param rgba: tuple containing rgba values
-    :type rgba: (int, int, int, int)
-    :param modifier: which color should be modified, eg. 'r', 'g', 'b'
-    :type modifier: str
+    :type rgba: Rgba
+    :param modifier: which color should be modified; options: 'r', 'g', 'b'
+    :type modifier: Literal['r', 'g', 'b']
     :param intensity: intensity to be added to selected color
     :type intensity: float
     :return: tuple containing rgba values
-    :rtype: (int, int, int, int)
+    :rtype: Rgba
     """
     rgba = list(rgba)
     index = {'r': 0,
@@ -748,18 +839,20 @@ def transform_colors_mpl_str(colors, no_alpha=False):
     Transforms rgba color values to a matplotlib string of color values with a range of [0, 1]
     :param colors: tuple of rgba color values ranging from [0, 255]
     :type colors: (float, float, float, float)
-    :param no_alpha: Wether to return a alpha value in the matplotlib color string
+    :param no_alpha: Whether to return an alpha value in the matplotlib color string
     :type no_alpha: bool
     :return: String containing r, g, b values and alpha value if no_alpha is False (default)
     :rtype: str
+
+    >>> transform_colors_mpl_str((255., 255., 255., 255.), True)
+    '(1.0, 1.0, 1.0)'
+    >>> transform_colors_mpl_str((255., 255., 255., 255.))
+    '(1.0, 1.0, 1.0, 1.0)'
     """
-    colors = list(colors)
-    colors_mpl = tuple([color / 255. for color in colors])
     if no_alpha:
-        colors_mpl = '({}, {}, {})'.format(*colors_mpl)
+        return '({}, {}, {})'.format(*transform_colors_mpl(colors))
     else:
-        colors_mpl = '({}, {}, {}, {})'.format(*colors_mpl)
-    return colors_mpl
+        return '({}, {}, {}, {})'.format(*transform_colors_mpl(colors))
 
 
 def transform_colors_mpl(colors):
@@ -769,27 +862,16 @@ def transform_colors_mpl(colors):
     :type colors: (float, float, float, float)
     :return: tuple of rgba color values ranging from [0, 1]
     :rtype: (float, float, float, float)
+
+    >>> transform_colors_mpl((127.5, 0., 63.75, 255.))
+    (0.5, 0.0, 0.25, 1.0)
+    >>> transform_colors_mpl(())
     """
     colors = list(colors)
     colors_mpl = tuple([color / 255. for color in colors])
     return colors_mpl
 
 
-def remove_underscores(data):
-    """
-    takes a `obspy.core.stream.Stream` object and removes all underscores
-    from station names
-    :param data: stream of seismic data
-    :type data: `~obspy.core.stream.Stream`
-    :return: data stream
-    :rtype: `~obspy.core.stream.Stream`
-    """
-    # for tr in data:
-    #    # remove underscores
-    #    tr.stats.station = tr.stats.station.strip('_')
-    return data
-
-
 def trim_station_components(data, trim_start=True, trim_end=True):
     """
     cut a stream so only the part common to all three traces is kept to avoid dealing with offsets
@@ -818,19 +900,6 @@ def trim_station_components(data, trim_start=True, trim_end=True):
     return data
 
 
-def merge_stream(stream):
-    gaps = stream.get_gaps()
-    if gaps:
-        # list of merged stations (seed_ids)
-        merged = ['{}.{}.{}.{}'.format(*gap[:4]) for gap in gaps]
-        stream.merge(method=1)
-        print('Merged the following stations because of gaps:')
-        for merged_station in merged:
-            print(merged_station)
-
-    return stream, gaps
-
-
 def check4gapsAndRemove(data):
     """
     check for gaps in Stream and remove them
@@ -851,12 +920,12 @@ def check4gapsAndRemove(data):
     return data
 
 
-def check4gapsAndMerge(data):
+def check_for_gaps_and_merge(data):
     """
     check for gaps in Stream and merge if gaps are found
     :param data: stream of seismic data
     :type data: `~obspy.core.stream.Stream`
-    :return: data stream
+    :return: data stream, gaps returned from obspy get_gaps
     :rtype: `~obspy.core.stream.Stream`
     """
     gaps = data.get_gaps()
@@ -867,7 +936,7 @@ def check4gapsAndMerge(data):
         for merged_station in merged:
             print(merged_station)
 
-    return data
+    return data, gaps
 
 
 def check4doubled(data):
@@ -897,13 +966,53 @@ def check4doubled(data):
     return data
 
 
+def check_for_nan(data, nan_value=0.):
+    """
+    Replace all NaNs in data with nan_value (in place)
+    :param data: stream of seismic data
+    :type data: `~obspy.core.stream.Stream`
+    :param nan_value: value which all NaNs are set to
+    :type nan_value: float, int
+    :return: None
+    """
+    if not data:
+        return
+    for trace in data:
+        np.nan_to_num(trace.data, copy=False, nan=nan_value)
+
+
+def get_pylot_eventfile_with_extension(event, fext):
+    if hasattr(event, 'path'):
+        eventpath = event.path
+    else:
+        logging.warning('No attribute path found for event.')
+        return
+    eventname = event.pylot_id #path.split('/')[-1]  # or event.pylot_id
+    filename = os.path.join(eventpath, 'PyLoT_' + eventname + fext)
+    if os.path.isfile(filename):
+        return filename
+
+
+def get_possible_pylot_eventfile_extensions(event, fext):
+    if hasattr(event, 'path'):
+        eventpath = event.path
+    else:
+        logging.warning('No attribute path found for event.')
+        return []
+    eventname = event.pylot_id
+    filename = os.path.join(eventpath, 'PyLoT_' + eventname + fext)
+    filenames = glob.glob(filename)
+    extensions = [os.path.split(path)[-1].split('PyLoT_' + eventname)[-1] for path in filenames]
+    return extensions
+
+
 def get_stations(data):
     """
-    Get list of all station names in data stream
+    Get list of all station names in data-stream
     :param data: stream containing seismic traces
     :type data: `~obspy.core.stream.Stream`
     :return: list of all station names in data, no duplicates
-    :rtype: list of str
+    :rtype: List(str)
     """
     stations = []
     for tr in data:
@@ -930,66 +1039,87 @@ def check4rotated(data, metadata=None, verbosity=1):
     :rtype: `~obspy.core.stream.Stream`
     """
 
-    def rotate_components(wfstream, metadata=None):
+    def rotation_required(trace_ids):
+        """
+        Derive if any rotation is required from the orientation code of the input.
+
+        :param trace_ids: string identifier of waveform data trace
+        :type trace_ids: List(str)
+        :return: boolean representing if rotation is necessary for any of the traces
+        :rtype: bool
+        """
+        orientations = [trace_id[-1] for trace_id in trace_ids]
+        return any([orientation.isnumeric() for orientation in orientations])
+
+    def rotate_components(wfs_in, metadata=None):
         """
         Rotate components if orientation code is numeric (= non traditional orientation).
 
         Azimut and dip are fetched from metadata. To be rotated, traces of a station have to be cut to the same length.
         Returns unrotated traces of no metadata is provided
-        :param wfstream: stream containing seismic traces of a station
-        :type wfstream: `~obspy.core.stream.Stream`
+        :param wfs_in: stream containing seismic traces of a station
+        :type wfs_in: `~obspy.core.stream.Stream`
         :param metadata: tuple containing metadata type string and metadata parser object
         :type metadata: (str, `~obspy.io.xseed.parser.Parser`)
         :return: stream object with traditionally oriented traces (ZNE)
         :rtype: `~obspy.core.stream.Stream`
         """
 
-        # check if any traces in this station need to be rotated
-        trace_ids = [trace.id for trace in wfstream]
-        orientations = [trace_id[-1] for trace_id in trace_ids]
-        rotation_required = [orientation.isnumeric() for orientation in orientations]
-        if any(rotation_required):
-            t_start = full_range(wfstream)
-            try:
-                azimuts = []
-                dips = []
-                for tr_id in trace_ids:
-                    azimuts.append(metadata.get_coordinates(tr_id, t_start)['azimuth'])
-                    dips.append(metadata.get_coordinates(tr_id, t_start)['dip'])
-            except (KeyError, TypeError) as e:
-                print('Failed to rotate trace {}, no azimuth or dip available in metadata'.format(tr_id))
-                return wfstream
-            if len(wfstream) < 3:
-                print('Failed to rotate Stream {}, not enough components available.'.format(wfstream))
-                return wfstream
-            # to rotate all traces must have same length, so trim them
-            wfstream = trim_station_components(wfstream, trim_start=True, trim_end=True)
-            try:
-                z, n, e = rotate2zne(wfstream[0], azimuts[0], dips[0],
-                                     wfstream[1], azimuts[1], dips[1],
-                                     wfstream[2], azimuts[2], dips[2])
-                print('check4rotated: rotated trace {} to ZNE'.format(trace_ids))
-                # replace old data with rotated data, change the channel code to ZNE
-                z_index = dips.index(min(
-                    dips))  # get z-trace index, z has minimum dip of -90 (dip is measured from 0 to -90, with -90 being vertical)
-                wfstream[z_index].data = z
-                wfstream[z_index].stats.channel = wfstream[z_index].stats.channel[0:-1] + 'Z'
-                del trace_ids[z_index]
-                for trace_id in trace_ids:
-                    coordinates = metadata.get_coordinates(trace_id, t_start)
-                    dip, az = coordinates['dip'], coordinates['azimuth']
-                    trace = wfstream.select(id=trace_id)[0]
-                    if az > 315 or az <= 45 or az > 135 and az <= 225:
-                        trace.data = n
-                        trace.stats.channel = trace.stats.channel[0:-1] + 'N'
-                    elif az > 45 and az <= 135 or az > 225 and az <= 315:
-                        trace.data = e
-                        trace.stats.channel = trace.stats.channel[0:-1] + 'E'
-            except (ValueError) as e:
-                print(e)
-                return wfstream
+        if len(wfs_in) < 3:
+            print(f"Stream {wfs_in=}, has not enough components to rotate.")
+            return wfs_in
 
-        return wfstream
+        # check if any traces in this station need to be rotated
+        trace_ids = [trace.id for trace in wfs_in]
+        if not rotation_required(trace_ids):
+            logging.debug(f"Stream does not need any rotation: Traces are {trace_ids=}")
+            return wfs_in
+
+        # check metadata quality
+        t_start = full_range(wfs_in)
+        try:
+            azimuths = []
+            dips = []
+            for tr_id in trace_ids:
+                azimuths.append(metadata.get_coordinates(tr_id, t_start)['azimuth'])
+                dips.append(metadata.get_coordinates(tr_id, t_start)['dip'])
+        except (KeyError, TypeError) as err:
+            logging.error(f"{type(err)=} occurred: {err=} Rotating not possible, not all azimuth and dip information "
+                  f"available in metadata. Stream remains unchanged.")
+            return wfs_in
+        except Exception as err:
+            print(f"Unexpected {err=}, {type(err)=}")
+            raise
+
+        # to rotate all traces must have same length, so trim them
+        wfs_out = trim_station_components(wfs_in, trim_start=True, trim_end=True)
+        try:
+            z, n, e = rotate2zne(wfs_out[0], azimuths[0], dips[0],
+                                 wfs_out[1], azimuths[1], dips[1],
+                                 wfs_out[2], azimuths[2], dips[2])
+            print('check4rotated: rotated trace {} to ZNE'.format(trace_ids))
+            # replace old data with rotated data, change the channel code to ZNE
+            z_index = dips.index(min(
+                dips))  # get z-trace index, z has minimum dip of -90 (dip is measured from 0 to -90, with -90
+            # being vertical)
+            wfs_out[z_index].data = z
+            wfs_out[z_index].stats.channel = wfs_out[z_index].stats.channel[0:-1] + 'Z'
+            del trace_ids[z_index]
+            for trace_id in trace_ids:
+                coordinates = metadata.get_coordinates(trace_id, t_start)
+                dip, az = coordinates['dip'], coordinates['azimuth']
+                trace = wfs_out.select(id=trace_id)[0]
+                if az > 315 or az <= 45 or 135 < az <= 225:
+                    trace.data = n
+                    trace.stats.channel = trace.stats.channel[0:-1] + 'N'
+                elif 45 < az <= 135 or 225 < az <= 315:
+                    trace.data = e
+                    trace.stats.channel = trace.stats.channel[0:-1] + 'E'
+        except ValueError as err:
+            print(f"{err=} Rotation failed. Stream remains unchanged.")
+            return wfs_in
+
+        return wfs_out
 
     if metadata is None:
         if verbosity:
@@ -1003,38 +1133,6 @@ def check4rotated(data, metadata=None, verbosity=1):
     return data
 
 
-def scaleWFData(data, factor=None, components='all'):
-    """
-    produce scaled waveforms from given waveform data and a scaling factor,
-    waveform may be selected by their components name
-    :param data: waveform data to be scaled
-    :type data: `~obspy.core.stream.Stream` object
-    :param factor: scaling factor
-    :type factor: float
-    :param components: components labels for the traces in data to be scaled by
-     the scaling factor (optional, default: 'all')
-    :type components: tuple
-    :return:  scaled waveform data
-    :rtype: `~obspy.core.stream.Stream` object
-    """
-    if components != 'all':
-        for comp in components:
-            if factor is None:
-                max_val = np.max(np.abs(data.select(component=comp)[0].data))
-                data.select(component=comp)[0].data /= 2 * max_val
-            else:
-                data.select(component=comp)[0].data /= 2 * factor
-    else:
-        for tr in data:
-            if factor is None:
-                max_val = float(np.max(np.abs(tr.data)))
-                tr.data /= 2 * max_val
-            else:
-                tr.data /= 2 * factor
-
-    return data
-
-
 def runProgram(cmd, parameter=None):
     """
     run an external program specified by cmd with parameters input returning the
@@ -1106,6 +1204,7 @@ def identifyPhase(phase):
         return False
 
 
+@lru_cache
 def identifyPhaseID(phase):
     """
     Returns phase id (capital P or S)

pylot/core/util/widgets.py

@@ -7,6 +7,7 @@ Created on Wed Mar 19 11:27:35 2014
 import copy
 import datetime
 import getpass
+import glob
 import multiprocessing
 import os
 import subprocess
@@ -16,6 +17,7 @@ import traceback
 
 import matplotlib
 import numpy as np
+from pylot.core.io.phases import getQualitiesfromxml
 
 matplotlib.use('QT5Agg')
 
@@ -36,7 +38,7 @@ from PySide2.QtWidgets import QAction, QApplication, QCheckBox, QComboBox, \
     QGridLayout, QLabel, QLineEdit, QMessageBox, \
     QTabWidget, QToolBar, QVBoxLayout, QHBoxLayout, QWidget, \
     QPushButton, QFileDialog, QInputDialog
-from PySide2.QtCore import QSettings, Qt, QUrl, Signal
+from PySide2.QtCore import QSettings, Qt, QUrl, Signal, QTimer
 from PySide2.QtWebEngineWidgets import QWebEngineView as QWebView
 from obspy import Stream, Trace, UTCDateTime
 from obspy.core.util import AttribDict
@@ -49,12 +51,12 @@ from pylot.core.pick.utils import getSNR, earllatepicker, getnoisewin, \
 from pylot.core.pick.compare import Comparison
 from pylot.core.pick.autopick import fmpicker
 from pylot.core.util.defaults import OUTPUTFORMATS, FILTERDEFAULTS
-from pylot.core.util.utils import prepTimeAxis, full_range, demeanTrace, isSorted, findComboBoxIndex, clims, \
+from pylot.core.util.utils import prep_time_axis, full_range, demeanTrace, isSorted, findComboBoxIndex, clims, \
     pick_linestyle_plt, pick_color_plt, \
-    check4rotated, check4doubled, merge_stream, identifyPhase, \
+    check4rotated, check4doubled, check_for_gaps_and_merge, check_for_nan, identifyPhase, \
     loopIdentifyPhase, trim_station_components, transformFilteroptions2String, \
-    identifyPhaseID, get_bool, get_None, pick_color, getAutoFilteroptions, SetChannelComponents, \
-    station_id_remove_channel
+    identifyPhaseID, get_bool, get_none, pick_color, getAutoFilteroptions, SetChannelComponents, \
+    station_id_remove_channel, get_pylot_eventfile_with_extension, get_possible_pylot_eventfile_extensions
 from autoPyLoT import autoPyLoT
 from pylot.core.util.thread import Thread
 from pylot.core.util.dataprocessing import Metadata
@@ -864,9 +866,24 @@ class WaveformWidgetPG(QtWidgets.QWidget):
 
     def plotWFData(self, wfdata, wfsyn=None, title=None, scaleddata=False, mapping=True,
                    component='*', nth_sample=1, verbosity=0, method='normal', gain=1., shift_syn=0.2):
+        def station_sort(nslc):
+            """Try to sort after station integer in case of a line array (e.g. active seismics)"""
+            try:
+                rval = sorted(nslc, key=lambda x: int(x.split('.')[1]))
+                return rval
+            except ValueError as e:
+                # this is the standard case for seismological stations
+                pass
+            except Exception as e:
+                print(f'Sorting by station integer failed with unknown exception: {e}')
+
+            # fallback to default sorting
+            return sorted(nslc)
+
         if not wfdata:
             print('Nothing to plot.')
             return
+
         self.title = title
         self.clearPlotDict()
         self.wfstart, self.wfend = full_range(wfdata)
@@ -884,14 +901,14 @@ class WaveformWidgetPG(QtWidgets.QWidget):
         else:
             st_select = wfdata
 
-        st_select, gaps = merge_stream(st_select)
+        # st_select, gaps = check_for_gaps_and_merge(st_select) #MP MP commented because probably done twice
 
         # list containing tuples of network, station, channel (for sorting)
         nslc = []
         for trace in st_select:
             nslc.append(
                 trace.get_id())  # (trace.stats.network, trace.stats.station, trace.stats.location trace.stats.channel))
-        nslc.sort()
+        nslc = station_sort(nslc)
         nslc.reverse()
         plots = []
 
@@ -923,10 +940,10 @@ class WaveformWidgetPG(QtWidgets.QWidget):
                 msg = 'plotting %s channel of station %s' % (channel, station)
                 print(msg)
             stime = trace.stats.starttime - self.wfstart
-            time_ax = prepTimeAxis(stime, trace)
+            time_ax = prep_time_axis(stime, trace)
             if st_syn:
                 stime_syn = trace_syn.stats.starttime - self.wfstart
-                time_ax_syn = prepTimeAxis(stime_syn, trace_syn)
+                time_ax_syn = prep_time_axis(stime_syn, trace_syn)
 
             if method == 'fast':
                 trace.data, time_ax = self.minMax(trace, time_ax)
@@ -955,7 +972,7 @@ class WaveformWidgetPG(QtWidgets.QWidget):
         self.ylabel = ''
         self.setXLims([0, self.wfend - self.wfstart])
         self.setYLims([0.5, nmax + 0.5])
-        return plots, gaps
+        return plots
 
     def minMax(self, trace, time_ax):
         '''
@@ -977,7 +994,7 @@ class WaveformWidgetPG(QtWidgets.QWidget):
         min_ = data.min(axis=1)
         max_ = data.max(axis=1)
         if remaining_samples:
-            extreme_values = np.empty((npixel + 1, 2), dtype=np.float)
+            extreme_values = np.empty((npixel + 1, 2), dtype=float)
             extreme_values[:-1, 0] = min_
             extreme_values[:-1, 1] = max_
             extreme_values[-1, 0] = \
@@ -985,7 +1002,7 @@ class WaveformWidgetPG(QtWidgets.QWidget):
             extreme_values[-1, 1] = \
                 trace.data[-remaining_samples:].max()
         else:
-            extreme_values = np.empty((npixel, 2), dtype=np.float)
+            extreme_values = np.empty((npixel, 2), dtype=float)
             extreme_values[:, 0] = min_
             extreme_values[:, 1] = max_
         data = extreme_values.flatten()
@@ -1427,7 +1444,7 @@ class PylotCanvas(FigureCanvas):
                     msg = 'plotting %s channel of station %s' % (channel, station)
                     print(msg)
                 stime = trace.stats.starttime - wfstart
-                time_ax = prepTimeAxis(stime, trace)
+                time_ax = prep_time_axis(stime, trace)
                 if time_ax is not None:
                     if scaleToChannel:
                         st_scale = wfdata.select(channel=scaleToChannel)
@@ -1467,7 +1484,7 @@ class PylotCanvas(FigureCanvas):
                 if not scaleddata:
                     trace.detrend('constant')
                     trace.normalize(np.max(np.abs(trace.data)) * 2)
-                time_ax = prepTimeAxis(stime, trace)
+                time_ax = prep_time_axis(stime, trace)
                 times = [time for index, time in enumerate(time_ax) if not index % nth_sample]
                 p_data = compare_stream[0].data
                 # #normalize
@@ -1575,6 +1592,178 @@ class PylotCanvas(FigureCanvas):
         self.draw()
 
 
+class SearchFileByExtensionDialog(QtWidgets.QDialog):
+    def __init__(self, parent=None, label='Text: ', default_text='.xml', events=None):
+        super(SearchFileByExtensionDialog, self).__init__(parent)
+        self.events = events
+        self.filepaths = []
+        self.file_extensions = []
+        self.check_all_state = True
+        self.merge_strategy = None
+        self.default_text = default_text
+        self.label = label
+        self.setButtons()
+        self.setupUi()
+        self.connectSignals()
+        self.showPaths()
+        self.refreshSelectionBox()
+        # self.refresh_timer = QTimer(self)
+        # self.refresh_timer.timeout.connect(self.showPaths)
+        # self.refresh_timer.start(10000)
+
+        self.resize(800, 450)
+
+    def setupUi(self):
+        ncol = 4
+        self.main_layout = QtWidgets.QVBoxLayout()
+        self.header_layout = QtWidgets.QHBoxLayout()
+        self.footer_layout = QtWidgets.QHBoxLayout()
+        #
+        self.setLayout(self.main_layout)
+
+        # widgets inside the dialog
+        self.textLabel = QtWidgets.QLabel(self.label)
+        self.comboBox = QtWidgets.QComboBox()
+        self.comboBox.addItem(self.default_text)
+        self.comboBox.setEditable(True)
+
+        # optional search button, currently disabled. List refreshed when text changes
+        self.searchButton = QtWidgets.QPushButton('Search')
+        self.searchButton.setVisible(False)
+
+        # check/uncheck button for table
+        self.checkAllButton = QtWidgets.QPushButton('Check/Uncheck all')
+
+        # radiobutton for merge selection
+        self.mergeRadioButtonGroup = QtWidgets.QButtonGroup()
+        self.merge_button = QtWidgets.QRadioButton('Merge')
+        self.overwrite_button = QtWidgets.QRadioButton('Overwrite')
+        self.mergeRadioButtonGroup.addButton(self.merge_button)
+        self.mergeRadioButtonGroup.addButton(self.overwrite_button)
+        self.merge_button.setChecked(True)
+        self.merge_strategy = self.merge_button.text()
+
+        # table
+        self.tableWidget = QtWidgets.QTableWidget()
+        tableWidget = self.tableWidget
+        tableWidget.setColumnCount(ncol)
+        tableWidget.setRowCount(len(self.events))
+        tableWidget.setHorizontalHeaderLabels(('', 'Event ID', 'Filename', 'Last modified'))
+        tableWidget.setEditTriggers(tableWidget.NoEditTriggers)
+        tableWidget.setSortingEnabled(True)
+        header = tableWidget.horizontalHeader()
+        header.setSectionResizeMode(QtWidgets.QHeaderView.ResizeToContents)
+        header.setStretchLastSection(True)
+
+        self.statusText = QtWidgets.QLabel()
+
+        self.header_layout.addWidget(self.textLabel)
+        self.header_layout.addWidget(self.comboBox)
+        self.header_layout.addWidget(self.searchButton)
+
+        self.footer_layout.addWidget(self.checkAllButton)
+        self.footer_layout.addWidget(self.statusText)
+        self.footer_layout.addWidget(self.merge_button)
+        self.footer_layout.addWidget(self.overwrite_button)
+
+        self.footer_layout.setStretch(0, 0)
+        self.footer_layout.setStretch(1, 1)
+
+        self.main_layout.addLayout(self.header_layout)
+        self.main_layout.addWidget(self.tableWidget)
+        self.main_layout.addLayout(self.footer_layout)
+        self.main_layout.addWidget(self._buttonbox)
+
+    def showPaths(self):
+        self.filepaths = []
+        fext = self.comboBox.currentText()
+        self.tableWidget.clearContents()
+        for index, event in enumerate(self.events):
+            filename = get_pylot_eventfile_with_extension(event, fext)
+            pf_selected_item = QtWidgets.QTableWidgetItem()
+            check_state = QtCore.Qt.Checked if filename else QtCore.Qt.Unchecked
+            pf_selected_item.setCheckState(check_state)
+            self.tableWidget.setItem(index, 0, pf_selected_item)
+            self.tableWidget.setItem(index, 1, QtWidgets.QTableWidgetItem(f'{event.pylot_id}'))
+            if filename:
+                self.filepaths.append(filename)
+                ts = int(os.path.getmtime(filename))
+
+                # create QTableWidgetItems of filepath and last modification time
+                fname_item = QtWidgets.QTableWidgetItem(f'{os.path.split(filename)[-1]}')
+                fname_item.setData(3, filename)
+                ts_item = QtWidgets.QTableWidgetItem(f'{datetime.datetime.fromtimestamp(ts)}')
+                self.tableWidget.setItem(index, 2, fname_item)
+                self.tableWidget.setItem(index, 3, ts_item)
+
+        self.update_status()
+
+    def refreshSelectionBox(self):
+        fext = self.comboBox.currentText()
+        self.file_extensions = [fext]
+
+        for event in self.events:
+            extensions = get_possible_pylot_eventfile_extensions(event, '*.xml')
+            for ext in extensions:
+                if not ext in self.file_extensions:
+                    self.file_extensions.append(ext)
+
+        self.comboBox.clear()
+        for ext in sorted(self.file_extensions):
+            self.comboBox.addItem(ext)
+
+    def setButtons(self):
+        self._buttonbox = QDialogButtonBox(QDialogButtonBox.Ok |
+                                           QDialogButtonBox.Cancel)
+
+    def toggleCheckAll(self):
+        self.check_all_state = not self.check_all_state
+        self.checkAll(self.check_all_state)
+
+    def checkAll(self, state):
+        state = QtCore.Qt.Checked if state else QtCore.Qt.Unchecked
+        for row_ind in range(self.tableWidget.rowCount()):
+            item = self.tableWidget.item(row_ind, 0)
+            item.setCheckState(state)
+
+    def getChecked(self):
+        filepaths = []
+        for row_ind in range(self.tableWidget.rowCount()):
+            item_check = self.tableWidget.item(row_ind, 0)
+            if item_check.checkState() == QtCore.Qt.Checked:
+                item_fname = self.tableWidget.item(row_ind, 2)
+                if item_fname:
+                    filepath = item_fname.data(3)
+                    filepaths.append(filepath)
+        return filepaths
+
+    def update_status(self, row=None, col=None):
+        if col is not None and col != 0:
+            return
+        filepaths = self.getChecked()
+        if len(filepaths) > 0:
+            status_text = f"Found {len(filepaths)} eventfile{'s' if len(filepaths) > 1 else ''}. Do you want to load them?"
+        else:
+            status_text = 'Did not find/select any files for specified file mask.'
+        self.statusText.setText(status_text)
+
+    def update_merge_strategy(self):
+        for button in (self.merge_button, self.overwrite_button):
+            if button.isChecked():
+                self.merge_strategy = button.text()
+
+    def connectSignals(self):
+        self._buttonbox.accepted.connect(self.accept)
+        self._buttonbox.rejected.connect(self.reject)
+        self.comboBox.editTextChanged.connect(self.showPaths)
+        self.searchButton.clicked.connect(self.showPaths)
+        self.checkAllButton.clicked.connect(self.toggleCheckAll)
+        self.checkAllButton.clicked.connect(self.update_status)
+        self.tableWidget.cellClicked.connect(self.update_status)
+        self.merge_button.clicked.connect(self.update_merge_strategy)
+        self.overwrite_button.clicked.connect(self.update_merge_strategy)
+
+
 class SingleTextLineDialog(QtWidgets.QDialog):
     def __init__(self, parent=None, label='Text: ', default_text='.xml'):
         super(SingleTextLineDialog, self).__init__(parent)
@@ -1681,13 +1870,14 @@ class PickDlg(QDialog):
 
     def __init__(self, parent=None, data=None, data_compare=None, station=None, network=None, location=None, picks=None,
                  autopicks=None, rotate=False, parameter=None, embedded=False, metadata=None, show_comp_data=False,
-                 event=None, filteroptions=None, model=None, wftype=None):
+                 event=None, filteroptions=None, wftype=None):
         super(PickDlg, self).__init__(parent, Qt.Window)
         self.orig_parent = parent
         self.setAttribute(Qt.WA_DeleteOnClose)
 
         # initialize attributes
         self.parameter = parameter
+        model = self.parameter.get('taup_model')
         self._embedded = embedded
         self.showCompData = show_comp_data
         self.station = station
@@ -2064,10 +2254,12 @@ class PickDlg(QDialog):
         station_id = trace.get_id()
         starttime = trace.stats.starttime
         station_coords = self.metadata.get_coordinates(station_id, starttime)
+        if not station_coords:
+            print('get_arrivals: No station coordinates found. Return!')
+            return
         origins = self.pylot_event.origins
         if phases == ['None', 'None']:
-            print("get_arrivals: Creation info (manual or auto) not available!")
-            print("Return!")
+            print("get_arrivals: Creation info (manual or auto) not available! Return!")
             return
         if origins:
             source_origin = origins[0]
@@ -2078,8 +2270,8 @@ class PickDlg(QDialog):
         arrivals = func[plot](source_origin.depth,
                               source_origin.latitude,
                               source_origin.longitude,
-                              station_coords['latitude'],
-                              station_coords['longitude'],
+                              station_coords.get('latitude'),
+                              station_coords.get('longitude'),
                               phases)
         self.arrivals = arrivals
 
@@ -2179,7 +2371,7 @@ class PickDlg(QDialog):
 
                 # create action and add to menu
                 # phase name transferred using lambda function
-                slot = lambda phase=phase, phaseID=phaseID: phaseSelect[phaseID](phase)
+                slot = lambda ph=phase, phID=phaseID: phaseSelect[phID](ph)
                 picksAction = createAction(parent=self, text=phase,
                                            slot=slot,
                                            shortcut=shortcut)
@@ -2598,7 +2790,7 @@ class PickDlg(QDialog):
         for wfd in [wfdata, wfdata_comp]:
             if wfd:
                 for trace in wfd:
-                    t = prepTimeAxis(trace.stats.starttime - stime, trace)
+                    t = prep_time_axis(trace.stats.starttime - stime, trace)
                     inoise = getnoisewin(t, ini_pick, noise_win, gap_win)
                     trace = demeanTrace(trace, inoise)
                     # upscale trace wfdata in a way that each trace is vertically zoomed to noiselevel*factor
@@ -3624,8 +3816,9 @@ class TuneAutopicker(QWidget):
             # wfdat = remove_underscores(wfdat)
             # rotate misaligned stations to ZNE
             # check for gaps and doubled channels
-            wfdat, gaps = merge_stream(wfdat)
-            # check4gaps(wfdat)
+            wfdat, _ = check_for_gaps_and_merge(wfdat)
+            # check for nans
+            check_for_nan(wfdat)
             check4doubled(wfdat)
             wfdat = check4rotated(wfdat, self.parent().metadata, verbosity=0)
             # trim station components to same start value
@@ -3782,6 +3975,7 @@ class TuneAutopicker(QWidget):
         st = self.data.getWFData()
         tr = st.select(station=self.get_current_station())[0]
         starttime = tr.stats.starttime
+        # create two lists with figure names and subindices (for subplots) to get the correct axes
         p_axes = [
             ('mainFig', 0),
             ('aicFig', 0),
@@ -4729,8 +4923,8 @@ class InputsTab(PropTab):
         self.tstopBox = QSpinBox()
         for spinbox in [self.tstartBox, self.tstopBox]:
             spinbox.setRange(-99999, 99999)
-        self.tstartBox.setValue(float(settings.value('tstart')) if get_None(settings.value('tstart')) else 0)
-        self.tstopBox.setValue(float(settings.value('tstop')) if get_None(settings.value('tstop')) else 1e6)
+        self.tstartBox.setValue(float(settings.value('tstart')) if get_none(settings.value('tstart')) else 0)
+        self.tstopBox.setValue(float(settings.value('tstop')) if get_none(settings.value('tstop')) else 1e6)
         self.cuttimesLayout.addWidget(self.tstartBox, 10)
         self.cuttimesLayout.addWidget(QLabel('[s] and'), 0)
         self.cuttimesLayout.addWidget(self.tstopBox, 10)
@@ -5726,7 +5920,7 @@ class ChooseWaveFormWindow(QWidget):
         #self.currentSpectro = self.traces[
         #    self.chooseBoxTraces.currentText()[3:]][self.chooseBoxComponent.currentText()].spectrogram(show=False, title=t)
         #self.currentSpectro.show()
-        applyFFT()
+        self.applyFFT()
 
     def applyFFT(self, trace):
         tra = self.traces[self.chooseBoxTraces.currentText()[3:]]['Z']

pylot/correlation/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+#

pylot/correlation/parameters_adriaarray.yaml

@@ -0,0 +1,90 @@
+#############################  correlation parameters  #####################################
+# min_corr_stacking: minimum correlation coefficient for building beam trace
+# min_corr_export: minimum correlation coefficient for pick export
+# min_stack: minimum number of stations for building beam trace
+# t_before: correlation window before pick
+# t_after: correlation window after pick#
+# cc_maxlag: maximum shift for initial correlation
+# cc_maxlag2: maximum shift for second (final) correlation (also for calculating pick uncertainty)
+# initial_pick_outlier_threshold: (hopefully) threshold for excluding large outliers of initial (AIC) picks
+# export_threshold: automatically exclude all onsets which deviate more than this threshold from corrected taup onsets
+# min_picks_export: minimum number of correlated picks for export
+# min_picks_autopylot: minimum number of reference autopicks picks to continue with event
+# check_RMS: do RMS check to search for restitution errors (very experimental)
+# use_taupy_onsets: use taupy onsets as reference picks instead of external picks
+# station_list: use the following stations as reference for stacking
+# use_stacked_trace: use existing stacked trace if found (spare re-computation)
+# data_dir: obspyDMT data subdirectory (e.g. 'raw', 'processed')
+# pickfile_extension: use quakeML files (PyLoT output) with the following extension, e.g. '_autopylot' for pickfiles
+#                     such as 'PyLoT_20170501_141822_autopylot.xml'
+
+logging: info
+pick_phases: ['P', 'S']
+
+# P-phase
+P:
+  min_corr_stacking: 0.8
+  min_corr_export: 0.6
+  min_stack: 20
+  t_before: 30.
+  t_after: 50.
+  cc_maxlag: 50.
+  cc_maxlag2: 5.
+  initial_pick_outlier_threshold: 30.
+  export_threshold: 2.5
+  min_picks_export: 100
+  min_picks_autopylot: 50
+  check_RMS: True
+  use_taupy_onsets: False
+  station_list: ['HU.MORH', 'HU.TIH',  'OX.FUSE', 'OX.BAD']
+  use_stacked_trace: False
+  data_dir: 'processed'
+  pickfile_extension: '_autopylot'
+  dt_stacking: [250, 250]
+
+  # filter for first correlation (rough)
+  filter_options:
+    freqmax: 0.5
+    freqmin: 0.03
+  # filter for second correlation (fine)
+  filter_options_final:
+    freqmax: 0.5
+    freqmin: 0.03
+
+  filter_type: bandpass
+  sampfreq: 20.0
+
+# S-phase
+S:
+  min_corr_stacking: 0.7
+  min_corr_export: 0.6
+  min_stack: 20
+  t_before: 60.
+  t_after: 60.
+  cc_maxlag: 100.
+  cc_maxlag2: 25.
+  initial_pick_outlier_threshold: 30.
+  export_threshold: 5.0
+  min_picks_export: 200
+  min_picks_autopylot: 50
+  check_RMS: True
+  use_taupy_onsets: False
+  station_list: ['HU.MORH','HU.TIH',  'OX.FUSE', 'OX.BAD']
+  use_stacked_trace: False
+  data_dir: 'processed'
+  pickfile_extension: '_autopylot'
+  dt_stacking: [250, 250]
+
+  # filter for first correlation (rough)
+  filter_options:
+    freqmax: 0.1
+    freqmin: 0.01
+
+  # filter for second correlation (fine)
+  filter_options_final:
+    freqmax: 0.2
+    freqmin: 0.01
+
+  filter_type: bandpass
+  sampfreq: 20.0
+

pylot/correlation/submit_pick_corr_correction.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+#ulimit -s 8192
+#ulimit -v $(ulimit -v | awk '{printf("%d",$1*0.95)}')
+#ulimit -v
+
+#655360
+
+source /opt/anaconda3/etc/profile.d/conda.sh
+conda activate pylot_311
+NSLOTS=20
+
+#qsub -l low -cwd -l "os=*stretch" -pe smp 40 submit_pick_corr_correction.sh
+#$ -l low
+#$ -l h_vmem=6G
+#$ -cwd
+#$ -pe smp 20
+#$ -N corr_pick
+
+
+export PYTHONPATH="$PYTHONPATH:/home/marcel/git/pylot_tools/"
+export PYTHONPATH="$PYTHONPATH:/home/marcel/git/"
+export PYTHONPATH="$PYTHONPATH:/home/marcel/git/pylot/"
+
+#export MKL_NUM_THREADS=${NSLOTS:=1}
+#export NUMEXPR_NUM_THREADS=${NSLOTS:=1}
+#export OMP_NUM_THREADS=${NSLOTS:=1}
+
+#python pick_correlation_correction.py '/data/AlpArray_Data/dmt_database_mantle_M5.8-6.0' '/home/marcel/.pylot/pylot_alparray_mantle_corr_stack_0.03-0.5.in' -pd -n ${NSLOTS:=1} -istart 0 -istop 100
+#python pick_correlation_correction.py '/data/AlpArray_Data/dmt_database_mantle_M5.8-6.0' '/home/marcel/.pylot/pylot_alparray_mantle_corr_stack_0.03-0.5.in' -pd -n ${NSLOTS:=1} -istart 100 -istop 200
+#python pick_correlation_correction.py '/data/AlpArray_Data/dmt_database_mantle_M6.0-6.5' '/home/marcel/.pylot/pylot_alparray_mantle_corr_stack_0.03-0.5.in' -pd -n ${NSLOTS:=1} -istart 0 -istop 100
+#python pick_correlation_correction.py '/data/AlpArray_Data/dmt_database_mantle_M5.8-6.0' '/home/marcel/.pylot/pylot_alparray_mantle_corr_stack_0.03-0.5.in' -pd -n ${NSLOTS:=1} -istart 100 -istop 200
+#python pick_correlation_correction.py 'H:\sciebo\dmt_database' 'H:\Sciebo\dmt_database\pylot_alparray_mantle_corr_S_0.01-0.2.in' -pd -n 4 -t
+
+pylot_infile='/home/marcel/.pylot/pylot_alparray_syn_fwi_mk6_it3.in'
+#pylot_infile='/home/marcel/.pylot/pylot_adriaarray_corr_P_and_S.in'
+
+# THIS SCRIPT SHOLD BE CALLED BY "submit_to_grid_engine.py" using the following line:
+python pick_correlation_correction.py $1 $pylot_infile -pd -n ${NSLOTS:=1} -istart $2 --params 'parameters_fwi_mk6_it3.yaml'
+#--event_blacklist eventlist.txt

pylot/correlation/submit_to_grid_engine.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+import subprocess
+
+fnames = [
+    ('/data/AlpArray_Data/dmt_database_synth_model_mk6_it3_no_rotation', 0),
+          ]
+
+#fnames = [('/data/AlpArray_Data/dmt_database_mantle_0.01-0.2_SKS-phase', 0),
+#          ('/data/AlpArray_Data/dmt_database_mantle_0.01-0.2_S-phase', 0),]
+
+####
+script_location = '/home/marcel/VersionCtrl/git/code_base/correlation_picker/submit_pick_corr_correction.sh'
+####
+
+for fnin, istart in fnames:
+    input_cmds = f'qsub -q low.q@minos15,low.q@minos14,low.q@minos13,low.q@minos12,low.q@minos11 {script_location} {fnin} {istart}'
+
+    print(input_cmds)
+    print(subprocess.check_output(input_cmds.split()))
+
+
+

pylot/correlation/utils.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import glob
+import json
+
+from obspy import read_events
+
+from pylot.core.util.dataprocessing import Metadata
+from pylot.core.util.obspyDMT_interface import qml_from_obspyDMT
+
+
+def get_event_obspy_dmt(eventdir):
+    event_pkl_file = os.path.join(eventdir, 'info', 'event.pkl')
+    if not os.path.exists(event_pkl_file):
+        raise IOError('Could not find event path for event: {}'.format(eventdir))
+    event = qml_from_obspyDMT(event_pkl_file)
+    return event
+
+
+def get_event_pylot(eventdir, extension=''):
+    event_id = get_event_id(eventdir)
+    filename = os.path.join(eventdir, 'PyLoT_{}{}.xml'.format(event_id, extension))
+    if not os.path.isfile(filename):
+        return
+    cat = read_events(filename)
+    return cat[0]
+
+
+def get_event_id(eventdir):
+    event_id = os.path.split(eventdir)[-1]
+    return event_id
+
+
+def get_picks(eventdir, extension=''):
+    event_id = get_event_id(eventdir)
+    filename = 'PyLoT_{}{}.xml'
+    filename = filename.format(event_id, extension)
+    fpath = os.path.join(eventdir, filename)
+    fpaths = glob.glob(fpath)
+    if len(fpaths) == 1:
+        cat = read_events(fpaths[0])
+        picks = cat[0].picks
+        return picks
+    elif len(fpaths) == 0:
+        print('get_picks: File not found: {}'.format(fpath))
+        return
+    print(f'WARNING: Ambiguous pick file specification. Found the following pick files {fpaths}\nFilemask: {fpath}')
+    return
+
+
+def write_json(object, fname):
+    with open(fname, 'w') as outfile:
+        json.dump(object, outfile, sort_keys=True, indent=4)
+
+
+def get_metadata(eventdir):
+    metadata_path = os.path.join(eventdir, 'resp')
+    metadata = Metadata(inventory=metadata_path, verbosity=0)
+    return metadata

requirements.txt

@@ -1,12 +1,7 @@
-# This file may be used to create an environment using:
-# $ conda create --name <env> --file <this file>
-# platform: win-64
-cartopy=0.20.2
-matplotlib-base=3.3.4
-numpy=1.22.3
-obspy=1.3.0
-pyqtgraph=0.12.4
-pyside2=5.13.2
-python=3.8.12
-qt=5.12.9
-scipy=1.8.0
+Cartopy==0.23.0
+joblib==1.4.2
+obspy==1.4.1
+pyaml==24.7.0
+pyqtgraph==0.13.7
+PySide2==5.15.8
+pytest==8.3.2

tests/test_autopicker/pylot_alparray_mantle_corr_stack_0.03-0.5.in

@@ -0,0 +1,101 @@
+%This is a parameter input file for PyLoT/autoPyLoT.
+%All main and special settings regarding data handling
+%and picking are to be set here!
+%Parameters are optimized for %extent data sets!
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+#main settings#
+                                                   #rootpath#       %project path                                                                                        
+                                                   #datapath#       %data path                                                                                           
+                                                   #database#       %name of data base                                                                                   
+20171010_063224.a                                  #eventID#        %event ID for single event processing (* for all events found in database)                           
+                                                   #invdir#         %full path to  inventory or dataless-seed file                                                       
+PILOT                                              #datastructure#  %choose data structure                                                                               
+True                                               #apverbose#      %choose 'True' or 'False' for terminal output                                                        
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+#NLLoc settings#
+None                                               #nllocbin#       %path to NLLoc executable                                                                            
+None                                               #nllocroot#      %root of NLLoc-processing directory                                                                  
+None                                               #phasefile#      %name of autoPyLoT-output phase file for NLLoc                                                       
+None                                               #ctrfile#        %name of autoPyLoT-output control file for NLLoc                                                     
+ttime                                              #ttpatter#       %pattern of NLLoc ttimes from grid                                                                   
+AUTOLOC_nlloc                                      #outpatter#      %pattern of NLLoc-output file                                                                        
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+#parameters for seismic moment estimation#
+3530.0                                             #vp#             %average P-wave velocity                                                                             
+2500.0                                             #rho#            %average rock density [kg/m^3]                                                                       
+300.0 0.8                                          #Qp#             %quality factor for P waves (Qp*f^a); list(Qp, a)                                                    
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+#settings local magnitude#
+1.0 1.0 1.0                                        #WAscaling#      %Scaling relation (log(Ao)+Alog(r)+Br+C) of Wood-Anderson amplitude Ao [nm]                           If zeros are set, original Richter magnitude is calculated!
+1.0 1.0                                            #magscaling#     %Scaling relation for derived local magnitude [a*Ml+b].                            If zeros are set, no scaling of network magnitude is applied!
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+#filter settings#
+0.03 0.03                                          #minfreq#        %Lower filter frequency [P, S]                                                                       
+0.5 0.5                                            #maxfreq#        %Upper filter frequency [P, S]                                                                       
+4 4                                                #filter_order#   %filter order [P, S]                                                                                 
+bandpass bandpass                                  #filter_type#    %filter type (bandpass, bandstop, lowpass, highpass) [P, S]                                          
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+#common settings picker#
+global                                             #extent#         %extent of array ("local", "regional" or "global")                                                   
+-100.0                                             #pstart#         %start time [s] for calculating CF for P-picking (if TauPy: seconds relative to estimated onset)     
+50.0                                               #pstop#          %end time [s] for calculating CF for P-picking (if TauPy: seconds relative to estimated onset)       
+-50.0                                              #sstart#         %start time [s] relative to P-onset for calculating CF for S-picking                                 
+50.0                                               #sstop#          %end time [s] after P-onset for calculating CF for S-picking                                         
+True                                               #use_taup#       %use estimated traveltimes from TauPy for calculating windows for CF                                 
+ak135                                              #taup_model#     %Define TauPy model for traveltime estimation. Possible values: 1066a, 1066b, ak135, ak135f, herrin, iasp91, jb, prem, pwdk, sp6
+P,Pdiff,S,SKS                                      #taup_phases#    %Specify possible phases for TauPy (comma separated). See Obspy TauPy documentation for possible values.
+0.03 0.5                                           #bpz1#           %lower/upper corner freq. of first band pass filter Z-comp. [Hz]                                     
+0.01 0.5                                           #bpz2#           %lower/upper corner freq. of second band pass filter Z-comp. [Hz]                                    
+0.03 0.5                                           #bph1#           %lower/upper corner freq. of first band pass filter H-comp. [Hz]                                     
+0.01 0.5                                           #bph2#           %lower/upper corner freq. of second band pass filter z-comp. [Hz]                                    
+#special settings for calculating CF#
+%!!Edit the following only if you know what you are doing!!%
+#Z-component#
+HOS                                                #algoP#          %choose algorithm for P-onset determination (HOS, ARZ, or AR3)                                       
+300.0                                              #tlta#           %for HOS-/AR-AIC-picker, length of LTA window [s]                                                    
+4                                                  #hosorder#       %for HOS-picker, order of Higher Order Statistics                                                    
+2                                                  #Parorder#       %for AR-picker, order of AR process of Z-component                                                   
+16.0                                               #tdet1z#         %for AR-picker, length of AR determination window [s] for Z-component, 1st pick                      
+10.0                                               #tpred1z#        %for AR-picker, length of AR prediction window [s] for Z-component, 1st pick                         
+12.0                                               #tdet2z#         %for AR-picker, length of AR determination window [s] for Z-component, 2nd pick                      
+6.0                                                #tpred2z#        %for AR-picker, length of AR prediction window [s] for Z-component, 2nd pick                         
+0.001                                              #addnoise#       %add noise to seismogram for stable AR prediction                                                    
+60.0 5.0 20.0 12.0                                 #tsnrz#          %for HOS/AR, window lengths for SNR-and slope estimation [tnoise, tsafetey, tsignal, tslope] [s]     
+50.0                                               #pickwinP#       %for initial AIC pick, length of P-pick window [s]                                                   
+30.0                                               #Precalcwin#     %for HOS/AR, window length [s] for recalculation of CF (relative to 1st pick)                        
+2.0                                                #aictsmooth#     %for HOS/AR, take average of samples for smoothing of AIC-function [s]                               
+2.0                                                #tsmoothP#       %for HOS/AR, take average of samples in this time window for smoothing CF [s]                        
+0.006                                              #ausP#           %for HOS/AR, artificial uplift of samples (aus) of CF (P)                                            
+2.0                                                #nfacP#          %for HOS/AR, noise factor for noise level determination (P)                                          
+#H-components#
+ARH                                                #algoS#          %choose algorithm for S-onset determination (ARH or AR3)                                             
+12.0                                               #tdet1h#         %for HOS/AR, length of AR-determination window [s], H-components, 1st pick                           
+6.0                                                #tpred1h#        %for HOS/AR, length of AR-prediction window [s], H-components, 1st pick                              
+8.0                                                #tdet2h#         %for HOS/AR, length of AR-determinaton window [s], H-components, 2nd pick                            
+4.0                                                #tpred2h#        %for HOS/AR, length of AR-prediction window [s], H-components, 2nd pick                              
+4                                                  #Sarorder#       %for AR-picker, order of AR process of H-components                                                  
+100.0                                              #Srecalcwin#     %for AR-picker, window length [s] for recalculation of CF (2nd pick) (H)                             
+195.0                                              #pickwinS#       %for initial AIC pick, length of S-pick window [s]                                                   
+60.0 10.0 30.0 12.0                                #tsnrh#          %for ARH/AR3, window lengths for SNR-and slope estimation [tnoise, tsafetey, tsignal, tslope] [s]    
+22.0                                               #aictsmoothS#    %for AIC-picker, take average of samples in this time window for smoothing of AIC-function [s]       
+20.0                                               #tsmoothS#       %for AR-picker, take average of samples for smoothing CF [s] (S)                                     
+0.001                                              #ausS#           %for HOS/AR, artificial uplift of samples (aus) of CF (S)                                            
+2.0                                                #nfacS#          %for AR-picker, noise factor for noise level determination (S)                                       
+#first-motion picker#
+1                                                  #minfmweight#    %minimum required P weight for first-motion determination                                            
+3.0                                                #minFMSNR#       %miniumum required SNR for first-motion determination                                                
+10.0                                               #fmpickwin#      %pick window [s] around P onset for calculating zero crossings                                       
+#quality assessment#
+0.1 0.2 0.4 0.8                                    #timeerrorsP#    %discrete time errors [s] corresponding to picking weights [0 1 2 3] for P                           
+4.0 8.0 16.0 32.0                                  #timeerrorsS#    %discrete time errors [s] corresponding to picking weights [0 1 2 3] for S                           
+0.005                                              #minAICPslope#   %below this slope [counts/s] the initial P pick is rejected                                          
+1.1                                                #minAICPSNR#     %below this SNR the initial P pick is rejected                                                       
+0.002                                              #minAICSslope#   %below this slope [counts/s] the initial S pick is rejected                                          
+1.3                                                #minAICSSNR#     %below this SNR the initial S pick is rejected                                                       
+20.0                                               #minsiglength#   %length of signal part for which amplitudes must exceed noiselevel [s]                               
+1.0                                                #noisefactor#    %noiselevel*noisefactor=threshold                                                                    
+10.0                                               #minpercent#     %required percentage of amplitudes exceeding threshold                                               
+0.1                                                #zfac#           %P-amplitude must exceed at least zfac times RMS-S amplitude                                         
+100.0                                              #mdttolerance#   %maximum allowed deviation of P picks from median [s]                                                
+50.0                                               #wdttolerance#   %maximum allowed deviation from Wadati-diagram                                                       
+25.0                                               #jackfactor#     %pick is removed if the variance of the subgroup with the pick removed is larger than the mean variance of all subgroups times safety factor

tests/test_autopicker/test_autopylot.py

@@ -0,0 +1,27 @@
+import os
+import pytest
+
+from autoPyLoT import autoPyLoT
+
+
+class TestAutopickerGlobal():
+    def init(self):
+        self.params_infile = 'pylot_alparray_mantle_corr_stack_0.03-0.5.in'
+        self.test_event_dir = 'dmt_database_test'
+
+        if not os.path.isfile(self.params_infile):
+            print(f'Test input file {os.path.abspath(self.params_infile)} not found.')
+            return False
+
+        if not os.path.exists(self.test_event_dir):
+            print(
+                f'Test event directory not found at location "{os.path.abspath(self.test_event_dir)}". '
+                f'Make sure to load it from the website first.'
+            )
+            return False
+
+        return True
+
+    def test_autopicker(self):
+        assert self.init(), 'Initialization failed due to missing input files.'
+        #autoPyLoT(inputfile=self.params_infile, eventid='20171010_063224.a')

tests/test_pick_correlation_correction/test_pick_correlation_correction.py

@@ -0,0 +1,76 @@
+import pytest
+from obspy import read, Trace, UTCDateTime
+
+from pylot.correlation.pick_correlation_correction import XCorrPickCorrection
+
+
+class TestXCorrPickCorrection():
+    def setup(self):
+        self.make_test_traces()
+        self.make_test_picks()
+        self.t_before = 2.
+        self.t_after = 2.
+        self.cc_maxlag = 0.5
+
+    def make_test_traces(self):
+        # take first trace of test Stream from obspy
+        tr1 = read()[0]
+        # filter trace
+        tr1.filter('bandpass', freqmin=1, freqmax=20)
+        # make a copy and shift the copy by 0.1 s
+        tr2 = tr1.copy()
+        tr2.stats.starttime += 0.1
+
+        self.trace1 = tr1
+        self.trace2 = tr2
+
+    def make_test_picks(self):
+        # create an artificial reference pick on reference trace (trace1) and another one on the 0.1 s shifted trace
+        self.tpick1 = UTCDateTime('2009-08-24T00:20:07.7')
+        # shift the second pick by 0.2 s, the correction should be around 0.1 s now
+        self.tpick2 = self.tpick1 + 0.2
+
+    def test_slice_trace_okay(self):
+
+        self.setup()
+        xcpc = XCorrPickCorrection(UTCDateTime(), Trace(), UTCDateTime(), Trace(),
+                                   t_before=self.t_before, t_after=self.t_after, cc_maxlag=self.cc_maxlag)
+
+        test_trace = self.trace1
+        pick_time = self.tpick2
+
+        sliced_trace = xcpc.slice_trace(test_trace, pick_time)
+        assert ((sliced_trace.stats.starttime == pick_time - self.t_before - self.cc_maxlag / 2)
+                and (sliced_trace.stats.endtime == pick_time + self.t_after + self.cc_maxlag / 2))
+
+    def test_slice_trace_fails(self):
+        self.setup()
+
+        test_trace = self.trace1
+        pick_time = self.tpick1
+
+        with pytest.raises(ValueError):
+            xcpc = XCorrPickCorrection(UTCDateTime(), Trace(), UTCDateTime(), Trace(),
+                                       t_before=self.t_before + 20, t_after=self.t_after, cc_maxlag=self.cc_maxlag)
+            xcpc.slice_trace(test_trace, pick_time)
+
+        with pytest.raises(ValueError):
+            xcpc = XCorrPickCorrection(UTCDateTime(), Trace(), UTCDateTime(), Trace(),
+                                       t_before=self.t_before, t_after=self.t_after + 50, cc_maxlag=self.cc_maxlag)
+            xcpc.slice_trace(test_trace, pick_time)
+
+    def test_cross_correlation(self):
+        self.setup()
+
+        # create XCorrPickCorrection object
+        xcpc = XCorrPickCorrection(self.tpick1, self.trace1, self.tpick2, self.trace2, t_before=self.t_before,
+                                   t_after=self.t_after, cc_maxlag=self.cc_maxlag)
+
+        # execute correlation
+        correction, cc_max, uncert, fwfm = xcpc.cross_correlation(False, '', '')
+
+        # define awaited test result
+        test_result = (-0.09983091718314982, 0.9578431835689154, 0.0015285160561610929, 0.03625786256084631)
+
+        # check results
+        assert pytest.approx(test_result, rel=1e-6) == (correction, cc_max, uncert, fwfm)