[initial] first implementation of residual plotting (WIP)

# Conflicts:
#	PyLoT.py
#	README.md
#	pylot/core/util/widgets.py

PyLoT.py

@@ -716,14 +716,14 @@ class MainWindow(QMainWindow):
         self.tabs.addTab(wf_tab, 'Waveform Plot')
         self.tabs.addTab(array_tab, 'Array Map')
         self.tabs.addTab(events_tab, 'Eventlist')
-        self.tabs.addTab(spectro_tab, 'Spectro')
+        #self.tabs.addTab(spectro_tab, 'Spectro')
 
         self.wf_layout.addWidget(self.no_data_label)
         self.wf_layout.addWidget(self.wf_scroll_area)
         self.wf_scroll_area.setWidgetResizable(True)
         self.init_array_tab()
         self.init_event_table()
-        self.init_spectro_tab()
+        #self.init_spectro_tab()
         self.tabs.setCurrentIndex(0)
 
         self.eventLabel = QLabel()
@@ -1537,8 +1537,8 @@ class MainWindow(QMainWindow):
             return True
         return not bool(os.listdir(wf_path))
 
-    def filename_from_action(self, action):
-        if action.data() is None:
+    def filename_from_action(self, action=None):
+        if not action or action.data() is None:
             filt = "Supported file formats" \
                    " (*.mat *.qml *.xml *.kor *.evt)"
             caption = "Open an event file"
@@ -2196,7 +2196,8 @@ class MainWindow(QMainWindow):
         if event.pylot_autopicks:
             self.drawPicks(picktype='auto')
         if event.pylot_picks or event.pylot_autopicks:
-            self.locateEventAction.setEnabled(True)
+            if not self._inputs.get('extent') == 'global':
+                self.locateEventAction.setEnabled(True)
             self.qualities_action.setEnabled(True)
             self.eventlist_xml_action.setEnabled(True)
 
@@ -2631,7 +2632,6 @@ class MainWindow(QMainWindow):
                           picks=self.getPicksOnStation(station, 'manual'),
                           autopicks=self.getPicksOnStation(station, 'auto'),
                           metadata=self.metadata, event=event,
-                          model=self.inputs.get('taup_model'),
                           filteroptions=self.filteroptions, wftype=wftype,
                           show_comp_data=self.dataPlot.comp_checkbox.isChecked())
         if self.filterActionP.isChecked():

README.md

@@ -54,33 +54,14 @@ In order to run PyLoT you need to install:
 
 #### Some handwork:
 
-PyLoT needs a properties folder on your system to work. It should be situated in your home directory
-(on Windows usually C:/Users/*username*):
-
-    mkdir ~/.pylot
-
-In the next step you have to copy some files to this directory:
-
-*for local distance seismicity*
-
-    cp path-to-pylot/inputs/pylot_local.in ~/.pylot/pylot.in
-
-*for regional distance seismicity*
-
-    cp path-to-pylot/inputs/pylot_regional.in ~/.pylot/pylot.in
-
-*for global distance seismicity*
-
-    cp path-to-pylot/inputs/pylot_global.in ~/.pylot/pylot.in
-
-and some extra information on error estimates (just needed for reading old PILOT data) and the Richter magnitude scaling
+Some extra information on error estimates (just needed for reading old PILOT data) and the Richter magnitude scaling
 relation
 
     cp path-to-pylot/inputs/PILOT_TimeErrors.in path-to-pylot/inputs/richter_scaling.data ~/.pylot/
 
 You may need to do some modifications to these files. Especially folder names should be reviewed.
 
-PyLoT has been tested on Mac OSX (10.11), Debian Linux 8 and on Windows 10.
+PyLoT has been tested on Mac OSX (10.11), Debian Linux 8 and on Windows 10/11.
 
 ## Release notes
 
@@ -89,6 +70,7 @@ PyLoT has been tested on Mac OSX (10.11), Debian Linux 8 and on Windows 10.
 - event organisation in project files and waveform visualisation
 - consistent manual phase picking through predefined SNR dependant zoom level
 - consistent automatic phase picking routines using Higher Order Statistics, AIC and Autoregression
+- pick correlation correction for teleseismic waveforms
 - interactive tuning of auto-pick parameters
 - uniform uncertainty estimation from waveform's properties for automatic and manual picks
 - pdf representation and comparison of picks taking the uncertainty intrinsically into account
@@ -97,7 +79,7 @@ PyLoT has been tested on Mac OSX (10.11), Debian Linux 8 and on Windows 10.
 
 #### Known issues:
 
-We hope to solve these with the next release.
+Current release is still in development progress and has several issues. We are currently lacking manpower, but hope to assess many of the issues in the near future.
 
 ## Staff
 
@@ -110,4 +92,4 @@ Others: A. Bruestle, T. Meier, W. Friederich
 
 [ObsPy]: http://github.com/obspy/obspy/wiki
 
-August 2024
+September 2024

pylot/__init__.py

@@ -9,7 +9,7 @@ PyLoT - the Python picking and Localization Tool
 
 This python library contains a graphical user interfaces for picking
 seismic phases. This software needs ObsPy (http://github.com/obspy/obspy/wiki)
-and the Qt4 libraries to be installed first.
+and the Qt libraries to be installed first.
 
 PILOT has been developed in Mathworks' MatLab. In order to distribute
 PILOT without facing portability problems, it has been decided to re-

pylot/core/io/default_parameters.py

@@ -501,7 +501,7 @@ defaults = {'datapath': {'type': str,
 
             'taup_model': {'type': str,
                            'tooltip': 'Define TauPy model for traveltime estimation. Possible values: 1066a, 1066b, ak135, ak135f, herrin, iasp91, jb, prem, pwdk, sp6',
-                           'value': None,
+                           'value': 'iasp91',
                            'namestring': 'TauPy model'},
 
             'taup_phases': {'type': str,

pylot/core/io/phases.py

@@ -218,12 +218,14 @@ def picksdict_from_obs(fn):
     return picks
 
 
-def picksdict_from_picks(evt, parameter=None):
+def picksdict_from_picks(evt, parameter=None, nwst_id: bool = False):
     """
     Takes an Event object and return the pick dictionary commonly used within
     PyLoT
     :param evt: Event object contain all available information
     :type evt: `~obspy.core.event.Event`
+    :param nwst_id: determines if network and station id are used or only station id
+    :type nwst_id: bool
     :return: pick dictionary (auto and manual)
     """
     picksdict = {
@@ -233,7 +235,10 @@ def picksdict_from_picks(evt, parameter=None):
     for pick in evt.picks:
         errors = None
         phase = {}
-        station = pick.waveform_id.station_code
+        if not nwst_id:
+            station_or_nwst = pick.waveform_id.station_code
+        else:
+            station_or_nwst = f'{pick.waveform_id.station_code}.{pick.waveform_id.network_code}'
         if pick.waveform_id.channel_code is None:
             channel = ''
         else:
@@ -254,7 +259,7 @@ def picksdict_from_picks(evt, parameter=None):
         if pick_method == 'None':
             pick_method = 'manual'
         try:
-            onsets = picksdict[pick_method][station]
+            onsets = picksdict[pick_method][station_or_nwst]
         except KeyError as e:
             # print(e)
             onsets = {}
@@ -301,7 +306,7 @@ def picksdict_from_picks(evt, parameter=None):
         phase['filter_id'] = filter_id if filter_id is not None else ''
 
         onsets[pick.phase_hint] = phase.copy()
-        picksdict[pick_method][station] = onsets.copy()
+        picksdict[pick_method][station_or_nwst] = onsets.copy()
     return picksdict
 
 

pylot/core/pick/charfuns.py

@@ -21,6 +21,7 @@ try:
     from scipy.signal import tukey
 except ImportError:
     from scipy.signal.windows import tukey
+
 from obspy.core import Stream
 
 from pylot.core.pick.utils import PickingFailedException

pylot/core/util/array_map.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-
+import copy
 import traceback
 
 import cartopy.crs as ccrs
@@ -16,6 +16,8 @@ from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
 from mpl_toolkits.axes_grid1.inset_locator import inset_axes
 from obspy import UTCDateTime
 
+from pylot.core.io.data import Data
+from pylot.core.io.phases import picksdict_from_picks
 from pylot.core.util.utils import identifyPhaseID
 from scipy.interpolate import griddata
 
@@ -41,6 +43,7 @@ class MplCanvas(FigureCanvas):
 class Array_map(QtWidgets.QWidget):
     def __init__(self, parent, metadata, parameter=None, axes=None, annotate=True, pointsize=25.,
                  linewidth=1.5, width=5e6, height=2e6):
+
         QtWidgets.QWidget.__init__(self, parent=parent)
 
         assert (parameter is not None or parent is not None), 'either parent or parameter has to be set'
@@ -58,6 +61,8 @@ class Array_map(QtWidgets.QWidget):
         self.uncertainties = None
         self.autopicks_dict = None
         self.hybrids_dict = None
+        self.picks_dict_ref = None
+        self.autopicks_dict_ref = None
         self.eventLoc = None
         self.parameter = parameter if parameter else parent._inputs
 
@@ -108,8 +113,10 @@ class Array_map(QtWidgets.QWidget):
         self.status_label = QtWidgets.QLabel()
         self.map_reset_button = QtWidgets.QPushButton('Reset Map View')
         self.save_map_button = QtWidgets.QPushButton('Save Map')
+        self.load_reference_picks_button = QtWidgets.QPushButton('Load reference picks.')
         self.go2eq_button = QtWidgets.QPushButton('Go to Event Location')
         self.subtract_mean_cb = QtWidgets.QCheckBox('Subtract mean')
+        self.subtract_ref_cb = QtWidgets.QCheckBox('Subtract reference onsets')
 
         self.main_box = QtWidgets.QVBoxLayout()
         self.setLayout(self.main_box)
@@ -156,7 +163,9 @@ class Array_map(QtWidgets.QWidget):
         self.bot_row.addWidget(self.map_reset_button, 2)
         self.bot_row.addWidget(self.go2eq_button, 2)
         self.bot_row.addWidget(self.save_map_button, 2)
+        self.bot_row.addWidget(self.load_reference_picks_button, 2)
         self.bot_row.addWidget(self.subtract_mean_cb, 0)
+        self.bot_row.addWidget(self.subtract_ref_cb, 0)
         self.bot_row.addWidget(self.status_label, 5)
 
     def init_colormap(self):
@@ -217,7 +226,9 @@ class Array_map(QtWidgets.QWidget):
         self.map_reset_button.clicked.connect(self.org_map_view)
         self.go2eq_button.clicked.connect(self.go2eq)
         self.save_map_button.clicked.connect(self.saveFigure)
+        self.load_reference_picks_button.clicked.connect(self.load_reference_picks)
         self.subtract_mean_cb.stateChanged.connect(self.toggle_subtract_mean)
+        self.subtract_ref_cb.stateChanged.connect(self.toggle_subtract_ref)
 
         self.plotWidget.mpl_connect('motion_notify_event', self.mouse_moved)
         self.plotWidget.mpl_connect('scroll_event', self.mouse_scroll)
@@ -374,8 +385,11 @@ class Array_map(QtWidgets.QWidget):
     def get_max_from_stations(self, key):
         return self._from_dict(max, key)
 
+    def get_selected_pick_type(self):
+        return self.comboBox_am.currentText().split(' ')[0]
+
     def current_picks_dict(self):
-        picktype = self.comboBox_am.currentText().split(' ')[0]
+        picktype = self.get_selected_pick_type()
         auto_manu = {'auto': self.autopicks_dict,
                      'manual': self.picks_dict,
                      'hybrid': self.hybrids_dict}
@@ -447,6 +461,9 @@ class Array_map(QtWidgets.QWidget):
         self.cmaps_box.setCurrentIndex(self.cmaps_box.findText(cmap))
         self._refresh_drawings()
 
+    def toggle_subtract_ref(self):
+        self._refresh_drawings()
+
     def init_lat_lon_dimensions(self):
         # init minimum and maximum lon and lat dimensions
         self.londim = self.lonmax - self.lonmin
@@ -459,7 +476,10 @@ class Array_map(QtWidgets.QWidget):
         self.longrid, self.latgrid = np.meshgrid(lonaxis, lataxis)
 
     def init_picksgrid(self):
-        picks, uncertainties, lats, lons = self.get_picks_lat_lon()
+        rval = self.get_picks_lat_lon()
+        if not rval:
+            return
+        picks, uncertainties, lats, lons = rval
         try:
             self.picksgrid_active = griddata((lats, lons), picks, (self.latgrid, self.longrid), method='linear')
         except Exception as e:
@@ -477,15 +497,20 @@ class Array_map(QtWidgets.QWidget):
 
     def get_picks_lat_lon(self):
         picks_rel = self.picks_rel_mean_corrected if self.subtract_mean_cb.isChecked() else self.picks_rel
+        picks_rel = self.picks_rel_ref_corrected if self.subtract_ref_cb.isChecked() else picks_rel
+        if not picks_rel:
+            return
         picks = []
         uncertainties = []
         latitudes = []
         longitudes = []
-        for st_id, pick in picks_rel.items():
+        for nwst_id, pick in picks_rel.items():
+            if nwst_id not in self.uncertainties or nwst_id not in self.stations_dict:
+                continue
             picks.append(pick)
-            uncertainties.append(self.uncertainties.get(st_id))
-            latitudes.append(self.stations_dict[st_id]['latitude'])
-            longitudes.append(self.stations_dict[st_id]['longitude'])
+            uncertainties.append(self.uncertainties.get(nwst_id))
+            latitudes.append(self.stations_dict[nwst_id]['latitude'])
+            longitudes.append(self.stations_dict[nwst_id]['longitude'])
         return picks, uncertainties, latitudes, longitudes
 
     # plotting -----------------------------------------------------
@@ -521,7 +546,6 @@ class Array_map(QtWidgets.QWidget):
                                   picks=self._parent.get_current_event().getPick(station),
                                   autopicks=self._parent.get_current_event().getAutopick(station),
                                   filteroptions=self._parent.filteroptions, metadata=self.metadata,
-                                  model=self.parameter.get('taup_model'),
                                   event=pyl_mw.get_current_event())
             except Exception as e:
                 message = 'Could not generate Plot for station {st}.\n {er}'.format(st=station, er=e)
@@ -583,7 +607,10 @@ class Array_map(QtWidgets.QWidget):
                                             transform=ccrs.PlateCarree())
 
     def scatter_picked_stations(self):
-        picks, uncertainties, lats, lons = self.get_picks_lat_lon()
+        rval = self.get_picks_lat_lon()
+        if not rval:
+            return
+        picks, uncertainties, lats, lons = rval
         if len(lons) < 1 and len(lats) < 1:
             return
 
@@ -738,6 +765,52 @@ class Array_map(QtWidgets.QWidget):
                 fname += '.png'
             self.canvas.fig.savefig(fname)
 
+    def load_reference_picks(self):
+        fname = self._parent.filename_from_action()
+        data_ref = Data(parent=self._parent, evtdata=str(fname))
+        evt_ref = data_ref.get_evt_data()
+        if not evt_ref:
+            return
+        picks_ref = evt_ref.picks
+        if not picks_ref:
+            return
+        picksdict_ref = picksdict_from_picks(evt_ref, nwst_id=False)
+        self.autopicks_dict = picksdict_ref['manual']
+        self.autopicks_dict_ref = picksdict_ref['auto']
+        return True
+
+    @property
+    def picks_rel_ref_corrected(self):
+        picktype = self.get_selected_pick_type()
+        if picktype in ['auto', 'hybrid']:
+            picks_dict = self.autopicks_dict_ref
+        elif picktype == 'manual':
+            picks_dict = self.autopicks_dict
+        else:
+            return
+        if picks_dict:
+            return self.subtract_picks(picks_dict)
+        else:
+            if self.load_reference_picks():
+                return self.picks_rel_ref_corrected
+
+    def subtract_picks(self, picks_dict):
+        current_picks = self.current_picks_dict()
+        subtracted_picks = {}
+        for station, ps_dict in current_picks.items():
+            for phase, pick in ps_dict.items():
+                pick_ref_ps = picks_dict.get(station)
+                if not pick_ref_ps:
+                    continue
+                pick_ref = pick_ref_ps.get(phase)
+                if not pick_ref:
+                    continue
+                mpp = pick['mpp'] - pick_ref['mpp']
+                nwst_id = f'{pick["network"]}.{station}'
+                subtracted_picks[nwst_id] = mpp
+        return subtracted_picks
+
+
     def _warn(self, message):
         self.qmb = QtWidgets.QMessageBox(QtWidgets.QMessageBox.Icon.Warning, 'Warning', message)
         self.qmb.show()

pylot/core/util/dataprocessing.py

@@ -59,6 +59,8 @@ class Metadata(object):
         :type path_to_inventory: str
         :return: None
         """
+        path_to_inventory = path_to_inventory.replace('\\', '/')
+        path_to_inventory = os.path.abspath(path_to_inventory)
         assert (os.path.isdir(path_to_inventory)), '{} is no directory'.format(path_to_inventory)
         if path_to_inventory not in self.inventories:
             self.inventories.append(path_to_inventory)

pylot/core/util/widgets.py

@@ -1871,13 +1871,14 @@ class PickDlg(QDialog):
 
     def __init__(self, parent=None, data=None, data_compare=None, station=None, network=None, location=None, picks=None,
                  autopicks=None, rotate=False, parameter=None, embedded=False, metadata=None, show_comp_data=False,
-                 event=None, filteroptions=None, model=None, wftype=None):
+                 event=None, filteroptions=None, wftype=None):
         super(PickDlg, self).__init__(parent, Qt.Window)
         self.orig_parent = parent
         self.setAttribute(Qt.WA_DeleteOnClose)
 
         # initialize attributes
         self.parameter = parameter
+        model = self.parameter.get('taup_model')
         self._embedded = embedded
         self.showCompData = show_comp_data
         self.station = station
@@ -2270,8 +2271,8 @@ class PickDlg(QDialog):
         arrivals = func[plot](source_origin.depth,
                               source_origin.latitude,
                               source_origin.longitude,
-                              station_coords['latitude'],
-                              station_coords['longitude'],
+                              station_coords.get('latitude'),
+                              station_coords.get('longitude'),
                               phases)
         self.arrivals = arrivals
 

pylot/correlation/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+#

pylot/correlation/parameters_adriaarray.yaml

@@ -0,0 +1,90 @@
+#############################  correlation parameters  #####################################
+# min_corr_stacking: minimum correlation coefficient for building beam trace
+# min_corr_export: minimum correlation coefficient for pick export
+# min_stack: minimum number of stations for building beam trace
+# t_before: correlation window before pick
+# t_after: correlation window after pick#
+# cc_maxlag: maximum shift for initial correlation
+# cc_maxlag2: maximum shift for second (final) correlation (also for calculating pick uncertainty)
+# initial_pick_outlier_threshold: (hopefully) threshold for excluding large outliers of initial (AIC) picks
+# export_threshold: automatically exclude all onsets which deviate more than this threshold from corrected taup onsets
+# min_picks_export: minimum number of correlated picks for export
+# min_picks_autopylot: minimum number of reference autopicks picks to continue with event
+# check_RMS: do RMS check to search for restitution errors (very experimental)
+# use_taupy_onsets: use taupy onsets as reference picks instead of external picks
+# station_list: use the following stations as reference for stacking
+# use_stacked_trace: use existing stacked trace if found (spare re-computation)
+# data_dir: obspyDMT data subdirectory (e.g. 'raw', 'processed')
+# pickfile_extension: use quakeML files (PyLoT output) with the following extension, e.g. '_autopylot' for pickfiles
+#                     such as 'PyLoT_20170501_141822_autopylot.xml'
+
+logging: info
+pick_phases: ['P', 'S']
+
+# P-phase
+P:
+  min_corr_stacking: 0.8
+  min_corr_export: 0.6
+  min_stack: 20
+  t_before: 30.
+  t_after: 50.
+  cc_maxlag: 50.
+  cc_maxlag2: 5.
+  initial_pick_outlier_threshold: 30.
+  export_threshold: 2.5
+  min_picks_export: 100
+  min_picks_autopylot: 50
+  check_RMS: True
+  use_taupy_onsets: False
+  station_list: ['HU.MORH', 'HU.TIH',  'OX.FUSE', 'OX.BAD']
+  use_stacked_trace: False
+  data_dir: 'processed'
+  pickfile_extension: '_autopylot'
+  dt_stacking: [250, 250]
+
+  # filter for first correlation (rough)
+  filter_options:
+    freqmax: 0.5
+    freqmin: 0.03
+  # filter for second correlation (fine)
+  filter_options_final:
+    freqmax: 0.5
+    freqmin: 0.03
+
+  filter_type: bandpass
+  sampfreq: 20.0
+
+# S-phase
+S:
+  min_corr_stacking: 0.7
+  min_corr_export: 0.6
+  min_stack: 20
+  t_before: 60.
+  t_after: 60.
+  cc_maxlag: 100.
+  cc_maxlag2: 25.
+  initial_pick_outlier_threshold: 30.
+  export_threshold: 5.0
+  min_picks_export: 200
+  min_picks_autopylot: 50
+  check_RMS: True
+  use_taupy_onsets: False
+  station_list: ['HU.MORH','HU.TIH',  'OX.FUSE', 'OX.BAD']
+  use_stacked_trace: False
+  data_dir: 'processed'
+  pickfile_extension: '_autopylot'
+  dt_stacking: [250, 250]
+
+  # filter for first correlation (rough)
+  filter_options:
+    freqmax: 0.1
+    freqmin: 0.01
+
+  # filter for second correlation (fine)
+  filter_options_final:
+    freqmax: 0.2
+    freqmin: 0.01
+
+  filter_type: bandpass
+  sampfreq: 20.0
+

pylot/correlation/submit_pick_corr_correction.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+#ulimit -s 8192
+#ulimit -v $(ulimit -v | awk '{printf("%d",$1*0.95)}')
+#ulimit -v
+
+#655360
+
+source /opt/anaconda3/etc/profile.d/conda.sh
+conda activate pylot_311
+NSLOTS=20
+
+#qsub -l low -cwd -l "os=*stretch" -pe smp 40 submit_pick_corr_correction.sh
+#$ -l low
+#$ -l h_vmem=6G
+#$ -cwd
+#$ -pe smp 20
+#$ -N corr_pick
+
+
+export PYTHONPATH="$PYTHONPATH:/home/marcel/git/pylot_tools/"
+export PYTHONPATH="$PYTHONPATH:/home/marcel/git/"
+export PYTHONPATH="$PYTHONPATH:/home/marcel/git/pylot/"
+
+#export MKL_NUM_THREADS=${NSLOTS:=1}
+#export NUMEXPR_NUM_THREADS=${NSLOTS:=1}
+#export OMP_NUM_THREADS=${NSLOTS:=1}
+
+#python pick_correlation_correction.py '/data/AlpArray_Data/dmt_database_mantle_M5.8-6.0' '/home/marcel/.pylot/pylot_alparray_mantle_corr_stack_0.03-0.5.in' -pd -n ${NSLOTS:=1} -istart 0 -istop 100
+#python pick_correlation_correction.py '/data/AlpArray_Data/dmt_database_mantle_M5.8-6.0' '/home/marcel/.pylot/pylot_alparray_mantle_corr_stack_0.03-0.5.in' -pd -n ${NSLOTS:=1} -istart 100 -istop 200
+#python pick_correlation_correction.py '/data/AlpArray_Data/dmt_database_mantle_M6.0-6.5' '/home/marcel/.pylot/pylot_alparray_mantle_corr_stack_0.03-0.5.in' -pd -n ${NSLOTS:=1} -istart 0 -istop 100
+#python pick_correlation_correction.py '/data/AlpArray_Data/dmt_database_mantle_M5.8-6.0' '/home/marcel/.pylot/pylot_alparray_mantle_corr_stack_0.03-0.5.in' -pd -n ${NSLOTS:=1} -istart 100 -istop 200
+#python pick_correlation_correction.py 'H:\sciebo\dmt_database' 'H:\Sciebo\dmt_database\pylot_alparray_mantle_corr_S_0.01-0.2.in' -pd -n 4 -t
+
+pylot_infile='/home/marcel/.pylot/pylot_alparray_syn_fwi_mk6_it3.in'
+#pylot_infile='/home/marcel/.pylot/pylot_adriaarray_corr_P_and_S.in'
+
+# THIS SCRIPT SHOLD BE CALLED BY "submit_to_grid_engine.py" using the following line:
+python pick_correlation_correction.py $1 $pylot_infile -pd -n ${NSLOTS:=1} -istart $2 --params 'parameters_fwi_mk6_it3.yaml'
+#--event_blacklist eventlist.txt

pylot/correlation/submit_to_grid_engine.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+import subprocess
+
+fnames = [
+    ('/data/AlpArray_Data/dmt_database_synth_model_mk6_it3_no_rotation', 0),
+          ]
+
+#fnames = [('/data/AlpArray_Data/dmt_database_mantle_0.01-0.2_SKS-phase', 0),
+#          ('/data/AlpArray_Data/dmt_database_mantle_0.01-0.2_S-phase', 0),]
+
+####
+script_location = '/home/marcel/VersionCtrl/git/code_base/correlation_picker/submit_pick_corr_correction.sh'
+####
+
+for fnin, istart in fnames:
+    input_cmds = f'qsub -q low.q@minos15,low.q@minos14,low.q@minos13,low.q@minos12,low.q@minos11 {script_location} {fnin} {istart}'
+
+    print(input_cmds)
+    print(subprocess.check_output(input_cmds.split()))
+
+
+

pylot/correlation/utils.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import glob
+import json
+
+from obspy import read_events
+
+from pylot.core.util.dataprocessing import Metadata
+from pylot.core.util.obspyDMT_interface import qml_from_obspyDMT
+
+
+def get_event_obspy_dmt(eventdir):
+    event_pkl_file = os.path.join(eventdir, 'info', 'event.pkl')
+    if not os.path.exists(event_pkl_file):
+        raise IOError('Could not find event path for event: {}'.format(eventdir))
+    event = qml_from_obspyDMT(event_pkl_file)
+    return event
+
+
+def get_event_pylot(eventdir, extension=''):
+    event_id = get_event_id(eventdir)
+    filename = os.path.join(eventdir, 'PyLoT_{}{}.xml'.format(event_id, extension))
+    if not os.path.isfile(filename):
+        return
+    cat = read_events(filename)
+    return cat[0]
+
+
+def get_event_id(eventdir):
+    event_id = os.path.split(eventdir)[-1]
+    return event_id
+
+
+def get_picks(eventdir, extension=''):
+    event_id = get_event_id(eventdir)
+    filename = 'PyLoT_{}{}.xml'
+    filename = filename.format(event_id, extension)
+    fpath = os.path.join(eventdir, filename)
+    fpaths = glob.glob(fpath)
+    if len(fpaths) == 1:
+        cat = read_events(fpaths[0])
+        picks = cat[0].picks
+        return picks
+    elif len(fpaths) == 0:
+        print('get_picks: File not found: {}'.format(fpath))
+        return
+    print(f'WARNING: Ambiguous pick file specification. Found the following pick files {fpaths}\nFilemask: {fpath}')
+    return
+
+
+def write_json(object, fname):
+    with open(fname, 'w') as outfile:
+        json.dump(object, outfile, sort_keys=True, indent=4)
+
+
+def get_metadata(eventdir):
+    metadata_path = os.path.join(eventdir, 'resp')
+    metadata = Metadata(inventory=metadata_path, verbosity=0)
+    return metadata

tests/test_autopickstation/test_autopickstation.py

@@ -25,7 +25,7 @@ class HidePrints:
         if self.hide:
             self._original_stdout = sys.stdout
             devnull = open(os.devnull, "w")
-            sys.stdout = devnull
+            #sys.stdout = devnull
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         if self.hide:
@@ -271,7 +271,7 @@ class TestAutopickStation(unittest.TestCase):
                   'fm': 'N', 'channel': None}}
         with HidePrints():
             result, station = autopickstation(wfstream=wfstream, pickparam=self.pickparam_taupy_disabled,
-                                              metadata=(None, None))
+                                              metadata=(None, None), iplot=2)
         compare_dicts(expected=expected['P'], result=result['P'], hint='P-')
         compare_dicts(expected=expected['S'], result=result['S'], hint='S-')
         self.assertEqual('GRA1', station)

tests/test_pick_correlation_correction/test_pick_correlation_correction.py

@@ -0,0 +1,76 @@
+import pytest
+from obspy import read, Trace, UTCDateTime
+
+from pylot.correlation.pick_correlation_correction import XCorrPickCorrection
+
+
+class TestXCorrPickCorrection():
+    def setup(self):
+        self.make_test_traces()
+        self.make_test_picks()
+        self.t_before = 2.
+        self.t_after = 2.
+        self.cc_maxlag = 0.5
+
+    def make_test_traces(self):
+        # take first trace of test Stream from obspy
+        tr1 = read()[0]
+        # filter trace
+        tr1.filter('bandpass', freqmin=1, freqmax=20)
+        # make a copy and shift the copy by 0.1 s
+        tr2 = tr1.copy()
+        tr2.stats.starttime += 0.1
+
+        self.trace1 = tr1
+        self.trace2 = tr2
+
+    def make_test_picks(self):
+        # create an artificial reference pick on reference trace (trace1) and another one on the 0.1 s shifted trace
+        self.tpick1 = UTCDateTime('2009-08-24T00:20:07.7')
+        # shift the second pick by 0.2 s, the correction should be around 0.1 s now
+        self.tpick2 = self.tpick1 + 0.2
+
+    def test_slice_trace_okay(self):
+
+        self.setup()
+        xcpc = XCorrPickCorrection(UTCDateTime(), Trace(), UTCDateTime(), Trace(),
+                                   t_before=self.t_before, t_after=self.t_after, cc_maxlag=self.cc_maxlag)
+
+        test_trace = self.trace1
+        pick_time = self.tpick2
+
+        sliced_trace = xcpc.slice_trace(test_trace, pick_time)
+        assert ((sliced_trace.stats.starttime == pick_time - self.t_before - self.cc_maxlag / 2)
+                and (sliced_trace.stats.endtime == pick_time + self.t_after + self.cc_maxlag / 2))
+
+    def test_slice_trace_fails(self):
+        self.setup()
+
+        test_trace = self.trace1
+        pick_time = self.tpick1
+
+        with pytest.raises(ValueError):
+            xcpc = XCorrPickCorrection(UTCDateTime(), Trace(), UTCDateTime(), Trace(),
+                                       t_before=self.t_before + 20, t_after=self.t_after, cc_maxlag=self.cc_maxlag)
+            xcpc.slice_trace(test_trace, pick_time)
+
+        with pytest.raises(ValueError):
+            xcpc = XCorrPickCorrection(UTCDateTime(), Trace(), UTCDateTime(), Trace(),
+                                       t_before=self.t_before, t_after=self.t_after + 50, cc_maxlag=self.cc_maxlag)
+            xcpc.slice_trace(test_trace, pick_time)
+
+    def test_cross_correlation(self):
+        self.setup()
+
+        # create XCorrPickCorrection object
+        xcpc = XCorrPickCorrection(self.tpick1, self.trace1, self.tpick2, self.trace2, t_before=self.t_before,
+                                   t_after=self.t_after, cc_maxlag=self.cc_maxlag)
+
+        # execute correlation
+        correction, cc_max, uncert, fwfm = xcpc.cross_correlation(False, '', '')
+
+        # define awaited test result
+        test_result = (-0.09983091718314982, 0.9578431835689154, 0.0015285160561610929, 0.03625786256084631)
+
+        # check results
+        assert pytest.approx(test_result, rel=1e-6) == (correction, cc_max, uncert, fwfm)