huge structural rearrangement to resolve circular import problems

[add] new feature added to QtPyLoT capable of automatically picking an event from overview window

QtPyLoT.py

@@ -34,17 +34,23 @@ from PySide.QtCore import QCoreApplication, QSettings, Signal, QFile, \
 from PySide.QtGui import QMainWindow, QInputDialog, QIcon, QFileDialog, \
     QWidget, QHBoxLayout, QStyle, QKeySequence, QLabel, QFrame, QAction, \
     QDialog, QErrorMessage, QApplication, QPixmap, QMessageBox, QSplashScreen, \
-    QActionGroup, QListWidget
+    QActionGroup, QListWidget, QDockWidget
 import numpy as np
 from obspy.core import UTCDateTime
 
-from pylot.core.read import Data, FilterOptions, AutoPickParameter
+from pylot.core.read.data import Data
-from pylot.core.util import _getVersionString, FILTERDEFAULTS, fnConstructor, \
+from pylot.core.read.inputs import FilterOptions, AutoPickParameter
-    checkurl, FormatError, FilterOptionsDialog, \
+from pylot.core.pick.autopick import autopickevent
-    NewEventDlg, createEvent, MPLWidget, PropertiesDlg, HelpForm, \
+from pylot.core.util.defaults import FILTERDEFAULTS
-    DatastructureError, createAction, getLogin, createCreationInfo, PickDlg
+from pylot.core.util.errors import FormatError, DatastructureError
-from pylot.core.util.thread import WorkerThread
+from pylot.core.util.connection import checkurl
+from pylot.core.util.utils import fnConstructor, createEvent, getLogin,\
+    createCreationInfo
+from pylot.core.util.widgets import FilterOptionsDialog, NewEventDlg,\
+    MPLWidget, PropertiesDlg, HelpForm, createAction, PickDlg
 from pylot.core.util.structure import DATASTRUCTURE
+from pylot.core.util.thread import WorkerThread
+from pylot.core.util.version import get_git_version as _getVersionString
 import icons_rc
 
 # Version information
@@ -586,6 +592,13 @@ class MainWindow(QMainWindow):
 
     def autoPick(self):
         list = QListWidget()
+        logDockWidget = QDockWidget("AutoPickLog", self)
+        logDockWidget.setObjectName("LogDockWidget")
+        logDockWidget.setAllowedAreas(Qt.LeftDockWidgetArea)
+        logDockWidget.setWidget(list)
+        logDockWidget.show()
+        logDockWidget.setFloating(False)
+        list.addItem('loading default values for local data ...')
         autopick_parameter = AutoPickParameter('autoPyLoT_local.in')
         list.addItem(str(autopick_parameter))
 
@@ -597,6 +610,8 @@ class MainWindow(QMainWindow):
         self.thread.message.connect(list.addItem)
         self.thread.start()
 
+        self.drawPicks()
+
 
     def addPicks(self, station, picks):
         stat_picks = self.getPicksOnStation(station)

autoPyLoT.py

@@ -9,10 +9,10 @@ import glob
 import matplotlib.pyplot as plt
 from obspy.core import read
 from pylot.core.util import _getVersionString
-from pylot.core.read import Data, AutoPickParameter
+from pylot.core.read.data import Data
-from pylot.core.pick.run_autopicking import run_autopicking
+from pylot.core.read.inputs import AutoPickParameter
 from pylot.core.util.structure import DATASTRUCTURE
-from pylot.core.pick.utils import wadaticheck, checkPonsets
+from pylot.core.pick.autopick import autopickevent
 
 __version__ = _getVersionString()
 
@@ -49,12 +49,6 @@ def autoPyLoT(inputfile):
 
     parameter = AutoPickParameter(inputfile)
 
-    # get some parameters for quality control from 
-    # parameter input file (usually autoPyLoT.in).
-    wdttolerance = parameter.getParam('wdttolerance')
-    mdttolerance = parameter.getParam('mdttolerance')
-    iplot = parameter.getParam('iplot')
-
     data = Data()
 
     # getting information on data structure
@@ -74,43 +68,19 @@ def autoPyLoT(inputfile):
         datastructure.modifyFields(**dsfields)
         datastructure.setExpandFields(exf)
 
-        # multiple event processing 
+        # multiple event processing
         # read each event in database
         datapath = datastructure.expandDataPath()
         if not parameter.hasParam('eventID'):
             for event in [events for events in glob.glob(os.path.join(datapath, '*')) if os.path.isdir(events)]:
                 data.setWFData(glob.glob(os.path.join(datapath, event, '*')))
-                print 'Working on event %s' %event 
+                print 'Working on event %s' %event
                 print data
 
                 wfdat = data.getWFData() # all available streams
                 ##########################################################
-                # !automated picking starts here!   
+                # !automated picking starts here!
-                procstats = []
+                picks = autopickevent(wfdat, parameter)
-                # initialize dictionary for onsets
-                picks = None
-                station = wfdat[0].stats.station
-                allonsets = {station: picks}
-                for i in range(len(wfdat)):
-                    stationID = wfdat[i].stats.station
-                    # check if station has already been processed
-                    if stationID not in procstats:
-                        procstats.append(stationID)
-                        # find corresponding streams
-                        statdat = wfdat.select(station=stationID)
-                        ######################################################
-                        # get onset times and corresponding picking errors
-                        picks = run_autopicking(statdat, parameter)
-                        ######################################################
-                        # add station and corresponding onsets to dictionary
-                        station = stationID
-                        allonsets[station] = picks
-
-                # quality control
-                # median check and jackknife on P-onset times
-                checkedonsetsjk = checkPonsets(allonsets, mdttolerance, iplot)
-                # check S-P times (Wadati) 
-                checkedonsetwd = wadaticheck(checkedonsetsjk, wdttolerance, iplot)
 
                 print '------------------------------------------'
                 print '-----Finished event %s!-----' % event
@@ -121,41 +91,16 @@ def autoPyLoT(inputfile):
             data.setWFData(glob.glob(os.path.join(datapath, parameter.getParam('eventID'), '*')))
             print 'Working on event ', parameter.getParam('eventID')
             print data
-       
+
             wfdat = data.getWFData() # all available streams
             ##########################################################
-            # !automated picking starts here!   
+            # !automated picking starts here!
-            procstats = []
+            picks = autopickevent(wfdat, parameter)
-            # initialize dictionary for onsets
-            picks = None
-            station = wfdat[0].stats.station
-            allonsets = {station: picks}
-            for i in range(len(wfdat)):
-            #for i in range(0,10):
-                stationID = wfdat[i].stats.station
-                #check if station has already been processed
-                if stationID not in procstats:
-                    procstats.append(stationID)
-                    # find corresponding streams
-                    statdat = wfdat.select(station=stationID)
-                    ######################################################
-                    # get onset times and corresponding picking parameters
-                    picks = run_autopicking(statdat, parameter)
-                    ######################################################
-                    # add station and corresponding onsets to dictionary
-                    station = stationID
-                    allonsets[station] = picks
-
-            # quality control
-            # median check and jackknife on P-onset times
-            checkedonsetsjk = checkPonsets(allonsets, mdttolerance, iplot)
-            # check S-P times (Wadati) 
-            checkedonsetswd = wadaticheck(checkedonsetsjk, wdttolerance, iplot)
 
             print '------------------------------------------'
             print '-------Finished event %s!-------' % parameter.getParam('eventID')
             print '------------------------------------------'
-            
+
     print '####################################'
     print '************************************'
     print '*********autoPyLoT terminates*******'
@@ -165,7 +110,7 @@ def autoPyLoT(inputfile):
 if __name__ == "__main__":
     # parse arguments
     parser = argparse.ArgumentParser(
-        description='''autoPyLoT automatically picks phase onset times using higher order statistics, 
+        description='''autoPyLoT automatically picks phase onset times using higher order statistics,
                        autoregressive prediction and AIC''')
 
     parser.add_argument('-i', '-I', '--inputfile', type=str,

pylot/core/pick/Picker.py

@@ -16,11 +16,11 @@ The picks with the above described algorithms are assumed to be the most likely
 For each most likely pick the corresponding earliest and latest possible picks are
 calculated after Diehl & Kissling (2009).
 
-:author: MAGS2 EP3 working group / Ludger Kueperkoch	
+:author: MAGS2 EP3 working group / Ludger Kueperkoch
 """
 import numpy as np
 import matplotlib.pyplot as plt
-from pylot.core.pick.utils import *
+from pylot.core.pick.utils import getnoisewin, getsignalwin
 from pylot.core.pick.CharFuns import CharacteristicFunction
 import warnings
 
@@ -28,7 +28,7 @@ class AutoPicking(object):
     '''
     Superclass of different, automated picking algorithms applied on a CF determined
     using AIC, HOS, or AR prediction.
-    ''' 
+    '''
 
     warnings.simplefilter('ignore')
 
@@ -84,9 +84,9 @@ class AutoPicking(object):
                    PickWindow=self.getPickWindow(),
                    aus=self.getaus(),
                    Tsmooth=self.getTsmooth(),
-                   Pick1=self.getpick1())     
+                   Pick1=self.getpick1())
+
 
- 
     def getTSNR(self):
         return self.TSNR
 
@@ -116,7 +116,7 @@ class AutoPicking(object):
 
     def getSNR(self):
         return self.SNR
-   
+
     def getSlope(self):
         return self.slope
 
@@ -140,12 +140,12 @@ class AICPicker(AutoPicking):
     '''
     Method to derive the onset time of an arriving phase based on CF
     derived from AIC. In order to get an impression of the quality of this inital pick,
-    a quality assessment is applied based on SNR and slope determination derived from the CF, 
+    a quality assessment is applied based on SNR and slope determination derived from the CF,
     from which the AIC has been calculated.
     '''
 
     def calcPick(self):
- 
+
         print 'AICPicker: Get initial onset time (pick) from AIC-CF ...'
 
         self.Pick = None
@@ -179,15 +179,15 @@ class AICPicker(AutoPicking):
         #find NaN's
         nn = np.isnan(diffcf)
         if len(nn) > 1:
-           diffcf[nn] = 0 
+           diffcf[nn] = 0
         #taper CF to get rid off side maxima
         tap = np.hanning(len(diffcf))
         diffcf = tap * diffcf * max(abs(aicsmooth))
         icfmax = np.argmax(diffcf)
-        
+
         #find minimum in AIC-CF front of maximum
         lpickwindow = int(round(self.PickWindow / self.dt))
-        for i in range(icfmax - 1, max([icfmax - lpickwindow, 2]), -1): 
+        for i in range(icfmax - 1, max([icfmax - lpickwindow, 2]), -1):
            if aicsmooth[i - 1] >= aicsmooth[i]:
               self.Pick = self.Tcf[i]
               break
@@ -198,7 +198,7 @@ class AICPicker(AutoPicking):
               if diffcf[i -1] >= diffcf[i]:
                  self.Pick = self.Tcf[i]
                  break
-        
+
         # quality assessment using SNR and slope from CF
         if self.Pick is not None:
            # get noise window
@@ -275,7 +275,7 @@ class AICPicker(AutoPicking):
 
         if self.Pick == None:
            print 'AICPicker: Could not find minimum, picking window too short?'
-  
+
 
 class PragPicker(AutoPicking):
     '''
@@ -287,7 +287,7 @@ class PragPicker(AutoPicking):
         if self.getpick1() is not None:
            print 'PragPicker: Get most likely pick from HOS- or AR-CF using pragmatic picking algorithm ...'
 
-           self.Pick = None 
+           self.Pick = None
            self.SNR = None
            self.slope = None
            pickflag = 0
@@ -333,7 +333,7 @@ class PragPicker(AutoPicking):
            for i in range(max(np.insert(ipick, 0, 2)), min([ipick1 + lpickwindow + 1, len(self.cf) - 1])):
               if self.cf[i + 1] > self.cf[i] and self.cf[i - 1] >= self.cf[i]:
                  if cfsmooth[i - 1] * (1 + aus1) >= cfsmooth[i]:
-                    if cfpick1 >= self.cf[i]:     
+                    if cfpick1 >= self.cf[i]:
                     	pick_r = self.Tcf[i]
                     	self.Pick = pick_r
                     	flagpick_l = 1
@@ -344,7 +344,7 @@ class PragPicker(AutoPicking):
            for i in range(ipick1, max([ipick1 - lpickwindow + 1, 2]), -1):
               if self.cf[i + 1] > self.cf[i] and self.cf[i - 1] >= self.cf[i]:
                  if cfsmooth[i - 1] * (1 + aus1) >= cfsmooth[i]:
-                    if cfpick1 >= self.cf[i]:     
+                    if cfpick1 >= self.cf[i]:
                     	pick_l = self.Tcf[i]
                     	self.Pick = pick_l
                     	flagpick_r = 1
@@ -360,7 +360,7 @@ class PragPicker(AutoPicking):
            	pickflag = 1
            elif flagpick_l == 0 and flagpick_r > 0 and cfpick_l >= cfpick_r:
                 self.Pick = pick_l
-                pickflag = 1 
+                pickflag = 1
            else:
            	print 'PragPicker: Could not find reliable onset!'
            	self.Pick = None
@@ -372,7 +372,7 @@ class PragPicker(AutoPicking):
               p2, = plt.plot(Tcfpick,cfsmoothipick, 'r')
               if pickflag > 0:
               	p3, = plt.plot([self.Pick, self.Pick], [min(cfipick), max(cfipick)], 'b', linewidth=2)
-              	plt.legend([p1, p2, p3], ['CF', 'Smoothed CF', 'Pick']) 
+              	plt.legend([p1, p2, p3], ['CF', 'Smoothed CF', 'Pick'])
               plt.xlabel('Time [s] since %s' % self.Data[0].stats.starttime)
               plt.yticks([])
               plt.title(self.Data[0].stats.station)
@@ -380,7 +380,7 @@ class PragPicker(AutoPicking):
               raw_input()
               plt.close(p)
 
-        else: 
+        else:
            print 'PragPicker: No initial onset time given! Check input!'
            self.Pick = None
            return

pylot/core/pick/autopick.py

@@ -11,9 +11,37 @@ function conglomerate utils.
 
 import matplotlib.pyplot as plt
 import numpy as np
-from pylot.core.pick.Picker import *
+from pylot.core.pick.Picker import AICPicker, PragPicker
-from pylot.core.pick.CharFuns import *
+from pylot.core.pick.CharFuns import HOScf, AICcf, ARZcf, ARHcf, AR3Ccf
+from pylot.core.pick.utils import checksignallength, checkZ4S, earllatepicker,\
+    getSNR, fmpicker, checkPonsets, wadaticheck
 
+def autopickevent(data, param):
+    stations = []
+    all_onsets = {}
+
+    # get some parameters for quality control from
+    # parameter input file (usually autoPyLoT.in).
+    wdttolerance = param.getParam('wdttolerance')
+    mdttolerance = param.getParam('mdttolerance')
+    iplot = param.getParam('iplot')
+
+    for n in range(len(data)):
+        station = data[n].stats.station
+        if station not in stations:
+            stations.append(station)
+        else:
+            continue
+
+    for station in stations:
+        topick = data.select(station=station)
+        all_onsets[station] = run_autopicking(topick, param)
+
+    # quality control
+    # median check and jackknife on P-onset times
+    jk_checked_onsets = checkPonsets(all_onsets, mdttolerance, iplot)
+    # check S-P times (Wadati)
+    return wadaticheck(jk_checked_onsets, wdttolerance, iplot)
 
 def run_autopicking(wfstream, pickparam):
     """

pylot/core/pick/utils.py

@@ -12,24 +12,9 @@ import numpy as np
 import scipy as sc
 import matplotlib.pyplot as plt
 from obspy.core import Stream, UTCDateTime
-from pylot.core.pick.run_autopicking import run_autopicking
 import warnings
 import pdb
 
-def autopickevent(data, param):
-    stations = []
-
-    for n in len(data):
-        station = data[n].stats.station
-        if station not in stations:
-            stations.append(station)
-        else:
-            continue
-
-    for station in stations:
-        topick = data.select(station=station)
-
-        stat_picks = run_autopicking(topick, param)
 
 def earllatepicker(X, nfac, TSNR, Pick1, iplot=None):
     '''
@@ -538,7 +523,7 @@ def wadaticheck(pickdic, dttolerance, iplot):
         	# calculate vp/vs ratio after check
         	cvpvsr = p2[0] + 1
         	print 'wadaticheck: Average Vp/Vs ratio after check:', cvpvsr
-                print 'wadatacheck: Skipped %d S pick(s).' % ibad 
+                print 'wadatacheck: Skipped %d S pick(s).' % ibad
         else:
                 print '###############################################'
         	print 'wadatacheck: Not enough checked S-P times available!'

pylot/core/read/__init__.py

@@ -1,4 +1 @@
-from pylot.core.read.inputs import AutoPickParameter, FilterOptions
+
-from pylot.core.read.io import readPILOTEvent
-from pylot.core.read.data import GenericDataStructure, SeiscompDataStructure, \
-    PilotDataStructure, Data

pylot/core/read/data.py

@@ -7,9 +7,9 @@ from obspy.core import (read, Stream, UTCDateTime)
 from obspy import readEvents, read_inventory
 from obspy.core.event import (Event, Catalog)
 
-from pylot.core.read import readPILOTEvent
+from pylot.core.read.io import readPILOTEvent
-from pylot.core.util import fnConstructor, FormatError, \
+from pylot.core.util.utils import fnConstructor, getGlobalTimes
-    getGlobalTimes
+from pylot.core.util.errors import FormatError
 
 
 class Data(object):

pylot/core/read/io.py

@@ -7,9 +7,8 @@ import scipy.io as sio
 import obspy.core.event as ope
 from obspy.core import UTCDateTime
 
-from pylot.core.util import getOwner, createPick, createArrival, createEvent, \
+from pylot.core.util.utils import getOwner, createPick, createArrival,\
-    createOrigin, createMagnitude
+    createEvent, createOrigin, createMagnitude
-
 
 def readPILOTEvent(phasfn=None, locfn=None, authority_id=None, **kwargs):
     """

pylot/core/util/__init__.py

@@ -1,10 +1 @@
-from pylot.core.util.connection import checkurl
-from pylot.core.util.defaults import FILTERDEFAULTS
-from pylot.core.util.errors import OptionsError, FormatError, DatastructureError
-from pylot.core.util.utils import fnConstructor, createArrival, createEvent,\
-    createPick, createAmplitude, createOrigin, createMagnitude, getOwner, \
-    getHash, getLogin, createCreationInfo, createResourceID, prepTimeAxis, \
-    getGlobalTimes, scaleWFData, demeanWFData
-from pylot.core.util.widgets import PickDlg, HelpForm, FilterOptionsDialog,\
-    PropertiesDlg, NewEventDlg, MPLWidget, createAction
 from pylot.core.util.version import get_git_version as _getVersionString

pylot/core/util/structure.py

@@ -6,7 +6,7 @@ Created on Wed Jan 26 17:47:25 2015
 @author: sebastianw
 """
 
-from pylot.core.read import SeiscompDataStructure, PilotDataStructure
+from pylot.core.read.data import SeiscompDataStructure, PilotDataStructure
 
 DATASTRUCTURE = {'PILOT': PilotDataStructure, 'SeisComP': SeiscompDataStructure,
                  None: None}

pylot/core/util/thread.py

@@ -17,7 +17,8 @@ class WorkerThread(QThread):
         picks = self.func(self.data, self.param)
 
         try:
-            self.parent().addPicks(picks)
+            for station in picks:
+                self.parent().addPicks(station, picks[station])
         except AttributeError:
             print picks
 

pylot/core/util/utils.py

@@ -10,7 +10,6 @@ import hashlib
 import numpy as np
 from obspy.core import UTCDateTime
 import obspy.core.event as ope
-from pylot.core.pick.utils import getnoisewin
 
 def runProgram(cmd, parameter=None):
     """
@@ -119,28 +118,20 @@ def scaleWFData(data, factor=None, components='all'):
 
     return data
 
-def demeanWFData(data, t0, win, gap):
+def demeanTrace(trace, window):
     """
     returns the DATA where each trace is demean by the average value within
-    a desired time window WIN before T0 and a GAP
+    WINDOW
-    :param data: waveform stream object
+    :param trace: waveform trace object
-    :type data: `~obspy.core.stream.Stream`
+    :type trace: `~obspy.core.stream.Trace`
-    :param t0: time before which the noise
+    :param inoise: range of indices of DATA within the WINDOW
-    :type t0: float
+    :type window: tuple
-    :param win: time window for average calculation
+    :return: trace
-    :type win: tuple
+    :rtype: `~obspy.core.stream.Trace`
-    :param gap: gap window to avoid polluting the average
-    :type gap: float
-    :return: data
-    :rtype: `~obspy.core.stream.Stream`
     """
-    starttime = getGlobalTimes(data)[0]
+    trace.data -= trace.data[window].mean()
-    for tr in data:
+    return trace
-        stime = tr.stats.starttime - starttime
+
-        t = prepTimeAxis(stime, tr)
-        inoise = getnoisewin(t, t0, win, gap)
-        tr.data -= tr.data[inoise].mean()
-    return data
 
 def getGlobalTimes(stream):
     min_start = UTCDateTime()

pylot/core/util/widgets.py

@@ -19,12 +19,12 @@ from PySide.QtGui import QAction, QApplication, QComboBox, QDateTimeEdit, \
 from PySide.QtCore import QSettings, Qt, QUrl, Signal, Slot
 from PySide.QtWebKit import QWebView
 from obspy import Stream, UTCDateTime
-from pylot.core.read import FilterOptions
+from pylot.core.read.inputs import FilterOptions
 from pylot.core.pick.utils import getSNR, earllatepicker, getnoisewin,\
     getResolutionWindow
 from pylot.core.util.defaults import OUTPUTFORMATS, FILTERDEFAULTS
-from pylot.core.util import prepTimeAxis, getGlobalTimes, scaleWFData, \
+from pylot.core.util.utils import prepTimeAxis, getGlobalTimes, scaleWFData, \
-    demeanWFData
+    demeanTrace
 
 
 def createAction(parent, text, slot=None, shortcut=None, icon=None,
@@ -191,6 +191,8 @@ class PickDlg(QDialog):
         else:
             self.data = data
 
+        self.stime, self.etime = getGlobalTimes(self.getWFData())
+
         # initialize plotting widget
         self.multicompfig = MPLWidget(self)
 
@@ -336,6 +338,12 @@ class PickDlg(QDialog):
             self.cidrelease = self.connectReleaseEvent(self.panRelease)
             self.cidscroll = self.connectScrollEvent(self.scrollZoom)
 
+    def getStartTime(self):
+        return self.stime
+
+    def getEndTime(self):
+        return self.etime
+
     def getComponents(self):
         return self.components
 
@@ -446,13 +454,16 @@ class PickDlg(QDialog):
 
         x_res = getResolutionWindow(snr)
 
-        data = demeanWFData(data=self.getWFData().copy(), t0=ini_pick,
+        # remove mean noise level from waveforms
-                                 win=noise_win, gap=gap_win)
+        for trace in wfdata:
+            t = prepTimeAxis(self.getStartTime(), trace)
+            inoise = getnoisewin(t, ini_pick, noise_win, gap_win)
+            trace = demeanTrace(trace=trace, window=inoise)
 
         self.setXLims([ini_pick - x_res, ini_pick + x_res])
         self.setYLims(np.array([-noiselevel * 2.5, noiselevel * 2.5]) +
                       trace_number)
-        self.getPlotWidget().plotWFData(wfdata=data,
+        self.getPlotWidget().plotWFData(wfdata=wfdata,
                                         title=self.getStation() +
                                               ' picking mode',
                                         zoomx=self.getXLims(),
@@ -482,7 +493,10 @@ class PickDlg(QDialog):
         filteroptions = self.getFilterOptions(phase).parseFilterOptions()
         data.filter(**filteroptions)
 
-        data = demeanWFData(data=data, t0=ini_pick, win=noise_win, gap=gap_win)
+        for trace in data:
+            t = prepTimeAxis(self.getStartTime(), trace)
+            inoise = getnoisewin(t, ini_pick, noise_win, gap_win)
+            trace = demeanTrace(trace, inoise)
 
         horiz_comp = ('n', 'e')
 
@@ -576,9 +590,9 @@ class PickDlg(QDialog):
         else:
             return
 
-        mpp = picks['mpp']
+        mpp = picks['mpp'] - self.getStartTime()
-        epp = picks['epp']
+        epp = picks['epp'] - self.getStartTime()
-        lpp = picks['lpp']
+        lpp = picks['lpp'] - self.getStartTime()
         spe = picks['spe']
 
         ax.fill_between([epp, lpp], ylims[0], ylims[1],