Merge branch 'develop' into feature/documentation

PyLoT.py

@@ -24,12 +24,11 @@ https://www.iconfinder.com/iconsets/flavour
 """
 
 import argparse
+import matplotlib
 import os
 import platform
 import sys
 
-import matplotlib
-
 matplotlib.use('Qt4Agg')
 matplotlib.rcParams['backend.qt4'] = 'PySide'
 matplotlib.rcParams['savefig.dpi'] = 300
@@ -40,8 +39,8 @@ from PySide.QtCore import QCoreApplication, QSettings, Signal, QFile, \
     QFileInfo, Qt, QSize
 from PySide.QtGui import QMainWindow, QInputDialog, QIcon, QFileDialog, \
     QWidget, QHBoxLayout, QVBoxLayout, QStyle, QKeySequence, QLabel, QFrame, QAction, \
-    QDialog, QErrorMessage, QApplication, QPixmap, QMessageBox, QSplashScreen, \
+    QDialog, QApplication, QPixmap, QMessageBox, QSplashScreen, \
-    QActionGroup, QListWidget, QLineEdit, QListView, QAbstractItemView, \
+    QActionGroup, QListWidget, QListView, QAbstractItemView, \
     QTreeView, QComboBox, QTabWidget, QPushButton, QGridLayout
 import numpy as np
 from obspy import UTCDateTime
@@ -56,7 +55,6 @@ try:
     from matplotlib.backends.backend_qt4agg import FigureCanvas
 except ImportError:
     from matplotlib.backends.backend_qt4agg import FigureCanvasQTAgg as FigureCanvas
-from matplotlib.backends.backend_qt4agg import NavigationToolbar2QT as NavigationToolbar
 from matplotlib.figure import Figure
 
 from pylot.core.analysis.magnitude import LocalMagnitude, MomentMagnitude
@@ -64,24 +62,23 @@ from pylot.core.io.data import Data
 from pylot.core.io.inputs import FilterOptions, PylotParameter
 from autoPyLoT import autoPyLoT
 from pylot.core.pick.compare import Comparison
-from pylot.core.pick.utils import symmetrize_error, getQualityFromUncertainty, getPickQuality
+from pylot.core.pick.utils import getQualityFromUncertainty
 from pylot.core.io.phases import picksdict_from_picks
 import pylot.core.loc.nll as nll
-from pylot.core.util.defaults import FILTERDEFAULTS
 from pylot.core.util.errors import DatastructureError, \
     OverwriteError
 from pylot.core.util.connection import checkurl
 from pylot.core.util.dataprocessing import Metadata, restitute_data
 from pylot.core.util.utils import fnConstructor, getLogin, \
-    full_range, readFilterInformation, trim_station_components, check4gaps, make_pen, pick_color_plt, \
+    full_range, readFilterInformation, make_pen, pick_color_plt, \
-    pick_linestyle_plt, remove_underscores, check4doubled, identifyPhaseID, excludeQualityClasses, \
+    pick_linestyle_plt, identifyPhaseID, excludeQualityClasses, \
-    check4rotated, transform_colors_mpl, transform_colors_mpl_str, getAutoFilteroptions, check_all_obspy, \
+    transform_colors_mpl, transform_colors_mpl_str, getAutoFilteroptions, check_all_obspy, \
     check_all_pylot, real_Bool, SetChannelComponents
 from pylot.core.util.event import Event
 from pylot.core.io.location import create_creation_info, create_event
 from pylot.core.util.widgets import FilterOptionsDialog, NewEventDlg, \
     PylotCanvas, WaveformWidgetPG, PropertiesDlg, HelpForm, createAction, PickDlg, \
-    getDataType, ComparisonWidget, TuneAutopicker, PylotParaBox, AutoPickDlg, CanvasWidget, AutoPickWidget, \
+    ComparisonWidget, TuneAutopicker, PylotParaBox, AutoPickDlg, CanvasWidget, AutoPickWidget, \
     CompareEventsWidget, ProgressBarWidget, AddMetadataWidget
 from pylot.core.util.array_map import Array_map
 from pylot.core.util.structure import DATASTRUCTURE
@@ -97,6 +94,9 @@ elif sys.version_info.major == 2:
 else:
     raise ImportError('Could not determine python version.')
 
+# workaround to prevent PyCharm from deleting icons_rc import when optimizing imports
+icons_rc = icons_rc
+
 locateTool = dict(nll=nll)
 
 
@@ -155,6 +155,9 @@ class MainWindow(QMainWindow):
             self.data._new = False
         self.autodata = Data(self)
 
+        self.fnames = None
+        self._stime = None
+
         while True:
             try:
                 if settings.value("user/FullName", None) is None:
@@ -166,6 +169,27 @@ class MainWindow(QMainWindow):
                                                   "Enter authority/institution name:",
                                                   "Authority")
                     settings.setValue("agency_id", agency)
+                structure_setting = settings.value("data/Structure", "PILOT")
+                if not structure_setting:
+                    structure_setting = 'PILOT'
+                self.dataStructure = DATASTRUCTURE[structure_setting]()
+                self.seismicPhase = str(settings.value("phase", "P"))
+                if settings.value("data/dataRoot", None) is None:
+                    dirname = QFileDialog().getExistingDirectory(
+                        caption='Choose data root ...')
+                    settings.setValue("data/dataRoot", dirname)
+                if settings.value('compclass', None) is None:
+                    settings.setValue('compclass', SetChannelComponents())
+                if settings.value('useGuiFilter') is None:
+                    settings.setValue('useGuiFilter', False)
+                if settings.value('output/Format', None) is None:
+                    outformat = QInputDialog.getText(self,
+                                                     "Enter output format (*.xml, *.cnv, *.obs):",
+                                                     "Format")
+                    settings.setValue("output/Format", outformat)
+                if settings.value('autoFilter', None) is None:
+                    settings.setValue('autoFilter', True)
+                settings.sync()
                 break
             except Exception as e:
                 qmb = QMessageBox(self, icon=QMessageBox.Question,
@@ -181,30 +205,6 @@ class MainWindow(QMainWindow):
                     sys.exit()
                 print('Settings cleared!')
 
-        self.fnames = None
-        self._stime = None
-        structure_setting = settings.value("data/Structure", "PILOT")
-        if not structure_setting:
-            structure_setting = 'PILOT'
-        self.dataStructure = DATASTRUCTURE[structure_setting]()
-        self.seismicPhase = str(settings.value("phase", "P"))
-        if settings.value("data/dataRoot", None) is None:
-            dirname = QFileDialog().getExistingDirectory(
-                caption='Choose data root ...')
-            settings.setValue("data/dataRoot", dirname)
-        if settings.value('compclass', None) is None:
-            settings.setValue('compclass', SetChannelComponents())
-        if settings.value('useGuiFilter') is None:
-            settings.setValue('useGuiFilter', False)
-        if settings.value('output/Format', None) is None:
-            outformat = QInputDialog.getText(self,
-                                             "Enter output format (*.xml, *.cnv, *.obs):",
-                                             "Format")
-            settings.setValue("output/Format", outformat)
-        if settings.value('autoFilter', None) is None:
-            settings.setValue('autoFilter', True)
-        settings.sync()
-
         # setup UI
         self.setupUi()
 
@@ -542,6 +542,7 @@ class MainWindow(QMainWindow):
         self.updateFileMenu()
 
         self.editMenu = self.menuBar().addMenu('&Edit')
+
         editActions = (self.filterActionP, self.filterActionS, filterEditAction, None,
                        # self.selectPAction, self.selectSAction, None,
                        self.inventoryAction, self.initMapAction, None,
@@ -553,6 +554,7 @@ class MainWindow(QMainWindow):
         self.autoPickMenu = self.pickMenu.addMenu(self.autopicksicon_small, 'Automatic picking')
         self.autoPickMenu.setEnabled(False)
 
+
         autoPickActions = (self.auto_pick, self.auto_pick_local, self.auto_pick_sge)
 
         self.helpMenu = self.menuBar().addMenu('&Help')
@@ -685,6 +687,7 @@ class MainWindow(QMainWindow):
 
         self.setCentralWidget(_widget)
 
+
     def init_wfWidget(self):
         xlab = self.startTime.strftime('seconds since %Y/%m/%d %H:%M:%S (%Z)')
         plottitle = None  # "Overview: {0} components ".format(self.getComponent())
@@ -830,8 +833,9 @@ class MainWindow(QMainWindow):
                                                  s_filter['order'])}
 
     def updateFileMenu(self):
-
+        settings = QSettings()
         self.fileMenu.clear()
+        self.recentProjectsMenu = self.fileMenu.addMenu('Recent Projects')
         for action in self.fileMenuActions[:-1]:
             if action is None:
                 self.fileMenu.addSeparator()
@@ -848,7 +852,6 @@ class MainWindow(QMainWindow):
                 recentEvents.append(eventID)
                 recentEvents.reverse()
                 self.recentfiles = recentEvents[0:5]
-                settings = QSettings()
                 settings.setValue()
         if recentEvents:
             for i, eventID in enumerate(recentEvents):
@@ -864,6 +867,18 @@ class MainWindow(QMainWindow):
         self.fileMenu.addSeparator()
         self.fileMenu.addAction(self.fileMenuActions[-1])
 
+        # add recent projects
+        recentProjects = settings.value('recentProjects', [])
+        if not type(recentProjects) == list:
+            recentProjects = [recentProjects]
+        for project in reversed(recentProjects):
+            action = self.createAction(self, project,
+                                       lambda fnm=project: self.loadProject(fnm),
+                                       None, None)
+
+            self.recentProjectsMenu.addAction(action)
+
+
     @property
     def inputs(self):
         return self._inputs
@@ -3273,10 +3288,13 @@ class MainWindow(QMainWindow):
             return
         if not fnm:
             dlg = QFileDialog(parent=self)
-            fnm = dlg.getOpenFileName(self, 'Open project file...', filter='Pylot project (*.plp)')
+            fnm = dlg.getOpenFileName(self, 'Open project file...', filter='Pylot project (*.plp)')[0]
             if not fnm:
                 return
-            fnm = fnm[0]
+        if not os.path.exists(fnm):
+            QMessageBox.warning(self, 'Could not open file',
+                                'Could not open project file {}. File does not exist.'.format(fnm))
+            return
         if fnm:
             self.project = Project.load(fnm)
             if hasattr(self.project, 'parameter'):
@@ -3292,8 +3310,24 @@ class MainWindow(QMainWindow):
             self.setDirty(False)
             self.init_metadata()
 
+            message = 'Opened project file {}.'.format(fnm)
+            print(message)
+            self.update_status(message)
+
             self.init_array_tab()
             self.set_metadata()
+            self.add2recentProjects(fnm)
+
+    def add2recentProjects(self, fnm):
+        settings = QtCore.QSettings()
+        recent = settings.value('recentProjects', [])
+        if not type(recent) == list:
+            recent = [recent]
+        if not fnm in recent:
+            recent.append(fnm)
+        new_recent = recent[-5:]
+        settings.setValue('recentProjects', new_recent)
+        settings.sync()
 
     def saveProjectAs(self, exists=False):
         '''
@@ -3312,6 +3346,7 @@ class MainWindow(QMainWindow):
         self.setDirty(False)
         self.saveProjectAsAction.setEnabled(True)
         self.update_status('Saved new project to {}'.format(filename), duration=5000)
+        self.add2recentProjects(filename)
         return True
 
     def saveProject(self, new=False):
@@ -3445,7 +3480,7 @@ class Project(object):
             eventID, date, time, mag, lat, lon, depth = line.split(separator)[:7]
             # skip first line
             try:
-                month, day, year = date.split('/')
+                day, month, year = date.split('/')
             except:
                 continue
             year = int(year)
@@ -3459,7 +3494,7 @@ class Project(object):
                 print(e, datetime, filename)
                 continue
             for event in self.eventlist:
-                if eventID in str(event.resource_id) or event.origins:
+                if eventID in str(event.resource_id) or eventID in event.origins:
                     if event.origins:
                         origin = event.origins[0]  # should have only one origin
                         if origin.time == datetime:

autoPyLoT.py

@@ -8,6 +8,8 @@ import datetime
 import glob
 import os
 import traceback
+from obspy import read_events
+from obspy.core.event import ResourceIdentifier
 
 import pylot.core.loc.focmec as focmec
 import pylot.core.loc.hash as hash
@@ -16,18 +18,16 @@ import pylot.core.loc.hypodd as hypodd
 import pylot.core.loc.hyposat as hyposat
 import pylot.core.loc.nll as nll
 import pylot.core.loc.velest as velest
-from obspy import read_events
-from obspy.core.event import ResourceIdentifier
 # from PySide.QtGui import QWidget, QInputDialog
 from pylot.core.analysis.magnitude import MomentMagnitude, LocalMagnitude
 from pylot.core.io.data import Data
 from pylot.core.io.inputs import PylotParameter
 from pylot.core.pick.autopick import autopickevent, iteratepicker
-from pylot.core.util.dataprocessing import restitute_data, read_metadata, Metadata
+from pylot.core.util.dataprocessing import restitute_data, Metadata
 from pylot.core.util.defaults import SEPARATOR
 from pylot.core.util.event import Event
 from pylot.core.util.structure import DATASTRUCTURE
-from pylot.core.util.utils import real_None, remove_underscores, trim_station_components, check4gaps, check4doubled, \
+from pylot.core.util.utils import real_None, trim_station_components, check4gaps, check4doubled, \
     check4rotated
 from pylot.core.util.version import get_git_version as _getVersionString
 

pylot/core/analysis/magnitude.py

@@ -11,11 +11,12 @@ import matplotlib.pyplot as plt
 import numpy as np
 import obspy.core.event as ope
 from obspy.geodetics import degrees2kilometers
+from scipy import integrate, signal
+from scipy.optimize import curve_fit
+
 from pylot.core.pick.utils import getsignalwin, crossings_nonzero_all, \
     select_for_phase
 from pylot.core.util.utils import common_range, fit_curve
-from scipy import integrate, signal
-from scipy.optimize import curve_fit
 
 
 def richter_magnitude_scaling(delta):

pylot/core/io/data.py

@@ -3,19 +3,18 @@
 
 import copy
 import os
-
 from obspy import read_events
 from obspy.core import read, Stream, UTCDateTime
 from obspy.core.event import Event as ObsPyEvent
 from obspy.io.sac import SacIOError
+
+import pylot.core.loc.velest as velest
 from pylot.core.io.phases import readPILOTEvent, picks_from_picksdict, \
     picksdict_from_pilot, merge_picks
 from pylot.core.util.errors import FormatError, OverwriteError
 from pylot.core.util.event import Event
-from pylot.core.util.utils import fnConstructor, full_range, remove_underscores, check4gaps, check4doubled, \
-    check4rotated, trim_station_components
-import pylot.core.loc.velest as velest
 from pylot.core.util.obspyDMT_interface import qml_from_obspyDMT
+from pylot.core.util.utils import fnConstructor, full_range, check4rotated, trim_station_components
 
 
 class Data(object):
@@ -405,7 +404,7 @@ class Data(object):
 
         # various pre-processing steps:
         # remove possible underscores in station names
-        #self.wfdata = remove_underscores(self.wfdata)
+        # self.wfdata = remove_underscores(self.wfdata)
         # check for stations with rotated components
         if checkRotated and metadata is not None:
             self.wfdata = check4rotated(self.wfdata, metadata, verbosity=0)

pylot/core/io/phases.py

@@ -2,22 +2,22 @@
 # -*- coding: utf-8 -*-
 
 import glob
-import os
-import warnings
-
 import matplotlib.pyplot as plt
 import numpy as np
 import obspy.core.event as ope
+import os
 import scipy.io as sio
+import warnings
 from obspy.core import UTCDateTime
 from obspy.core.event import read_events
 from obspy.core.util import AttribDict
+
 from pylot.core.io.inputs import PylotParameter
 from pylot.core.io.location import create_event, \
     create_magnitude
 from pylot.core.pick.utils import select_for_phase
-from pylot.core.util.utils import getOwner, full_range, four_digits, transformFilteroptions2String, \
+from pylot.core.util.utils import getOwner, full_range, four_digits, transformFilterString4Export, \
-    transformFilterString4Export, backtransformFilterString
+    backtransformFilterString
 
 
 def add_amplitudes(event, amplitudes):

pylot/core/loc/nll.py

@@ -4,8 +4,8 @@
 import glob
 import os
 import subprocess
-
 from obspy import read_events
+
 from pylot.core.io.phases import writephases
 from pylot.core.util.utils import getPatternLine, runProgram, which
 from pylot.core.util.version import get_git_version as _getVersionString

pylot/core/pick/autopick.py

@@ -8,10 +8,11 @@ function conglomerate utils.
 
 :author: MAGS2 EP3 working group / Ludger Kueperkoch
 """
-import traceback
-
 import matplotlib.pyplot as plt
 import numpy as np
+import traceback
+from obspy.taup import TauPyModel
+
 from pylot.core.pick.charfuns import CharacteristicFunction
 from pylot.core.pick.charfuns import HOScf, AICcf, ARZcf, ARHcf, AR3Ccf
 from pylot.core.pick.picker import AICPicker, PragPicker
@@ -20,8 +21,6 @@ from pylot.core.pick.utils import checksignallength, checkZ4S, earllatepicker, \
 from pylot.core.util.utils import getPatternLine, gen_Pool, \
     real_Bool, identifyPhaseID
 
-from obspy.taup import TauPyModel
-
 
 def autopickevent(data, param, iplot=0, fig_dict=None, fig_dict_wadatijack=None, ncores=0, metadata=None, origin=None):
     """

pylot/core/pick/compare.py

@@ -2,14 +2,12 @@
 # -*- coding: utf-8 -*-
 
 import copy
-import operator
-import os
-
 import matplotlib.pyplot as plt
 import numpy as np
-from obspy import read_events
+import operator
+import os
 from obspy.core import AttribDict
-from pylot.core.io.phases import picksdict_from_picks
+
 from pylot.core.util.pdf import ProbabilityDensityFunction
 from pylot.core.util.utils import find_in_list
 from pylot.core.util.version import get_git_version as _getVersionString

pylot/core/pick/picker.py

@@ -19,11 +19,11 @@ calculated after Diehl & Kissling (2009).
 :author: MAGS2 EP3 working group / Ludger Kueperkoch
 """
 
-import warnings
-
 import matplotlib.pyplot as plt
 import numpy as np
+import warnings
 from scipy.signal import argrelmax, argrelmin
+
 from pylot.core.pick.charfuns import CharacteristicFunction
 from pylot.core.pick.utils import getnoisewin, getsignalwin
 
@@ -265,7 +265,7 @@ class AICPicker(AutoPicker):
             else:
                 islope = np.where((self.Tcf <= min([self.Pick + tslope, self.Tcf[-1]])) \
                                   & (
-                                              self.Tcf >= self.Pick + tsafety))  # TODO: put this in a seperate function like getsignalwin
+                                          self.Tcf >= self.Pick + tsafety))  # TODO: put this in a seperate function like getsignalwin
             # find maximum within slope determination window
             # 'cause slope should be calculated up to first local minimum only!
             try:

pylot/core/pick/utils.py

@@ -8,11 +8,11 @@
    :author: Ludger Kueperkoch, BESTEC GmbH
 """
 
-import warnings
-
 import matplotlib.pyplot as plt
 import numpy as np
+import warnings
 from obspy.core import Stream, UTCDateTime
+
 from pylot.core.util.utils import real_Bool, real_None, SetChannelComponents
 
 

pylot/core/util/array_map.py

@@ -1,17 +1,18 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-import traceback
 import matplotlib.pyplot as plt
 import numpy as np
 import obspy
+import traceback
 from PySide import QtGui
-from mpl_toolkits.basemap import Basemap
 from matplotlib.figure import Figure
 from mpl_toolkits.axes_grid1.inset_locator import inset_axes
-from pylot.core.util.widgets import PickDlg, PylotCanvas
+from mpl_toolkits.basemap import Basemap
 from scipy.interpolate import griddata
 
+from pylot.core.util.widgets import PickDlg, PylotCanvas
+
 plt.interactive(False)
 
 
@@ -30,6 +31,8 @@ class Array_map(QtGui.QWidget):
         self.autopicks_dict = None
         self.eventLoc = None
         self.figure = figure
+        self.picks_rel = {}
+        self.marked_stations = []
         self.init_graphics()
         self.init_stations()
         self.init_basemap(resolution='l')
@@ -45,7 +48,6 @@ class Array_map(QtGui.QWidget):
                 hybrids_dict[station] = pick
         return hybrids_dict
 
-
     def init_map(self):
         self.init_lat_lon_dimensions()
         self.init_lat_lon_grid()
@@ -57,8 +59,56 @@ class Array_map(QtGui.QWidget):
     def onpick(self, event):
         ind = event.ind
         button = event.mouseevent.button
-        if ind == [] or not button == 1:
+        if ind == []:
             return
+        if button == 1:
+            self.openPickDlg(ind)
+        elif button == 2:
+            self.deletePick(ind)
+        elif button == 3:
+            self.pickInfo(ind)
+
+    def deletePick(self, ind):
+        for index in ind:
+            network, station = self._station_onpick_ids[index].split('.')[:2]
+            try:
+                phase = self.comboBox_phase.currentText()
+                picks = self.current_picks_dict()[station]
+                pick = picks.get(phase)
+                if pick:
+                    picker = pick['picker']
+                    message = 'Deleted {} pick for phase {}, station {}.{} at timestamp {}'
+                    message = message.format(picker, phase, network, station,
+                                             pick['mpp'])
+                    if picker == 'auto':
+                        del (self.autopicks_dict[station])
+                    elif picker == 'manual':
+                        del (self.picks_dict[station])
+                    else:
+                        raise TypeError('Unknown "picker" {}'.format(picker))
+                    print(message)
+                    pyl_mw = self._parent
+                    pyl_mw.setDirty(True)
+                    pyl_mw.update_status(message)
+                    self._refresh_drawings()
+                    pyl_mw.drawPicks(station)
+                    pyl_mw.draw()
+            except Exception as e:
+                print('Could not delete pick for station {}.{}: {}'.format(network, station, e))
+
+    def pickInfo(self, ind):
+        for index in ind:
+            network, station = self._station_onpick_ids[index].split('.')[:2]
+            dic = self.current_picks_dict()[station]
+            for phase, picks in dic.items():
+                # because of wadati...
+                if phase == 'SPt':
+                    continue
+                print('{} - Pick:'.format(phase))
+                for key, info in picks.items():
+                    print('{}: {}'.format(key, info))
+
+    def openPickDlg(self, ind):
         data = self._parent.get_data().getWFData()
         for index in ind:
             network, station = self._station_onpick_ids[index].split('.')[:2]
@@ -297,10 +347,50 @@ class Array_map(QtGui.QWidget):
         return picks, latitudes, longitudes
 
     def draw_contour_filled(self, nlevel='50'):
+        # self.test_gradient()
+
         levels = np.linspace(self.get_min_from_picks(), self.get_max_from_picks(), nlevel)
         self.contourf = self.basemap.contourf(self.longrid, self.latgrid, self.picksgrid_active,
                                               levels, latlon=True, zorder=9, alpha=0.5)
 
+    def test_gradient(self):
+        st_ids = self.picks_rel.keys()
+        x, y = np.gradient(self.picksgrid_active)
+        gradient_modulus = np.sqrt(x ** 2 + y ** 2)
+        global_mean_gradient = np.nanmean(gradient_modulus)
+        delta_gradient = []
+        for st_id in st_ids:
+            pick_item = self.picks_rel.pop(st_id)
+            self.init_picksgrid()
+            x, y = np.gradient(self.picksgrid_active)
+            gradient_modulus = np.sqrt(x ** 2 + y ** 2)
+            mean_gradient = np.nanmean(gradient_modulus)
+            dgradient = global_mean_gradient - mean_gradient
+            # print('station: {}, mean gradient: {}'.format(st_id, dgradient))
+            delta_gradient.append(dgradient)
+            self.picks_rel[st_id] = pick_item
+        global_std_gradient = np.nanstd(delta_gradient)
+        marked_stations = []
+        for st_id, dg in zip(st_ids, delta_gradient):
+            if abs(dg) > global_std_gradient:
+                marked_stations.append(st_id)
+        self.marked_stations = marked_stations
+        self.init_picksgrid()
+
+        # fig = plt.figure()
+        # x = list(range(len(st_ids)))
+        # gradients = zip(x, delta_gradient)
+        # gradients.sort(key=lambda a: a[1])
+        # plt.plot(gradients[0], gradients[1])
+
+        # global_var_gradient = np.nanvar(delta_gradient)
+        # plt.plot(x, delta_gradient)
+        # plt.axhline(global_std_gradient, color='green')
+        # plt.axhline(2 * global_std_gradient, color='blue')
+        # plt.axhline(global_var_gradient, color='red')
+        # plt.xticks(x, st_ids)
+        # plt.show()
+
     def scatter_all_stations(self):
         stations, lats, lons = self.get_st_lat_lon_for_plot()
         self.sc = self.basemap.scatter(lons, lats, s=50, facecolor='none', latlon=True,
@@ -318,21 +408,32 @@ class Array_map(QtGui.QWidget):
             return
         # workaround because of an issue with latlon transformation of arrays with len <3
         if len(lons) <= 2 and len(lats) <= 2:
-            self.sc_picked = self.basemap.scatter(lons[0], lats[0], s=50, facecolor='white',
+            self.sc_picked = self.basemap.scatter(lons[0], lats[0], s=50, edgecolors='white',
                                                   c=picks[0], latlon=True, zorder=11, label='Picked')
         if len(lons) == 2 and len(lats) == 2:
-            self.sc_picked = self.basemap.scatter(lons[1], lats[1], s=50, facecolor='white',
+            self.sc_picked = self.basemap.scatter(lons[1], lats[1], s=50, edgecolors='white',
                                                   c=picks[1], latlon=True, zorder=11)
         else:
-            self.sc_picked = self.basemap.scatter(lons, lats, s=50, facecolor='white',
+            self.sc_picked = self.basemap.scatter(lons, lats, s=50, edgecolors='white',
                                                   c=picks, latlon=True, zorder=11, label='Picked')
 
     def annotate_ax(self):
         self.annotations = []
         stations, xs, ys = self.get_st_x_y_for_plot()
+        # MP MP testing station highlighting if they have high impact on mean gradient of color map
+        # if self.picks_rel:
+        #    self.test_gradient()
+        color_marked = {True: 'red',
+                        False: 'white'}
         for st, x, y in zip(stations, xs, ys):
+            if st in self.picks_rel:
+                color = 'white'
+            else:
+                color = 'lightgrey'
+            if st in self.marked_stations:
+                color = 'red'
             self.annotations.append(self.main_ax.annotate(' %s' % st, xy=(x, y),
-                                                          fontsize='x-small', color='white', zorder=12))
+                                                          fontsize='x-small', color=color, zorder=14))
         self.legend = self.main_ax.legend(loc=1)
         self.legend.get_frame().set_facecolor((1, 1, 1, 0.75))
 
@@ -381,6 +482,7 @@ class Array_map(QtGui.QWidget):
         self.canvas.draw()
 
     def remove_drawings(self):
+        self.remove_annotations()
         if hasattr(self, 'cbar'):
             self.cbar.remove()
             self.cbax_bg.remove()

pylot/core/util/dataprocessing.py

@@ -2,14 +2,14 @@
 # -*- coding: utf-8 -*-
 
 import glob
+import numpy as np
 import os
 import sys
-
-import numpy as np
 from obspy import UTCDateTime, read_inventory, read
 from obspy.io.xseed import Parser
+
 from pylot.core.util.utils import key_for_set_value, find_in_list, \
-    remove_underscores, gen_Pool
+    gen_Pool
 
 
 class Metadata(object):
@@ -274,6 +274,7 @@ class Metadata(object):
         """
         # functions used to read metadata for different file endings (or file types)
         read_functions = {'dless': self._read_dless,
+                          'dataless': self._read_dless,
                           'dseed': self._read_dless,
                           'xml': self._read_inventory_file,
                           'resp': self._read_inventory_file}
@@ -281,6 +282,7 @@ class Metadata(object):
         if file_ending in read_functions.keys():
             robj, exc = read_functions[file_ending](path_to_inventory_filename)
             if exc is not None:
+                print("Nicht None")
                 raise exc
             return file_ending, robj
         # in case file endings did not match the above keys, try and error
@@ -616,7 +618,7 @@ def restitute_data(data, metadata, unit='VEL', force=False, ncores=0):
 
     restflag = list()
 
-    #data = remove_underscores(data)
+    # data = remove_underscores(data)
 
     # loop over traces
     input_tuples = []

pylot/core/util/defaults.py

@@ -9,12 +9,12 @@ Created on Wed Feb 26 12:31:25 2014
 import os
 import platform
 
-from pylot.core.util.utils import readDefaultFilterInformation
 from pylot.core.loc import hypo71
 from pylot.core.loc import hypodd
 from pylot.core.loc import hyposat
 from pylot.core.loc import nll
 from pylot.core.loc import velest
+from pylot.core.util.utils import readDefaultFilterInformation
 
 # determine system dependent path separator
 system_name = platform.system()
@@ -39,5 +39,3 @@ OUTPUTFORMATS = {'.xml': 'QUAKEML',
                  '.obs': 'NLLOC_OBS'}
 
 LOCTOOLS = dict(nll=nll, hyposat=hyposat, velest=velest, hypo71=hypo71, hypodd=hypodd)
-
-

pylot/core/util/event.py

@@ -2,10 +2,10 @@
 # -*- coding: utf-8 -*-
 
 import os
-
 from obspy import UTCDateTime
 from obspy.core.event import Event as ObsPyEvent
 from obspy.core.event import Origin, ResourceIdentifier
+
 from pylot.core.io.phases import picks_from_picksdict
 from pylot.core.util.obspyDMT_interface import qml_from_obspyDMT
 

pylot/core/util/pdf.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-import warnings
-
 import numpy as np
+import warnings
 from obspy import UTCDateTime
+
 from pylot.core.util.utils import fit_curve, clims
 from pylot.core.util.version import get_git_version as _getVersionString
 

pylot/core/util/thread.py

@@ -1,8 +1,11 @@
 # -*- coding: utf-8 -*-
-import sys, os, traceback
 import multiprocessing
+import os
+import sys
+import traceback
+
 from PySide.QtCore import QThread, Signal, Qt, Slot, QRunnable, QObject
-from PySide.QtGui import QDialog, QProgressBar, QLabel, QHBoxLayout, QPushButton
+from PySide.QtGui import QDialog, QProgressBar, QLabel, QHBoxLayout
 
 
 class Thread(QThread):
@@ -40,7 +43,6 @@ class Thread(QThread):
 
     def showProgressbar(self):
         if self.progressText:
-
             # # generate widget if not given in init
             # if not self.pb_widget:
             #     self.pb_widget = ProgressBarWidget(self.parent())

pylot/core/util/utils.py

@@ -2,27 +2,23 @@
 # -*- coding: utf-8 -*-
 
 import hashlib
+import numpy as np
 import os
 import platform
+import pyqtgraph as pg
 import re
 import subprocess
 import warnings
-
+from PySide import QtCore
-import numpy as np
 from obspy import UTCDateTime, read
 from obspy.core import AttribDict
 from obspy.signal.rotate import rotate2zne
-from obspy.io.xseed.utils import SEEDParserException
+from scipy.interpolate import splrep, splev
-
-from pylot.core.util.obspyDMT_interface import check_obspydmt_eventfolder
 
 from pylot.core.io.inputs import PylotParameter, FilterOptions
+from pylot.core.util.obspyDMT_interface import check_obspydmt_eventfolder
 from pylot.styles import style_settings
 
-from scipy.interpolate import splrep, splev
-from PySide import QtCore, QtGui
-
-import pyqtgraph as pg
 
 def _pickle_method(m):
     if m.im_self is None:
@@ -835,7 +831,7 @@ def remove_underscores(data):
     :return: data stream
     :rtype: `~obspy.core.stream.Stream`
     """
-    #for tr in data:
+    # for tr in data:
     #    # remove underscores
     #    tr.stats.station = tr.stats.station.strip('_')
     return data
@@ -874,7 +870,7 @@ def merge_stream(stream):
     if gaps:
         # list of merged stations (seed_ids)
         merged = ['{}.{}.{}.{}'.format(*gap[:4]) for gap in gaps]
-        stream.merge()
+        stream.merge(method=1)
         print('Merged the following stations because of gaps:')
         for merged_station in merged:
             print(merged_station)

pylot/core/util/widgets.py

@@ -8,16 +8,14 @@ Created on Wed Mar 19 11:27:35 2014
 import copy
 import datetime
 import getpass
+import matplotlib
 import multiprocessing
+import numpy as np
 import os
 import subprocess
 import sys
 import time
 
-import numpy as np
-
-import matplotlib
-
 matplotlib.use('QT4Agg')
 
 from matplotlib.figure import Figure
@@ -28,8 +26,6 @@ except ImportError:
     from matplotlib.backends.backend_qt4agg import FigureCanvasQTAgg as FigureCanvas
 from matplotlib.backends.backend_qt4agg import NavigationToolbar2QT
 from matplotlib.widgets import MultiCursor
-from matplotlib.tight_layout import get_renderer, get_subplotspec_list, get_tight_layout_figure
-from scipy.signal import argrelmin, argrelmax
 from obspy import read
 
 from PySide import QtCore, QtGui
@@ -38,7 +34,7 @@ from PySide.QtGui import QAction, QApplication, QCheckBox, QComboBox, \
     QGridLayout, QIcon, QLabel, QLineEdit, QMessageBox, \
     QPixmap, QSpinBox, QTabWidget, QToolBar, QVBoxLayout, QHBoxLayout, QWidget, \
     QPushButton, QFileDialog, QInputDialog, QKeySequence
-from PySide.QtCore import QSettings, Qt, QUrl, Signal, Slot
+from PySide.QtCore import QSettings, Qt, QUrl, Signal
 from PySide.QtWebKit import QWebView
 from obspy import Stream, Trace, UTCDateTime
 from obspy.core.util import AttribDict
@@ -50,9 +46,9 @@ from pylot.core.pick.utils import getSNR, earllatepicker, getnoisewin, \
     getResolutionWindow, getQualityFromUncertainty
 from pylot.core.pick.compare import Comparison
 from pylot.core.util.defaults import OUTPUTFORMATS, FILTERDEFAULTS
-from pylot.core.util.utils import prepTimeAxis, full_range, scaleWFData, \
+from pylot.core.util.utils import prepTimeAxis, full_range, demeanTrace, isSorted, findComboBoxIndex, clims, \
-    demeanTrace, isSorted, findComboBoxIndex, clims, pick_linestyle_plt, pick_color_plt, \
+    pick_linestyle_plt, pick_color_plt, \
-    check4rotated, check4doubled, check4gaps, merge_stream, remove_underscores, find_horizontals, identifyPhase, \
+    check4rotated, check4doubled, merge_stream, identifyPhase, \
     loopIdentifyPhase, trim_station_components, transformFilteroptions2String, \
     identifyPhaseID, real_Bool, pick_color, getAutoFilteroptions, SetChannelComponents
 from autoPyLoT import autoPyLoT
@@ -66,6 +62,9 @@ elif sys.version_info.major == 2:
 else:
     raise ImportError('Could not determine python version.')
 
+# workaround to prevent PyCharm from deleting icons_rc import when optimizing imports
+icons_rc = icons_rc
+
 
 def getDataType(parent):
     type = QInputDialog().getItem(parent, "Select phases type", "Type:",
@@ -148,7 +147,7 @@ class AddMetadataWidget(QWidget):
 
         self.center()
         self.show()
-        #self.__test__()
+        # self.__test__()
 
     def __test__(self):
         self.add_item(r'/rscratch/minos14/marcel/git/pylot/tests')
@@ -529,7 +528,7 @@ class ComparisonWidget(QWidget):
             legend.draggable()
 
             for ax in axes_dict['P'].values():
-                ax.set_ylabel('Frequency [-]')
+                ax.set_ylabel('number of picks [-]')
 
             self.canvas.draw()
         else:
@@ -739,7 +738,8 @@ class WaveformWidgetPG(QtGui.QWidget):
         # list containing tuples of network, station, channel (for sorting)
         nslc = []
         for trace in st_select:
-            nslc.append(trace.get_id())#(trace.stats.network, trace.stats.station, trace.stats.location trace.stats.channel))
+            nslc.append(
+                trace.get_id())  # (trace.stats.network, trace.stats.station, trace.stats.location trace.stats.channel))
         nslc.sort()
         nslc.reverse()
         plots = []
@@ -1203,7 +1203,8 @@ class PylotCanvas(FigureCanvas):
     def plotWFData(self, wfdata, title=None, zoomx=None, zoomy=None,
                    noiselevel=None, scaleddata=False, mapping=True,
                    component='*', nth_sample=1, iniPick=None, verbosity=0,
-                   plot_additional=False, additional_channel=None, scaleToChannel=None):
+                   plot_additional=False, additional_channel=None, scaleToChannel=None,
+                   snr=None):
         ax = self.axes[0]
         ax.cla()
 
@@ -1232,7 +1233,7 @@ class PylotCanvas(FigureCanvas):
         gaps = st_select.get_gaps()
         if gaps:
             merged = ['{}.{}.{}.{}'.format(*gap[:4]) for gap in gaps]
-            st_select.merge()
+            st_select.merge(method=1)
             print('Merged the following stations because of gaps:')
             for merged_station in merged:
                 print(merged_station)
@@ -1323,6 +1324,14 @@ class PylotCanvas(FigureCanvas):
             self.setXLims(ax, zoomx)
         if zoomy is not None:
             self.setYLims(ax, zoomy)
+        if snr is not None:
+            if snr < 2:
+                warning = 'LOW SNR'
+                if snr < 1.5:
+                    warning = 'VERY LOW SNR'
+                ax.text(0.1, 0.9, 'WARNING - {}'.format(warning), ha='center', va='center', transform=ax.transAxes,
+                        color='red')
+
         self.draw()
 
     @staticmethod
@@ -2280,6 +2289,12 @@ class PickDlg(QDialog):
             filterphase = 'S'
         return filterphase
 
+    def getNoiseWin(self, phase):
+        twins_phase = {'P': 'tsnrz',
+                       'S': 'tsnrh'}
+
+        return self.parameter.get(twins_phase[phase])[:3]
+
     def setIniPickP(self, gui_event):
         self.setIniPickPS(gui_event, phase='P')
 
@@ -2291,17 +2306,12 @@ class PickDlg(QDialog):
 
         nfac_phase = {'P': 'nfacP',
                       'S': 'nfacS'}
-        twins_phase = {'P': 'tsnrz',
-                       'S': 'tsnrh'}
 
         parameter = self.parameter
         ini_pick = gui_event.xdata
 
         nfac = parameter.get(nfac_phase[phase])
-        twins = parameter.get(twins_phase[phase])
+        noise_win, gap_win, signal_win = self.getNoiseWin(phase)
-        noise_win = twins[0]
-        gap_win = twins[1]
-        signal_win = twins[2]
 
         stime = self.getStartTime()
 
@@ -2355,12 +2365,14 @@ class PickDlg(QDialog):
             trace.data *= noiseScaleFactor
             noiselevels[channel] *= noiseScaleFactor
 
-        x_res = getResolutionWindow(np.mean(snr), parameter.get('extent'))
+        mean_snr = np.mean(snr)
+        x_res = getResolutionWindow(mean_snr, parameter.get('extent'))
 
         xlims = [ini_pick - x_res, ini_pick + x_res]
         ylims = list(np.array([-.5, .5]) + [0, len(data) - 1])
 
         title = self.getStation() + ' picking mode'
+        title += ' | SNR: {}'.format(mean_snr)
         if filterphase:
             filtops_str = transformFilteroptions2String(filteroptions)
             title += ' | Filteroptions: {}'.format(filtops_str)
@@ -2375,7 +2387,8 @@ class PickDlg(QDialog):
                                      scaleddata=True,
                                      iniPick=ini_pick,
                                      plot_additional=plot_additional,
-                                     additional_channel=additional_channel)
+                                     additional_channel=additional_channel,
+                                     snr=mean_snr)
 
     def setPick(self, gui_event):
 
@@ -2447,6 +2460,11 @@ class PickDlg(QDialog):
         self.zoomAction.setEnabled(True)
         # self.pick_block = self.togglPickBlocker()
         # self.resetZoom()
+        noise_win, gap_win, signal_win = self.getNoiseWin(phase)
+        snr, snrDB, noiselevel = getSNR(wfdata, (noise_win, gap_win, signal_win), pick - stime_diff)
+        print('SNR of final pick: {}'.format(snr))
+        if snr < 1.5:
+            QMessageBox.warning(self, 'SNR too low', 'WARNING! SNR of final pick below 1.5! SNR = {}'.format(snr))
         self.leave_picking_mode()
 
     def savePick(self, phase, phasepicks):
@@ -2520,25 +2538,25 @@ class PickDlg(QDialog):
             linestyle_mpp, width_mpp = pick_linestyle_plt(picktype, 'mpp')
             vl = ax.axvline(mpp, ylims[0], ylims[1], color=color, linestyle=linestyle_mpp, linewidth=width_mpp,
                             label='{}-{}-Pick (quality: {})'.format(phase, picktype, quality), picker=5,
-                            zorder=baseorder+9)
+                            zorder=baseorder + 9)
             phaseLineKey = '{}-{}'.format(phase, picktype)
             self.phaseLines[phaseLineKey] = vl
             if spe:
                 ax.fill_between([mpp - spe, mpp + spe], ylims[0], ylims[1],
-                                alpha=.25, color=color, label='{}-{}-SPE'.format(phase, picktype), zorder=baseorder+1)
+                                alpha=.25, color=color, label='{}-{}-SPE'.format(phase, picktype), zorder=baseorder + 1)
             if picks['epp']:
                 linestyle_epp, width_epp = pick_linestyle_plt(picktype, 'epp')
                 ax.axvline(epp, ylims[0], ylims[1], color=color, linestyle=linestyle_epp,
-                           linewidth=width_epp, label='{}-{}-EPP'.format(phase, picktype), zorder=baseorder+2)
+                           linewidth=width_epp, label='{}-{}-EPP'.format(phase, picktype), zorder=baseorder + 2)
             if picks['lpp']:
                 linestyle_lpp, width_lpp = pick_linestyle_plt(picktype, 'lpp')
                 ax.axvline(lpp, ylims[0], ylims[1], color=color, linestyle=linestyle_lpp,
-                           linewidth=width_lpp, label='{}-{}-LPP'.format(phase, picktype), zorder=baseorder+2)
+                           linewidth=width_lpp, label='{}-{}-LPP'.format(phase, picktype), zorder=baseorder + 2)
             if picktype == 'auto':
-                ax.plot(mpp, ylims[1], color=color, marker='v', zorder=baseorder+3)
+                ax.plot(mpp, ylims[1], color=color, marker='v', zorder=baseorder + 3)
-                ax.plot(mpp, ylims[0], color=color, marker='^', zorder=baseorder+3)
+                ax.plot(mpp, ylims[0], color=color, marker='^', zorder=baseorder + 3)
         # append phase text (if textOnly: draw with current ylims)
-        self.phaseText.append(ax.text(mpp, ylims[1], phase, color=color, zorder=baseorder+10))
+        self.phaseText.append(ax.text(mpp, ylims[1], phase, color=color, zorder=baseorder + 10))
         ax.legend(loc=1)
 
     def connect_mouse_motion(self):
@@ -3326,11 +3344,11 @@ class TuneAutopicker(QWidget):
             self.data.setWFData(fnames)
             wfdat = self.data.getWFData()  # all available streams
             # remove possible underscores in station names
-            #wfdat = remove_underscores(wfdat)
+            # wfdat = remove_underscores(wfdat)
             # rotate misaligned stations to ZNE
             # check for gaps and doubled channels
             wfdat, gaps = merge_stream(wfdat)
-            #check4gaps(wfdat)
+            # check4gaps(wfdat)
             check4doubled(wfdat)
             wfdat = check4rotated(wfdat, self.parent().metadata, verbosity=0)
             # trim station components to same start value
@@ -3398,6 +3416,7 @@ class TuneAutopicker(QWidget):
         return str(self.stationBox.currentText()).split('.')[1]
 
     def get_current_station_id(self):
+        print(self.stationBox, self.stationBox.currentText())
         return str(self.stationBox.currentText())
 
     @staticmethod
@@ -3415,7 +3434,15 @@ class TuneAutopicker(QWidget):
             self.pdlg_widget = None
             return
         self.load_wf_data()
-        network, station, location, channel = self.get_current_station_id()
+        try:
+            network, station, location, channel = self.get_current_station_id()
+        except ValueError as e:
+            vmsg = '{0}'.format(e)
+            print(vmsg)
+            station = self.get_current_station()
+            location = None
+            network = None
+
         wfdata = self.data.getWFData()
         metadata = self.parent().metadata
         event = self.get_current_event()
@@ -4587,7 +4614,7 @@ class GraphicsTab(PropTab):
         super(GraphicsTab, self).__init__(parent)
         self.pylot_mainwindow = parent._pylot_mainwindow
         self.init_layout()
-        #self.add_pg_cb()
+        # self.add_pg_cb()
         self.add_nth_sample()
         self.add_style_settings()
         self.setLayout(self.main_layout)

tests/test_Metadata/test_Metadata.py

@@ -1,9 +1,9 @@
-import unittest
 import os
-
+import unittest
 from obspy import UTCDateTime
-from obspy.io.xseed.utils import SEEDParserException
 from obspy.io.xseed import Parser
+from obspy.io.xseed.utils import SEEDParserException
+
 from pylot.core.util.dataprocessing import Metadata
 from tests.utils import HidePrints
 

tests/utils.py

@@ -3,8 +3,8 @@
 
 """Utilities/helpers for testing"""
 
-import sys
 import os
+import sys
 
 
 class HidePrints: