Merge branch 'feature/obspy_dmt_interface' into develop

.gitignore

@@ -1,3 +1,4 @@
 *.pyc
 *~
 pylot/RELEASE-VERSION
+*.idea

PyLoT.py

@@ -138,6 +138,7 @@ class MainWindow(QMainWindow):
         self._eventChanged = [False, False]
         self.apd_local = None
         self.apd_sge = None
+        self.stations_highlighted = []
 
         self.poS_id = None
         self.ae_id = None
@@ -615,6 +616,10 @@ class MainWindow(QMainWindow):
 
         # add scroll area used in case number of traces gets too high
         self.wf_scroll_area = QtGui.QScrollArea(self)
+        self.wf_scroll_area.setVisible(False)
+        self.no_data_label = QLabel('No Data')
+        self.no_data_label.setStyleSheet('color: red')
+        self.no_data_label.setAlignment(Qt.AlignCenter)
 
         # create central matplotlib figure canvas widget
         self.init_wfWidget()
@@ -637,6 +642,7 @@ class MainWindow(QMainWindow):
         self.tabs.addTab(array_tab, 'Array Map')
         self.tabs.addTab(events_tab, 'Eventlist')
 
+        self.wf_layout.addWidget(self.no_data_label)
         self.wf_layout.addWidget(self.wf_scroll_area)
         self.wf_scroll_area.setWidgetResizable(True)
         self.init_array_tab()
@@ -1249,6 +1255,17 @@ class MainWindow(QMainWindow):
             event_ref = event.isRefEvent()
             event_test = event.isTestEvent()
 
+            time = lat = lon = depth = mag = None
+            if len(event.origins) == 1:
+                origin = event.origins[0]
+                time = origin.time + 0 # add 0 because there was an exception for time = 0s
+                lat = origin.latitude
+                lon = origin.longitude
+                depth = origin.depth
+            if len(event.magnitudes) == 1:
+                magnitude = event.magnitudes[0]
+                mag = magnitude.mag
+
             # text = '{path:{plen}} | manual: [{p:3d}] | auto: [{a:3d}]'
             # text = text.format(path=event_path,
             #                    plen=plmax,
@@ -1256,6 +1273,11 @@ class MainWindow(QMainWindow):
             #                    a=event_nautopicks)
 
             item_path = QtGui.QStandardItem('{path:{plen}}'.format(path=event_path, plen=plmax))
+            item_time = QtGui.QStandardItem('{}'.format(time))
+            item_lat = QtGui.QStandardItem('{}'.format(lat))
+            item_lon = QtGui.QStandardItem('{}'.format(lon))
+            item_depth = QtGui.QStandardItem('{}'.format(depth))
+            item_mag = QtGui.QStandardItem('{}'.format(mag))
             item_nmp = QtGui.QStandardItem(str(ma_count['manual']))
             item_nmp.setIcon(self.manupicksicon_small)
             item_nap = QtGui.QStandardItem(str(ma_count['auto']))
@@ -1281,8 +1303,11 @@ class MainWindow(QMainWindow):
             # item.setFont(font)
             # item2.setForeground(QtGui.QColor('black'))
             # item2.setFont(font)
-            itemlist = [item_path, item_nmp, item_nap, item_ref, item_test, item_notes]
+            itemlist = [item_path, item_time, item_lat, item_lon, item_depth,
-            if event_test and select_events == 'ref':
+                        item_mag, item_nmp, item_nap, item_ref, item_test, item_notes]
+            for item in itemlist:
+                item.setTextAlignment(Qt.AlignCenter)
+            if event_test and select_events == 'ref' or self.isEmpty(event_path):
                 for item in itemlist:
                     item.setEnabled(False)
             model.appendRow(itemlist)
@@ -1296,6 +1321,23 @@ class MainWindow(QMainWindow):
         eventBox.setCurrentIndex(index)
         self.refreshRefTestButtons()
 
+    def isEmpty(self, event_path):
+        wf_stat = {True: 'processed',
+                   False: 'raw',
+                   None: None}
+
+        # self.data.processed is only None for PyLoT datastructure, else True or False
+        wf_dir = wf_stat[self.data.processed]
+        if wf_dir is not None:
+            wf_path = os.path.join(event_path, wf_dir)
+            if wf_dir is 'processed' and not os.path.exists(wf_path):
+                wf_path = os.path.join(event_path, 'raw')
+        else:
+            wf_path = event_path
+        if not os.path.exists(wf_path):
+            return True
+        return not bool(os.listdir(wf_path))
+
     def filename_from_action(self, action):
         if action.data() is None:
             filt = "Supported file formats" \
@@ -1666,6 +1708,21 @@ class MainWindow(QMainWindow):
                             metadata=self.metadata,
                             obspy_dmt=obspy_dmt)
 
+    def setWFstatus(self):
+        '''
+        show status of current data, can be either 'raw' or 'processed'
+        :param status:
+        :return:
+        '''
+        status = self.data.processed
+        wf_stat_color = {True: 'green',
+                         False: 'black',
+                         None: None}
+        wf_stat = {True: 'processed',
+                   False: 'raw',
+                   None: None}
+        self.dataPlot.setPermTextRight(wf_stat[status], wf_stat_color[status])
+
     def check_plot_quantity(self):
         """
         Check the amount of samples to be plotted and ask user to reduce the amount if it is too large.
@@ -1757,7 +1814,10 @@ class MainWindow(QMainWindow):
 
     def finish_pg_plot(self):
         self.getPlotWidget().updateWidget()
-        plots = self.wfp_thread.data
+        plots, gaps = self.wfp_thread.data
+        # do not show plot if no data are given
+        self.wf_scroll_area.setVisible(len(plots) > 0)
+        self.no_data_label.setVisible(not len(plots) > 0)
         for times, data, times_syn, data_syn in plots:
             self.dataPlot.plotWidget.getPlotItem().plot(times, data,
                                                         pen=self.dataPlot.pen_linecolor)
@@ -1765,8 +1825,7 @@ class MainWindow(QMainWindow):
                 self.dataPlot.plotWidget.getPlotItem().plot(times_syn, data_syn,
                                                             pen=self.dataPlot.pen_linecolor_syn)
         self.dataPlot.reinitMoveProxy()
-        self.dataPlot.plotWidget.showAxis('left')
+        self.highlight_stations()
-        self.dataPlot.plotWidget.showAxis('bottom')
 
     def finishWaveformDataPlot(self):
         self.comparable = self.checkEvents4comparison()
@@ -1808,6 +1867,7 @@ class MainWindow(QMainWindow):
         if True in self.comparable.values():
             self.compare_action.setEnabled(True)
         self.draw()
+        self.setWFstatus()
 
     def checkEvent4comparison(self, event):
         if event.pylot_picks and event.pylot_autopicks:
@@ -1842,12 +1902,10 @@ class MainWindow(QMainWindow):
             comparable[event.pylot_id] = self.checkEvent4comparison(event)
         return comparable
 
-    def clearWaveformDataPlot(self):
+    def clearWaveformDataPlot(self, refresh_plot=False):
         self.disconnectWFplotEvents()
         if self.pg:
             self.dataPlot.plotWidget.getPlotItem().clear()
-            self.dataPlot.plotWidget.hideAxis('bottom')
-            self.dataPlot.plotWidget.hideAxis('left')
         else:
             for ax in self.dataPlot.axes:
                 ax.cla()
@@ -1863,6 +1921,9 @@ class MainWindow(QMainWindow):
         self.openEventAction.setEnabled(False)
         self.openEventsAutoAction.setEnabled(False)
         self.loadpilotevent.setEnabled(False)
+        if not refresh_plot:
+            self.wf_scroll_area.setVisible(False)
+            self.no_data_label.setVisible(True)
         self.disableSaveEventAction()
         self.draw()
 
@@ -1871,7 +1932,7 @@ class MainWindow(QMainWindow):
         Open a modal thread to plot current waveform data.
         '''
         self.check_plot_quantity()
-        self.clearWaveformDataPlot()
+        self.clearWaveformDataPlot(refresh_plot=True)
         self.wfp_thread = Thread(self, self.plotWaveformData,
                                  arg=filter,
                                  progressText='Plotting waveform data...',
@@ -1904,9 +1965,10 @@ class MainWindow(QMainWindow):
             self.plot_method = 'fast'
         else:
             self.plot_method = 'normal'
-        plots = plotWidget.plotWFData(wfdata=wfst, wfsyn=wfsyn, title=title, mapping=False, component=comp,
+        rval = plotWidget.plotWFData(wfdata=wfst, wfsyn=wfsyn, title=title, mapping=False, component=comp,
-                                      nth_sample=int(nth_sample), method=self.plot_method)
+                                     nth_sample=int(nth_sample), method=self.plot_method)
-        return plots
+        plots, gaps = rval if rval else ([], [])
+        return plots, gaps
 
     def adjustPlotHeight(self):
         if self.pg:
@@ -2135,10 +2197,12 @@ class MainWindow(QMainWindow):
 
     def pickOnStation(self, gui_event):
         if self.pg:
-            if not gui_event.button() == 1:
+            button = gui_event.button()
+            if not button in [1, 4]:
                 return
         else:
-            if not gui_event.button == 1:
+            button = gui_event.button
+            if not button == 1:
                 return
 
         if self.pg:
@@ -2149,12 +2213,42 @@ class MainWindow(QMainWindow):
         wfID = self.getWFID(ycoord)
 
         if wfID is None: return
-        self.pickDialog(wfID)
 
-    def pickDialog(self, wfID, nextStation=False):
-        station = self.getStationName(wfID)
         network = self.getNetworkName(wfID)
+        station = self.getStationName(wfID)
+        if button == 1:
+            self.pickDialog(wfID, network, station)
+        elif button == 4:
+            self.toggle_station_color(wfID, network, station)
+
+    def toggle_station_color(self, wfID, network, station):
+        black_pen = pg.mkPen((0, 0, 0))
+        red_pen = pg.mkPen((200, 50, 50))
+        line_item = self.dataPlot.plotWidget.getPlotItem().listDataItems()[wfID - 1]
+        current_pen = line_item.opts['pen']
+        nwst = '{}.{}'.format(network, station)
+        if current_pen == black_pen:
+            line_item.setPen(red_pen)
+            if not nwst in self.stations_highlighted:
+                self.stations_highlighted.append(nwst)
+        else:
+            line_item.setPen(black_pen)
+            if nwst in self.stations_highlighted:
+                self.stations_highlighted.pop(self.stations_highlighted.index(nwst))
+
+    def highlight_stations(self):
+        for wfID, value in self.getPlotWidget().getPlotDict().items():
+            station, channel, network = value
+            nwst = '{}.{}'.format(network, station)
+            if nwst in self.stations_highlighted:
+                self.toggle_station_color(wfID, network, station)
+
+    def pickDialog(self, wfID, network=None, station=None, nextStation=False):
+        if not network:
+            network = self.getNetworkName(wfID)
         if not station:
+            station = self.getStationName(wfID)
+        if not station or not network:
             return
         self.update_status('picking on station {0}'.format(station))
         data = self.get_data().getOriginalWFData().copy()
@@ -2894,7 +2988,7 @@ class MainWindow(QMainWindow):
             if hasattr(event, 'origins'):
                 if event.origins:
                     origin = event.origins[0]
-                    item_time.setText(str(origin.time).split('.')[0])
+                    item_time.setText(str(origin.time + 0).split('.')[0]) # +0 as workaround in case time=0s
                     item_lon.setText(str(origin.longitude))
                     item_lat.setText(str(origin.latitude))
                     item_depth.setText(str(origin.depth))
@@ -2931,6 +3025,9 @@ class MainWindow(QMainWindow):
             if index%2:
                 set_background_color(column, QtGui.QColor(*(245, 245, 245, 255)))
 
+            if self.isEmpty(event.path):
+                set_foreground_color(column, QtGui.QColor(*(180, 180, 180, 255)))
+
             if event == current_event:
                 set_foreground_color(column, QtGui.QColor(*(0, 143, 143, 255)))
 
@@ -3197,9 +3294,9 @@ class MainWindow(QMainWindow):
         self.fill_eventbox()
         self.getPlotWidget().draw()
         if self.plot_method == 'fast':
-            self.dataPlot.setPermText('MIN/MAX plot', color='red')
+            self.dataPlot.setPermTextMid('MIN/MAX plot', color='red')
         else:
-            self.dataPlot.setPermText()
+            self.dataPlot.setPermTextMid()
 
     def _setDirty(self):
         self.setDirty(True)

icons_rc_2.py

@@ -7,7 +7,7 @@
 #
 # WARNING! All changes made in this file will be lost!
 
-from PyQt4 import QtCore
+from PySide import QtCore
 
 qt_resource_data = "\
 \x00\x00\x9e\x04\

icons_rc_3.py

@@ -7,7 +7,7 @@
 #
 # WARNING! All changes made in this file will be lost!
 
-from PyQt4 import QtCore
+from PySide import QtCore
 
 qt_resource_data = "\
 \x00\x00\x9e\x04\

pylot/core/io/data.py

@@ -15,7 +15,7 @@ from pylot.core.util.event import Event
 from pylot.core.util.utils import fnConstructor, full_range, remove_underscores, check4gaps, check4doubled, \
     check4rotated, trim_station_components
 import pylot.core.loc.velest as velest
-
+from pylot.core.util.obspyDMT_interface import qml_from_obspyDMT
 
 class Data(object):
     """
@@ -60,6 +60,8 @@ class Data(object):
                         raise NotImplementedError('PILOT location information '
                                                   'read support not yet '
                                                   'implemeted.')
+                    elif 'event.pkl' in evtdata:
+                        evtdata = qml_from_obspyDMT(evtdata)
                     else:
                         raise e
                 else:
@@ -72,6 +74,7 @@ class Data(object):
         self.wforiginal = None
         self.cuttimes = None
         self.dirty = False
+        self.processed = None
 
     def __str__(self):
         return str(self.wfdata)
@@ -379,8 +382,14 @@ class Data(object):
         self.wfsyn = Stream()
         wffnames = None
         wffnames_syn = None
-        wfdir = 'processed' if 'processed' in [fname.split('/')[-1] for fname in fnames] else 'raw'
         if obspy_dmt:
+            wfdir = 'raw'
+            self.processed = False
+            for fname in fnames:
+                if fname.endswith('processed'):
+                    wfdir = 'processed'
+                    self.processed = True
+                    break
             for fpath in fnames:
                 if fpath.endswith(wfdir):
                     wffnames = [os.path.join(fpath, fname) for fname in os.listdir(fpath)]
@@ -398,9 +407,6 @@ class Data(object):
         # various pre-processing steps:
         # remove possible underscores in station names
         self.wfdata = remove_underscores(self.wfdata)
-        # check for gaps and doubled channels
-        check4gaps(self.wfdata)
-        check4doubled(self.wfdata)
         # check for stations with rotated components
         if checkRotated and metadata is not None:
             self.wfdata = check4rotated(self.wfdata, metadata, verbosity=0)

pylot/core/util/event.py

@@ -7,6 +7,7 @@ from obspy import UTCDateTime
 from obspy.core.event import Event as ObsPyEvent
 from obspy.core.event import Origin, ResourceIdentifier
 from pylot.core.io.phases import picks_from_picksdict
+from pylot.core.util.obspyDMT_interface import qml_from_obspyDMT
 
 
 class Event(ObsPyEvent):
@@ -33,6 +34,7 @@ class Event(ObsPyEvent):
         self._testEvent = False
         self._refEvent = False
         self.get_notes()
+        self.get_obspy_event_info()
 
     def get_notes_path(self):
         """
@@ -43,6 +45,18 @@ class Event(ObsPyEvent):
         notesfile = os.path.join(self.path, 'notes.txt')
         return notesfile
 
+    def get_obspy_event_info(self):
+        infile_pickle = os.path.join(self.path, 'info/event.pkl')
+        if not os.path.isfile(infile_pickle):
+            return
+        try:
+            event_dmt = qml_from_obspyDMT(infile_pickle)
+        except Exception as e:
+            print('Could not get obspy event info: {}'.format(e))
+            return
+        self.magnitudes = event_dmt.magnitudes
+        self.origins = event_dmt.origins
+
     def get_notes(self):
         """
         set self.note attribute to content of notes file

pylot/core/util/obspyDMT_interface.py

@@ -25,4 +25,20 @@ def check_obspydmt_eventfolder(folder):
     except Exception as e:
         return False, e
 
-check_obspydmt_eventfolder('20110311_054623.a')
+def qml_from_obspyDMT(path):
+    import pickle
+    from obspy.core.event import Event, Magnitude, Origin
+
+    if not os.path.exists(path):
+        return IOError('Could not find Event at {}'.format(path))
+    infile = open(path, 'rb')
+    event_dmt = pickle.load(infile)
+    ev = Event(resource_id=event_dmt['event_id'])
+    origin = Origin(resource_id=event_dmt['origin_id'], time=event_dmt['datetime'], longitude=event_dmt['longitude'],
+                    latitude=event_dmt['latitude'], depth=event_dmt['depth'])
+    mag = Magnitude(mag=event_dmt['magnitude'], magnitude_type=event_dmt['magnitude_type'],
+                    origin_id=event_dmt['origin_id'])
+    ev.magnitudes.append(mag)
+    ev.origins.append(origin)
+    return ev
+

pylot/core/util/widgets.py

@@ -447,23 +447,25 @@ class WaveformWidgetPG(QtGui.QWidget):
         self.orig_parent = parent
         # attribute plotdict is a dictionary connecting position and a name
         self.plotdict = dict()
+        # init labels
+        self.xlabel = None
+        self.ylabel = None
+        self.title = None
         # create plot
         self.main_layout = QtGui.QVBoxLayout()
         self.label_layout = QtGui.QHBoxLayout()
-        self.status_label = QtGui.QLabel()
+        self.add_labels()
-        self.perm_label = QtGui.QLabel()
-        self.setLayout(self.main_layout)
         self.plotWidget = self.pg.PlotWidget(self.parent(), title=title)
         self.main_layout.addWidget(self.plotWidget)
         self.main_layout.addLayout(self.label_layout)
         self.label_layout.addWidget(self.status_label)
-        self.label_layout.addWidget(self.perm_label)
+        self.label_layout.addWidget(self.perm_label_mid)
+        self.label_layout.addWidget(self.perm_label_right)
         self.plotWidget.showGrid(x=False, y=True, alpha=0.3)
-        self.plotWidget.hideAxis('bottom')
-        self.plotWidget.hideAxis('left')
         self.wfstart, self.wfend = 0, 0
         self.pen_multicursor = self.pg.mkPen(self.parent()._style['multicursor']['rgba'])
         self.pen_linecolor = self.pg.mkPen(self.parent()._style['linecolor']['rgba'])
+        self.pen_linecolor_highlight = self.pg.mkPen((255, 100, 100, 255))
         self.pen_linecolor_syn = self.pg.mkPen((100, 0, 255, 255))
         self.reinitMoveProxy()
         self._proxy = self.pg.SignalProxy(self.plotWidget.scene().sigMouseMoved, rateLimit=60, slot=self.mouseMoved)
@@ -489,12 +491,24 @@ class WaveformWidgetPG(QtGui.QWidget):
             self.vLine.setPos(mousePoint.x())
             self.hLine.setPos(mousePoint.y())
 
+    def add_labels(self):
+        self.status_label = QtGui.QLabel()
+        self.perm_label_mid = QtGui.QLabel()
+        self.perm_label_mid.setAlignment(4)
+        self.perm_label_right = QtGui.QLabel()
+        self.perm_label_right.setAlignment(2)
+        self.setLayout(self.main_layout)
+
     def getPlotDict(self):
         return self.plotdict
 
-    def setPermText(self, text=None, color='black'):
+    def setPermTextMid(self, text=None, color='black'):
-        self.perm_label.setText(text)
+        self.perm_label_mid.setText(text)
-        self.perm_label.setStyleSheet('color: {}'.format(color))
+        self.perm_label_mid.setStyleSheet('color: {}'.format(color))
+
+    def setPermTextRight(self, text=None, color='black'):
+        self.perm_label_right.setText(text)
+        self.perm_label_right.setStyleSheet('color: {}'.format(color))
 
     def setPlotDict(self, key, value):
         self.plotdict[key] = value
@@ -529,6 +543,14 @@ class WaveformWidgetPG(QtGui.QWidget):
         else:
             st_select = wfdata
 
+        gaps = st_select.get_gaps()
+        if gaps:
+            merged = ['{}.{}.{}.{}'.format(*gap[:4]) for gap in gaps]
+            st_select.merge()
+            print('Merged the following stations because of gaps:')
+            for merged_station in merged:
+                print(merged_station)
+
         # list containing tuples of network, station, channel (for sorting)
         nsc = []
         for trace in st_select:
@@ -552,6 +574,8 @@ class WaveformWidgetPG(QtGui.QWidget):
             st_syn = wfsyn.select(network=network, station=station, channel=channel)
             if st_syn:
                 trace_syn = st_syn[0].copy()
+            else:
+                trace_syn = Trace()
             if mapping:
                 comp = channel[-1]
                 n = compclass.getPlotPosition(str(comp))
@@ -568,9 +592,11 @@ class WaveformWidgetPG(QtGui.QWidget):
                 time_ax_syn = prepTimeAxis(stime_syn, trace_syn)
 
             if method == 'fast':
-               trace.data, time_ax = self.minMax(trace, time_ax)
+                trace.data, time_ax = self.minMax(trace, time_ax)
+                if trace_syn:
+                    trace_syn.data, time_ax_syn = self.minMax(trace_syn, time_ax_syn)
 
-            if time_ax not in [None, []]:
+            if len(time_ax) > 0:
                 if not scaleddata:
                     trace.detrend('constant')
                     trace.normalize(np.max(np.abs(trace.data)) * 2)
@@ -581,16 +607,16 @@ class WaveformWidgetPG(QtGui.QWidget):
                 times = np.array([time for index, time in enumerate(time_ax) if not index % nth_sample])
                 times_syn = np.array([time for index, time in enumerate(time_ax_syn) if not index % nth_sample] if st_syn else [])
                 trace.data = np.array([datum + n for index, datum in enumerate(trace.data) if not index % nth_sample])
-                trace.data_syn = np.array([datum + n for index, datum in enumerate(trace.data_syn)
+                trace_syn.data = np.array([datum + n for index, datum in enumerate(trace_syn.data)
                             if not index % nth_sample] if st_syn else [])
                 plots.append((times, trace.data,
-                              times_syn, trace.data_syn))
+                              times_syn, trace_syn.data))
                 self.setPlotDict(n, (station, channel, network))
         self.xlabel = 'seconds since {0}'.format(self.wfstart)
         self.ylabel = ''
         self.setXLims([0, self.wfend - self.wfstart])
         self.setYLims([0.5, nmax + 0.5])
-        return plots
+        return plots, gaps
 
     def minMax(self, trace, time_ax):
         '''
@@ -1013,6 +1039,14 @@ class PylotCanvas(FigureCanvas):
         if mapping:
             plot_positions = self.calcPlotPositions(st_select, compclass)
 
+        gaps = st_select.get_gaps()
+        if gaps:
+            merged = ['{}.{}.{}.{}'.format(*gap[:4]) for gap in gaps]
+            st_select.merge()
+            print('Merged the following stations because of gaps:')
+            for merged_station in merged:
+                print(merged_station)
+
         # list containing tuples of network, station, channel and plot position (for sorting)
         nsc = []
         for plot_pos, trace in enumerate(st_select):