From 366db9aef01e16b278751f6ea7c16755f742fa7c Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 4 Apr 2018 14:57:54 +0200 Subject: [PATCH 01/11] [initial] first changes to supply an interface to an obspyDMT database --- PyLoT.py | 26 +++++++++++----- pylot/RELEASE-VERSION | 2 +- pylot/core/util/obspyDMT_interface.py | 28 +++++++++++++++++ pylot/core/util/structure.py | 2 +- pylot/core/util/utils.py | 43 +++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 9 deletions(-) create mode 100644 pylot/core/util/obspyDMT_interface.py diff --git a/PyLoT.py b/PyLoT.py index fc1cf229..7993a4e7 100755 --- a/PyLoT.py +++ b/PyLoT.py @@ -46,6 +46,9 @@ from obspy import UTCDateTime from obspy.core.event import Magnitude, Origin from obspy.core.util import AttribDict +from pylot.core.util.obspyDMT_interface import check_obspydmt_structure + + try: import pyqtgraph as pg except Exception as e: @@ -75,7 +78,8 @@ from pylot.core.util.dataprocessing import read_metadata, restitute_data from pylot.core.util.utils import fnConstructor, getLogin, \ full_range, readFilterInformation, trim_station_components, check4gaps, make_pen, pick_color_plt, \ pick_linestyle_plt, remove_underscores, check4doubled, identifyPhaseID, excludeQualityClasses, has_spe, \ - check4rotated, transform_colors_mpl, transform_colors_mpl_str, getAutoFilteroptions + check4rotated, transform_colors_mpl, transform_colors_mpl_str, getAutoFilteroptions, check_all_obspy, \ + check_all_pylot from pylot.core.util.event import Event from pylot.core.io.location import create_creation_info, create_event from pylot.core.util.widgets import FilterOptionsDialog, NewEventDlg, \ @@ -987,8 +991,14 @@ class MainWindow(QMainWindow): ''' Return waveform filenames from event in eventbox. ''' + # TODO: add dataStructure class for obspyDMT here, this is just a workaround! + eventpath = self.get_current_event_path(eventbox) + basepath = eventpath.split(os.path.basename(eventpath))[0] + if check_obspydmt_structure(basepath): + directory = os.path.join(eventpath, 'raw') + else: + directory = eventpath if self.dataStructure: - directory = self.get_current_event_path(eventbox) if not directory: return fnames = [os.path.join(directory, f) for f in os.listdir(directory)] @@ -1036,13 +1046,15 @@ class MainWindow(QMainWindow): ed = getExistingDirectories(self, 'Select event directories...') if ed.exec_(): eventlist = ed.selectedFiles() - # select only folders that start with 'e', containin two dots and have length 12 - eventlist = [item for item in eventlist if item.split('/')[-1].startswith('e') - and len(item.split('/')[-1].split('.')) == 3 - and len(item.split('/')[-1]) == 12] + basepath = eventlist[0].split(os.path.basename(eventlist[0]))[0] + if check_obspydmt_structure(basepath): + print('Recognized obspyDMT structure in selected files.') + eventlist = check_all_obspy(eventlist) + else: + eventlist = check_all_pylot(eventlist) if not eventlist: print('No events found! Expected structure for event folders: [eEVID.DOY.YR],\n' - ' e.g. eventID=1, doy=2, yr=2016: e0001.002.16') + ' e.g. eventID=1, doy=2, yr=2016: e0001.002.16 or obspyDMT database') return else: return diff --git a/pylot/RELEASE-VERSION b/pylot/RELEASE-VERSION index c687e0f2..ba9652dc 100644 --- a/pylot/RELEASE-VERSION +++ b/pylot/RELEASE-VERSION @@ -1 +1 @@ -39f92-dirty +7941-dirty diff --git a/pylot/core/util/obspyDMT_interface.py b/pylot/core/util/obspyDMT_interface.py new file mode 100644 index 00000000..661e4f5b --- /dev/null +++ b/pylot/core/util/obspyDMT_interface.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +from obspy import UTCDateTime + +def check_obspydmt_structure(path): + ''' + Check path for obspyDMT event structure. + :param path: + :return: + ''' + ev_info = os.path.join(path, 'EVENTS-INFO') + if os.path.isdir(ev_info): + if os.path.isfile(os.path.join(ev_info, 'logger_command.txt')): + return True + return False + +def check_obspydmt_eventfolder(folder): + try: + time = folder.split('.')[0] + time = time.replace('_', 'T') + time = UTCDateTime(time) + return True, time + except Exception as e: + return False, e + +check_obspydmt_eventfolder('20110311_054623.a') \ No newline at end of file diff --git a/pylot/core/util/structure.py b/pylot/core/util/structure.py index 68a16552..f556eb7c 100644 --- a/pylot/core/util/structure.py +++ b/pylot/core/util/structure.py @@ -9,4 +9,4 @@ Created on Wed Jan 26 17:47:25 2015 from pylot.core.io.data import SeiscompDataStructure, PilotDataStructure DATASTRUCTURE = {'PILOT': PilotDataStructure, 'SeisComP': SeiscompDataStructure, - None: None} + 'obspyDMT': None, None: None} diff --git a/pylot/core/util/utils.py b/pylot/core/util/utils.py index 2c93286e..ba6656f7 100644 --- a/pylot/core/util/utils.py +++ b/pylot/core/util/utils.py @@ -6,6 +6,7 @@ import os import platform import re import subprocess +import warnings import numpy as np from obspy import UTCDateTime, read @@ -13,6 +14,8 @@ from obspy.core import AttribDict from obspy.signal.rotate import rotate2zne from obspy.io.xseed.utils import SEEDParserException +from pylot.core.util.obspyDMT_interface import check_obspydmt_eventfolder + from pylot.core.io.inputs import PylotParameter, FilterOptions from pylot.styles import style_settings @@ -1216,6 +1219,46 @@ def has_spe(pick): return pick['spe'] +def check_all_obspy(eventlist): + ev_type = 'obspydmt' + return check_event_folders(eventlist, ev_type) + + +def check_all_pylot(eventlist): + ev_type = 'pylot' + return check_event_folders(eventlist, ev_type) + + +def check_event_folders(eventlist, ev_type): + checklist = [] + clean_eventlist = [] + for path in eventlist: + folder_check = check_event_folder(path) + if not folder_check: + warnings.warn('Unrecognized event folder: {}'.format(path)) + continue + checklist.append(folder_check == ev_type) + clean_eventlist.append(path) + if all(checklist) or len(checklist) == 0: + return clean_eventlist + else: + warnings.warn('Not all selected folders of type {}'.format(ev_type)) + return [] + + +def check_event_folder(path): + ev_type = None + folder = path.split('/')[-1] + # for pylot: select only folders that start with 'e', containin two dots and have length 12 + if (folder.startswith('e') + and len(folder.split('.')) == 3 + and len(folder) == 12): + ev_type = 'pylot' + elif check_obspydmt_eventfolder(folder)[0]: + ev_type = 'obspydmt' + return ev_type + + if __name__ == "__main__": import doctest From 4cf785a13586c3c61696fe161edbf56062ad8221 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 5 Apr 2018 15:33:40 +0200 Subject: [PATCH 02/11] [add] synthetic data plot (not yet flexible) --- .gitignore | 3 ++- PyLoT.py | 22 ++++++++++++--------- pylot/core/io/data.py | 32 ++++++++++++++++++++++++------- pylot/core/util/dataprocessing.py | 2 +- pylot/core/util/widgets.py | 20 ++++++++++++++++--- 5 files changed, 58 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 6add511e..c99398e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc -*~ \ No newline at end of file +*~ +pylot/RELEASE-VERSION diff --git a/PyLoT.py b/PyLoT.py index 7993a4e7..1fb629e9 100755 --- a/PyLoT.py +++ b/PyLoT.py @@ -994,14 +994,10 @@ class MainWindow(QMainWindow): # TODO: add dataStructure class for obspyDMT here, this is just a workaround! eventpath = self.get_current_event_path(eventbox) basepath = eventpath.split(os.path.basename(eventpath))[0] - if check_obspydmt_structure(basepath): - directory = os.path.join(eventpath, 'raw') - else: - directory = eventpath if self.dataStructure: - if not directory: + if not eventpath: return - fnames = [os.path.join(directory, f) for f in os.listdir(directory)] + fnames = [os.path.join(eventpath, f) for f in os.listdir(eventpath)] else: raise DatastructureError('not specified') return fnames @@ -1661,9 +1657,13 @@ class MainWindow(QMainWindow): # else: # ans = False self.fnames = self.getWFFnames_from_eventbox() + eventpath = self.get_current_event_path() + basepath = eventpath.split(os.path.basename(eventpath))[0] + obspy_dmt = check_obspydmt_structure(basepath) self.data.setWFData(self.fnames, checkRotated=True, - metadata=self.metadata) + metadata=self.metadata, + obspy_dmt=obspy_dmt) def connectWFplotEvents(self): ''' @@ -1719,9 +1719,12 @@ class MainWindow(QMainWindow): def finish_pg_plot(self): self.getPlotWidget().updateWidget() plots = self.wfp_thread.data - for times, data in plots: + for times, data, times_syn, data_syn in plots: self.dataPlot.plotWidget.getPlotItem().plot(times, data, pen=self.dataPlot.pen_linecolor) + if data_syn: + self.dataPlot.plotWidget.getPlotItem().plot(times_syn, data_syn, + pen=self.dataPlot.pen_linecolor_syn) self.dataPlot.reinitMoveProxy() self.dataPlot.plotWidget.showAxis('left') self.dataPlot.plotWidget.showAxis('bottom') @@ -1848,6 +1851,7 @@ class MainWindow(QMainWindow): comp = self.getComponent() title = 'section: {0} components'.format(zne_text[comp]) wfst = self.get_data().getWFData() + wfsyn = self.get_data().getSynWFData() if self.filterActionP.isChecked() and filter: self.filterWaveformData(plot=False, phase='P') elif self.filterActionS.isChecked() and filter: @@ -1856,7 +1860,7 @@ class MainWindow(QMainWindow): # wfst += self.get_data().getWFData().select(component=alter_comp) plotWidget = self.getPlotWidget() self.adjustPlotHeight() - plots = plotWidget.plotWFData(wfdata=wfst, title=title, mapping=False, component=comp, + plots = plotWidget.plotWFData(wfdata=wfst, wfsyn=wfsyn, title=title, mapping=False, component=comp, nth_sample=int(nth_sample)) return plots diff --git a/pylot/core/io/data.py b/pylot/core/io/data.py index 60d017d1..24626fe5 100644 --- a/pylot/core/io/data.py +++ b/pylot/core/io/data.py @@ -368,7 +368,7 @@ class Data(object): data.filter(**kwargs) self.dirty = True - def setWFData(self, fnames, checkRotated=False, metadata=None): + def setWFData(self, fnames, checkRotated=False, metadata=None, obspy_dmt=False): """ Clear current waveform data and set given waveform data :param fnames: waveform data names to append @@ -376,8 +376,21 @@ class Data(object): """ self.wfdata = Stream() self.wforiginal = None - if fnames is not None: - self.appendWFData(fnames) + self.wfsyn = Stream() + wffnames = None + wffnames_syn = None + if obspy_dmt: + for fpath in fnames: + if fpath.endswith('raw'): + wffnames = [os.path.join(fpath, fname) for fname in os.listdir(fpath)] + if 'syngine' in fpath.split('/')[-1]: + wffnames_syn = [os.path.join(fpath, fname) for fname in os.listdir(fpath)] + else: + wffnames = fnames + if wffnames is not None: + self.appendWFData(wffnames) + if wffnames_syn is not None: + self.appendWFData(wffnames_syn, synthetic=True) else: return False @@ -399,8 +412,7 @@ class Data(object): return True - - def appendWFData(self, fnames): + def appendWFData(self, fnames, synthetic=False): """ Read waveform data from fnames and append it to current wf data :param fnames: waveform data to append @@ -413,13 +425,16 @@ class Data(object): if self.dirty: self.resetWFData() + real_or_syn_data = {True: self.wfsyn, + False: self.wfdata} + warnmsg = '' for fname in fnames: try: - self.wfdata += read(fname) + real_or_syn_data[synthetic] += read(fname) except TypeError: try: - self.wfdata += read(fname, format='GSE2') + real_or_syn_data[synthetic] += read(fname, format='GSE2') except Exception as e: warnmsg += '{0}\n{1}\n'.format(fname, e) except SacIOError as se: @@ -434,6 +449,9 @@ class Data(object): def getOriginalWFData(self): return self.wforiginal + def getSynWFData(self): + return self.wfsyn + def resetWFData(self): """ Set waveform data to original waveform data diff --git a/pylot/core/util/dataprocessing.py b/pylot/core/util/dataprocessing.py index 1d13e952..6b952164 100644 --- a/pylot/core/util/dataprocessing.py +++ b/pylot/core/util/dataprocessing.py @@ -192,7 +192,7 @@ def read_metadata(path_to_inventory): robj = inv[invtype] else: print("Reading metadata information from inventory-xml file ...") - robj = inv[invtype] + robj = read_inventory(inv[invtype]) return invtype, robj diff --git a/pylot/core/util/widgets.py b/pylot/core/util/widgets.py index a7fec228..98b50fb1 100644 --- a/pylot/core/util/widgets.py +++ b/pylot/core/util/widgets.py @@ -450,7 +450,7 @@ class WaveformWidgetPG(QtGui.QWidget): self.main_layout = QtGui.QVBoxLayout() self.label = QtGui.QLabel() self.setLayout(self.main_layout) - self.plotWidget = self.pg.PlotWidget(self.parent(), title=title, autoDownsample=True) + self.plotWidget = self.pg.PlotWidget(self.parent(), title=title) self.main_layout.addWidget(self.plotWidget) self.main_layout.addWidget(self.label) self.plotWidget.showGrid(x=False, y=True, alpha=0.3) @@ -459,8 +459,10 @@ class WaveformWidgetPG(QtGui.QWidget): self.wfstart, self.wfend = 0, 0 self.pen_multicursor = self.pg.mkPen(self.parent()._style['multicursor']['rgba']) self.pen_linecolor = self.pg.mkPen(self.parent()._style['linecolor']['rgba']) + self.pen_linecolor_syn = self.pg.mkPen((100, 0, 255, 255)) self.reinitMoveProxy() self._proxy = self.pg.SignalProxy(self.plotWidget.scene().sigMouseMoved, rateLimit=60, slot=self.mouseMoved) + self.plotWidget.getPlotItem().setDownsampling(auto=True) def reinitMoveProxy(self): self.vLine = self.pg.InfiniteLine(angle=90, movable=False, pen=self.pen_multicursor) @@ -491,7 +493,7 @@ class WaveformWidgetPG(QtGui.QWidget): def clearPlotDict(self): self.plotdict = dict() - def plotWFData(self, wfdata, title=None, zoomx=None, zoomy=None, + def plotWFData(self, wfdata, wfsyn=None, title=None, zoomx=None, zoomy=None, noiselevel=None, scaleddata=False, mapping=True, component='*', nth_sample=1, iniPick=None, verbosity=0): if not wfdata: @@ -536,7 +538,10 @@ class WaveformWidgetPG(QtGui.QWidget): for n, (network, station, channel) in enumerate(nsc): n+=1 st = st_select.select(network=network, station=station, channel=channel) + st_syn = wfsyn.select(network=network, station=station, channel=channel) trace = st[0] + if st_syn: + trace_syn = st_syn[0] if mapping: comp = channel[-1] n = compclass.getPlotPosition(str(comp)) @@ -548,13 +553,22 @@ class WaveformWidgetPG(QtGui.QWidget): print(msg) stime = trace.stats.starttime - self.wfstart time_ax = prepTimeAxis(stime, trace) + if st_syn: + stime_syn = trace_syn.stats.starttime - self.wfstart + time_ax_syn = prepTimeAxis(stime_syn, trace_syn) if time_ax is not None: if not scaleddata: trace.detrend('constant') trace.normalize(np.max(np.abs(trace.data)) * 2) + if st_syn: + trace_syn.detrend('constant') + trace_syn.normalize(np.max(np.abs(trace_syn.data)) * 2) times = [time for index, time in enumerate(time_ax) if not index % nth_sample] + times_syn = [time for index, time in enumerate(time_ax_syn) if not index % nth_sample] if st_syn else [] data = [datum + n for index, datum in enumerate(trace.data) if not index % nth_sample] - plots.append((times, data)) + data_syn = [datum + n for index, datum in enumerate(trace_syn.data) + if not index % nth_sample] if st_syn else [] + plots.append((times, data, times_syn, data_syn)) self.setPlotDict(n, (station, channel, network)) self.xlabel = 'seconds since {0}'.format(self.wfstart) self.ylabel = '' From ba37d587a631e0e454d34692d996cee0484dc1b7 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 13 Apr 2018 10:56:52 +0200 Subject: [PATCH 03/11] [update] testing min/max plot (WIP) --- PyLoT.py | 7 ++++--- pylot/core/io/data.py | 3 ++- pylot/core/util/widgets.py | 42 ++++++++++++++++++++++++++++++-------- 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/PyLoT.py b/PyLoT.py index 86375c78..4e2ddd41 100755 --- a/PyLoT.py +++ b/PyLoT.py @@ -1667,8 +1667,8 @@ class MainWindow(QMainWindow): def check_plot_quantity(self): settings = QSettings() - nth_sample = settings.value("nth_sample") if settings.value("nth_sample") else 1 - npts_max = 1e6 + nth_sample = int(settings.value("nth_sample")) if settings.value("nth_sample") else 1 + npts_max = 1e7 npts = self.get_npts_to_plot() npts2plot = npts/nth_sample if npts2plot < npts_max: @@ -1677,7 +1677,8 @@ class MainWindow(QMainWindow): message = "You are about to plot a huge dataset with {npts} datapoints. With a current setting of " \ "nth_sample = {nth_sample} a total of {npts2plot} points will be plotted which is more " \ "than the maximum setting of {npts_max}. " \ - "PyLoT recommends to raise nth_sample from {nth_sample} to {nth_sample_new}. Continue?" + "PyLoT recommends to raise nth_sample from {nth_sample} to {nth_sample_new}. Do you want "\ + "to change nth_sample to {nth_sample_new} now?" ans = QMessageBox.question(self, self.tr("Optimize plot performance..."), self.tr(message.format(npts=npts, diff --git a/pylot/core/io/data.py b/pylot/core/io/data.py index 24626fe5..a7cb85d5 100644 --- a/pylot/core/io/data.py +++ b/pylot/core/io/data.py @@ -379,9 +379,10 @@ class Data(object): self.wfsyn = Stream() wffnames = None wffnames_syn = None + wfdir = 'processed' if 'processed' in [fname.split('/')[-1] for fname in fnames] else 'raw' if obspy_dmt: for fpath in fnames: - if fpath.endswith('raw'): + if fpath.endswith(wfdir): wffnames = [os.path.join(fpath, fname) for fname in os.listdir(fpath)] if 'syngine' in fpath.split('/')[-1]: wffnames_syn = [os.path.join(fpath, fname) for fname in os.listdir(fpath)] diff --git a/pylot/core/util/widgets.py b/pylot/core/util/widgets.py index 98b50fb1..637f178c 100644 --- a/pylot/core/util/widgets.py +++ b/pylot/core/util/widgets.py @@ -25,6 +25,7 @@ except ImportError: from matplotlib.backends.backend_qt4agg import NavigationToolbar2QT from matplotlib.widgets import MultiCursor from matplotlib.tight_layout import get_renderer, get_subplotspec_list, get_tight_layout_figure +from scipy.signal import argrelmin, argrelmax from PySide import QtCore, QtGui from PySide.QtGui import QAction, QApplication, QCheckBox, QComboBox, \ @@ -539,9 +540,9 @@ class WaveformWidgetPG(QtGui.QWidget): n+=1 st = st_select.select(network=network, station=station, channel=channel) st_syn = wfsyn.select(network=network, station=station, channel=channel) - trace = st[0] + trace = st[0].copy() if st_syn: - trace_syn = st_syn[0] + trace_syn = st_syn[0].copy() if mapping: comp = channel[-1] n = compclass.getPlotPosition(str(comp)) @@ -552,23 +553,46 @@ class WaveformWidgetPG(QtGui.QWidget): msg = 'plotting %s channel of station %s' % (channel, station) print(msg) stime = trace.stats.starttime - self.wfstart + time_ax = prepTimeAxis(stime, trace) if st_syn: stime_syn = trace_syn.stats.starttime - self.wfstart time_ax_syn = prepTimeAxis(stime_syn, trace_syn) - if time_ax is not None: + + # TEST TEST max/min plot + minsamples = argrelmin(trace.data) + maxsamples = argrelmax(trace.data) + plot_samples = np.sort(np.append(minsamples, maxsamples)) + trace.data = trace.data[plot_samples] + time_ax = time_ax[plot_samples] + if st_syn: + minsamples_syn = argrelmin(trace_syn.data) + maxsamples_syn = argrelmax(trace_syn.data) + plot_samples_syn = np.sort(np.append(minsamples_syn, maxsamples_syn)) + trace_syn.data = trace_syn.data[plot_samples_syn] + time_ax_syn = time_ax_syn[plot_samples_syn] + + # TEST TEST Remove middle of seismograms for overview plot + # trace.data = trace.data[abs(trace.data) > 0.1 * max(abs(trace.data))] + # if st_syn: + # trace.data_syn = trace.data_syn[abs(trace.data_syn) > 0.1 * max(abs(trace.data))] + # TEST TEST ------ + + if time_ax not in [None, []]: if not scaleddata: trace.detrend('constant') trace.normalize(np.max(np.abs(trace.data)) * 2) if st_syn: trace_syn.detrend('constant') trace_syn.normalize(np.max(np.abs(trace_syn.data)) * 2) - times = [time for index, time in enumerate(time_ax) if not index % nth_sample] - times_syn = [time for index, time in enumerate(time_ax_syn) if not index % nth_sample] if st_syn else [] - data = [datum + n for index, datum in enumerate(trace.data) if not index % nth_sample] - data_syn = [datum + n for index, datum in enumerate(trace_syn.data) - if not index % nth_sample] if st_syn else [] - plots.append((times, data, times_syn, data_syn)) + # TODO: change this to numpy operations instead of lists? + times = np.array([time for index, time in enumerate(time_ax) if not index % nth_sample]) + times_syn = np.array([time for index, time in enumerate(time_ax_syn) if not index % nth_sample] if st_syn else []) + trace.data = np.array([datum + n for index, datum in enumerate(trace.data) if not index % nth_sample]) + trace.data_syn = np.array([datum + n for index, datum in enumerate(trace.data_syn) + if not index % nth_sample] if st_syn else []) + plots.append((times, trace.data, + times_syn, trace.data_syn)) self.setPlotDict(n, (station, channel, network)) self.xlabel = 'seconds since {0}'.format(self.wfstart) self.ylabel = '' From f0b6897053c11a9e27db439a0ba0365d6d278c5b Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 18 Apr 2018 16:01:39 +0200 Subject: [PATCH 04/11] [bugfix] identity check of np.array --- PyLoT.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyLoT.py b/PyLoT.py index 4e2ddd41..74faabba 100755 --- a/PyLoT.py +++ b/PyLoT.py @@ -1752,7 +1752,7 @@ class MainWindow(QMainWindow): for times, data, times_syn, data_syn in plots: self.dataPlot.plotWidget.getPlotItem().plot(times, data, pen=self.dataPlot.pen_linecolor) - if data_syn: + if len(data_syn) > 0: self.dataPlot.plotWidget.getPlotItem().plot(times_syn, data_syn, pen=self.dataPlot.pen_linecolor_syn) self.dataPlot.reinitMoveProxy() From 5fcb07e6476fb6a7ae0984a7d57c9ed60e2ffe4a Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 18 Apr 2018 16:17:16 +0200 Subject: [PATCH 05/11] [change] prepTimeAxis using linspace (is that ok?) --- pylot/core/util/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylot/core/util/utils.py b/pylot/core/util/utils.py index ba6656f7..79872e31 100644 --- a/pylot/core/util/utils.py +++ b/pylot/core/util/utils.py @@ -592,7 +592,7 @@ def prepTimeAxis(stime, trace, verbosity=0): srate = trace.stats.sampling_rate tincr = trace.stats.delta etime = stime + nsamp / srate - time_ax = np.arange(stime, etime, tincr) + time_ax = np.linspace(stime, etime, nsamp) if len(time_ax) < nsamp: if verbosity: print('elongate time axes by one datum') From e72111a6fb1538b959e411fefc6bcd168ec65e4d Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 18 Apr 2018 16:17:54 +0200 Subject: [PATCH 06/11] [add] obspy-like min/max function, removed testing area --- pylot/core/util/widgets.py | 58 +++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/pylot/core/util/widgets.py b/pylot/core/util/widgets.py index 637f178c..37d5e004 100644 --- a/pylot/core/util/widgets.py +++ b/pylot/core/util/widgets.py @@ -35,7 +35,7 @@ from PySide.QtGui import QAction, QApplication, QCheckBox, QComboBox, \ QPushButton, QFileDialog, QInputDialog, QKeySequence from PySide.QtCore import QSettings, Qt, QUrl, Signal, Slot from PySide.QtWebKit import QWebView -from obspy import Stream, UTCDateTime +from obspy import Stream, Trace, UTCDateTime from obspy.core.util import AttribDict from obspy.taup import TauPyModel from obspy.taup.utils import get_phase_names @@ -540,7 +540,6 @@ class WaveformWidgetPG(QtGui.QWidget): n+=1 st = st_select.select(network=network, station=station, channel=channel) st_syn = wfsyn.select(network=network, station=station, channel=channel) - trace = st[0].copy() if st_syn: trace_syn = st_syn[0].copy() if mapping: @@ -553,31 +552,11 @@ class WaveformWidgetPG(QtGui.QWidget): msg = 'plotting %s channel of station %s' % (channel, station) print(msg) stime = trace.stats.starttime - self.wfstart - time_ax = prepTimeAxis(stime, trace) if st_syn: stime_syn = trace_syn.stats.starttime - self.wfstart time_ax_syn = prepTimeAxis(stime_syn, trace_syn) - # TEST TEST max/min plot - minsamples = argrelmin(trace.data) - maxsamples = argrelmax(trace.data) - plot_samples = np.sort(np.append(minsamples, maxsamples)) - trace.data = trace.data[plot_samples] - time_ax = time_ax[plot_samples] - if st_syn: - minsamples_syn = argrelmin(trace_syn.data) - maxsamples_syn = argrelmax(trace_syn.data) - plot_samples_syn = np.sort(np.append(minsamples_syn, maxsamples_syn)) - trace_syn.data = trace_syn.data[plot_samples_syn] - time_ax_syn = time_ax_syn[plot_samples_syn] - - # TEST TEST Remove middle of seismograms for overview plot - # trace.data = trace.data[abs(trace.data) > 0.1 * max(abs(trace.data))] - # if st_syn: - # trace.data_syn = trace.data_syn[abs(trace.data_syn) > 0.1 * max(abs(trace.data))] - # TEST TEST ------ - if time_ax not in [None, []]: if not scaleddata: trace.detrend('constant') @@ -600,6 +579,41 @@ class WaveformWidgetPG(QtGui.QWidget): self.setYLims([0.5, nmax + 0.5]) return plots + def minMax(self, trace, time_ax): + ''' + create min/max array for fast plotting (approach based on obspy __plot_min_max function) + :returns data, time_ax + ''' + npixel = self.width() + ndata = len(trace.data) + pts_per_pixel = ndata/npixel + if pts_per_pixel < 2: + return trace.data, time_ax + remaining_samples = ndata%pts_per_pixel + npixel = ndata//pts_per_pixel + if remaining_samples: + data = trace.data[:-remaining_samples] + else: + data = trace.data + data = data.reshape(npixel, pts_per_pixel) + min_ = data.min(axis=1) + max_ = data.max(axis=1) + if remaining_samples: + extreme_values = np.empty((npixel + 1, 2), dtype=np.float) + extreme_values[:-1, 0] = min_ + extreme_values[:-1, 1] = max_ + extreme_values[-1, 0] = \ + trace.data[-remaining_samples:].min() + extreme_values[-1, 1] = \ + trace.data[-remaining_samples:].max() + else: + extreme_values = np.empty((npixel, 2), dtype=np.float) + extreme_values[:, 0] = min_ + extreme_values[:, 1] = max_ + data = extreme_values.flatten() + time_ax = np.linspace(time_ax[0], time_ax[-1], num=len(data)) + return data, time_ax + # def getAxes(self): # return self.axes From 726210daeb450370dfa6ec7ce0bc5b8f0929f7d4 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 18 Apr 2018 16:21:27 +0200 Subject: [PATCH 07/11] [add] plot-method = 'fast' for min/max plot --- pylot/core/util/widgets.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pylot/core/util/widgets.py b/pylot/core/util/widgets.py index 37d5e004..5d3afb0f 100644 --- a/pylot/core/util/widgets.py +++ b/pylot/core/util/widgets.py @@ -496,7 +496,8 @@ class WaveformWidgetPG(QtGui.QWidget): def plotWFData(self, wfdata, wfsyn=None, title=None, zoomx=None, zoomy=None, noiselevel=None, scaleddata=False, mapping=True, - component='*', nth_sample=1, iniPick=None, verbosity=0): + component='*', nth_sample=1, iniPick=None, verbosity=0, + method='normal'): if not wfdata: print('Nothing to plot.') return @@ -557,6 +558,9 @@ class WaveformWidgetPG(QtGui.QWidget): stime_syn = trace_syn.stats.starttime - self.wfstart time_ax_syn = prepTimeAxis(stime_syn, trace_syn) + if method == 'fast': + trace.data, time_ax = self.minMax(trace, time_ax) + if time_ax not in [None, []]: if not scaleddata: trace.detrend('constant') From 349715d13c5997c808f2ff8fe7c586bddbe148c9 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 18 Apr 2018 16:21:57 +0200 Subject: [PATCH 08/11] [removed] nth_sample dialog --- PyLoT.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/PyLoT.py b/PyLoT.py index 74faabba..82d46ae5 100755 --- a/PyLoT.py +++ b/PyLoT.py @@ -1673,24 +1673,24 @@ class MainWindow(QMainWindow): npts2plot = npts/nth_sample if npts2plot < npts_max: return - nth_sample_new = int(np.ceil(npts/npts_max)) - message = "You are about to plot a huge dataset with {npts} datapoints. With a current setting of " \ - "nth_sample = {nth_sample} a total of {npts2plot} points will be plotted which is more " \ - "than the maximum setting of {npts_max}. " \ - "PyLoT recommends to raise nth_sample from {nth_sample} to {nth_sample_new}. Do you want "\ - "to change nth_sample to {nth_sample_new} now?" - - ans = QMessageBox.question(self, self.tr("Optimize plot performance..."), - self.tr(message.format(npts=npts, - nth_sample=nth_sample, - npts_max=npts_max, - nth_sample_new=nth_sample_new, - npts2plot=npts2plot)), - QMessageBox.Yes | QMessageBox.No, - QMessageBox.Yes) - if ans == QMessageBox.Yes: - settings.setValue("nth_sample", nth_sample_new) - settings.sync() + # nth_sample_new = int(np.ceil(npts/npts_max)) + # message = "You are about to plot a huge dataset with {npts} datapoints. With a current setting of " \ + # "nth_sample = {nth_sample} a total of {npts2plot} points will be plotted which is more " \ + # "than the maximum setting of {npts_max}. " \ + # "PyLoT recommends to raise nth_sample from {nth_sample} to {nth_sample_new}. Do you want "\ + # "to change nth_sample to {nth_sample_new} now?" + # + # ans = QMessageBox.question(self, self.tr("Optimize plot performance..."), + # self.tr(message.format(npts=npts, + # nth_sample=nth_sample, + # npts_max=npts_max, + # nth_sample_new=nth_sample_new, + # npts2plot=npts2plot)), + # QMessageBox.Yes | QMessageBox.No, + # QMessageBox.Yes) + # if ans == QMessageBox.Yes: + # settings.setValue("nth_sample", nth_sample_new) + # settings.sync() def get_npts_to_plot(self): return sum(trace.stats.npts for trace in self.data.getWFData()) From af54cb0d4bc65df1ecd38aa631da3771a87baea6 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 18 Apr 2018 16:44:33 +0200 Subject: [PATCH 09/11] [add] auto set plotmethod 'fast' for large dataset --- PyLoT.py | 14 +++++++++++--- pylot/core/util/widgets.py | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/PyLoT.py b/PyLoT.py index 82d46ae5..8ee4def3 100755 --- a/PyLoT.py +++ b/PyLoT.py @@ -79,7 +79,7 @@ from pylot.core.util.utils import fnConstructor, getLogin, \ full_range, readFilterInformation, trim_station_components, check4gaps, make_pen, pick_color_plt, \ pick_linestyle_plt, remove_underscores, check4doubled, identifyPhaseID, excludeQualityClasses, has_spe, \ check4rotated, transform_colors_mpl, transform_colors_mpl_str, getAutoFilteroptions, check_all_obspy, \ - check_all_pylot + check_all_pylot, real_Bool from pylot.core.util.event import Event from pylot.core.io.location import create_creation_info, create_event from pylot.core.util.widgets import FilterOptionsDialog, NewEventDlg, \ @@ -1672,7 +1672,11 @@ class MainWindow(QMainWindow): npts = self.get_npts_to_plot() npts2plot = npts/nth_sample if npts2plot < npts_max: - return + settings.setValue('large_dataset', False) + else: + settings.setValue('large_dataset', True) + self.update_status('Dataset is very large. Using fast plotting method (MIN/MAX)', 10000) + settings.sync() # nth_sample_new = int(np.ceil(npts/npts_max)) # message = "You are about to plot a huge dataset with {npts} datapoints. With a current setting of " \ # "nth_sample = {nth_sample} a total of {npts2plot} points will be plotted which is more " \ @@ -1891,8 +1895,12 @@ class MainWindow(QMainWindow): # wfst += self.get_data().getWFData().select(component=alter_comp) plotWidget = self.getPlotWidget() self.adjustPlotHeight() + if real_Bool(settings.value('large_dataset')) == True: + method = 'fast' + else: + method = 'normal' plots = plotWidget.plotWFData(wfdata=wfst, wfsyn=wfsyn, title=title, mapping=False, component=comp, - nth_sample=int(nth_sample)) + nth_sample=int(nth_sample), method=method) return plots def adjustPlotHeight(self): diff --git a/pylot/core/util/widgets.py b/pylot/core/util/widgets.py index 5d3afb0f..46ecb1dc 100644 --- a/pylot/core/util/widgets.py +++ b/pylot/core/util/widgets.py @@ -540,6 +540,7 @@ class WaveformWidgetPG(QtGui.QWidget): for n, (network, station, channel) in enumerate(nsc): n+=1 st = st_select.select(network=network, station=station, channel=channel) + trace = st[0].copy() st_syn = wfsyn.select(network=network, station=station, channel=channel) if st_syn: trace_syn = st_syn[0].copy() From 964aa9ce6cc65c8d5ea00f8a0e059b2d5383b7d3 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 18 Apr 2018 16:46:21 +0200 Subject: [PATCH 10/11] [change] set downsampling to false again (creating errors when plotting picks) --- pylot/core/util/widgets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylot/core/util/widgets.py b/pylot/core/util/widgets.py index 46ecb1dc..c17bdb4f 100644 --- a/pylot/core/util/widgets.py +++ b/pylot/core/util/widgets.py @@ -463,7 +463,7 @@ class WaveformWidgetPG(QtGui.QWidget): self.pen_linecolor_syn = self.pg.mkPen((100, 0, 255, 255)) self.reinitMoveProxy() self._proxy = self.pg.SignalProxy(self.plotWidget.scene().sigMouseMoved, rateLimit=60, slot=self.mouseMoved) - self.plotWidget.getPlotItem().setDownsampling(auto=True) + #self.plotWidget.getPlotItem().setDownsampling(auto=True) def reinitMoveProxy(self): self.vLine = self.pg.InfiniteLine(angle=90, movable=False, pen=self.pen_multicursor) From a8763843384314e0efb4b9d0dea58f79129ad6d3 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 19 Apr 2018 13:38:17 +0200 Subject: [PATCH 11/11] [add] info for user on min/max plot --- PyLoT.py | 11 ++++++++--- pylot/core/util/widgets.py | 14 +++++++++++--- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/PyLoT.py b/PyLoT.py index 8ee4def3..cd3c849d 100755 --- a/PyLoT.py +++ b/PyLoT.py @@ -147,6 +147,7 @@ class MainWindow(QMainWindow): # default factor for dataplot e.g. enabling/disabling scrollarea self.height_factor = 12 + self.plot_method = 'normal' # UI has to be set up before(!) children widgets are about to show up self.createAction = createAction @@ -1896,11 +1897,11 @@ class MainWindow(QMainWindow): plotWidget = self.getPlotWidget() self.adjustPlotHeight() if real_Bool(settings.value('large_dataset')) == True: - method = 'fast' + self.plot_method = 'fast' else: - method = 'normal' + self.plot_method = 'normal' plots = plotWidget.plotWFData(wfdata=wfst, wfsyn=wfsyn, title=title, mapping=False, component=comp, - nth_sample=int(nth_sample), method=method) + nth_sample=int(nth_sample), method=self.plot_method) return plots def adjustPlotHeight(self): @@ -3174,6 +3175,10 @@ class MainWindow(QMainWindow): def draw(self): self.fill_eventbox() self.getPlotWidget().draw() + if self.plot_method == 'fast': + self.dataPlot.setPermText('MIN/MAX plot', color='red') + else: + self.dataPlot.setPermText() def _setDirty(self): self.setDirty(True) diff --git a/pylot/core/util/widgets.py b/pylot/core/util/widgets.py index c17bdb4f..56dc16dd 100644 --- a/pylot/core/util/widgets.py +++ b/pylot/core/util/widgets.py @@ -449,11 +449,15 @@ class WaveformWidgetPG(QtGui.QWidget): self.plotdict = dict() # create plot self.main_layout = QtGui.QVBoxLayout() - self.label = QtGui.QLabel() + self.label_layout = QtGui.QHBoxLayout() + self.status_label = QtGui.QLabel() + self.perm_label = QtGui.QLabel() self.setLayout(self.main_layout) self.plotWidget = self.pg.PlotWidget(self.parent(), title=title) self.main_layout.addWidget(self.plotWidget) - self.main_layout.addWidget(self.label) + self.main_layout.addLayout(self.label_layout) + self.label_layout.addWidget(self.status_label) + self.label_layout.addWidget(self.perm_label) self.plotWidget.showGrid(x=False, y=True, alpha=0.3) self.plotWidget.hideAxis('bottom') self.plotWidget.hideAxis('left') @@ -481,13 +485,17 @@ class WaveformWidgetPG(QtGui.QWidget): station = self.orig_parent.getStationName(wfID) abstime = self.wfstart + x if self.orig_parent.get_current_event(): - self.label.setText("station = {}, T = {}, t = {} [s]".format(station, abstime, x)) + self.status_label.setText("station = {}, T = {}, t = {} [s]".format(station, abstime, x)) self.vLine.setPos(mousePoint.x()) self.hLine.setPos(mousePoint.y()) def getPlotDict(self): return self.plotdict + def setPermText(self, text=None, color='black'): + self.perm_label.setText(text) + self.perm_label.setStyleSheet('color: {}'.format(color)) + def setPlotDict(self, key, value): self.plotdict[key] = value