From f13542840fb1a28f11caebd5df8564e5830b3a0a Mon Sep 17 00:00:00 2001 From: Marcel Paffrath Date: Mon, 31 Jul 2017 10:30:53 +0200 Subject: [PATCH 1/3] [add] MultiThread class added --- pylot/core/util/thread.py | 65 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/pylot/core/util/thread.py b/pylot/core/util/thread.py index 5928ace3..81836da0 100644 --- a/pylot/core/util/thread.py +++ b/pylot/core/util/thread.py @@ -96,3 +96,68 @@ class Thread(QThread): def flush(self): pass + +class MultiThread(QThread): + finished = Signal(str) + message = Signal(str) + + def __init__(self, parent, func, args, ncores=1, + progressText=None, pb_widget=None, redirect_stdout=False): + QThread.__init__(self, parent) + self.func = func + self.args = args + self.progressText = progressText + self.pb_widget = pb_widget + self.redirect_stdout = redirect_stdout + self.finished.connect(self.hideProgressbar) + self.showProgressbar() + + def run(self): + if self.redirect_stdout: + sys.stdout = self + try: + pool = multiprocessing.Pool(self.ncores) + self.data = pool.map_async(self.func, self.args, callback=self.emitDone) + #self.data = pool.apply_async(self.func, self.shotlist, callback=self.emitDone) #emit each time returned + pool.close() + self._executed = True + return result + except Exception as e: + self._executed = False + self._executedError = e + exc_type, exc_obj, exc_tb = sys.exc_info() + fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] + print('Exception: {}, file: {}, line: {}'.format(exc_type, fname, exc_tb.tb_lineno)) + sys.stdout = sys.__stdout__ + + def __del__(self): + self.wait() + + def showProgressbar(self): + if self.progressText: + if not self.pb_widget: + self.pb_widget = QDialog(self.parent()) + self.pb_widget.setWindowFlags(Qt.SplashScreen) + self.pb_widget.setModal(True) + hl = QHBoxLayout() + pb = QProgressBar() + pb.setRange(0, 0) + hl.addWidget(pb) + hl.addWidget(QLabel(self.progressText)) + self.pb_widget.setLayout(hl) + self.pb_widget.show() + + def hideProgressbar(self): + if self.pb_widget: + self.pb_widget.hide() + + def write(self, text): + self.message.emit(text) + + def flush(self): + pass + + def emitDone(self, result): + print('emitDone!') + self.finished.emit('Done thread!') + self.hideProgressBar() From 121ef4357f8f2085749f5519c86b2c268c279db3 Mon Sep 17 00:00:00 2001 From: Marcel Paffrath Date: Mon, 31 Jul 2017 13:43:06 +0200 Subject: [PATCH 2/3] [WIP] using multiThread to call autopylot --- QtPyLoT.py | 82 +++++++++++++++++++++++++++++++-------- autoPyLoT.py | 4 +- pylot/RELEASE-VERSION | 2 +- pylot/core/util/thread.py | 50 ++++++++++++++---------- 4 files changed, 99 insertions(+), 39 deletions(-) diff --git a/QtPyLoT.py b/QtPyLoT.py index 7a18a9d1..34a7e22f 100755 --- a/QtPyLoT.py +++ b/QtPyLoT.py @@ -81,7 +81,7 @@ from pylot.core.util.widgets import FilterOptionsDialog, NewEventDlg, \ getDataType, ComparisonDialog, TuneAutopicker, PylotParaBox from pylot.core.util.map_projection import map_projection from pylot.core.util.structure import DATASTRUCTURE -from pylot.core.util.thread import AutoPickThread, Thread +from pylot.core.util.thread import AutoPickThread, Thread, MultiThread from pylot.core.util.version import get_git_version as _getVersionString if sys.version_info.major == 3: @@ -1838,6 +1838,36 @@ class MainWindow(QMainWindow): self.canvas_dict[key] = FigureCanvas(self.fig_dict[key]) self.tap.fill_tabs(picked=True) + # def autoPick(self): + # self.autosave = QFileDialog().getExistingDirectory(caption='Select autoPyLoT output') + # if not os.path.exists(self.autosave): + # QMessageBox.warning(self, "PyLoT Warning", + # "No autoPyLoT output declared!") + # return + # self.listWidget = QListWidget() + # self.setDirty(True) + # self.logDockWidget = QDockWidget("AutoPickLog", self) + # self.logDockWidget.setObjectName("LogDockWidget") + # self.logDockWidget.setAllowedAreas( + # Qt.LeftDockWidgetArea | Qt.RightDockWidgetArea) + # self.logDockWidget.setWidget(self.listWidget) + # self.addDockWidget(Qt.LeftDockWidgetArea, self.logDockWidget) + # self.addListItem('Loading default values from PyLoT-input file %s' + # % self.infile) + # autopick_parameter = self._inputs + # self.addListItem(str(autopick_parameter)) + # receventid = self.get_current_event_path() + # self.thread = AutoPickThread(parent=self, + # func=autoPyLoT, + # infile=self.infile, + # fnames=self.fnames, + # eventid=receventid, + # savepath=self.autosave) + + # self.thread.message.connect(self.addListItem) + # self.thread.start() + # self.thread.finished.connect(self.finalizeAutoPick) + def autoPick(self): self.autosave = QFileDialog().getExistingDirectory(caption='Select autoPyLoT output') if not os.path.exists(self.autosave): @@ -1852,26 +1882,44 @@ class MainWindow(QMainWindow): Qt.LeftDockWidgetArea | Qt.RightDockWidgetArea) self.logDockWidget.setWidget(self.listWidget) self.addDockWidget(Qt.LeftDockWidgetArea, self.logDockWidget) - self.addListItem('Loading default values from PyLoT-input file %s' - % self.infile) - autopick_parameter = self._inputs - self.addListItem(str(autopick_parameter)) - receventid = self.get_current_event_path() - self.thread = AutoPickThread(parent=self, - func=autoPyLoT, - infile=self.infile, - fnames=self.fnames, - eventid=receventid, - savepath=self.autosave) + # self.addListItem('Loading default values from PyLoT-input file %s' + # % self.infile) - self.thread.message.connect(self.addListItem) - self.thread.start() - self.thread.finished.connect(self.finalizeAutoPick) + stations = [] + # catch all station names + for trace in self.data.getWFData(): + station = trace.stats.station + if not station in stations: + stations.append(station) + + mp_args = [] + # create input_dict for each station in a list for multiprocessing.Pool iteration + for station in stations: + args = {'parameter': self._inputs, + 'station': station, + 'fnames': 'None', + 'eventid': self.get_current_event_path (), + 'iplot': 0, + 'fig_dict': None, + 'locflag': 0} + mp_args.append(args) + + self.mp_thread = MultiThread (self, autoPyLoT, args=mp_args, + ncores=0, + progressText='Picking event...', + pb_widget=None, + redirect_stdout=True) + + self.addListItem(str(self._inputs)) + + self.mp_thread.message.connect(self.addListItem) + self.mp_thread.start() + self.mp_thread.finished.connect(self.finalizeAutoPick) def finalizeAutoPick(self): self.drawPicks(picktype='auto') self.draw() - self.thread.quit() + self.mp_thread.quit() def addPicks(self, station, picks, type='manual'): stat_picks = self.getPicksOnStation(station, type) @@ -2218,7 +2266,7 @@ class MainWindow(QMainWindow): # generate delete icon del_icon = QIcon() del_icon.addPixmap(QPixmap(':/icons/delete.png')) - + # remove old table if hasattr(self, 'event_table'): self.event_table.setParent(None) diff --git a/autoPyLoT.py b/autoPyLoT.py index a02ff306..6f39424e 100755 --- a/autoPyLoT.py +++ b/autoPyLoT.py @@ -66,6 +66,8 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even station = input_dict['station'] if input_dict.has_key('fnames'): fnames = input_dict['fnames'] + if input_dict.has_key('eventid'): + eventid = input_dict['eventid'] if input_dict.has_key('iplot'): iplot = input_dict['iplot'] if input_dict.has_key('locflag'): @@ -155,7 +157,7 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even events = [] events.append(os.path.join(datapath, parameter['database'], - parameter['eventID'])) + eventid)) if not events: print('autoPyLoT: No events given. Return!') diff --git a/pylot/RELEASE-VERSION b/pylot/RELEASE-VERSION index 78f9d87f..518a60d7 100644 --- a/pylot/RELEASE-VERSION +++ b/pylot/RELEASE-VERSION @@ -1 +1 @@ -04d4-dirty +f135-dirty diff --git a/pylot/core/util/thread.py b/pylot/core/util/thread.py index 81836da0..26c0c508 100644 --- a/pylot/core/util/thread.py +++ b/pylot/core/util/thread.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- import sys, os +import multiprocessing from PySide.QtCore import QThread, Signal, Qt -from PySide.QtGui import QDialog, QProgressBar, QLabel, QHBoxLayout +from PySide.QtGui import QDialog, QProgressBar, QLabel, QHBoxLayout, QPushButton class AutoPickThread(QThread): @@ -69,20 +70,27 @@ class Thread(QThread): print('Exception: {}, file: {}, line: {}'.format(exc_type, fname, exc_tb.tb_lineno)) sys.stdout = sys.__stdout__ - def __del__(self): - self.wait() + # def __del__(self): + # self.wait() def showProgressbar(self): if self.progressText: + + # generate widget if not given in init if not self.pb_widget: self.pb_widget = QDialog(self.parent()) self.pb_widget.setWindowFlags(Qt.SplashScreen) self.pb_widget.setModal(True) + + # add button + delete_button = QPushButton('X') + delete_button.clicked.connect(self.exit) hl = QHBoxLayout() pb = QProgressBar() pb.setRange(0, 0) hl.addWidget(pb) hl.addWidget(QLabel(self.progressText)) + hl.addWidget(delete_button) self.pb_widget.setLayout(hl) self.pb_widget.show() @@ -101,11 +109,12 @@ class MultiThread(QThread): finished = Signal(str) message = Signal(str) - def __init__(self, parent, func, args, ncores=1, + def __init__(self, parent, func, args, ncores=0, progressText=None, pb_widget=None, redirect_stdout=False): QThread.__init__(self, parent) self.func = func self.args = args + self.ncores = ncores self.progressText = progressText self.pb_widget = pb_widget self.redirect_stdout = redirect_stdout @@ -113,21 +122,22 @@ class MultiThread(QThread): self.showProgressbar() def run(self): - if self.redirect_stdout: - sys.stdout = self - try: - pool = multiprocessing.Pool(self.ncores) - self.data = pool.map_async(self.func, self.args, callback=self.emitDone) - #self.data = pool.apply_async(self.func, self.shotlist, callback=self.emitDone) #emit each time returned - pool.close() - self._executed = True - return result - except Exception as e: - self._executed = False - self._executedError = e - exc_type, exc_obj, exc_tb = sys.exc_info() - fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] - print('Exception: {}, file: {}, line: {}'.format(exc_type, fname, exc_tb.tb_lineno)) + # if self.redirect_stdout: + # sys.stdout = self + # #try: + if not self.ncores: + self.ncores = multiprocessing.cpu_count() + pool = multiprocessing.Pool(self.ncores) + self.data = pool.map_async(self.func, self.args, callback=self.emitDone) + #self.data = pool.apply_async(self.func, self.shotlist, callback=self.emitDone) #emit each time returned + pool.close() + self._executed = True + # except Exception as e: + # self._executed = False + # self._executedError = e + # exc_type, exc_obj, exc_tb = sys.exc_info() + # fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] + # print('Exception: {}, file: {}, line: {}'.format(exc_type, fname, exc_tb.tb_lineno)) sys.stdout = sys.__stdout__ def __del__(self): @@ -160,4 +170,4 @@ class MultiThread(QThread): def emitDone(self, result): print('emitDone!') self.finished.emit('Done thread!') - self.hideProgressBar() + self.hideProgressbar() From 7ca6509611074384ae020a6169607d7fdb7e51d1 Mon Sep 17 00:00:00 2001 From: marcel Date: Mon, 31 Jul 2017 14:56:59 +0200 Subject: [PATCH 3/3] [bugfix] has_keys not a method of dict in python3 --- pylot/core/io/inputs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylot/core/io/inputs.py b/pylot/core/io/inputs.py index f49d9405..1bb1d2a8 100644 --- a/pylot/core/io/inputs.py +++ b/pylot/core/io/inputs.py @@ -107,7 +107,7 @@ class PylotParameter(object): yield key, value def hasParam(self, parameter): - if self.__parameter.has_key(parameter): + if parameter in self.__parameter.keys(): return True return False