From 154e7b459f2db8bb8a8a4192e9c72c34391d3462 Mon Sep 17 00:00:00 2001 From: marcel Date: Thu, 3 Aug 2017 11:49:15 +0200 Subject: [PATCH] [add] --ncores flag added to autoPyLoT call, also enabled in GUI widget --- autoPyLoT.py | 32 ++++++++++++++++++++++++------- pylot/core/pick/autopick.py | 4 ++-- pylot/core/util/dataprocessing.py | 4 ++-- pylot/core/util/utils.py | 8 ++++++-- pylot/core/util/widgets.py | 13 +++++++------ 5 files changed, 42 insertions(+), 19 deletions(-) diff --git a/autoPyLoT.py b/autoPyLoT.py index 10bc4489..f3ec8fd5 100755 --- a/autoPyLoT.py +++ b/autoPyLoT.py @@ -34,7 +34,7 @@ __version__ = _getVersionString() def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, eventid=None, savepath=None, station='all', - iplot=0): + iplot=0, ncores=0): """ Determine phase onsets automatically utilizing the automatic picking algorithms by Kueperkoch et al. 2010/2012. @@ -48,6 +48,16 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even .. rubric:: Example """ + + if ncores == 1: + sp_info = 'autoPyLoT is running serial on 1 cores.' + else: + if ncores == 0: + ncores_readable = 'all available' + else: + ncores_readable = ncores + sp_info = 'autoPyLoT is running in parallel on {} cores.'.format(ncores_readable) + splash = '''************************************\n *********autoPyLoT starting*********\n The Python picking and Location Tool\n @@ -57,9 +67,13 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even L. Kueperkoch (BESTEC GmbH, Landau i. d. Pfalz)\n S. Wehling-Benatelli (Ruhr-Universitaet Bochum)\n M. Paffrath (Ruhr-Universitaet Bochum)\n - ***********************************'''.format(version=_getVersionString()) + + {sp} + ***********************************'''.format(version=_getVersionString(), + sp=sp_info) print(splash) + parameter = real_None(parameter) inputfile = real_None(inputfile) eventid = real_None(eventid) @@ -84,7 +98,7 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even if not parameter: if inputfile: parameter = PylotParameter(inputfile) - iplot = parameter['iplot'] + #iplot = parameter['iplot'] else: infile = os.path.join(os.path.expanduser('~'), '.pylot', 'pylot.in') print('Using default input file {}'.format(infile)) @@ -216,7 +230,7 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even wfdat = remove_underscores(wfdat) metadata = read_metadata(parameter.get('invdir')) print("Restitute data ...") - corr_dat = restitute_data(wfdat.copy(), *metadata) + corr_dat = restitute_data(wfdat.copy(), *metadata, ncores=ncores) if not corr_dat and locflag: locflag = 2 print('Working on event %s. Stations: %s' % (event, station)) @@ -226,9 +240,9 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even if input_dict: if 'fig_dict' in input_dict: fig_dict = input_dict['fig_dict'] - picks = autopickevent(wfdat, parameter, iplot=iplot, fig_dict=fig_dict) + picks = autopickevent(wfdat, parameter, iplot=iplot, fig_dict=fig_dict, ncores=ncores) else: - picks = autopickevent(wfdat, parameter, iplot=iplot) + picks = autopickevent(wfdat, parameter, iplot=iplot, ncores=ncores) ########################################################## # locating if locflag > 0: @@ -442,6 +456,9 @@ if __name__ == "__main__": parser.add_argument('-s', '-S', '--spath', type=str, action='store', help='''optional, save path for autoPyLoT output''') + parser.add_argument('-c', '-C', '--ncores', type=int, + action='store', default=0, + help='''optional, number of CPU cores used for parallel processing (default: all available)''') # parser.add_argument('-v', '-V', '--version', action='version', # version='autoPyLoT ' + __version__, # help='show version information and exit') @@ -449,4 +466,5 @@ if __name__ == "__main__": cla = parser.parse_args() picks = autoPyLoT(inputfile=str(cla.inputfile), fnames=str(cla.fnames), - eventid=str(cla.eventid), savepath=str(cla.spath)) + eventid=str(cla.eventid), savepath=str(cla.spath), + ncores=cla.ncores) diff --git a/pylot/core/pick/autopick.py b/pylot/core/pick/autopick.py index 32d1e9a1..2f396dee 100644 --- a/pylot/core/pick/autopick.py +++ b/pylot/core/pick/autopick.py @@ -21,7 +21,7 @@ from pylot.core.pick.utils import checksignallength, checkZ4S, earllatepicker, \ from pylot.core.util.utils import getPatternLine, gen_Pool -def autopickevent(data, param, iplot=0, fig_dict=None): +def autopickevent(data, param, iplot=0, fig_dict=None, ncores=0): stations = [] all_onsets = {} input_tuples = [] @@ -50,7 +50,7 @@ def autopickevent(data, param, iplot=0, fig_dict=None): print('iPlot Flag active: NO MULTIPROCESSING possible.') return all_onsets - pool = gen_Pool() + pool = gen_Pool(ncores) result = pool.map(call_autopickstation, input_tuples) pool.close() diff --git a/pylot/core/util/dataprocessing.py b/pylot/core/util/dataprocessing.py index 3f6caf3a..1d13e952 100644 --- a/pylot/core/util/dataprocessing.py +++ b/pylot/core/util/dataprocessing.py @@ -269,7 +269,7 @@ def restitute_trace(input_tuple): return tr, remove_trace -def restitute_data(data, invtype, inobj, unit='VEL', force=False): +def restitute_data(data, invtype, inobj, unit='VEL', force=False, ncores=0): """ takes a data stream and a path_to_inventory and returns the corrected waveform data stream @@ -293,7 +293,7 @@ def restitute_data(data, invtype, inobj, unit='VEL', force=False): input_tuples.append((tr, invtype, inobj, unit, force)) data.remove(tr) - pool = gen_Pool() + pool = gen_Pool(ncores) result = pool.map(restitute_trace, input_tuples) pool.close() diff --git a/pylot/core/util/utils.py b/pylot/core/util/utils.py index 381e9271..c21f3e7a 100644 --- a/pylot/core/util/utils.py +++ b/pylot/core/util/utils.py @@ -33,10 +33,14 @@ def getindexbounds(f, eta): return mi, l, u -def gen_Pool(ncores='max'): +def gen_Pool(ncores=0): + ''' + :param ncores: number of CPU cores for multiprocessing.Pool, if ncores == 0 use all available + :return: multiprocessing.Pool object + ''' import multiprocessing - if ncores == 'max': + if ncores == 0: ncores = multiprocessing.cpu_count() pool = multiprocessing.Pool(ncores) diff --git a/pylot/core/util/widgets.py b/pylot/core/util/widgets.py index 2115bd08..0a8e5805 100644 --- a/pylot/core/util/widgets.py +++ b/pylot/core/util/widgets.py @@ -2869,7 +2869,6 @@ class AutoPickDlg(QDialog): maxcores = multiprocessing.cpu_count() self.sb_ncores.setMinimum(1) self.sb_ncores.setMaximum(maxcores) - self.sb_ncores.setEnabled(False) # not yet implemented in autoPyLoT self.layout_ncores.addWidget(self.label_ncores) self.layout_ncores.addWidget(self.sb_ncores) self.layout_ncores.setStretch(0, 1) @@ -2903,7 +2902,7 @@ class AutoPickDlg(QDialog): def accept(self): self.exportParameter() - self.job_widget.start(self.pp_export) + self.job_widget.start(self.pp_export, self.sb_ncores.value()) QDialog.accept(self) @@ -2931,7 +2930,7 @@ class Submit2Grid(QWidget): default_command = 'qsub -l low -cwd -q TARGET_MACHINE -pe mpi-fu NCORES' self.textedit.setText(default_command) - def start(self, pp_export): + def start(self, pp_export, ncores=None): self.genShellScript(pp_export) self.execute_script() @@ -2966,12 +2965,14 @@ class SubmitLocal(QWidget): def start(self): print('subprocess Popen') - def start(self, pp_export): - self.execute_command(pp_export) + def start(self, pp_export, ncores): + self.execute_command(pp_export, ncores) - def execute_command(self, pp_export): + def execute_command(self, pp_export, ncores): command = self.script_fn command.append(pp_export) + command.append('--ncores') + command.append(str(ncores)) cmd_str = str() for item in command: cmd_str += item + ' '