From be1e49fa575633abc72d0f5290ecf0b5cdd989d1 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 26 Nov 2019 11:10:06 +0100 Subject: [PATCH] [updates] generate_array_maps multiprocessing, added extension '_autopylot' to picks generated by autopylot to prevent overwriting on auto-save --- autoPyLoT.py | 2 +- pylot/core/util/generate_array_maps.py | 77 +++++++++++++++----------- pylot/core/util/widgets.py | 2 +- 3 files changed, 48 insertions(+), 33 deletions(-) diff --git a/autoPyLoT.py b/autoPyLoT.py index d5fe628b..f9d18289 100755 --- a/autoPyLoT.py +++ b/autoPyLoT.py @@ -483,7 +483,7 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even saveEvtPath = eventpath else: saveEvtPath = savepath - fnqml = '%s/PyLoT_%s' % (saveEvtPath, evID) + fnqml = '%s/PyLoT_%s_autopylot' % (saveEvtPath, evID) data.exportEvent(fnqml, fnext='.xml', fcheck=['auto', 'magnitude', 'origin']) if locflag == 1: # HYPO71 diff --git a/pylot/core/util/generate_array_maps.py b/pylot/core/util/generate_array_maps.py index 6651b67c..fc2dab51 100755 --- a/pylot/core/util/generate_array_maps.py +++ b/pylot/core/util/generate_array_maps.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import os +import multiprocessing from PyLoT import Project from pylot.core.util.dataprocessing import Metadata @@ -9,44 +10,58 @@ from pylot.core.util.array_map import Array_map import matplotlib.pyplot as plt -def main(project_file_path, manual=False, auto=True, file_format='png', f_ext=''): +def main(project_file_path, manual=False, auto=True, file_format='png', f_ext='', ncores=None): project = Project.load(project_file_path) nEvents = len(project.eventlist) + input_list = [] for index, event in enumerate(project.eventlist): - eventdir = event.path # MP MP TESTING +++ #if not eventdir.endswith('20170908_044946.a'): # continue # MP MP ---- - print('Working on event: {} ({}/{})'.format(eventdir, index + 1, nEvents)) - # check for picks - manualpicks = event.getPicks() - autopicks = event.getAutopicks() - # prepare event and get metadata - metadata_path = os.path.join(eventdir, 'resp') - metadata = Metadata(inventory=metadata_path, verbosity=0) - for pick_type in ['manual', 'auto']: - if pick_type == 'manual' and (not manualpicks or not manual): - continue - if pick_type == 'auto' and (not autopicks or not auto): - continue - # create figure to plot on - fig = plt.figure(figsize=(16,9)) - # create array map object - map = Array_map(None, metadata, figure=fig, width=2.13e6, height=1.2e6, pointsize=15., linewidth=1.0) - # set combobox to auto/manual to plot correct pick type - map.comboBox_am.setCurrentIndex(map.comboBox_am.findText(pick_type)) - # add picks to map and save file - map.refresh_drawings(manualpicks, autopicks) - fpath_out = os.path.join(eventdir, 'array_map_{}_{}{}.{}'.format(event.pylot_id, pick_type, f_ext, - file_format)) - fig.savefig(fpath_out, dpi=300.) - print('Wrote file: {}'.format(fpath_out)) + kwargs = dict(event=event, nEvents=nEvents, index=index, manual=manual, auto=auto, file_format=file_format, + f_ext=f_ext) + input_list.append(kwargs) + + if ncores == 1: + for item in input_list: + array_map_worker(item) + else: + pool = multiprocessing.Pool(ncores) + result = pool.map(array_map_worker, input_list) + pool.close() + pool.join() + +def array_map_worker(input_dict): + event = input_dict['event'] + eventdir = event.path + print('Working on event: {} ({}/{})'.format(eventdir, input_dict['index'] + 1, input_dict['nEvents'])) + # check for picks + manualpicks = event.getPicks() + autopicks = event.getAutopicks() + # prepare event and get metadata + metadata_path = os.path.join(eventdir, 'resp') + metadata = Metadata(inventory=metadata_path, verbosity=0) + for pick_type in ['manual', 'auto']: + if pick_type == 'manual' and (not manualpicks or not input_dict['manual']): + continue + if pick_type == 'auto' and (not autopicks or not input_dict['auto']): + continue + # create figure to plot on + fig = plt.figure(figsize=(16,9)) + # create array map object + map = Array_map(None, metadata, figure=fig, width=2.13e6, height=1.2e6, pointsize=15., linewidth=1.0) + # set combobox to auto/manual to plot correct pick type + map.comboBox_am.setCurrentIndex(map.comboBox_am.findText(pick_type)) + # add picks to map and save file + map.refresh_drawings(manualpicks, autopicks) + fpath_out = os.path.join(eventdir, 'array_map_{}_{}{}.{}'.format(event.pylot_id, pick_type, input_dict['f_ext'], + input_dict['file_format'])) + fig.savefig(fpath_out, dpi=300.) + print('Wrote file: {}'.format(fpath_out)) if __name__ == '__main__': - #main('/home/marcel/pylot_m7_mantle_correlated_ORIG_AUTOPYLOT.plp') - #main('/home/marcel/test_project_obs_highf.plp', f_ext='_highf') - #main('/home/marcel/test_project_obs_lowf.plp', f_ext='_lowf') - #main('/home/marcel/test_project_obs_hybrid.plp', f_ext='_hybrid') - main('/home/marcel/test_project_obs_uhf.plp', f_ext='_uhf') + #main('/home/marcel/pylot_m7_mantle_correlated.plp', f_ext='_0.5Hz') + main('/home/marcel/alparray_m7_mantle_correlated_v3.plp', f_ext='_correlated_0.5Hz') + #main('/home/marcel/alparray_m6.5-6.9_mantle_correlated_v3.plp', f_ext='_correlated_0.5Hz') diff --git a/pylot/core/util/widgets.py b/pylot/core/util/widgets.py index e132a0c8..d3e5a0f7 100644 --- a/pylot/core/util/widgets.py +++ b/pylot/core/util/widgets.py @@ -3793,7 +3793,7 @@ class TuneAutopicker(QWidget): 'savexml': False, 'obspyDMT_wfpath': wfpath} event = self.get_current_event() - self.parent().saveData(event, event.path, '.xml') + #self.parent().saveData(event, event.path, '.xml') MP MP uncommented because overwriting pick files in tune mode for key in self.fig_dict.keys(): if not key == 'plot_style': self.fig_dict[key].clear()