From 7a13288c85b02cf96ea9f67c73e962fe5781ca40 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 27 Aug 2024 17:45:15 +0200 Subject: [PATCH] [major] getting rid of unused/unnecessary "rootpath" and "database" structure. Testing required. --- PyLoT.py | 83 ++++++++----------- autoPyLoT.py | 10 +-- docs/gui.md | 2 - inputs/pylot_global.in | 6 +- inputs/pylot_local.in | 6 +- inputs/pylot_regional.in | 6 +- pylot/core/io/default_parameters.py | 20 +---- pylot/core/io/inputs.py | 30 ++++++- pylot/core/io/phases.py | 4 +- pylot/core/util/event.py | 3 - pylot/core/util/widgets.py | 19 +++-- .../autoPyLoT_global_taupy_false.in | 6 +- .../autoPyLoT_global_taupy_true.in | 6 +- 13 files changed, 90 insertions(+), 111 deletions(-) diff --git a/PyLoT.py b/PyLoT.py index ad29d954..c5c10899 100755 --- a/PyLoT.py +++ b/PyLoT.py @@ -83,7 +83,7 @@ from pylot.core.util.event import Event from pylot.core.io.location import create_creation_info, create_event from pylot.core.util.widgets import FilterOptionsDialog, NewEventDlg, \ PylotCanvas, WaveformWidgetPG, PropertiesDlg, HelpForm, createAction, PickDlg, \ - ComparisonWidget, TuneAutopicker, PylotParaBox, AutoPickDlg, CanvasWidget, AutoPickWidget, \ + ComparisonWidget, TuneAutopicker, PylotParameterWidget, AutoPickDlg, CanvasWidget, AutoPickWidget, \ CompareEventsWidget, ProgressBarWidget, AddMetadataWidget, SingleTextLineDialog, LogWidget, PickQualitiesFromXml, \ SpectrogramTab, SearchFileByExtensionDialog from pylot.core.util.array_map import Array_map @@ -136,7 +136,7 @@ class MainWindow(QMainWindow): self.project.parameter = self._inputs self.tap = None self.apw = None - self.paraBox = None + self.parameterWidget = None self.array_map = None self._metadata = Metadata(verbosity=0) self._eventChanged = [False, False] @@ -188,7 +188,6 @@ class MainWindow(QMainWindow): self.table_headers = ['', 'Event', 'Time', 'Lat', 'Lon', 'Depth', 'Ml', 'Mw', '[N] MP', '[N] AP', 'Tuning Set', 'Test Set', 'Notes'] - # TODO: refactor rootpath to datapath while True: try: if settings.value("user/FullName", None) is None: @@ -1210,7 +1209,7 @@ class MainWindow(QMainWindow): with open(eventlist_file, 'r') as infile: eventlist_subset = [os.path.join(basepath, filename.split('\n')[0]) for filename in infile.readlines()] - msg = 'Found file "eventlist.txt" in database path. WILL ONLY USE SELECTED EVENTS out of {} events ' \ + msg = 'Found file "eventlist.txt" in datapath. WILL ONLY USE SELECTED EVENTS out of {} events ' \ 'contained in this subset' print(msg.format(len(eventlist_subset))) eventlist = [eventname for eventname in eventlist if eventname in eventlist_subset] @@ -1235,49 +1234,34 @@ class MainWindow(QMainWindow): # get path from first event in list and split them path = eventlist[0] try: - system_name = platform.system() - if system_name in ["Linux", "Darwin"]: - dirs = { - 'database': path.split('/')[-2], - 'datapath': os.path.split(path)[0], # path.split('/')[-3], - 'rootpath': '/' + os.path.join(*path.split('/')[:-3]) - } - elif system_name == "Windows": - rootpath = path.split('/')[:-3] - rootpath[0] += '/' - dirs = { - # TODO: Arrange path to meet Win standards - 'database': path.split('/')[-2], - 'datapath': path.split('/')[-3], - 'rootpath': os.path.join(*rootpath) - } + datapath = os.path.split(path)[0] + dirs = { + 'datapath': datapath, + } except Exception as e: dirs = { - 'database': '', 'datapath': '', - 'rootpath': '' } print('Warning: Could not automatically init folder structure. ({})'.format(e)) settings = QSettings() - settings.setValue("data/dataRoot", dirs['datapath']) # d irs['rootpath']) + settings.setValue("data/dataRoot", dirs['datapath']) settings.sync() if not self.project.eventlist: # init parameter object self.setParameter(show=False) # hide all parameter (show all needed parameter later) - self.paraBox.hide_parameter() + self.parameterWidget.hide_parameter() for directory in dirs.keys(): # set parameter - box = self.paraBox.boxes[directory] - self.paraBox.setValue(box, dirs[directory]) + box = self.parameterWidget.boxes[directory] + self.parameterWidget.setValue(box, dirs[directory]) # show needed parameter in box - self.paraBox.show_parameter(directory) - dirs_box = self.paraBox.get_groupbox_dialog('Directories') + self.parameterWidget.show_parameter(directory) + dirs_box = self.parameterWidget.get_groupbox_dialog('Directories') if not dirs_box.exec_(): return - self.project.rootpath = dirs['rootpath'] self.project.datapath = dirs['datapath'] else: if hasattr(self.project, 'datapath'): @@ -1286,7 +1270,6 @@ class MainWindow(QMainWindow): 'Datapath missmatch to current project!') return else: - self.project.rootpath = dirs['rootpath'] self.project.datapath = dirs['datapath'] self.project.add_eventlist(eventlist) @@ -1374,11 +1357,10 @@ class MainWindow(QMainWindow): return True def modify_project_path(self, new_rootpath): - # TODO: change root to datapath - self.project.rootpath = new_rootpath + self.project.datapath = new_rootpath for event in self.project.eventlist: - event.rootpath = new_rootpath - event.path = os.path.join(event.rootpath, event.datapath, event.database, event.pylot_id) + event.datapath = new_rootpath + event.path = os.path.join(event.datapath, event.pylot_id) event.path = event.path.replace('\\', '/') event.path = event.path.replace('//', '/') @@ -1572,7 +1554,7 @@ class MainWindow(QMainWindow): self.set_fname(self.get_data().getEventFileName(), type) return self.get_fnames(type) - def saveData(self, event=None, directory=None, outformats=['.xml', '.cnv', '.obs', '_focmec.in', '.pha']): + def saveData(self, event=None, directory=None, outformats=None): ''' Save event data to directory with specified output formats. :param event: PyLoT Event, if not set current event will be used @@ -1580,6 +1562,8 @@ class MainWindow(QMainWindow): :param outformats: str/list of output formats :return: ''' + if outformats is None: + outformats = ['.xml', '.cnv', '.obs', '_focmec.in', '.pha'] if not event: event = self.get_current_event() if not type(outformats) == list: @@ -1715,7 +1699,7 @@ class MainWindow(QMainWindow): # WIP JG def eventlistXml(self): - path = self._inputs['rootpath'] + '/' + self._inputs['datapath'] + '/' + self._inputs['database'] + path = self._inputs['datapath'] outpath = self.project.location[:self.project.location.rfind('/')] geteventlistfromxml(path, outpath) return @@ -2441,7 +2425,7 @@ class MainWindow(QMainWindow): filterS = filteroptions['S'] minP, maxP = filterP.getFreq() minS, maxS = filterS.getFreq() - self.paraBox.params_to_gui() + self.parameterWidget.params_to_gui() def getFilterOptions(self): return self.filteroptions @@ -3190,8 +3174,8 @@ class MainWindow(QMainWindow): ttt = parameter['ttpatter'] outfile = parameter['outpatter'] eventname = self.get_current_event_name() - obsdir = os.path.join(self._inputs['rootpath'], self._inputs['datapath'], self._inputs['database'], eventname) - self.saveData(event=self.get_current_event(), directory=obsdir, outformats='.obs') + obsdir = os.path.join(self._inputs['datapath'], eventname) + self.saveData(event=self.get_current_event(), directory=obsdir, outformats=['.obs']) filename = 'PyLoT_' + eventname locpath = os.path.join(locroot, 'loc', filename) phasefile = os.path.join(obsdir, filename + '.obs') @@ -3748,6 +3732,7 @@ class MainWindow(QMainWindow): if self.project.parameter: # do this step to update default parameter on older PyLoT projects self.project.parameter.reinit_default_parameters() + PylotParameter.check_deprecated_parameters(self.project.parameter) self._inputs = self.project.parameter self.updateFilteroptions() @@ -3865,13 +3850,13 @@ class MainWindow(QMainWindow): def setParameter(self, checked=0, show=True): if checked: pass # dummy argument to receive trigger signal (checked) if called by QAction - if not self.paraBox: - self.paraBox = PylotParaBox(self._inputs, parent=self, windowflag=Qt.Window) - self.paraBox.accepted.connect(self._setDirty) - self.paraBox.accepted.connect(self.filterOptionsFromParameter) + if not self.parameterWidget: + self.parameterWidget = PylotParameterWidget(self._inputs, parent=self, windowflag=Qt.Window) + self.parameterWidget.accepted.connect(self._setDirty) + self.parameterWidget.accepted.connect(self.filterOptionsFromParameter) if show: - self.paraBox.params_to_gui() - self.paraBox.show() + self.parameterWidget.params_to_gui() + self.parameterWidget.show() def deleteAllAutopicks(self): qmb = QMessageBox(self, icon=QMessageBox.Question, @@ -3918,16 +3903,18 @@ class Project(object): Pickable class containing information of a PyLoT project, like event lists and file locations. ''' - # TODO: remove rootpath def __init__(self): self.eventlist = [] self.location = None - self.rootpath = None self.datapath = None self.dirty = False self.parameter = None self._table = None + @property + def rootpath(self): + return self.datapath + def add_eventlist(self, eventlist): ''' Add events from an eventlist containing paths to event directories. @@ -3937,8 +3924,6 @@ class Project(object): return for item in eventlist: event = Event(item) - event.rootpath = self.parameter['rootpath'] - event.database = self.parameter['database'] event.datapath = self.parameter['datapath'] if not event.path in self.getPaths(): self.eventlist.append(event) diff --git a/autoPyLoT.py b/autoPyLoT.py index 506025c5..4746c963 100755 --- a/autoPyLoT.py +++ b/autoPyLoT.py @@ -136,11 +136,9 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even if parameter.hasParam('datastructure'): # getting information on data structure datastructure = DATASTRUCTURE[parameter.get('datastructure')]() - dsfields = {'root': parameter.get('rootpath'), - 'dpath': parameter.get('datapath'), - 'dbase': parameter.get('database')} + dsfields = {'dpath': parameter.get('datapath'),} - exf = ['root', 'dpath', 'dbase'] + exf = ['dpath'] if parameter['eventID'] != '*' and fnames == 'None': dsfields['eventID'] = parameter['eventID'] @@ -206,12 +204,10 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even locflag = 2 else: # started in tune or interactive mode - datapath = os.path.join(parameter['rootpath'], - parameter['datapath']) + datapath = parameter['datapath'] events = [] for eventID in eventid: events.append(os.path.join(datapath, - parameter['database'], eventID)) if not events: diff --git a/docs/gui.md b/docs/gui.md index 4fc735c1..018a3a81 100644 --- a/docs/gui.md +++ b/docs/gui.md @@ -203,8 +203,6 @@ The meaning of the header entries is: PyLoT GUI starts with an empty project. To add events, use the add event data button. Select one or multiple folders containing events. -[//]: <> (TODO: explain _Directories: Root path, Data path, Database path_) - ### Saving projects Save the current project from the menu with File->Save project or File->Save project as. PyLoT uses ``.plp`` files to diff --git a/inputs/pylot_global.in b/inputs/pylot_global.in index 6024b7d8..3bad82c4 100644 --- a/inputs/pylot_global.in +++ b/inputs/pylot_global.in @@ -4,10 +4,8 @@ %Parameters are optimized for %extent data sets! %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% #main settings# - #rootpath# %project path - #datapath# %data path - #database# %name of data base - #eventID# %event ID for single event processing (* for all events found in database) + #datapath# %data path + #eventID# %event ID for single event processing (* for all events found in datapath) #invdir# %full path to inventory or dataless-seed file PILOT #datastructure# %choose data structure True #apverbose# %choose 'True' or 'False' for terminal output diff --git a/inputs/pylot_local.in b/inputs/pylot_local.in index ccfeddd4..3f689374 100644 --- a/inputs/pylot_local.in +++ b/inputs/pylot_local.in @@ -4,10 +4,8 @@ %Parameters are optimized for %extent data sets! %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% #main settings# -/DATA/Insheim #rootpath# %project path -EVENT_DATA/LOCAL #datapath# %data path -2018.02_Insheim #database# %name of data base -e0006.038.18 #eventID# %event ID for single event processing (* for all events found in database) +/DATA/Insheim/EVENT_DATA/LOCAL/2018.02_Insheim #datapath# %data path +e0006.038.18 #eventID# %event ID for single event processing (* for all events found in datapath) /DATA/Insheim/STAT_INFO #invdir# %full path to inventory or dataless-seed file PILOT #datastructure# %choose data structure True #apverbose# %choose 'True' or 'False' for terminal output diff --git a/inputs/pylot_regional.in b/inputs/pylot_regional.in index fe019440..4e33d0a6 100644 --- a/inputs/pylot_regional.in +++ b/inputs/pylot_regional.in @@ -4,10 +4,8 @@ %Parameters are optimized for %extent data sets! %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% #main settings# - #rootpath# %project path - #datapath# %data path - #database# %name of data base - #eventID# %event ID for single event processing (* for all events found in database) + #datapath# %data path + #eventID# %event ID for single event processing (* for all events found in datapath) #invdir# %full path to inventory or dataless-seed file PILOT #datastructure# %choose data structure True #apverbose# %choose 'True' or 'False' for terminal output diff --git a/pylot/core/io/default_parameters.py b/pylot/core/io/default_parameters.py index 6e216939..f6f717e4 100644 --- a/pylot/core/io/default_parameters.py +++ b/pylot/core/io/default_parameters.py @@ -6,24 +6,14 @@ import numpy as np Default parameters used for picking """ -defaults = {'rootpath': {'type': str, - 'tooltip': 'project path', - 'value': '', - 'namestring': 'Root path'}, - - 'datapath': {'type': str, - 'tooltip': 'data path', +defaults = {'datapath': {'type': str, + 'tooltip': 'path to eventfolders', 'value': '', 'namestring': 'Data path'}, - 'database': {'type': str, - 'tooltip': 'name of data base', - 'value': '', - 'namestring': 'Database path'}, - 'eventID': {'type': str, - 'tooltip': 'event ID for single event processing (* for all events found in database)', - 'value': '', + 'tooltip': 'event ID for single event processing (* for all events found in datapath)', + 'value': '*', 'namestring': 'Event ID'}, 'extent': {'type': str, @@ -522,9 +512,7 @@ defaults = {'rootpath': {'type': str, settings_main = { 'dirs': [ - 'rootpath', 'datapath', - 'database', 'eventID', 'invdir', 'datastructure', diff --git a/pylot/core/io/inputs.py b/pylot/core/io/inputs.py index 6f2528bd..71888642 100644 --- a/pylot/core/io/inputs.py +++ b/pylot/core/io/inputs.py @@ -1,5 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import logging +import os from pylot.core.io import default_parameters from pylot.core.util.errors import ParameterError @@ -88,10 +90,10 @@ class PylotParameter(object): return bool(self.__parameter) def __getitem__(self, key): - try: + if key in self.__parameter: return self.__parameter[key] - except: - return None + else: + logging.warning(f'{key} not found in PylotParameter') def __setitem__(self, key, value): try: @@ -418,6 +420,28 @@ class PylotParameter(object): line = value + name + ttip fid.write(line) + @staticmethod + def check_deprecated_parameters(parameters): + if parameters.hasParam('database') and parameters.hasParam('rootpath'): + parameters['datapath'] = os.path.join(parameters['rootpath'], parameters['datapath'], + parameters['database']) + logging.warning( + f'Parameters database and rootpath are deprecated. ' + f'Tried to merge them to now path: {parameters["datapath"]}.' + ) + + remove_keys = [] + for key in parameters: + if not key in default_parameters.defaults.keys(): + remove_keys.append(key) + logging.warning(f'Removing deprecated parameter: {key}') + + for key in remove_keys: + del parameters[key] + + parameters._settings_main = default_parameters.settings_main + parameters._settings_special_pick = default_parameters.settings_special_pick + class FilterOptions(object): ''' diff --git a/pylot/core/io/phases.py b/pylot/core/io/phases.py index 3c50d131..93043536 100644 --- a/pylot/core/io/phases.py +++ b/pylot/core/io/phases.py @@ -512,7 +512,7 @@ def writephases(arrivals, fformat, filename, parameter=None, eventinfo=None): fid = open("%s" % filename, 'w') # write header fid.write('# EQEVENT: %s Label: EQ%s Loc: X 0.00 Y 0.00 Z 10.00 OT 0.00 \n' % - (parameter.get('database'), parameter.get('eventID'))) + (parameter.get('datapath'), parameter.get('eventID'))) arrivals = chooseArrivals(arrivals) # MP MP what is chooseArrivals? It is not defined anywhere for key in arrivals: # P onsets @@ -665,7 +665,7 @@ def writephases(arrivals, fformat, filename, parameter=None, eventinfo=None): print("Writing phases to %s for HYPOSAT" % filename) fid = open("%s" % filename, 'w') # write header - fid.write('%s, event %s \n' % (parameter.get('database'), parameter.get('eventID'))) + fid.write('%s, event %s \n' % (parameter.get('datapath'), parameter.get('eventID'))) arrivals = chooseArrivals(arrivals) # MP MP what is chooseArrivals? It is not defined anywhere for key in arrivals: # P onsets diff --git a/pylot/core/util/event.py b/pylot/core/util/event.py index ede1c9f9..5745bb9f 100644 --- a/pylot/core/util/event.py +++ b/pylot/core/util/event.py @@ -22,14 +22,11 @@ class Event(ObsPyEvent): :param path: path to event directory :type path: str """ - # TODO: remove rootpath and database self.pylot_id = path.split('/')[-1] # initialize super class super(Event, self).__init__(resource_id=ResourceIdentifier('smi:local/' + self.pylot_id)) self.path = path - self.database = path.split('/')[-2] self.datapath = os.path.split(path)[0] # path.split('/')[-3] - self.rootpath = '/' + os.path.join(*path.split('/')[:-3]) self.pylot_autopicks = {} self.pylot_picks = {} self.notes = '' diff --git a/pylot/core/util/widgets.py b/pylot/core/util/widgets.py index 438f7d0d..3eb66984 100644 --- a/pylot/core/util/widgets.py +++ b/pylot/core/util/widgets.py @@ -8,6 +8,7 @@ import copy import datetime import getpass import glob +import logging import multiprocessing import os import subprocess @@ -3835,7 +3836,7 @@ class TuneAutopicker(QWidget): self.stb_names = ['aicARHfig', 'refSpick', 'el_S1pick', 'el_S2pick'] def add_parameters(self): - self.paraBox = PylotParaBox(self.parameter, parent=self, windowflag=Qt.Widget) + self.paraBox = PylotParameterWidget(self.parameter, parent=self, windowflag=Qt.Widget) self.paraBox.set_tune_mode(True) self.update_eventID() self.parameter_layout.addWidget(self.paraBox) @@ -4202,7 +4203,7 @@ class TuneAutopicker(QWidget): self.qmb.show() -class PylotParaBox(QtWidgets.QWidget): +class PylotParameterWidget(QtWidgets.QWidget): accepted = QtCore.Signal(str) rejected = QtCore.Signal(str) @@ -4316,6 +4317,11 @@ class PylotParaBox(QtWidgets.QWidget): grid = QtWidgets.QGridLayout() for index1, name in enumerate(parameter_names): + if name in ['rootpath', 'database']: + logging.warning( + f'Deprecated parameter loaded: {name}. Check if datapath is still correct in parameter widget.' + ) + continue default_item = self.parameter.get_defaults()[name] tooltip = default_item['tooltip'] tooltip += ' | type: {}'.format(default_item['type']) @@ -4885,7 +4891,7 @@ class PropTab(QWidget): def getValues(self): return None - def resetValues(self, infile=None): + def resetValues(self, infile): return None @@ -4982,12 +4988,7 @@ class InputsTab(PropTab): else: index = 2 datapath = para.get('datapath') if not para.get('datapath') is None else '' - rootpath = para.get('rootpath') if not para.get('rootpath') is None else '' - database = para.get('database') if not para.get('database') is None else '' - if isinstance(database, int): - database = str(database) - path = os.path.join(os.path.expanduser('~'), rootpath, datapath, database) - values = {"data/dataRoot": self.dataDirEdit.setText("%s" % path), + values = {"data/dataRoot": self.dataDirEdit.setText("%s" % datapath), "user/FullName": self.fullNameEdit.text(), "data/Structure": self.structureSelect.setCurrentIndex(index), "tstart": self.tstartBox.setValue(0), diff --git a/tests/test_autopickstation/autoPyLoT_global_taupy_false.in b/tests/test_autopickstation/autoPyLoT_global_taupy_false.in index 6aa2edcc..51eb51bb 100644 --- a/tests/test_autopickstation/autoPyLoT_global_taupy_false.in +++ b/tests/test_autopickstation/autoPyLoT_global_taupy_false.in @@ -4,10 +4,8 @@ %Parameters are optimized for %extent data sets! %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% #main settings# -/home/darius #rootpath# %project path -alparray #datapath# %data path -waveforms_used #database# %name of data base -e0093.173.16 #eventID# %event ID for single event processing (* for all events found in database) +/home/darius/alparray/waveforms_used #datapath# %data path +e0093.173.16 #eventID# %event ID for single event processing (* for all events found in datapath) /home/darius/alparray/metadata #invdir# %full path to inventory or dataless-seed file PILOT #datastructure# %choose data structure True #apverbose# %choose 'True' or 'False' for terminal output diff --git a/tests/test_autopickstation/autoPyLoT_global_taupy_true.in b/tests/test_autopickstation/autoPyLoT_global_taupy_true.in index eefaa6c2..2bdf29fc 100644 --- a/tests/test_autopickstation/autoPyLoT_global_taupy_true.in +++ b/tests/test_autopickstation/autoPyLoT_global_taupy_true.in @@ -4,10 +4,8 @@ %Parameters are optimized for %extent data sets! %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% #main settings# -/home/darius #rootpath# %project path -alparray #datapath# %data path -waveforms_used #database# %name of data base -e0093.173.16 #eventID# %event ID for single event processing (* for all events found in database) +/home/darius/alparray/waveforms_used #datapath# %data path +e0093.173.16 #eventID# %event ID for single event processing (* for all events found in datapath) /home/darius/alparray/metadata #invdir# %full path to inventory or dataless-seed file PILOT #datastructure# %choose data structure True #apverbose# %choose 'True' or 'False' for terminal output