[major] getting rid of unused/unnecessary "rootpath" and "database" structure. Testing required.

This commit is contained in:
Marcel Paffrath 2024-08-27 17:45:15 +02:00
parent 3f97097bf6
commit 7a13288c85
13 changed files with 90 additions and 111 deletions

View File

@ -83,7 +83,7 @@ from pylot.core.util.event import Event
from pylot.core.io.location import create_creation_info, create_event
from pylot.core.util.widgets import FilterOptionsDialog, NewEventDlg, \
PylotCanvas, WaveformWidgetPG, PropertiesDlg, HelpForm, createAction, PickDlg, \
ComparisonWidget, TuneAutopicker, PylotParaBox, AutoPickDlg, CanvasWidget, AutoPickWidget, \
ComparisonWidget, TuneAutopicker, PylotParameterWidget, AutoPickDlg, CanvasWidget, AutoPickWidget, \
CompareEventsWidget, ProgressBarWidget, AddMetadataWidget, SingleTextLineDialog, LogWidget, PickQualitiesFromXml, \
SpectrogramTab, SearchFileByExtensionDialog
from pylot.core.util.array_map import Array_map
@ -136,7 +136,7 @@ class MainWindow(QMainWindow):
self.project.parameter = self._inputs
self.tap = None
self.apw = None
self.paraBox = None
self.parameterWidget = None
self.array_map = None
self._metadata = Metadata(verbosity=0)
self._eventChanged = [False, False]
@ -188,7 +188,6 @@ class MainWindow(QMainWindow):
self.table_headers = ['', 'Event', 'Time', 'Lat', 'Lon', 'Depth', 'Ml', 'Mw', '[N] MP', '[N] AP', 'Tuning Set',
'Test Set', 'Notes']
# TODO: refactor rootpath to datapath
while True:
try:
if settings.value("user/FullName", None) is None:
@ -1210,7 +1209,7 @@ class MainWindow(QMainWindow):
with open(eventlist_file, 'r') as infile:
eventlist_subset = [os.path.join(basepath, filename.split('\n')[0]) for filename in
infile.readlines()]
msg = 'Found file "eventlist.txt" in database path. WILL ONLY USE SELECTED EVENTS out of {} events ' \
msg = 'Found file "eventlist.txt" in datapath. WILL ONLY USE SELECTED EVENTS out of {} events ' \
'contained in this subset'
print(msg.format(len(eventlist_subset)))
eventlist = [eventname for eventname in eventlist if eventname in eventlist_subset]
@ -1235,49 +1234,34 @@ class MainWindow(QMainWindow):
# get path from first event in list and split them
path = eventlist[0]
try:
system_name = platform.system()
if system_name in ["Linux", "Darwin"]:
datapath = os.path.split(path)[0]
dirs = {
'database': path.split('/')[-2],
'datapath': os.path.split(path)[0], # path.split('/')[-3],
'rootpath': '/' + os.path.join(*path.split('/')[:-3])
}
elif system_name == "Windows":
rootpath = path.split('/')[:-3]
rootpath[0] += '/'
dirs = {
# TODO: Arrange path to meet Win standards
'database': path.split('/')[-2],
'datapath': path.split('/')[-3],
'rootpath': os.path.join(*rootpath)
'datapath': datapath,
}
except Exception as e:
dirs = {
'database': '',
'datapath': '',
'rootpath': ''
}
print('Warning: Could not automatically init folder structure. ({})'.format(e))
settings = QSettings()
settings.setValue("data/dataRoot", dirs['datapath']) # d irs['rootpath'])
settings.setValue("data/dataRoot", dirs['datapath'])
settings.sync()
if not self.project.eventlist:
# init parameter object
self.setParameter(show=False)
# hide all parameter (show all needed parameter later)
self.paraBox.hide_parameter()
self.parameterWidget.hide_parameter()
for directory in dirs.keys():
# set parameter
box = self.paraBox.boxes[directory]
self.paraBox.setValue(box, dirs[directory])
box = self.parameterWidget.boxes[directory]
self.parameterWidget.setValue(box, dirs[directory])
# show needed parameter in box
self.paraBox.show_parameter(directory)
dirs_box = self.paraBox.get_groupbox_dialog('Directories')
self.parameterWidget.show_parameter(directory)
dirs_box = self.parameterWidget.get_groupbox_dialog('Directories')
if not dirs_box.exec_():
return
self.project.rootpath = dirs['rootpath']
self.project.datapath = dirs['datapath']
else:
if hasattr(self.project, 'datapath'):
@ -1286,7 +1270,6 @@ class MainWindow(QMainWindow):
'Datapath missmatch to current project!')
return
else:
self.project.rootpath = dirs['rootpath']
self.project.datapath = dirs['datapath']
self.project.add_eventlist(eventlist)
@ -1374,11 +1357,10 @@ class MainWindow(QMainWindow):
return True
def modify_project_path(self, new_rootpath):
# TODO: change root to datapath
self.project.rootpath = new_rootpath
self.project.datapath = new_rootpath
for event in self.project.eventlist:
event.rootpath = new_rootpath
event.path = os.path.join(event.rootpath, event.datapath, event.database, event.pylot_id)
event.datapath = new_rootpath
event.path = os.path.join(event.datapath, event.pylot_id)
event.path = event.path.replace('\\', '/')
event.path = event.path.replace('//', '/')
@ -1572,7 +1554,7 @@ class MainWindow(QMainWindow):
self.set_fname(self.get_data().getEventFileName(), type)
return self.get_fnames(type)
def saveData(self, event=None, directory=None, outformats=['.xml', '.cnv', '.obs', '_focmec.in', '.pha']):
def saveData(self, event=None, directory=None, outformats=None):
'''
Save event data to directory with specified output formats.
:param event: PyLoT Event, if not set current event will be used
@ -1580,6 +1562,8 @@ class MainWindow(QMainWindow):
:param outformats: str/list of output formats
:return:
'''
if outformats is None:
outformats = ['.xml', '.cnv', '.obs', '_focmec.in', '.pha']
if not event:
event = self.get_current_event()
if not type(outformats) == list:
@ -1715,7 +1699,7 @@ class MainWindow(QMainWindow):
# WIP JG
def eventlistXml(self):
path = self._inputs['rootpath'] + '/' + self._inputs['datapath'] + '/' + self._inputs['database']
path = self._inputs['datapath']
outpath = self.project.location[:self.project.location.rfind('/')]
geteventlistfromxml(path, outpath)
return
@ -2441,7 +2425,7 @@ class MainWindow(QMainWindow):
filterS = filteroptions['S']
minP, maxP = filterP.getFreq()
minS, maxS = filterS.getFreq()
self.paraBox.params_to_gui()
self.parameterWidget.params_to_gui()
def getFilterOptions(self):
return self.filteroptions
@ -3190,8 +3174,8 @@ class MainWindow(QMainWindow):
ttt = parameter['ttpatter']
outfile = parameter['outpatter']
eventname = self.get_current_event_name()
obsdir = os.path.join(self._inputs['rootpath'], self._inputs['datapath'], self._inputs['database'], eventname)
self.saveData(event=self.get_current_event(), directory=obsdir, outformats='.obs')
obsdir = os.path.join(self._inputs['datapath'], eventname)
self.saveData(event=self.get_current_event(), directory=obsdir, outformats=['.obs'])
filename = 'PyLoT_' + eventname
locpath = os.path.join(locroot, 'loc', filename)
phasefile = os.path.join(obsdir, filename + '.obs')
@ -3748,6 +3732,7 @@ class MainWindow(QMainWindow):
if self.project.parameter:
# do this step to update default parameter on older PyLoT projects
self.project.parameter.reinit_default_parameters()
PylotParameter.check_deprecated_parameters(self.project.parameter)
self._inputs = self.project.parameter
self.updateFilteroptions()
@ -3865,13 +3850,13 @@ class MainWindow(QMainWindow):
def setParameter(self, checked=0, show=True):
if checked: pass # dummy argument to receive trigger signal (checked) if called by QAction
if not self.paraBox:
self.paraBox = PylotParaBox(self._inputs, parent=self, windowflag=Qt.Window)
self.paraBox.accepted.connect(self._setDirty)
self.paraBox.accepted.connect(self.filterOptionsFromParameter)
if not self.parameterWidget:
self.parameterWidget = PylotParameterWidget(self._inputs, parent=self, windowflag=Qt.Window)
self.parameterWidget.accepted.connect(self._setDirty)
self.parameterWidget.accepted.connect(self.filterOptionsFromParameter)
if show:
self.paraBox.params_to_gui()
self.paraBox.show()
self.parameterWidget.params_to_gui()
self.parameterWidget.show()
def deleteAllAutopicks(self):
qmb = QMessageBox(self, icon=QMessageBox.Question,
@ -3918,16 +3903,18 @@ class Project(object):
Pickable class containing information of a PyLoT project, like event lists and file locations.
'''
# TODO: remove rootpath
def __init__(self):
self.eventlist = []
self.location = None
self.rootpath = None
self.datapath = None
self.dirty = False
self.parameter = None
self._table = None
@property
def rootpath(self):
return self.datapath
def add_eventlist(self, eventlist):
'''
Add events from an eventlist containing paths to event directories.
@ -3937,8 +3924,6 @@ class Project(object):
return
for item in eventlist:
event = Event(item)
event.rootpath = self.parameter['rootpath']
event.database = self.parameter['database']
event.datapath = self.parameter['datapath']
if not event.path in self.getPaths():
self.eventlist.append(event)

View File

@ -136,11 +136,9 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even
if parameter.hasParam('datastructure'):
# getting information on data structure
datastructure = DATASTRUCTURE[parameter.get('datastructure')]()
dsfields = {'root': parameter.get('rootpath'),
'dpath': parameter.get('datapath'),
'dbase': parameter.get('database')}
dsfields = {'dpath': parameter.get('datapath'),}
exf = ['root', 'dpath', 'dbase']
exf = ['dpath']
if parameter['eventID'] != '*' and fnames == 'None':
dsfields['eventID'] = parameter['eventID']
@ -206,12 +204,10 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even
locflag = 2
else:
# started in tune or interactive mode
datapath = os.path.join(parameter['rootpath'],
parameter['datapath'])
datapath = parameter['datapath']
events = []
for eventID in eventid:
events.append(os.path.join(datapath,
parameter['database'],
eventID))
if not events:

View File

@ -203,8 +203,6 @@ The meaning of the header entries is:
PyLoT GUI starts with an empty project. To add events, use the add event data button. Select one or multiple folders
containing events.
[//]: <> (TODO: explain _Directories: Root path, Data path, Database path_)
### Saving projects
Save the current project from the menu with File->Save project or File->Save project as. PyLoT uses ``.plp`` files to

View File

@ -4,10 +4,8 @@
%Parameters are optimized for %extent data sets!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#main settings#
#rootpath# %project path
#datapath# %data path
#database# %name of data base
#eventID# %event ID for single event processing (* for all events found in database)
#eventID# %event ID for single event processing (* for all events found in datapath)
#invdir# %full path to inventory or dataless-seed file
PILOT #datastructure# %choose data structure
True #apverbose# %choose 'True' or 'False' for terminal output

View File

@ -4,10 +4,8 @@
%Parameters are optimized for %extent data sets!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#main settings#
/DATA/Insheim #rootpath# %project path
EVENT_DATA/LOCAL #datapath# %data path
2018.02_Insheim #database# %name of data base
e0006.038.18 #eventID# %event ID for single event processing (* for all events found in database)
/DATA/Insheim/EVENT_DATA/LOCAL/2018.02_Insheim #datapath# %data path
e0006.038.18 #eventID# %event ID for single event processing (* for all events found in datapath)
/DATA/Insheim/STAT_INFO #invdir# %full path to inventory or dataless-seed file
PILOT #datastructure# %choose data structure
True #apverbose# %choose 'True' or 'False' for terminal output

View File

@ -4,10 +4,8 @@
%Parameters are optimized for %extent data sets!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#main settings#
#rootpath# %project path
#datapath# %data path
#database# %name of data base
#eventID# %event ID for single event processing (* for all events found in database)
#eventID# %event ID for single event processing (* for all events found in datapath)
#invdir# %full path to inventory or dataless-seed file
PILOT #datastructure# %choose data structure
True #apverbose# %choose 'True' or 'False' for terminal output

View File

@ -6,24 +6,14 @@ import numpy as np
Default parameters used for picking
"""
defaults = {'rootpath': {'type': str,
'tooltip': 'project path',
'value': '',
'namestring': 'Root path'},
'datapath': {'type': str,
'tooltip': 'data path',
defaults = {'datapath': {'type': str,
'tooltip': 'path to eventfolders',
'value': '',
'namestring': 'Data path'},
'database': {'type': str,
'tooltip': 'name of data base',
'value': '',
'namestring': 'Database path'},
'eventID': {'type': str,
'tooltip': 'event ID for single event processing (* for all events found in database)',
'value': '',
'tooltip': 'event ID for single event processing (* for all events found in datapath)',
'value': '*',
'namestring': 'Event ID'},
'extent': {'type': str,
@ -522,9 +512,7 @@ defaults = {'rootpath': {'type': str,
settings_main = {
'dirs': [
'rootpath',
'datapath',
'database',
'eventID',
'invdir',
'datastructure',

View File

@ -1,5 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging
import os
from pylot.core.io import default_parameters
from pylot.core.util.errors import ParameterError
@ -88,10 +90,10 @@ class PylotParameter(object):
return bool(self.__parameter)
def __getitem__(self, key):
try:
if key in self.__parameter:
return self.__parameter[key]
except:
return None
else:
logging.warning(f'{key} not found in PylotParameter')
def __setitem__(self, key, value):
try:
@ -418,6 +420,28 @@ class PylotParameter(object):
line = value + name + ttip
fid.write(line)
@staticmethod
def check_deprecated_parameters(parameters):
if parameters.hasParam('database') and parameters.hasParam('rootpath'):
parameters['datapath'] = os.path.join(parameters['rootpath'], parameters['datapath'],
parameters['database'])
logging.warning(
f'Parameters database and rootpath are deprecated. '
f'Tried to merge them to now path: {parameters["datapath"]}.'
)
remove_keys = []
for key in parameters:
if not key in default_parameters.defaults.keys():
remove_keys.append(key)
logging.warning(f'Removing deprecated parameter: {key}')
for key in remove_keys:
del parameters[key]
parameters._settings_main = default_parameters.settings_main
parameters._settings_special_pick = default_parameters.settings_special_pick
class FilterOptions(object):
'''

View File

@ -512,7 +512,7 @@ def writephases(arrivals, fformat, filename, parameter=None, eventinfo=None):
fid = open("%s" % filename, 'w')
# write header
fid.write('# EQEVENT: %s Label: EQ%s Loc: X 0.00 Y 0.00 Z 10.00 OT 0.00 \n' %
(parameter.get('database'), parameter.get('eventID')))
(parameter.get('datapath'), parameter.get('eventID')))
arrivals = chooseArrivals(arrivals) # MP MP what is chooseArrivals? It is not defined anywhere
for key in arrivals:
# P onsets
@ -665,7 +665,7 @@ def writephases(arrivals, fformat, filename, parameter=None, eventinfo=None):
print("Writing phases to %s for HYPOSAT" % filename)
fid = open("%s" % filename, 'w')
# write header
fid.write('%s, event %s \n' % (parameter.get('database'), parameter.get('eventID')))
fid.write('%s, event %s \n' % (parameter.get('datapath'), parameter.get('eventID')))
arrivals = chooseArrivals(arrivals) # MP MP what is chooseArrivals? It is not defined anywhere
for key in arrivals:
# P onsets

View File

@ -22,14 +22,11 @@ class Event(ObsPyEvent):
:param path: path to event directory
:type path: str
"""
# TODO: remove rootpath and database
self.pylot_id = path.split('/')[-1]
# initialize super class
super(Event, self).__init__(resource_id=ResourceIdentifier('smi:local/' + self.pylot_id))
self.path = path
self.database = path.split('/')[-2]
self.datapath = os.path.split(path)[0] # path.split('/')[-3]
self.rootpath = '/' + os.path.join(*path.split('/')[:-3])
self.pylot_autopicks = {}
self.pylot_picks = {}
self.notes = ''

View File

@ -8,6 +8,7 @@ import copy
import datetime
import getpass
import glob
import logging
import multiprocessing
import os
import subprocess
@ -3835,7 +3836,7 @@ class TuneAutopicker(QWidget):
self.stb_names = ['aicARHfig', 'refSpick', 'el_S1pick', 'el_S2pick']
def add_parameters(self):
self.paraBox = PylotParaBox(self.parameter, parent=self, windowflag=Qt.Widget)
self.paraBox = PylotParameterWidget(self.parameter, parent=self, windowflag=Qt.Widget)
self.paraBox.set_tune_mode(True)
self.update_eventID()
self.parameter_layout.addWidget(self.paraBox)
@ -4202,7 +4203,7 @@ class TuneAutopicker(QWidget):
self.qmb.show()
class PylotParaBox(QtWidgets.QWidget):
class PylotParameterWidget(QtWidgets.QWidget):
accepted = QtCore.Signal(str)
rejected = QtCore.Signal(str)
@ -4316,6 +4317,11 @@ class PylotParaBox(QtWidgets.QWidget):
grid = QtWidgets.QGridLayout()
for index1, name in enumerate(parameter_names):
if name in ['rootpath', 'database']:
logging.warning(
f'Deprecated parameter loaded: {name}. Check if datapath is still correct in parameter widget.'
)
continue
default_item = self.parameter.get_defaults()[name]
tooltip = default_item['tooltip']
tooltip += ' | type: {}'.format(default_item['type'])
@ -4885,7 +4891,7 @@ class PropTab(QWidget):
def getValues(self):
return None
def resetValues(self, infile=None):
def resetValues(self, infile):
return None
@ -4982,12 +4988,7 @@ class InputsTab(PropTab):
else:
index = 2
datapath = para.get('datapath') if not para.get('datapath') is None else ''
rootpath = para.get('rootpath') if not para.get('rootpath') is None else ''
database = para.get('database') if not para.get('database') is None else ''
if isinstance(database, int):
database = str(database)
path = os.path.join(os.path.expanduser('~'), rootpath, datapath, database)
values = {"data/dataRoot": self.dataDirEdit.setText("%s" % path),
values = {"data/dataRoot": self.dataDirEdit.setText("%s" % datapath),
"user/FullName": self.fullNameEdit.text(),
"data/Structure": self.structureSelect.setCurrentIndex(index),
"tstart": self.tstartBox.setValue(0),

View File

@ -4,10 +4,8 @@
%Parameters are optimized for %extent data sets!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#main settings#
/home/darius #rootpath# %project path
alparray #datapath# %data path
waveforms_used #database# %name of data base
e0093.173.16 #eventID# %event ID for single event processing (* for all events found in database)
/home/darius/alparray/waveforms_used #datapath# %data path
e0093.173.16 #eventID# %event ID for single event processing (* for all events found in datapath)
/home/darius/alparray/metadata #invdir# %full path to inventory or dataless-seed file
PILOT #datastructure# %choose data structure
True #apverbose# %choose 'True' or 'False' for terminal output

View File

@ -4,10 +4,8 @@
%Parameters are optimized for %extent data sets!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#main settings#
/home/darius #rootpath# %project path
alparray #datapath# %data path
waveforms_used #database# %name of data base
e0093.173.16 #eventID# %event ID for single event processing (* for all events found in database)
/home/darius/alparray/waveforms_used #datapath# %data path
e0093.173.16 #eventID# %event ID for single event processing (* for all events found in datapath)
/home/darius/alparray/metadata #invdir# %full path to inventory or dataless-seed file
PILOT #datastructure# %choose data structure
True #apverbose# %choose 'True' or 'False' for terminal output