marcel/pylot
3
0
Fork 0
Code Issues 17 Pull Requests 1 Releases Wiki Activity

[major] getting rid of unused/unnecessary "rootpath" and "database" structure. Testing required.

Browse Source
This commit is contained in:
Marcel Paffrath 2024-08-27 17:45:15 +02:00
parent 3f97097bf6
commit 7a13288c85
13 changed files with 90 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
PyLoT.py
View File

@ -83,7 +83,7 @@ from pylot.core.util.event import Event
from pylot.core.io.location import create_creation_info, create_event
from pylot.core.util.widgets import FilterOptionsDialog, NewEventDlg, \
PylotCanvas, WaveformWidgetPG, PropertiesDlg, HelpForm, createAction, PickDlg, \
ComparisonWidget, TuneAutopicker, PylotParaBox, AutoPickDlg, CanvasWidget, AutoPickWidget, \
ComparisonWidget, TuneAutopicker, PylotParameterWidget, AutoPickDlg, CanvasWidget, AutoPickWidget, \
CompareEventsWidget, ProgressBarWidget, AddMetadataWidget, SingleTextLineDialog, LogWidget, PickQualitiesFromXml, \
SpectrogramTab, SearchFileByExtensionDialog
from pylot.core.util.array_map import Array_map
@ -136,7 +136,7 @@ class MainWindow(QMainWindow):
self.project.parameter = self._inputs
self.tap = None
self.apw = None
self.paraBox = None
self.parameterWidget = None
self.array_map = None
self._metadata = Metadata(verbosity=0)
self._eventChanged = [False, False]
@ -188,7 +188,6 @@ class MainWindow(QMainWindow):
self.table_headers = ['', 'Event', 'Time', 'Lat', 'Lon', 'Depth', 'Ml', 'Mw', '[N] MP', '[N] AP', 'Tuning Set',
'Test Set', 'Notes']
# TODO: refactor rootpath to datapath
while True:
try:
if settings.value("user/FullName", None) is None:
@ -1210,7 +1209,7 @@ class MainWindow(QMainWindow):
with open(eventlist_file, 'r') as infile:
eventlist_subset = [os.path.join(basepath, filename.split('\n')[0]) for filename in
infile.readlines()]
msg = 'Found file "eventlist.txt" in database path. WILL ONLY USE SELECTED EVENTS out of {} events ' \
msg = 'Found file "eventlist.txt" in datapath. WILL ONLY USE SELECTED EVENTS out of {} events ' \
'contained in this subset'
print(msg.format(len(eventlist_subset)))
eventlist = [eventname for eventname in eventlist if eventname in eventlist_subset]
@ -1235,49 +1234,34 @@ class MainWindow(QMainWindow):
# get path from first event in list and split them
path = eventlist[0]
try:
system_name = platform.system()
if system_name in ["Linux", "Darwin"]:
dirs = {
'database': path.split('/')[-2],
'datapath': os.path.split(path)[0], # path.split('/')[-3],
'rootpath': '/' + os.path.join(*path.split('/')[:-3])
}
elif system_name == "Windows":
rootpath = path.split('/')[:-3]
rootpath[0] += '/'
dirs = {
# TODO: Arrange path to meet Win standards
'database': path.split('/')[-2],
'datapath': path.split('/')[-3],
'rootpath': os.path.join(*rootpath)
}
datapath = os.path.split(path)[0]
dirs = {
'datapath': datapath,
}
except Exception as e:
dirs = {
'database': '',
'datapath': '',
'rootpath': ''
}
print('Warning: Could not automatically init folder structure. ({})'.format(e))
settings = QSettings()
settings.setValue("data/dataRoot", dirs['datapath']) # d irs['rootpath'])
settings.setValue("data/dataRoot", dirs['datapath'])
settings.sync()
if not self.project.eventlist:
# init parameter object
self.setParameter(show=False)
# hide all parameter (show all needed parameter later)
self.paraBox.hide_parameter()
self.parameterWidget.hide_parameter()
for directory in dirs.keys():
# set parameter
box = self.paraBox.boxes[directory]
self.paraBox.setValue(box, dirs[directory])
box = self.parameterWidget.boxes[directory]
self.parameterWidget.setValue(box, dirs[directory])
# show needed parameter in box
self.paraBox.show_parameter(directory)
dirs_box = self.paraBox.get_groupbox_dialog('Directories')
self.parameterWidget.show_parameter(directory)
dirs_box = self.parameterWidget.get_groupbox_dialog('Directories')
if not dirs_box.exec_():
return
self.project.rootpath = dirs['rootpath']
self.project.datapath = dirs['datapath']
else:
if hasattr(self.project, 'datapath'):
@ -1286,7 +1270,6 @@ class MainWindow(QMainWindow):
'Datapath missmatch to current project!')
return
else:
self.project.rootpath = dirs['rootpath']
self.project.datapath = dirs['datapath']
self.project.add_eventlist(eventlist)
@ -1374,11 +1357,10 @@ class MainWindow(QMainWindow):
return True
def modify_project_path(self, new_rootpath):
# TODO: change root to datapath
self.project.rootpath = new_rootpath
self.project.datapath = new_rootpath
for event in self.project.eventlist:
event.rootpath = new_rootpath
event.path = os.path.join(event.rootpath, event.datapath, event.database, event.pylot_id)
event.datapath = new_rootpath
event.path = os.path.join(event.datapath, event.pylot_id)
event.path = event.path.replace('\\', '/')
event.path = event.path.replace('//', '/')
@ -1572,7 +1554,7 @@ class MainWindow(QMainWindow):
self.set_fname(self.get_data().getEventFileName(), type)
return self.get_fnames(type)
def saveData(self, event=None, directory=None, outformats=['.xml', '.cnv', '.obs', '_focmec.in', '.pha']):
def saveData(self, event=None, directory=None, outformats=None):
'''
Save event data to directory with specified output formats.
:param event: PyLoT Event, if not set current event will be used
@ -1580,6 +1562,8 @@ class MainWindow(QMainWindow):
:param outformats: str/list of output formats
:return:
'''
if outformats is None:
outformats = ['.xml', '.cnv', '.obs', '_focmec.in', '.pha']
if not event:
event = self.get_current_event()
if not type(outformats) == list:
@ -1715,7 +1699,7 @@ class MainWindow(QMainWindow):
# WIP JG
def eventlistXml(self):
path = self._inputs['rootpath'] + '/' + self._inputs['datapath'] + '/' + self._inputs['database']
path = self._inputs['datapath']
outpath = self.project.location[:self.project.location.rfind('/')]
geteventlistfromxml(path, outpath)
return
@ -2441,7 +2425,7 @@ class MainWindow(QMainWindow):
filterS = filteroptions['S']
minP, maxP = filterP.getFreq()
minS, maxS = filterS.getFreq()
self.paraBox.params_to_gui()
self.parameterWidget.params_to_gui()
def getFilterOptions(self):
return self.filteroptions
@ -3190,8 +3174,8 @@ class MainWindow(QMainWindow):
ttt = parameter['ttpatter']
outfile = parameter['outpatter']
eventname = self.get_current_event_name()
obsdir = os.path.join(self._inputs['rootpath'], self._inputs['datapath'], self._inputs['database'], eventname)
self.saveData(event=self.get_current_event(), directory=obsdir, outformats='.obs')
obsdir = os.path.join(self._inputs['datapath'], eventname)
self.saveData(event=self.get_current_event(), directory=obsdir, outformats=['.obs'])
filename = 'PyLoT_' + eventname
locpath = os.path.join(locroot, 'loc', filename)
phasefile = os.path.join(obsdir, filename + '.obs')
@ -3748,6 +3732,7 @@ class MainWindow(QMainWindow):
if self.project.parameter:
# do this step to update default parameter on older PyLoT projects
self.project.parameter.reinit_default_parameters()
PylotParameter.check_deprecated_parameters(self.project.parameter)
self._inputs = self.project.parameter
self.updateFilteroptions()
@ -3865,13 +3850,13 @@ class MainWindow(QMainWindow):
def setParameter(self, checked=0, show=True):
if checked: pass # dummy argument to receive trigger signal (checked) if called by QAction
if not self.paraBox:
self.paraBox = PylotParaBox(self._inputs, parent=self, windowflag=Qt.Window)
self.paraBox.accepted.connect(self._setDirty)
self.paraBox.accepted.connect(self.filterOptionsFromParameter)
if not self.parameterWidget:
self.parameterWidget = PylotParameterWidget(self._inputs, parent=self, windowflag=Qt.Window)
self.parameterWidget.accepted.connect(self._setDirty)
self.parameterWidget.accepted.connect(self.filterOptionsFromParameter)
if show:
self.paraBox.params_to_gui()
self.paraBox.show()
self.parameterWidget.params_to_gui()
self.parameterWidget.show()
def deleteAllAutopicks(self):
qmb = QMessageBox(self, icon=QMessageBox.Question,
@ -3918,16 +3903,18 @@ class Project(object):
Pickable class containing information of a PyLoT project, like event lists and file locations.
'''
# TODO: remove rootpath
def __init__(self):
self.eventlist = []
self.location = None
self.rootpath = None
self.datapath = None
self.dirty = False
self.parameter = None
self._table = None
@property
def rootpath(self):
return self.datapath
def add_eventlist(self, eventlist):
'''
Add events from an eventlist containing paths to event directories.
@ -3937,8 +3924,6 @@ class Project(object):
return
for item in eventlist:
event = Event(item)
event.rootpath = self.parameter['rootpath']
event.database = self.parameter['database']
event.datapath = self.parameter['datapath']
if not event.path in self.getPaths():
self.eventlist.append(event)

10
autoPyLoT.py
View File

@ -136,11 +136,9 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even
if parameter.hasParam('datastructure'):
# getting information on data structure
datastructure = DATASTRUCTURE[parameter.get('datastructure')]()
dsfields = {'root': parameter.get('rootpath'),
'dpath': parameter.get('datapath'),
'dbase': parameter.get('database')}
dsfields = {'dpath': parameter.get('datapath'),}
exf = ['root', 'dpath', 'dbase']
exf = ['dpath']
if parameter['eventID'] != '*' and fnames == 'None':
dsfields['eventID'] = parameter['eventID']
@ -206,12 +204,10 @@ def autoPyLoT(input_dict=None, parameter=None, inputfile=None, fnames=None, even
locflag = 2
else:
# started in tune or interactive mode
datapath = os.path.join(parameter['rootpath'],
parameter['datapath'])
datapath = parameter['datapath']
events = []
for eventID in eventid:
events.append(os.path.join(datapath,
parameter['database'],
eventID))
if not events:

2
docs/gui.md
View File

@ -203,8 +203,6 @@ The meaning of the header entries is:
PyLoT GUI starts with an empty project. To add events, use the add event data button. Select one or multiple folders
containing events.
[//]: <> (TODO: explain _Directories: Root path, Data path, Database path_)
### Saving projects
Save the current project from the menu with File->Save project or File->Save project as. PyLoT uses ``.plp`` files to

6
inputs/pylot_global.in
View File

@ -4,10 +4,8 @@
%Parameters are optimized for %extent data sets!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#main settings#
#rootpath# %project path
#datapath# %data path
#database# %name of data base
#eventID# %event ID for single event processing (* for all events found in database)
#datapath# %data path
#eventID# %event ID for single event processing (* for all events found in datapath)
#invdir# %full path to inventory or dataless-seed file
PILOT #datastructure# %choose data structure
True #apverbose# %choose 'True' or 'False' for terminal output

6
inputs/pylot_local.in
View File

@ -4,10 +4,8 @@
%Parameters are optimized for %extent data sets!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#main settings#
/DATA/Insheim #rootpath# %project path
EVENT_DATA/LOCAL #datapath# %data path
2018.02_Insheim #database# %name of data base
e0006.038.18 #eventID# %event ID for single event processing (* for all events found in database)
/DATA/Insheim/EVENT_DATA/LOCAL/2018.02_Insheim #datapath# %data path
e0006.038.18 #eventID# %event ID for single event processing (* for all events found in datapath)
/DATA/Insheim/STAT_INFO #invdir# %full path to inventory or dataless-seed file
PILOT #datastructure# %choose data structure
True #apverbose# %choose 'True' or 'False' for terminal output

6
inputs/pylot_regional.in
View File

@ -4,10 +4,8 @@
%Parameters are optimized for %extent data sets!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#main settings#
#rootpath# %project path
#datapath# %data path
#database# %name of data base
#eventID# %event ID for single event processing (* for all events found in database)
#datapath# %data path
#eventID# %event ID for single event processing (* for all events found in datapath)
#invdir# %full path to inventory or dataless-seed file
PILOT #datastructure# %choose data structure
True #apverbose# %choose 'True' or 'False' for terminal output

20
pylot/core/io/default_parameters.py
View File

@ -6,24 +6,14 @@ import numpy as np
Default parameters used for picking
"""
defaults = {'rootpath': {'type': str,
'tooltip': 'project path',
'value': '',
'namestring': 'Root path'},
'datapath': {'type': str,
'tooltip': 'data path',
defaults = {'datapath': {'type': str,
'tooltip': 'path to eventfolders',
'value': '',
'namestring': 'Data path'},
'database': {'type': str,
'tooltip': 'name of data base',
'value': '',
'namestring': 'Database path'},
'eventID': {'type': str,
'tooltip': 'event ID for single event processing (* for all events found in database)',
'value': '',
'tooltip': 'event ID for single event processing (* for all events found in datapath)',
'value': '*',
'namestring': 'Event ID'},
'extent': {'type': str,
@ -522,9 +512,7 @@ defaults = {'rootpath': {'type': str,
settings_main = {
'dirs': [
'rootpath',
'datapath',
'database',
'eventID',
'invdir',
'datastructure',

30
pylot/core/io/inputs.py
View File

@ -1,5 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging
import os
from pylot.core.io import default_parameters
from pylot.core.util.errors import ParameterError
@ -88,10 +90,10 @@ class PylotParameter(object):
return bool(self.__parameter)
def __getitem__(self, key):
try:
if key in self.__parameter:
return self.__parameter[key]
except:
return None
else:
logging.warning(f'{key} not found in PylotParameter')
def __setitem__(self, key, value):
try:
@ -418,6 +420,28 @@ class PylotParameter(object):
line = value + name + ttip
fid.write(line)
@staticmethod
def check_deprecated_parameters(parameters):
if parameters.hasParam('database') and parameters.hasParam('rootpath'):
parameters['datapath'] = os.path.join(parameters['rootpath'], parameters['datapath'],
parameters['database'])
logging.warning(
f'Parameters database and rootpath are deprecated. '
f'Tried to merge them to now path: {parameters["datapath"]}.'
)
remove_keys = []
for key in parameters:
if not key in default_parameters.defaults.keys():
remove_keys.append(key)
logging.warning(f'Removing deprecated parameter: {key}')
for key in remove_keys:
del parameters[key]
parameters._settings_main = default_parameters.settings_main
parameters._settings_special_pick = default_parameters.settings_special_pick
class FilterOptions(object):
'''

4
pylot/core/io/phases.py
View File

@ -512,7 +512,7 @@ def writephases(arrivals, fformat, filename, parameter=None, eventinfo=None):
fid = open("%s" % filename, 'w')
# write header
fid.write('# EQEVENT: %s Label: EQ%s Loc: X 0.00 Y 0.00 Z 10.00 OT 0.00 \n' %
(parameter.get('database'), parameter.get('eventID')))
(parameter.get('datapath'), parameter.get('eventID')))
arrivals = chooseArrivals(arrivals) # MP MP what is chooseArrivals? It is not defined anywhere
for key in arrivals:
# P onsets
@ -665,7 +665,7 @@ def writephases(arrivals, fformat, filename, parameter=None, eventinfo=None):
print("Writing phases to %s for HYPOSAT" % filename)
fid = open("%s" % filename, 'w')
# write header
fid.write('%s, event %s \n' % (parameter.get('database'), parameter.get('eventID')))
fid.write('%s, event %s \n' % (parameter.get('datapath'), parameter.get('eventID')))
arrivals = chooseArrivals(arrivals) # MP MP what is chooseArrivals? It is not defined anywhere
for key in arrivals:
# P onsets

3
pylot/core/util/event.py
View File

@ -22,14 +22,11 @@ class Event(ObsPyEvent):
:param path: path to event directory
:type path: str
"""
# TODO: remove rootpath and database
self.pylot_id = path.split('/')[-1]
# initialize super class
super(Event, self).__init__(resource_id=ResourceIdentifier('smi:local/' + self.pylot_id))
self.path = path
self.database = path.split('/')[-2]
self.datapath = os.path.split(path)[0] # path.split('/')[-3]
self.rootpath = '/' + os.path.join(*path.split('/')[:-3])
self.pylot_autopicks = {}
self.pylot_picks = {}
self.notes = ''

19
pylot/core/util/widgets.py
View File

@ -8,6 +8,7 @@ import copy
import datetime
import getpass
import glob
import logging
import multiprocessing
import os
import subprocess
@ -3835,7 +3836,7 @@ class TuneAutopicker(QWidget):
self.stb_names = ['aicARHfig', 'refSpick', 'el_S1pick', 'el_S2pick']
def add_parameters(self):
self.paraBox = PylotParaBox(self.parameter, parent=self, windowflag=Qt.Widget)
self.paraBox = PylotParameterWidget(self.parameter, parent=self, windowflag=Qt.Widget)
self.paraBox.set_tune_mode(True)
self.update_eventID()
self.parameter_layout.addWidget(self.paraBox)
@ -4202,7 +4203,7 @@ class TuneAutopicker(QWidget):
self.qmb.show()
class PylotParaBox(QtWidgets.QWidget):
class PylotParameterWidget(QtWidgets.QWidget):
accepted = QtCore.Signal(str)
rejected = QtCore.Signal(str)
@ -4316,6 +4317,11 @@ class PylotParaBox(QtWidgets.QWidget):
grid = QtWidgets.QGridLayout()
for index1, name in enumerate(parameter_names):
if name in ['rootpath', 'database']:
logging.warning(
f'Deprecated parameter loaded: {name}. Check if datapath is still correct in parameter widget.'
)
continue
default_item = self.parameter.get_defaults()[name]
tooltip = default_item['tooltip']
tooltip += ' | type: {}'.format(default_item['type'])
@ -4885,7 +4891,7 @@ class PropTab(QWidget):
def getValues(self):
return None
def resetValues(self, infile=None):
def resetValues(self, infile):
return None
@ -4982,12 +4988,7 @@ class InputsTab(PropTab):
else:
index = 2
datapath = para.get('datapath') if not para.get('datapath') is None else ''
rootpath = para.get('rootpath') if not para.get('rootpath') is None else ''
database = para.get('database') if not para.get('database') is None else ''
if isinstance(database, int):
database = str(database)
path = os.path.join(os.path.expanduser('~'), rootpath, datapath, database)
values = {"data/dataRoot": self.dataDirEdit.setText("%s" % path),
values = {"data/dataRoot": self.dataDirEdit.setText("%s" % datapath),
"user/FullName": self.fullNameEdit.text(),
"data/Structure": self.structureSelect.setCurrentIndex(index),
"tstart": self.tstartBox.setValue(0),

6
tests/test_autopickstation/autoPyLoT_global_taupy_false.in
View File

@ -4,10 +4,8 @@
%Parameters are optimized for %extent data sets!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#main settings#
/home/darius #rootpath# %project path
alparray #datapath# %data path
waveforms_used #database# %name of data base
e0093.173.16 #eventID# %event ID for single event processing (* for all events found in database)
/home/darius/alparray/waveforms_used #datapath# %data path
e0093.173.16 #eventID# %event ID for single event processing (* for all events found in datapath)
/home/darius/alparray/metadata #invdir# %full path to inventory or dataless-seed file
PILOT #datastructure# %choose data structure
True #apverbose# %choose 'True' or 'False' for terminal output

6
tests/test_autopickstation/autoPyLoT_global_taupy_true.in
View File

@ -4,10 +4,8 @@
%Parameters are optimized for %extent data sets!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#main settings#
/home/darius #rootpath# %project path
alparray #datapath# %data path
waveforms_used #database# %name of data base
e0093.173.16 #eventID# %event ID for single event processing (* for all events found in database)
/home/darius/alparray/waveforms_used #datapath# %data path
e0093.173.16 #eventID# %event ID for single event processing (* for all events found in datapath)
/home/darius/alparray/metadata #invdir# %full path to inventory or dataless-seed file
PILOT #datastructure# %choose data structure
True #apverbose# %choose 'True' or 'False' for terminal output