diff --git a/QtPyLoT.py b/QtPyLoT.py index 6daaffa6..6deb6f16 100755 --- a/QtPyLoT.py +++ b/QtPyLoT.py @@ -51,7 +51,8 @@ from pylot.core.util.errors import FormatError, DatastructureError, \ OverwriteError from pylot.core.util.connection import checkurl from pylot.core.util.utils import fnConstructor, createEvent, getLogin, \ - createCreationInfo, getGlobalTimes + getGlobalTimes +from pylot.core.io.location import create_creation_info, create_event from pylot.core.util.widgets import FilterOptionsDialog, NewEventDlg, \ MPLWidget, PropertiesDlg, HelpForm, createAction, PickDlg from pylot.core.util.structure import DATASTRUCTURE @@ -829,8 +830,8 @@ class MainWindow(QMainWindow): new = NewEventDlg() if new.exec_() != QDialog.Rejected: evtpar = new.getValues() - cinfo = createCreationInfo(agency_id=self.agency) - event = createEvent(evtpar['origintime'], cinfo) + cinfo = create_creation_info(agency_id=self.agency) + event = create_event(evtpar['origintime'], cinfo) self.data = Data(self, evtdata=event) self.setDirty(True) diff --git a/pylot/RELEASE-VERSION b/pylot/RELEASE-VERSION index c35d452c..671949c9 100644 --- a/pylot/RELEASE-VERSION +++ b/pylot/RELEASE-VERSION @@ -1 +1 @@ -1508-dirty +7c5a-dirty diff --git a/pylot/core/active/seismicshot.py b/pylot/core/active/seismicshot.py index 27470fe0..16b402ad 100644 --- a/pylot/core/active/seismicshot.py +++ b/pylot/core/active/seismicshot.py @@ -395,7 +395,7 @@ class SeismicShot(object): self.picks[traceID]['spe']) = earllatepicker(self.getSingleStream(traceID), nfac, (tnoise, tgap, tsignal), self.getPickIncludeRemoved(traceID), - stealthMode=True) + stealth_mode=True) if self.picks[traceID]['epp'] < 0: self.picks[traceID]['epp'] diff --git a/pylot/core/io/inputs.py b/pylot/core/io/inputs.py index 431ac731..a200e056 100644 --- a/pylot/core/io/inputs.py +++ b/pylot/core/io/inputs.py @@ -36,7 +36,7 @@ class AutoPickParameter(object): ========== ========== ======================================= ''' - def __init__(self, fnin=None, fnout=None, **kwargs): + def __init__(self, fnin=None, fnout=None, verbosity=0, **kwargs): ''' Initialize parameter object: @@ -60,7 +60,8 @@ class AutoPickParameter(object): parspl = line.split('\t')[:2] parFileCont[parspl[0].strip()] = parspl[1] except IndexError as e: - self._printParameterError(e) + if verbosity > 0: + self._printParameterError(e) inputFile.seek(0) lines = inputFile.readlines() for line in lines: diff --git a/pylot/core/io/location.py b/pylot/core/io/location.py new file mode 100644 index 00000000..667b7f91 --- /dev/null +++ b/pylot/core/io/location.py @@ -0,0 +1,222 @@ +from obspy import UTCDateTime +from obspy.core import event as ope + +from pylot.core.util.utils import getLogin, getHash + + +def create_amplitude(pickID, amp, unit, category, cinfo): + ''' + + :param pickID: + :param amp: + :param unit: + :param category: + :param cinfo: + :return: + ''' + amplitude = ope.Amplitude() + amplitude.creation_info = cinfo + amplitude.generic_amplitude = amp + amplitude.unit = ope.AmplitudeUnit(unit) + amplitude.type = ope.AmplitudeCategory(category) + amplitude.pick_id = pickID + return amplitude + + +def create_arrival(pickresID, cinfo, phase, azimuth=None, dist=None): + ''' + create_arrival - function to create an Obspy Arrival + + :param pickresID: Resource identifier of the created pick + :type pickresID: :class: `~obspy.core.event.ResourceIdentifier` object + :param cinfo: An ObsPy :class: `~obspy.core.event.CreationInfo` object + holding information on the creation of the returned object + :type cinfo: :class: `~obspy.core.event.CreationInfo` object + :param phase: name of the arrivals seismic phase + :type phase: str + :param azimuth: azimuth between source and receiver + :type azimuth: float or int, optional + :param dist: distance between source and receiver + :type dist: float or int, optional + :return: An ObsPy :class: `~obspy.core.event.Arrival` object + ''' + arrival = ope.Arrival() + arrival.creation_info = cinfo + arrival.pick_id = pickresID + arrival.phase = phase + if azimuth is not None: + arrival.azimuth = float(azimuth) if azimuth > -180 else azimuth + 360. + else: + arrival.azimuth = azimuth + arrival.distance = dist + return arrival + + +def create_creation_info(agency_id=None, creation_time=None, author=None): + ''' + + :param agency_id: + :param creation_time: + :param author: + :return: + ''' + if author is None: + author = getLogin() + if creation_time is None: + creation_time = UTCDateTime() + return ope.CreationInfo(agency_id=agency_id, author=author, + creation_time=creation_time) + + +def create_event(origintime, cinfo, originloc=None, etype=None, resID=None, + authority_id=None): + ''' + create_event - funtion to create an ObsPy Event + + :param origintime: the events origintime + :type origintime: :class: `~obspy.core.utcdatetime.UTCDateTime` object + :param cinfo: An ObsPy :class: `~obspy.core.event.CreationInfo` object + holding information on the creation of the returned object + :type cinfo: :class: `~obspy.core.event.CreationInfo` object + :param originloc: tuple containing the location of the origin + (LAT, LON, DEP) affiliated with the event which is created + :type originloc: tuple, list + :param etype: Event type str object. converted via ObsPy to a valid event + type string. + :type etype: str + :param resID: Resource identifier of the created event + :type resID: :class: `~obspy.core.event.ResourceIdentifier` object, str + :param authority_id: name of the institution carrying out the processing + :type authority_id: str + :return: An ObsPy :class: `~obspy.core.event.Event` object + ''' + etype = ope.EventType(etype) + if originloc is not None: + o = create_origin(origintime, cinfo, + originloc[0], originloc[1], originloc[2]) + else: + o = None + if etype is None: + etype = ope.EventType('earthquake') # defaults to 'earthquake' + if not resID: + resID = create_resourceID(origintime, etype, authority_id) + elif isinstance(resID, str): + resID = create_resourceID(origintime, etype, authority_id, resID) + elif not isinstance(resID, ope.ResourceIdentifier): + raise TypeError("unsupported type(resID) for resource identifier " + "generation: %s" % type(resID)) + event = ope.Event(resource_id=resID) + event.creation_info = cinfo + event.event_type = etype + if o: + event.origins = [o] + return event + + +def create_magnitude(originID, cinfo): + ''' + create_magnitude - function to create an ObsPy Magnitude object + :param originID: + :type originID: + :param cinfo: + :type cinfo: + :return: + ''' + magnitude = ope.Magnitude() + magnitude.creation_info = cinfo + magnitude.origin_id = originID + return magnitude + + +def create_origin(origintime, cinfo, latitude, longitude, depth): + ''' + create_origin - function to create an ObsPy Origin + :param origintime: the origins time of occurence + :type origintime: :class: `~obspy.core.utcdatetime.UTCDateTime` object + :param cinfo: + :type cinfo: + :param latitude: latitude in decimal degree of the origins location + :type latitude: float + :param longitude: longitude in decimal degree of the origins location + :type longitude: float + :param depth: hypocentral depth of the origin + :type depth: float + :return: An ObsPy :class: `~obspy.core.event.Origin` object + ''' + + assert isinstance(origintime, UTCDateTime), "origintime has to be " \ + "a UTCDateTime object, but " \ + "actually is of type " \ + "'%s'" % type(origintime) + + origin = ope.Origin() + origin.time = origintime + origin.creation_info = cinfo + origin.latitude = latitude + origin.longitude = longitude + origin.depth = depth + return origin + + +def create_pick(origintime, picknum, picktime, eventnum, cinfo, phase, station, + wfseedstr, authority_id): + ''' + create_pick - function to create an ObsPy Pick + + :param origintime: + :type origintime: + :param picknum: number of the created pick + :type picknum: int + :param picktime: + :type picktime: + :param eventnum: human-readable event identifier + :type eventnum: str + :param cinfo: An ObsPy :class: `~obspy.core.event.CreationInfo` object + holding information on the creation of the returned object + :type cinfo: :class: `~obspy.core.event.CreationInfo` object + :param phase: name of the arrivals seismic phase + :type phase: str + :param station: name of the station at which the seismic phase has been + picked + :type station: str + :param wfseedstr: A SEED formatted string of the form + network.station.location.channel in order to set a referenced waveform + :type wfseedstr: str, SEED formatted + :param authority_id: name of the institution carrying out the processing + :type authority_id: str + :return: An ObsPy :class: `~obspy.core.event.Pick` object + ''' + pickID = eventnum + '_' + station.strip() + '/{0:03d}'.format(picknum) + pickresID = create_resourceID(origintime, 'pick', authority_id, pickID) + pick = ope.Pick() + pick.resource_id = pickresID + pick.time = picktime + pick.creation_info = cinfo + pick.phase_hint = phase + pick.waveform_id = ope.ResourceIdentifier(id=wfseedstr, prefix='file:/') + return pick + + +def create_resourceID(timetohash, restype, authority_id=None, hrstr=None): + ''' + + :param timetohash: + :type timetohash + :param restype: type of the resource, e.g. 'orig', 'earthquake' ... + :type restype: str + :param authority_id: name of the institution carrying out the processing + :type authority_id: str, optional + :param hrstr: + :type hrstr: + :return: + ''' + assert isinstance(timetohash, UTCDateTime), "'timetohash' is not an ObsPy" \ + "UTCDateTime object" + hid = getHash(timetohash) + if hrstr is None: + resID = ope.ResourceIdentifier(restype + '/' + hid[0:6]) + else: + resID = ope.ResourceIdentifier(restype + '/' + hrstr + '_' + hid[0:6]) + if authority_id is not None: + resID.convertIDToQuakeMLURI(authority_id=authority_id) + return resID \ No newline at end of file diff --git a/pylot/core/io/phases.py b/pylot/core/io/phases.py index 3924ddd3..38fa1aaa 100644 --- a/pylot/core/io/phases.py +++ b/pylot/core/io/phases.py @@ -1,16 +1,18 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import os import glob -import warnings -import scipy.io as sio import obspy.core.event as ope +import os +import scipy.io as sio +import warnings from obspy.core import UTCDateTime +from pylot.core.io.location import create_arrival, create_event, \ + create_magnitude, create_origin, create_pick from pylot.core.pick.utils import select_for_phase -from pylot.core.util.utils import getOwner, createPick, createArrival, \ - createEvent, createOrigin, createMagnitude, getGlobalTimes +from pylot.core.util.utils import getOwner, getGlobalTimes + def readPILOTEvent(phasfn=None, locfn=None, authority_id=None, **kwargs): """ @@ -75,14 +77,14 @@ def readPILOTEvent(phasfn=None, locfn=None, authority_id=None, **kwargs): stations = [stat for stat in phases['stat'][0:-1:3]] - event = createEvent(eventDate, loccinfo, etype='earthquake', resID=eventNum, - authority_id=authority_id) + event = create_event(eventDate, loccinfo, etype='earthquake', resID=eventNum, + authority_id=authority_id) lat = float(loc['LAT']) lon = float(loc['LON']) dep = float(loc['DEP']) - origin = createOrigin(eventDate, loccinfo, lat, lon, dep) + origin = create_origin(eventDate, loccinfo, lat, lon, dep) for n, pick in enumerate(phases['Ptime']): if pick[0] > 0: kwargs = {'year': int(pick[0]), @@ -115,15 +117,15 @@ def readPILOTEvent(phasfn=None, locfn=None, authority_id=None, **kwargs): wffn = os.path.join(sdir, '{0}*{1}*'.format( stations[n].strip(), '[ne]')) print(wffn) - pick = createPick(eventDate, np, picktime, eventNum, pickcinfo, - phase, stations[n], wffn, authority_id) + pick = create_pick(eventDate, np, picktime, eventNum, pickcinfo, + phase, stations[n], wffn, authority_id) event.picks.append(pick) pickID = pick.get('id') - arrival = createArrival(pickID, pickcinfo, phase) + arrival = create_arrival(pickID, pickcinfo, phase) origin.arrivals.append(arrival) np += 1 - magnitude = createMagnitude(origin.get('id'), loccinfo) + magnitude = create_magnitude(origin.get('id'), loccinfo) magnitude.mag = float(loc['Mnet']) magnitude.magnitude_type = 'Ml' @@ -232,7 +234,7 @@ def picksdict_from_picks(evt): def picks_from_picksdict(picks): picks_list = list() for station, onsets in picks.items(): - print('Reading picks on station %s' % station) + #print('Reading picks on station %s' % station) for label, phase in onsets.items(): if not isinstance(phase, dict) or len(phase) < 3: continue @@ -262,21 +264,28 @@ def picks_from_picksdict(picks): else: pick.polarity = 'undecidable' except KeyError as e: - print(e.message, 'No polarity information found for %s' % phase) + if 'fm' in e.message: # no polarity information found for this phase + pass + else: + raise e picks_list.append(pick) return picks_list -def reassess_pilot_db(root_dir, out_dir=None, fn_param=None): +def reassess_pilot_db(root_dir, db_dir, out_dir=None, fn_param=None, verbosity=0): import glob - evt_list = glob.glob1(root_dir,'e????.???.??') + db_root = os.path.join(root_dir, db_dir) + evt_list = glob.glob1(db_root,'e????.???.??') for evt in evt_list: - reassess_pilot_event(root_dir, evt, out_dir, fn_param) + if verbosity > 0: + print('Reassessing event {0}'.format(evt)) + reassess_pilot_event(root_dir, db_dir, evt, out_dir, fn_param, verbosity) -def reassess_pilot_event(root_dir, event_id, out_dir=None, fn_param=None, verbosity=0): + +def reassess_pilot_event(root_dir, db_dir, event_id, out_dir=None, fn_param=None, verbosity=0): from obspy import read from pylot.core.io.inputs import AutoPickParameter @@ -286,15 +295,19 @@ def reassess_pilot_event(root_dir, event_id, out_dir=None, fn_param=None, verbos import pylot.core.util.defaults as defaults fn_param = defaults.AUTOMATIC_DEFAULTS - default = AutoPickParameter(fn_param) + default = AutoPickParameter(fn_param, verbosity) - search_base = os.path.join(root_dir, event_id) + search_base = os.path.join(root_dir, db_dir, event_id) phases_file = glob.glob(os.path.join(search_base, 'PHASES.mat')) if not phases_file: return - print('Opening PILOT phases file: {fn}'.format(fn=phases_file[0])) + if verbosity > 1: + print('Opening PILOT phases file: {fn}'.format(fn=phases_file[0])) picks_dict = picks_from_pilot(phases_file[0]) - print('Dictionary read from PHASES.mat:\n{0}'.format(picks_dict)) + if verbosity > 0: + print('Dictionary read from PHASES.mat:\n{0}'.format(picks_dict)) + datacheck = list() + info = None for station in picks_dict.keys(): fn_pattern = os.path.join(search_base, '{0}*'.format(station)) try: @@ -302,6 +315,21 @@ def reassess_pilot_event(root_dir, event_id, out_dir=None, fn_param=None, verbos except TypeError as e: print(e.message) st = read(fn_pattern) + except ValueError as e: + if e.message == 'second must be in 0..59': + info = 'A known Error was raised. Please find the list of corrupted files and double-check these files.' + datacheck.append(fn_pattern + ' (time info)\n') + continue + else: + raise ValueError(e.message) + except Exception as e: + if 'No file matching file pattern:' in e.message: + if verbosity > 0: + warnings.warn('no waveform data found for station {station}'.format(station=station), RuntimeWarning) + datacheck.append(fn_pattern + ' (no data)\n') + continue + else: + raise e for phase in picks_dict[station].keys(): try: mpp = picks_dict[station][phase]['mpp'] @@ -310,10 +338,9 @@ def reassess_pilot_event(root_dir, event_id, out_dir=None, fn_param=None, verbos continue sel_st = select_for_phase(st, phase) if not sel_st: - raise warnings.formatwarning( - 'no waveform data found for station {station}'.format( - station=station), category=RuntimeWarning) - print(sel_st) + msg = 'no waveform data found for station {station}'.format(station=station) + warnings.warn(msg, RuntimeWarning) + continue stime, etime = getGlobalTimes(sel_st) rel_pick = mpp - stime epp, lpp, spe = earllatepicker(sel_st, @@ -321,7 +348,7 @@ def reassess_pilot_event(root_dir, event_id, out_dir=None, fn_param=None, verbos default.get('tsnrz' if phase == 'P' else 'tsnrh'), Pick1=rel_pick, iplot=None, - ) + stealth_mode=True) if epp is None or lpp is None: continue epp = stime + epp @@ -332,13 +359,24 @@ def reassess_pilot_event(root_dir, event_id, out_dir=None, fn_param=None, verbos if mpp - epp < min_diff: epp = mpp - min_diff picks_dict[station][phase] = dict(epp=epp, mpp=mpp, lpp=lpp, spe=spe) + if datacheck: + if info: + if verbosity > 0: + print(info + ': {0}'.format(search_base)) + fncheck = open(os.path.join(search_base, 'datacheck_list'), 'w') + fncheck.writelines(datacheck) + fncheck.close() + del datacheck # create Event object for export evt = ope.Event(resource_id=event_id) evt.picks = picks_from_picksdict(picks_dict) # write phase information to file if not out_dir: - fnout_prefix = os.path.join(root_dir, event_id, '{0}.'.format(event_id)) + fnout_prefix = os.path.join(root_dir, db_dir, event_id, '{0}.'.format(event_id)) else: + out_dir = os.path.join(out_dir, db_dir) + if not os.path.isdir(out_dir): + os.makedirs(out_dir) fnout_prefix = os.path.join(out_dir, '{0}.'.format(event_id)) evt.write(fnout_prefix + 'xml', format='QUAKEML') #evt.write(fnout_prefix + 'cnv', format='VELEST') diff --git a/pylot/core/pick/utils.py b/pylot/core/pick/utils.py index e603cfa4..09405fb1 100644 --- a/pylot/core/pick/utils.py +++ b/pylot/core/pick/utils.py @@ -16,7 +16,7 @@ import numpy as np from obspy.core import Stream, UTCDateTime -def earllatepicker(X, nfac, TSNR, Pick1, iplot=None, stealthMode=False): +def earllatepicker(X, nfac, TSNR, Pick1, iplot=None, stealth_mode=False): ''' Function to derive earliest and latest possible pick after Diehl & Kissling (2009) as reasonable uncertainties. Latest possible pick is based on noise level, @@ -45,8 +45,9 @@ def earllatepicker(X, nfac, TSNR, Pick1, iplot=None, stealthMode=False): LPick = None EPick = None PickError = None - if stealthMode is False: - print 'earllatepicker: Get earliest and latest possible pick relative to most likely pick ...' + if stealth_mode is False: + print('earllatepicker: Get earliest and latest possible pick' + ' relative to most likely pick ...') x = X[0].data t = np.arange(0, X[0].stats.npts / X[0].stats.sampling_rate, @@ -62,8 +63,9 @@ def earllatepicker(X, nfac, TSNR, Pick1, iplot=None, stealthMode=False): ilup, = np.where(x[isignal] > nlevel) ildown, = np.where(x[isignal] < -nlevel) if not ilup.size and not ildown.size: - print ("earllatepicker: Signal lower than noise level!") - print ("Skip this trace!") + if stealth_mode is False: + print ("earllatepicker: Signal lower than noise level!\n" + "Skip this trace!") return LPick, EPick, PickError il = min(np.min(ilup) if ilup.size else float('inf'), np.min(ildown) if ildown.size else float('inf')) @@ -78,7 +80,7 @@ def earllatepicker(X, nfac, TSNR, Pick1, iplot=None, stealthMode=False): # if EPick stays NaN the signal window size will be doubled while np.isnan(EPick): if count > 0: - if stealthMode is False: + if stealth_mode is False: print("\nearllatepicker: Doubled signal window size %s time(s) " "because of NaN for earliest pick." % count) isigDoubleWinStart = pis[-1] + 1 @@ -87,7 +89,8 @@ def earllatepicker(X, nfac, TSNR, Pick1, iplot=None, stealthMode=False): if (isigDoubleWinStart + len(pis)) < X[0].data.size: pis = np.concatenate((pis, isignalDoubleWin)) else: - print("Could not double signal window. Index out of bounds.") + if stealth_mode is False: + print("Could not double signal window. Index out of bounds.") break count += 1 # determine all zero crossings in signal window (demeaned) diff --git a/pylot/core/util/dataprocessing.py b/pylot/core/util/dataprocessing.py index f91e2ba2..1a4eff92 100644 --- a/pylot/core/util/dataprocessing.py +++ b/pylot/core/util/dataprocessing.py @@ -4,31 +4,151 @@ import os import glob from obspy import UTCDateTime - -def gse_time_header(lines): - ''' - takes a path FILE to a GSE data file and returns the time header of the file - :param file: path to GSE data file - :type file: str - :return: time header from FILE - :rtype: str - - >>> gse_time_header('test.gse') - "WID2 2005/10/09 20:17:25.000 ATH SHZ NSP CM6 9000 50.000000 0.10E+01 1.000 NSP -1.0 0.0" - ''' - - return lines[1] +import sys def time_from_header(header): + """ + Function takes in the second line from a .gse file and takes out the date and time from that line. + :param header: second line from .gse file + :type header: string + :return: a list of integers of form [year, month, day, hour, minute, second, microsecond] + """ timeline = header.split(' ') time = timeline[1].split('/') + timeline[2].split(':') time = time[:-1] + time[-1].split('.') - time[-1] += '000' return [int(t) for t in time] -def check_time(time): + +def check_time(datetime): + """ + Function takes in date and time as list and validates it's values by trying to make an UTCDateTime object from it + :param datetime: list of integers [year, month, day, hour, minute, second, microsecond] + :type datetime: list + :return: returns True if Values are in supposed range, returns False otherwise + + >>> check_time([1999, 01, 01, 23, 59, 59, 999000]) + True + >>> check_time([1999, 01, 01, 23, 59, 60, 999000]) + False + >>> check_time([1999, 01, 01, 23, 59, 59, 1000000]) + False + >>> check_time([1999, 01, 01, 23, 60, 59, 999000]) + False + >>> check_time([1999, 01, 01, 23, 60, 59, 999000]) + False + >>> check_time([1999, 01, 01, 24, 59, 59, 999000]) + False + >>> check_time([1999, 01, 31, 23, 59, 59, 999000]) + True + >>> check_time([1999, 02, 30, 23, 59, 59, 999000]) + False + >>> check_time([1999, 02, 29, 23, 59, 59, 999000]) + False + >>> check_time([2000, 02, 29, 23, 59, 59, 999000]) + True + >>> check_time([2000, 13, 29, 23, 59, 59, 999000]) + False + """ try: - UTCDateTime(time) + UTCDateTime(*datetime) return True except ValueError: - return False \ No newline at end of file + return False + + +def get_file_list(root_dir): + """ + Function uses a directorie to get all the *.gse files from it. + :param root_dir: a directorie leading to the .gse files + :type root_dir: string + :return: returns a list of filenames (without path to them) + """ + file_list = glob.glob1(root_dir, '*.gse') + return file_list + + +def checks_station_second(datetime, file): + """ + Function uses the given list to check if the parameter 'second' is set to 60 by mistake + and sets the time correctly if so. Can only correct time if no date change would be necessary. + :param datetime: [year, month, day, hour, minute, second, microsecond] + :return: returns the input with the correct value for second + """ + if datetime[5] == 60: + if datetime[4] == 59: + if datetime[3] == 23: + err_msg = 'Date should be next day. ' \ + 'File not changed: {0}'.format(file) + raise ValueError(err_msg) + else: + datetime[3] += 1 + datetime[4] = 0 + datetime[5] = 0 + else: + datetime[4] += 1 + datetime[5] = 0 + return datetime + + +def make_time_line(line, datetime): + """ + Function takes in the original line from a .gse file and a list of date and + time values to make a new line with corrected date and time. + :param line: second line from .gse file. + :type line: string + :param datetime: list of integers [year, month, day, hour, minute, second, microsecond] + :type datetime: list + :return: returns a string to write it into a file. + """ + ins_form = '{0:02d}:{1:02d}:{2:02d}.{3:03d}' + insertion = ins_form.format(int(datetime[3]), + int(datetime[4]), + int(datetime[5]), + int(datetime[6] * 1e-3)) + newline = line[:16] + insertion + line[28:] + return newline + + +def evt_head_check(root_dir, out_dir = None): + """ + A function to make sure that an arbitrary number of .gse files have correct values in their header. + :param root_dir: a directory leading to the .gse files. + :type root_dir: string + :param out_dir: a directory to store the new files somwhere els. + :return: returns nothing + """ + if not out_dir: + print('WARNING files are going to be overwritten!') + inp = str(raw_input('Continue? [y/N]')) + if not inp == 'y': + sys.exit() + filelist = get_file_list(root_dir) + nfiles = 0 + for file in filelist: + infile = open(os.path.join(root_dir, file), 'r') + lines = infile.readlines() + infile.close() + datetime = time_from_header(lines[1]) + if check_time(datetime): + continue + else: + nfiles += 1 + datetime = checks_station_second(datetime, file) + print('writing ' + file) + # write File + lines[1] = make_time_line(lines[1], datetime) + if not out_dir: + out = open(os.path.join(root_dir, file), 'w') + out.writelines(lines) + out.close() + else: + out = open(os.path.join(out_dir, file), 'w') + out.writelines(lines) + out.close() + print(nfiles) + + +if __name__ == "__main__": + import doctest + + doctest.testmod() \ No newline at end of file diff --git a/pylot/core/util/utils.py b/pylot/core/util/utils.py index f2b5aae5..a7167c43 100644 --- a/pylot/core/util/utils.py +++ b/pylot/core/util/utils.py @@ -1,14 +1,14 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import os -import subprocess -import pwd -import re import hashlib import numpy as np +import os +import pwd +import re +import subprocess from obspy.core import UTCDateTime -import obspy.core.event as ope + def _pickle_method(m): if m.im_self is None: @@ -16,237 +16,21 @@ def _pickle_method(m): else: return getattr, (m.im_self, m.im_func.func_name) -def worker(func, input, cores = 'max', async = False): + +def worker(func, input, cores='max', async=False): + return result import multiprocessing if cores == 'max': cores = multiprocessing.cpu_count() - pool = multiprocessing.Pool(cores) + if async == True: result = pool.map_async(func, input) else: result = pool.map(func, input) pool.close() - return result - -def createAmplitude(pickID, amp, unit, category, cinfo): - ''' - - :param pickID: - :param amp: - :param unit: - :param category: - :param cinfo: - :return: - ''' - amplitude = ope.Amplitude() - amplitude.creation_info = cinfo - amplitude.generic_amplitude = amp - amplitude.unit = ope.AmplitudeUnit(unit) - amplitude.type = ope.AmplitudeCategory(category) - amplitude.pick_id = pickID - return amplitude - - -def createArrival(pickresID, cinfo, phase, azimuth=None, dist=None): - ''' - createArrival - function to create an Obspy Arrival - - :param pickresID: Resource identifier of the created pick - :type pickresID: :class: `~obspy.core.event.ResourceIdentifier` object - :param cinfo: An ObsPy :class: `~obspy.core.event.CreationInfo` object - holding information on the creation of the returned object - :type cinfo: :class: `~obspy.core.event.CreationInfo` object - :param phase: name of the arrivals seismic phase - :type phase: str - :param azimuth: azimuth between source and receiver - :type azimuth: float or int, optional - :param dist: distance between source and receiver - :type dist: float or int, optional - :return: An ObsPy :class: `~obspy.core.event.Arrival` object - ''' - arrival = ope.Arrival() - arrival.creation_info = cinfo - arrival.pick_id = pickresID - arrival.phase = phase - if azimuth is not None: - arrival.azimuth = float(azimuth) if azimuth > -180 else azimuth + 360. - else: - arrival.azimuth = azimuth - arrival.distance = dist - return arrival - - -def createCreationInfo(agency_id=None, creation_time=None, author=None): - ''' - - :param agency_id: - :param creation_time: - :param author: - :return: - ''' - if author is None: - author = getLogin() - if creation_time is None: - creation_time = UTCDateTime() - return ope.CreationInfo(agency_id=agency_id, author=author, - creation_time=creation_time) - - -def createEvent(origintime, cinfo, originloc=None, etype=None, resID=None, - authority_id=None): - ''' - createEvent - funtion to create an ObsPy Event - - :param origintime: the events origintime - :type origintime: :class: `~obspy.core.utcdatetime.UTCDateTime` object - :param cinfo: An ObsPy :class: `~obspy.core.event.CreationInfo` object - holding information on the creation of the returned object - :type cinfo: :class: `~obspy.core.event.CreationInfo` object - :param originloc: tuple containing the location of the origin - (LAT, LON, DEP) affiliated with the event which is created - :type originloc: tuple, list - :param etype: Event type str object. converted via ObsPy to a valid event - type string. - :type etype: str - :param resID: Resource identifier of the created event - :type resID: :class: `~obspy.core.event.ResourceIdentifier` object, str - :param authority_id: name of the institution carrying out the processing - :type authority_id: str - :return: An ObsPy :class: `~obspy.core.event.Event` object - ''' - etype = ope.EventType(etype) - if originloc is not None: - o = createOrigin(origintime, cinfo, - originloc[0], originloc[1], originloc[2]) - else: - o = None - if etype is None: - etype = ope.EventType('earthquake') # defaults to 'earthquake' - if not resID: - resID = createResourceID(origintime, etype, authority_id) - elif isinstance(resID, str): - resID = createResourceID(origintime, etype, authority_id, resID) - elif not isinstance(resID, ope.ResourceIdentifier): - raise TypeError("unsupported type(resID) for resource identifier " - "generation: %s" % type(resID)) - event = ope.Event(resource_id=resID) - event.creation_info = cinfo - event.event_type = etype - if o: - event.origins = [o] - return event - - -def createMagnitude(originID, cinfo): - ''' - createMagnitude - function to create an ObsPy Magnitude object - :param originID: - :type originID: - :param cinfo: - :type cinfo: - :return: - ''' - magnitude = ope.Magnitude() - magnitude.creation_info = cinfo - magnitude.origin_id = originID - return magnitude - - -def createOrigin(origintime, cinfo, latitude, longitude, depth): - ''' - createOrigin - function to create an ObsPy Origin - :param origintime: the origins time of occurence - :type origintime: :class: `~obspy.core.utcdatetime.UTCDateTime` object - :param cinfo: - :type cinfo: - :param latitude: latitude in decimal degree of the origins location - :type latitude: float - :param longitude: longitude in decimal degree of the origins location - :type longitude: float - :param depth: hypocentral depth of the origin - :type depth: float - :return: An ObsPy :class: `~obspy.core.event.Origin` object - ''' - - assert isinstance(origintime, UTCDateTime), "origintime has to be " \ - "a UTCDateTime object, but " \ - "actually is of type " \ - "'%s'" % type(origintime) - - origin = ope.Origin() - origin.time = origintime - origin.creation_info = cinfo - origin.latitude = latitude - origin.longitude = longitude - origin.depth = depth - return origin - - -def createPick(origintime, picknum, picktime, eventnum, cinfo, phase, station, - wfseedstr, authority_id): - ''' - createPick - function to create an ObsPy Pick - - :param origintime: - :type origintime: - :param picknum: number of the created pick - :type picknum: int - :param picktime: - :type picktime: - :param eventnum: human-readable event identifier - :type eventnum: str - :param cinfo: An ObsPy :class: `~obspy.core.event.CreationInfo` object - holding information on the creation of the returned object - :type cinfo: :class: `~obspy.core.event.CreationInfo` object - :param phase: name of the arrivals seismic phase - :type phase: str - :param station: name of the station at which the seismic phase has been - picked - :type station: str - :param wfseedstr: A SEED formatted string of the form - network.station.location.channel in order to set a referenced waveform - :type wfseedstr: str, SEED formatted - :param authority_id: name of the institution carrying out the processing - :type authority_id: str - :return: An ObsPy :class: `~obspy.core.event.Pick` object - ''' - pickID = eventnum + '_' + station.strip() + '/{0:03d}'.format(picknum) - pickresID = createResourceID(origintime, 'pick', authority_id, pickID) - pick = ope.Pick() - pick.resource_id = pickresID - pick.time = picktime - pick.creation_info = cinfo - pick.phase_hint = phase - pick.waveform_id = ope.ResourceIdentifier(id=wfseedstr, prefix='file:/') - return pick - - -def createResourceID(timetohash, restype, authority_id=None, hrstr=None): - ''' - - :param timetohash: - :type timetohash - :param restype: type of the resource, e.g. 'orig', 'earthquake' ... - :type restype: str - :param authority_id: name of the institution carrying out the processing - :type authority_id: str, optional - :param hrstr: - :type hrstr: - :return: - ''' - assert isinstance(timetohash, UTCDateTime), "'timetohash' is not an ObsPy" \ - "UTCDateTime object" - hid = getHash(timetohash) - if hrstr is None: - resID = ope.ResourceIdentifier(restype + '/' + hid[0:6]) - else: - resID = ope.ResourceIdentifier(restype + '/' + hrstr + '_' + hid[0:6]) - if authority_id is not None: - resID.convertIDToQuakeMLURI(authority_id=authority_id) - return resID - + def demeanTrace(trace, window): """ @@ -469,6 +253,7 @@ def runProgram(cmd, parameter=None): output = subprocess.check_output('{} | tee /dev/stderr'.format(cmd), shell=True) + if __name__ == "__main__": import doctest diff --git a/scripts/pylot-reasses-pilot-db.py b/scripts/pylot-reasses-pilot-db.py index 5bb4441d..7b55ce6a 100755 --- a/scripts/pylot-reasses-pilot-db.py +++ b/scripts/pylot-reasses-pilot-db.py @@ -19,15 +19,23 @@ if __name__ == '__main__': ) parser.add_argument( - 'dbroot', type=str, help='specifies the root directory (in ' - 'most cases PILOT database folder)' + 'root', type=str, help='specifies the root directory' ) parser.add_argument( - '--output', '-o', type=str, help='path to the output directory', dest='output' + 'db', type=str, help='specifies the database name' ) parser.add_argument( - '--parameterfile', '-p', type=str, help='full path to the parameterfile', dest='parfile' + '--output', '-o', type=str, help='path to the output directory', + dest='output' + ) + parser.add_argument( + '--parameterfile', '-p', type=str, + help='full path to the parameterfile', dest='parfile' + ) + parser.add_argument( + '--verbosity', '-v', action='count', help='increase output verbosity', + default=0, dest='verbosity' ) args = parser.parse_args() - reassess_pilot_db(args.dbroot, args.output, args.parfile) + reassess_pilot_db(args.root, args.db, args.output, args.parfile, args.verbosity) diff --git a/scripts/pylot-reasses-pilot-event.py b/scripts/pylot-reasses-pilot-event.py index 1852e680..b9fd3a39 100755 --- a/scripts/pylot-reasses-pilot-event.py +++ b/scripts/pylot-reasses-pilot-event.py @@ -19,8 +19,10 @@ if __name__ == '__main__': ) parser.add_argument( - 'dbroot', type=str, help='specifies the root directory (in ' - 'most cases PILOT database folder)' + 'root', type=str, help='specifies the root directory' + ) + parser.add_argument( + 'db', type=str, help='specifies the database name' ) parser.add_argument( 'id', type=str, help='PILOT event identifier' @@ -33,4 +35,4 @@ if __name__ == '__main__': ) args = parser.parse_args() - reassess_pilot_event(args.dbroot, args.id, args.output, args.parfile) + reassess_pilot_event(args.root, args.db, args.id, args.output, args.parfile)