[task] started to implement data processing step for checking corrupted GSE files

This commit is contained in:
sebastianp 2016-05-19 15:35:11 +02:00
parent 7c5aff0a27
commit 9b7db91037
3 changed files with 60 additions and 10 deletions

View File

@ -1 +1 @@
1508-dirty 7c5a-dirty

View File

@ -232,7 +232,7 @@ def picksdict_from_picks(evt):
def picks_from_picksdict(picks): def picks_from_picksdict(picks):
picks_list = list() picks_list = list()
for station, onsets in picks.items(): for station, onsets in picks.items():
print('Reading picks on station %s' % station) #print('Reading picks on station %s' % station)
for label, phase in onsets.items(): for label, phase in onsets.items():
if not isinstance(phase, dict) or len(phase) < 3: if not isinstance(phase, dict) or len(phase) < 3:
continue continue
@ -276,6 +276,7 @@ def reassess_pilot_db(root_dir, out_dir=None, fn_param=None):
reassess_pilot_event(root_dir, evt, out_dir, fn_param) reassess_pilot_event(root_dir, evt, out_dir, fn_param)
def reassess_pilot_event(root_dir, event_id, out_dir=None, fn_param=None, verbosity=0): def reassess_pilot_event(root_dir, event_id, out_dir=None, fn_param=None, verbosity=0):
from obspy import read from obspy import read
@ -292,9 +293,11 @@ def reassess_pilot_event(root_dir, event_id, out_dir=None, fn_param=None, verbos
phases_file = glob.glob(os.path.join(search_base, 'PHASES.mat')) phases_file = glob.glob(os.path.join(search_base, 'PHASES.mat'))
if not phases_file: if not phases_file:
return return
print('Opening PILOT phases file: {fn}'.format(fn=phases_file[0])) #print('Opening PILOT phases file: {fn}'.format(fn=phases_file[0]))
picks_dict = picks_from_pilot(phases_file[0]) picks_dict = picks_from_pilot(phases_file[0])
print('Dictionary read from PHASES.mat:\n{0}'.format(picks_dict)) #print('Dictionary read from PHASES.mat:\n{0}'.format(picks_dict))
datacheck = list()
info = None
for station in picks_dict.keys(): for station in picks_dict.keys():
fn_pattern = os.path.join(search_base, '{0}*'.format(station)) fn_pattern = os.path.join(search_base, '{0}*'.format(station))
try: try:
@ -302,6 +305,20 @@ def reassess_pilot_event(root_dir, event_id, out_dir=None, fn_param=None, verbos
except TypeError as e: except TypeError as e:
print(e.message) print(e.message)
st = read(fn_pattern) st = read(fn_pattern)
except ValueError as e:
if e.message == 'second must be in 0..59':
info = 'A known Error was raised. Please find the list of corrupted files and double-check these files.'
datacheck.append(fn_pattern + ' (time info)\n')
continue
else:
raise ValueError(e.message)
except Exception as e:
if 'No file matching file pattern:' in e.message:
warnings.warn('no waveform data found for station {station}'.format(station=station), RuntimeWarning)
datacheck.append(fn_pattern + ' (no data)\n')
continue
else:
raise e
for phase in picks_dict[station].keys(): for phase in picks_dict[station].keys():
try: try:
mpp = picks_dict[station][phase]['mpp'] mpp = picks_dict[station][phase]['mpp']
@ -310,10 +327,7 @@ def reassess_pilot_event(root_dir, event_id, out_dir=None, fn_param=None, verbos
continue continue
sel_st = select_for_phase(st, phase) sel_st = select_for_phase(st, phase)
if not sel_st: if not sel_st:
raise warnings.formatwarning( warnings.warn('no waveform data found for station {station}'.format(station=station), RuntimeWarning)
'no waveform data found for station {station}'.format(
station=station), category=RuntimeWarning)
print(sel_st)
stime, etime = getGlobalTimes(sel_st) stime, etime = getGlobalTimes(sel_st)
rel_pick = mpp - stime rel_pick = mpp - stime
epp, lpp, spe = earllatepicker(sel_st, epp, lpp, spe = earllatepicker(sel_st,
@ -321,7 +335,7 @@ def reassess_pilot_event(root_dir, event_id, out_dir=None, fn_param=None, verbos
default.get('tsnrz' if phase == 'P' else 'tsnrh'), default.get('tsnrz' if phase == 'P' else 'tsnrh'),
Pick1=rel_pick, Pick1=rel_pick,
iplot=None, iplot=None,
) stealthMode=True)
if epp is None or lpp is None: if epp is None or lpp is None:
continue continue
epp = stime + epp epp = stime + epp
@ -332,6 +346,13 @@ def reassess_pilot_event(root_dir, event_id, out_dir=None, fn_param=None, verbos
if mpp - epp < min_diff: if mpp - epp < min_diff:
epp = mpp - min_diff epp = mpp - min_diff
picks_dict[station][phase] = dict(epp=epp, mpp=mpp, lpp=lpp, spe=spe) picks_dict[station][phase] = dict(epp=epp, mpp=mpp, lpp=lpp, spe=spe)
if datacheck:
if info:
print(info + ': {0}'.format(search_base))
fncheck = open(os.path.join(search_base, 'datacheck_list'), 'w')
fncheck.writelines(datacheck)
fncheck.close()
del datacheck
# create Event object for export # create Event object for export
evt = ope.Event(resource_id=event_id) evt = ope.Event(resource_id=event_id)
evt.picks = picks_from_picksdict(picks_dict) evt.picks = picks_from_picksdict(picks_dict)

View File

@ -25,6 +25,29 @@ def check_time(datetime):
:param datetime: list of integers [year, month, day, hour, minute, second, microsecond] :param datetime: list of integers [year, month, day, hour, minute, second, microsecond]
:type datetime: list :type datetime: list
:return: returns True if Values are in supposed range, returns False otherwise :return: returns True if Values are in supposed range, returns False otherwise
>>> check_time([1999, 01, 01, 23, 59, 59, 999000])
True
>>> check_time([1999, 01, 01, 23, 59, 60, 999000])
False
>>> check_time([1999, 01, 01, 23, 59, 59, 1000000])
False
>>> check_time([1999, 01, 01, 23, 60, 59, 999000])
False
>>> check_time([1999, 01, 01, 23, 60, 59, 999000])
False
>>> check_time([1999, 01, 01, 24, 59, 59, 999000])
False
>>> check_time([1999, 01, 31, 23, 59, 59, 999000])
True
>>> check_time([1999, 02, 30, 23, 59, 59, 999000])
False
>>> check_time([1999, 02, 29, 23, 59, 59, 999000])
False
>>> check_time([2000, 02, 29, 23, 59, 59, 999000])
True
>>> check_time([2000, 13, 29, 23, 59, 59, 999000])
False
""" """
try: try:
UTCDateTime(*datetime) UTCDateTime(*datetime)
@ -123,3 +146,9 @@ def evt_head_check(root_dir, out_dir = None):
out.writelines(lines) out.writelines(lines)
out.close() out.close()
print(nfiles) print(nfiles)
if __name__ == "__main__":
import doctest
doctest.testmod()