diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 76d4a36..bd8cf3f 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,10 @@ +1.0.1b1: + - --fdsn_base_url can now be set for several urls, e.g. --fdsn_base_url "IRIS,ORFEUS" or it can be set to --fdsn_base_url all for which all the urls will be used. + - ArcLink master directory path is now corrected again. Everything seems to work without any problem in using both FDSN and ArcLink. + - --seismicity option is replaced by --plot_seismicity + - --event_url does not exist anymore to make the code more user-friendly. Instead, the user should only selects --event_catalog and the rest is done internally + - in case that no waveforms are requested, the SAC option is diabled to avoid unnecessary warnings + - Added a function to print all available FDSN web-service providers. Example: obspyDMT --fdsn_urls 1.0.0: - version 1.0.0 1.0.0rc1: diff --git a/README.rst b/README.rst index 03157a1..ef2fb40 100755 --- a/README.rst +++ b/README.rst @@ -10,7 +10,7 @@ This tool is developed mainly to address the following tasks automatically: 1. Retrieval of waveforms (MSEED or SAC), stationXML/response files and metadata from FDSN and ArcLink archives. This could be done in *serial* or in *parallel* for single or large requests. 2. Supports both event-based and continuous requests. -3. Extracting the information of all the events via user-defined options (time span, magnitude, depth and event location) from IRIS, NEIC and GCMT. Therefore, moment tensor information can also be retrieved. +3. Extracting the information of all the events via user-defined options (time span, magnitude, depth and event location) from NEIC, Global CMT, IRIS DMC, NCEDC, USGS, INGV and ISC. Therefore, moment tensor information can also be retrieved. 4. Updating existing archives (waveforms, stationXML/response files and metadata). 5. Processing the data in *serial* or in *parallel* (e.g. *removing the trends and means of the time series, tapering, filtering and Instrument correction*). 6. Management of large seismological datasets. @@ -862,16 +862,16 @@ The fourth figure is similar to the third one, but the time shifts of "bad stati Seismicity ---------- -Geographical and historical distribution of earthquake activities (seismicity) can be plotted using *--seismicity* option in obspyDMT. In this mode, the software finds the events according to the input parameters and generates an image in which the events are categorized based on depth and magnitude. +Geographical and historical distribution of earthquake activities (seismicity) can be plotted using *--plot_seismicity* option in obspyDMT. In this mode, the software finds the events according to the input parameters and generates an image in which the events are categorized based on depth and magnitude. **Example 1:** the command line to create *Japan* seismicity map from all the events available in IRIS with magnitude more than 5.0 since 2000 is as follow: :: - $ obspyDMT --datapath 'seismicity_japan' --seismicity --min_mag 5.0 --min_date 2000-01-01 --max_date 2014-12-31 --event_rect 120.0/155.0/25.0/55.0 + $ obspyDMT --datapath 'seismicity_japan' --plot_seismicity --min_mag 5.0 --min_date 2000-01-01 --max_date 2014-12-31 --event_rect 120.0/155.0/25.0/55.0 -*--datapath* is the address where the event catalog will be created, *--seismicity* enables the seismicity mode and *--min_mag*, *--min_date*, *--max_date* and *--event_rect* are event search parameters. +*--datapath* is the address where the event catalog will be created, *--plot_seismicity* enables the seismicity mode and *--min_mag*, *--min_date*, *--max_date* and *--event_rect* are event search parameters. .. image:: figures/seismicity_japan.png :scale: 50% @@ -883,7 +883,7 @@ follow: (27057 events) :: - $ obspyDMT --datapath 'seismicity_glob' --seismicity --min_mag 5.0 --min_date 2000-01-01 --max_date 2014-12-31 + $ obspyDMT --datapath 'seismicity_glob' --plot_seismicity --min_mag 5.0 --min_date 2000-01-01 --max_date 2014-12-31 .. image:: figures/seismicity_glob.png :scale: 50% @@ -956,7 +956,7 @@ is as follow: :: - $ obspyDMT --datapath seismicity_glob_neic --seismicity --min_mag 5.0 --min_date 1976-01-01 --max_date 2014-12-31 --event_catalog NEIC_USGS + $ obspyDMT --datapath seismicity_glob_neic --plot_seismicity --min_mag 5.0 --min_date 1976-01-01 --max_date 2014-12-31 --event_catalog NEIC_USGS .. image:: figures/seismicity_glob_neic_nonfocal.png @@ -995,7 +995,7 @@ Histogram of event depths: :: - $ obspyDMT --datapath 'seismicity_japan_gcmt' --seismicity --min_mag 5.0 --min_date 1976-01-01 --max_date 2013-12-31 --event_rect 120.0/155.0/25.0/55.0 --event_catalog GCMT_COMBO + $ obspyDMT --datapath 'seismicity_japan_gcmt' --plot_seismicity --min_mag 5.0 --min_date 1976-01-01 --max_date 2013-12-31 --event_rect 120.0/155.0/25.0/55.0 --event_catalog GCMT_COMBO .. image:: figures/seismicity_japan_gcmt.png :scale: 80% diff --git a/obspyDMT/obspyDMT.py b/obspyDMT/obspyDMT.py index b763ab4..7841fa1 100755 --- a/obspyDMT/obspyDMT.py +++ b/obspyDMT/obspyDMT.py @@ -17,7 +17,7 @@ from utils.arclink_handler import ARC_network from utils.event_handler import get_Events, create_tar_file -from utils.fdsn_handler import FDSN_network +from utils.fdsn_handler import FDSN_network, FDSN_urls from utils.input_handler import command_parse, read_input_command from utils.instrument_handler import FDSN_ARC_IC from utils.merge_handler import FDSN_ARC_merge @@ -44,6 +44,9 @@ def obspyDMT(**kwargs): (options, args, parser) = command_parse() # ------------------Read INPUT file (Parameters)-------------------- input_dics = read_input_command(parser, **kwargs) + # ------------------FDSN URLs-------------------- + if input_dics['fdsn_urls']: + FDSN_urls() # ------------------plot stationxml files-------------------- if input_dics['plotxml_dir']: plot_xml_response(input_dics) diff --git a/obspyDMT/tests/test_event_handler.py b/obspyDMT/tests/test_event_handler.py index d203351..ce2c0c2 100644 --- a/obspyDMT/tests/test_event_handler.py +++ b/obspyDMT/tests/test_event_handler.py @@ -41,7 +41,7 @@ def test_gmt_catalog(): evradmax = input_dics['evradmax'] evradmin = input_dics['evradmin'] - client_fdsn = Client_fdsn(base_url=input_dics['event_url']) + client_fdsn = Client_fdsn(base_url=input_dics['event_catalog']) events_QML = client_fdsn.get_events( minlatitude=evlatmin, maxlatitude=evlatmax, @@ -58,7 +58,7 @@ def test_gmt_catalog(): minmagnitude=input_dics['min_mag'], maxmagnitude=input_dics['max_mag'], orderby='time', - catalog=input_dics['event_catalog'], + catalog=None, magnitudetype=input_dics['mag_type']) assert events_QML[0].preferred_origin().latitude == 38.2963 diff --git a/obspyDMT/tests/test_input_handler.py b/obspyDMT/tests/test_input_handler.py index 9f1a6a1..d030690 100644 --- a/obspyDMT/tests/test_input_handler.py +++ b/obspyDMT/tests/test_input_handler.py @@ -35,10 +35,10 @@ def test_read_input_command(): assert len(parser.option_groups[1].option_list) == 2 assert len(parser.option_groups[2].option_list) == 13 assert len(parser.option_groups[3].option_list) == 3 - assert len(parser.option_groups[4].option_list) == 15 + assert len(parser.option_groups[4].option_list) == 14 assert len(parser.option_groups[5].option_list) == 7 assert len(parser.option_groups[6].option_list) == 8 - assert len(parser.option_groups[7].option_list) == 5 + assert len(parser.option_groups[7].option_list) == 6 assert len(parser.option_groups[8].option_list) == 3 assert len(parser.option_groups[9].option_list) == 14 assert len(parser.option_groups[10].option_list) == 3 @@ -62,8 +62,7 @@ def test_default_inputs(): UTCDateTime(input_dics['min_date']) < (60 * 60 * 24 * 5 + 1)) assert (UTCDateTime(input_dics['max_date']) - UTCDateTime(input_dics['min_date']) > (60 * 60 * 24 * 5 - 1)) - assert input_dics['event_url'] == 'IRIS' - assert input_dics['event_catalog'] is None + assert input_dics['event_catalog'] == 'IRIS' assert input_dics['mag_type'] is None assert input_dics['min_mag'] == 5.5 assert input_dics['max_mag'] == 9.9 diff --git a/obspyDMT/utils/arclink_handler.py b/obspyDMT/utils/arclink_handler.py index 8b17a66..ff91ade 100644 --- a/obspyDMT/utils/arclink_handler.py +++ b/obspyDMT/utils/arclink_handler.py @@ -30,6 +30,9 @@ from resample_handler import resample_all from utility_codes import read_list_stas, calculate_time_phase, getFolderSize +import warnings +warnings.filterwarnings("ignore", category=UserWarning) + # ##################### Arclink_network ################################# @@ -41,11 +44,9 @@ def ARC_network(input_dics, events): print '\n***********************************************************' print 'ArcLink -- Download waveforms, response files and meta-data' print '***********************************************************' - period = '{0:s}_{1:s}_{2:s}_{3:s}'.format( + period = '{0:s}_{1:s}'.format( input_dics['min_date'].split('T')[0], - input_dics['max_date'].split('T')[0], - str(input_dics['min_mag']), - str(input_dics['max_mag'])) + input_dics['max_date'].split('T')[0]) eventpath = os.path.join(input_dics['datapath'], period) if input_dics['FDSN'] != 'Y': @@ -136,11 +137,9 @@ def ARC_waveform(input_dics, events, Sta_req, i, req_type): add_event = [] if req_type == 'save': - period = '{0:s}_{1:s}_{2:s}_{3:s}'.\ + period = '{0:s}_{1:s}'.\ format(input_dics['min_date'].split('T')[0], - input_dics['max_date'].split('T')[0], - str(input_dics['min_mag']), - str(input_dics['max_mag'])) + input_dics['max_date'].split('T')[0]) eventpath = os.path.join(input_dics['datapath'], period) for k in range(len(events)): add_event.append(os.path.join(eventpath, events[k]['event_id'])) diff --git a/obspyDMT/utils/event_handler.py b/obspyDMT/utils/event_handler.py index 0b1a112..cf5e61f 100644 --- a/obspyDMT/utils/event_handler.py +++ b/obspyDMT/utils/event_handler.py @@ -150,22 +150,23 @@ def events_info(input_dics, request): evradmax = input_dics['evradmax'] event_switch = 'fdsn' + event_url = input_dics['event_catalog'] + event_fdsn_cat = None + if input_dics['event_catalog']: if input_dics['event_catalog'].lower() == 'gcmt_combo': event_switch = 'gcmt_combo' if input_dics['event_catalog'].lower() == 'neic_usgs': event_switch = 'neic_usgs' - else: - print 'Event(s) are based on:\t', - print input_dics['event_url'] - else: - print 'Event(s) are based on:\t', - print input_dics['event_url'] + if input_dics['event_catalog'].lower() == 'isc': + event_url = 'IRIS' + event_fdsn_cat = 'ISC' - print 'Specified catalog: \t%s\n' % input_dics['event_catalog'] + print 'Event(s) are based on:\t', + print input_dics['event_catalog'] if event_switch == 'fdsn': - client_fdsn = Client_fdsn(base_url=input_dics['event_url']) + client_fdsn = Client_fdsn(base_url=event_url) events_QML = client_fdsn.get_events( minlatitude=evlatmin, maxlatitude=evlatmax, @@ -182,7 +183,7 @@ def events_info(input_dics, request): minmagnitude=input_dics['min_mag'], maxmagnitude=input_dics['max_mag'], orderby='time', - catalog=input_dics['event_catalog'], + catalog=event_fdsn_cat, magnitudetype=input_dics['mag_type']) elif event_switch == 'gcmt_combo': events_QML = \ diff --git a/obspyDMT/utils/fdsn_handler.py b/obspyDMT/utils/fdsn_handler.py index c3fc01f..52a08f7 100644 --- a/obspyDMT/utils/fdsn_handler.py +++ b/obspyDMT/utils/fdsn_handler.py @@ -20,10 +20,13 @@ import multiprocessing try: from obspy.clients.fdsn import Client as Client_fdsn + from obspy.clients.fdsn import URL_MAPPINGS except Exception, e: from obspy.fdsn import Client as Client_fdsn + from obspy.fdsn.header import URL_MAPPINGS import os import pickle +import sys from event_handler import quake_info, create_folders_files from format_converter import writesac_all @@ -82,6 +85,19 @@ def FDSN_network(input_dics, events): if Stas_fdsn != [[]]: FDSN_waveform(input_dics, events, Stas_fdsn, i, req_type='save') + fdsn_url_orig = input_dics['fdsn_base_url'] + for cli_rest in input_dics['fdsn_base_url_rest']: + try: + from update_handler import FDSN_update + input_dics['fdsn_base_url'] = cli_rest + input_dics['fdsn_update'] = target_path + FDSN_update(input_dics, address=target_path) + except Exception, e: + print e + continue + if len(input_dics['fdsn_base_url_rest']) > 0: + input_dics['fdsn_base_url'] = fdsn_url_orig + input_dics['fdsn_update'] = 'N' else: print 'No available station in %s for your request and ' \ 'for event %s!' % (input_dics['fdsn_base_url'], str(i+1)) @@ -406,19 +422,28 @@ def FDSN_download_core(i, j, dic, len_events, events, add_event, if input_dics['response'] == 'Y': dummy = 'Response' - client_fdsn.get_stations(network=Sta_req[j][0], - station=Sta_req[j][1], - location=Sta_req[j][2], - channel=Sta_req[j][3], - # starttime=t_start, endtime=t_end, - filename=os.path.join(add_event[i], - 'Resp', - 'STXML.%s.%s.%s.%s' - % (Sta_req[j][0], - Sta_req[j][1], - Sta_req[j][2], - Sta_req[j][3])), - level='response') + if os.path.isfile( + os.path.join(add_event[i], + 'BH_RAW', + '%s.%s.%s.%s' % (Sta_req[j][0], + Sta_req[j][1], + Sta_req[j][2], + Sta_req[j][3]))): + client_fdsn.get_stations(network=Sta_req[j][0], + station=Sta_req[j][1], + location=Sta_req[j][2], + channel=Sta_req[j][3], + starttime=t_start, endtime=t_end, + filename=os.path.join( + add_event[i], 'Resp', + 'STXML.%s.%s.%s.%s' + % (Sta_req[j][0], + Sta_req[j][1], + Sta_req[j][2], + Sta_req[j][3])), + level='response') + else: + raise Exception("Waveform does not exist! (666)") print "%ssaving Response for: %s.%s.%s.%s ---> DONE" \ % (info_req, Sta_req[j][0], Sta_req[j][1], @@ -486,22 +511,24 @@ def FDSN_download_core(i, j, dic, len_events, events, add_event, time_file.close() if len(Sta_req[j]) != 0: - print '%s%s---%s.%s.%s.%s' % (info_req, dummy, - Sta_req[j][0], - Sta_req[j][1], - Sta_req[j][2], - Sta_req[j][3]) ee = 'fdsn -- %s---%s-%s---%s.%s.%s.%s---%s\n' \ % (dummy, i+1, j+1, - Sta_req[j][0], Sta_req[j][1], Sta_req[j][2], - Sta_req[j][3], e) + Sta_req[j][0], Sta_req[j][1], + Sta_req[j][2], Sta_req[j][3], e) + if not '666' in ee: + print '%s%s---%s.%s.%s.%s' % (info_req, dummy, + Sta_req[j][0], + Sta_req[j][1], + Sta_req[j][2], + Sta_req[j][3]) else: ee = 'There is no available station for this event.' Exception_file = open(os.path.join(add_event[i], 'info', 'exception'), 'a') Exception_file.writelines(ee) Exception_file.close() - print 'ERROR: %s' % ee + if not '666' in ee: + print 'ERROR: %s' % ee # ##################### FDSN_bulk_request ################################## @@ -586,6 +613,24 @@ def FDSN_reporter(i, add_event, events, input_dics, Sta_req, len_sta_ev, print 'Total Time: %s' % t_wave print "------------------------" +# ##################### FDSN_urls ################################## + + +def FDSN_urls(): + """ + Function to print available web service providers + """ + print "\nList of all shortcut names" + print "--------------------------\n" + for key in sorted(URL_MAPPINGS.keys()): + print("{0:<7} {1}".format(key, URL_MAPPINGS[key])) + print "\n============================================================" + print "This is the list of all shortcut names which can be used for" + print "--fdsn_base_url option." + print "However, the base URL can be entered directly as well." + print "============================================================" + sys.exit() + # -------------------------------- TRASH # parallel_len_req_fdsn = range(0, len_req_fdsn) diff --git a/obspyDMT/utils/input_handler.py b/obspyDMT/utils/input_handler.py index 48fb98a..f2cfdfa 100644 --- a/obspyDMT/utils/input_handler.py +++ b/obspyDMT/utils/input_handler.py @@ -268,12 +268,8 @@ def command_parse(): group_ev.add_option("--event_circle", action="store", dest="event_circle", help=helpmsg) - helpmsg = "event webservice (IRIS or NERIES). [Default: 'IRIS']" - group_ev.add_option("--event_url", action="store", - dest="event_url", help=helpmsg) - - helpmsg = "event catalog (GCMT_COMBO, IRIS, ISC, EMSC, GCMT, NEIC PDE). " \ - "[Default: IRIS]" + helpmsg = "event catalog (NEIC_USGS, GCMT_COMBO, IRIS, NCEDC, " \ + "USGS, INGV, ISC, NERIES). [Default: IRIS]" group_ev.add_option("--event_catalog", action="store", dest="event_catalog", help=helpmsg) @@ -383,7 +379,17 @@ def command_parse(): # --------------- FDSN group_fdsn = OptionGroup(parser, "8. FDSN") + + helpmsg = "Print available FDSN web service providers." + group_fdsn.add_option("--fdsn_urls", action="store_true", + dest="fdsn_urls", help=helpmsg) + helpmsg = "base_url for FDSN requests (waveform/response). " \ + "This option can be defined: --fdsn_base_url IRIS or " \ + "--fdsn_base_url 'IRIS,ORFEUS' for which all the stations of " \ + "IRIS will be downloaded and the directory will be updated for " \ + "ORFEUS. It is also possible to --fdsn_base_url all which will " \ + "download waveforms for all available FDSN urls. " \ "[Default: 'IRIS']" group_fdsn.add_option("--fdsn_base_url", action="store", dest="fdsn_base_url", help=helpmsg) @@ -573,7 +579,7 @@ def command_parse(): group_plt = OptionGroup(parser, "13. Plotting") helpmsg = "create a seismicity map according to " \ "the event and location specifications." - group_plt.add_option("--seismicity", action="store_true", + group_plt.add_option("--plot_seismicity", action="store_true", dest="seismicity", help=helpmsg) helpmsg = "depth bins for plotting the seismicity histrogram. " \ @@ -776,7 +782,6 @@ def read_input_command(parser, **kwargs): input_dics = {'datapath': 'obspyDMT-data', 'min_date': str(UTCDateTime() - 60 * 60 * 24 * 10 * 1), 'max_date': str(UTCDateTime() - 60 * 60 * 24 * 5 * 1), - 'event_url': 'IRIS', 'event_catalog': 'IRIS', 'mag_type': None, 'min_mag': 5.5, 'max_mag': 9.9, @@ -884,7 +889,7 @@ def read_input_command(parser, **kwargs): if options.version: print '\n\t\t' + '*********************************' print '\t\t' + '* obspyDMT version: *' - print '\t\t' + '*' + '\t\t' + '1.0.0' + '\t\t' + '*' + print '\t\t' + '*' + '\t\t' + '1.0.1b1' + '\t\t' + '*' print '\t\t' + '*********************************' print '\n' sys.exit(2) @@ -907,7 +912,6 @@ def read_input_command(parser, **kwargs): options.max_date = '2011-03-12' options.min_mag = '8.9' options.identity = 'TA.1*.*.BHZ' - options.event_url = 'IRIS' options.event_catalog = 'IRIS' options.req_parallel = True options.ArcLink = 'N' @@ -1048,12 +1052,9 @@ def read_input_command(parser, **kwargs): input_dics['cut_time_phase'] = False input_dics['min_date'] = str(UTCDateTime(options.min_date)) input_dics['max_date'] = str(UTCDateTime(options.max_date)) - input_dics['event_url'] = options.event_url.upper() if options.event_catalog: input_dics['event_catalog'] = options.event_catalog.upper() - if input_dics['event_catalog'].upper() == 'IRIS': - input_dics['event_catalog'] = None if options.read_catalog: input_dics['read_catalog'] = options.read_catalog @@ -1099,6 +1100,9 @@ def read_input_command(parser, **kwargs): input_dics['req_parallel'] = options.req_parallel input_dics['req_np'] = int(options.req_np) input_dics['list_stas'] = options.list_stas + if options.fdsn_urls: + options.fdsn_urls = 'Y' + input_dics['fdsn_urls'] = options.fdsn_urls if options.fdsn_bulk: options.fdsn_bulk = 'Y' input_dics['fdsn_bulk'] = options.fdsn_bulk @@ -1134,6 +1138,25 @@ def read_input_command(parser, **kwargs): input_dics['resample_corr'] = False input_dics['FDSN'] = options.FDSN input_dics['fdsn_base_url'] = options.fdsn_base_url + if input_dics['fdsn_base_url'].lower() == 'all': + input_dics['fdsn_base_url'] = \ + "LMU,GFZ,ETH,INGV,NIEP,IPGP,RESIF,ORFEUS,ODC,NERIES,BGR,KOERI," \ + "GEONET," \ + "USP," \ + "NCEDC,SCEDC,IRIS" + print "\n---------------------------------" + print "Waveforms will be retrieved from:" + print input_dics['fdsn_base_url'] + print "---------------------------------\n\n" + + input_dics['fdsn_base_url'] = \ + [x.strip() for x in input_dics['fdsn_base_url'].split(',')] + if len(input_dics['fdsn_base_url']) > 1: + input_dics['fdsn_base_url_rest'] = input_dics['fdsn_base_url'][1:] + input_dics['fdsn_base_url'] = input_dics['fdsn_base_url'][0] + else: + input_dics['fdsn_base_url_rest'] = [] + input_dics['fdsn_base_url'] = input_dics['fdsn_base_url'][0] input_dics['fdsn_user'] = options.fdsn_user input_dics['fdsn_pass'] = options.fdsn_pass input_dics['ArcLink'] = options.ArcLink @@ -1318,6 +1341,9 @@ def read_input_command(parser, **kwargs): input_dics['arc_merge_auto'] = 'N' input_dics['max_result'] = 1000000 + if input_dics['waveform'] == 'N': + input_dics['SAC'] = 'N' + if options.FDSN == 'N': input_dics['fdsn_ic_auto'] = 'N' input_dics['fdsn_merge_auto'] = 'N' diff --git a/obspyDMT/utils/instrument_handler.py b/obspyDMT/utils/instrument_handler.py index e0cb46f..88dff3b 100644 --- a/obspyDMT/utils/instrument_handler.py +++ b/obspyDMT/utils/instrument_handler.py @@ -43,6 +43,8 @@ def FDSN_ARC_IC(input_dics, clients): :param clients: :return: """ + if len(input_dics['fdsn_base_url_rest']) > 0: + clients = 'all_fdsn' print '\n*****************************' print '%s -- Instrument Correction' % clients print '*****************************'