Merge pull request #383 from eqcorrscan/develop

Version 0.4.1

.stickler.yml

@@ -1,2 +1,3 @@
 linters:
-  flake8: {  }
+  flake8:
+    python: 3

CHANGES.md

@@ -1,3 +1,36 @@
+## 0.4.1
+* core.match_filter
+  - BUG-FIX: Empty families are no longer run through lag-calc when using 
+    Party.lag_calc().  Previously this resulted in a "No matching data" error,
+    see #341.
+* core.template_gen
+  - BUG-FIX: Fix bug where events were incorrectly associated with templates
+    in `Tribe().construct()` if the given catalog contained events outside
+    of the time-range of the stream. See issue #381 and PR #382.
+* utils.catalog_to_dd
+  - Added ability to turn off parallel processing (this is turned off by 
+    default now) for `write_correlations` - parallel processing for moderate
+    to large datasets was copying far too much data and using lots of memory.
+    This is a short-term fix - ideally we will move filtering and resampling to
+    C functions with shared-memory parallelism and GIL releasing.
+    See PR #374.
+  - Moved parallelism for `_compute_dt_correlations` to the C functions to
+    reduce memory overhead. Using a generator to construct sub-catalogs rather
+    than making a list of lists in memory. See issue #361.
+* utils.mag_calc:
+  - `amp_pick_event` now works on a copy of the data by default
+  - `amp_pick_event` uses the appropriate digital filter gain to correct the
+    applied filter. See issue #376.
+  - `amp_pick_event` rewritten for simplicity.
+  - `amp_pick_event` now has simple synthetic tests for accuracy.
+  - `_sim_wa` uses the full response information to correct to velocity
+    this includes FIR filters (previously not used), and ensures that the
+    wood-anderson poles (with a single zero) are correctly applied to velocity
+    waveforms.
+  - `calc_max_curv` is now computed using the non-cumulative distribution.
+* Some problem solved in _match_filter_plot. Now it shows all new detections.
+* Add plotdir to eqcorrscan.core.lag_calc.lag_calc function to save the images.
+
 ## 0.4.0
 * Change resampling to use pyFFTW backend for FFT's.  This is an attempt to
   alleviate issue related to large-prime length transforms.  This requires an
@@ -97,6 +130,8 @@
   change the quality of correlations.
 * Removed depreciated `template_gen` functions and `bright_lights` and
   `seismo_logs`. See #315
+* BUG-FIX: `eqcorrscan.core.template_gen.py` fix conflict with special character on windows
+  output-filename. See issue #344
 
 ## 0.3.3
 * Make test-script more stable.

CONTRIBUTORS.md

@@ -7,3 +7,4 @@
 * Chris Scott
 * Felix Halpaap
 * Iman Kahbasi
+* eQ Halauwet

appveyor.yml

@@ -39,7 +39,8 @@ install:
   #      else
   #        PYFLAKES="pyflakes=0.9.0"
   #      fi
-  - "conda install -q --yes pip obspy numpy=1.14 scipy>=0.18 matplotlib mock flake8 pyflakes cython h5py pyproj bottleneck lxml pyfftw"
+  # prproj needs to be pinned before obspy 1.2.0
+  - "conda install -q --yes pip obspy numpy=1.14 scipy>=0.18 matplotlib mock flake8 pyflakes cython h5py bottleneck pyfftw"
   # additional dependecies
   - "pip install pyimgur"
 

eqcorrscan/__init__.py

@@ -25,7 +25,7 @@
 
 __all__ = ['core', 'utils', 'tutorials', 'tests']
 
-__version__ = '0.4.0'
+__version__ = '0.4.1'
 
 # Cope with changes to name-space to remove most of the camel-case
 _import_map = {}

eqcorrscan/core/lag_calc.py

@@ -112,10 +112,6 @@ def _concatenate_and_correlate(streams, template, cores):
     """
     UsedChannel = namedtuple("UsedChannel", "channel used")
 
-    channel_length = {tr.stats.npts for st in streams for tr in st}
-    assert len(channel_length) == 1, "Multiple lengths found."
-    channel_length = channel_length.pop()
-
     samp_rate = {tr.stats.sampling_rate for st in streams for tr in st}
     assert len(samp_rate) == 1, "Multiple sample rates found"
     samp_rate = samp_rate.pop()
@@ -124,6 +120,10 @@ def _concatenate_and_correlate(streams, template, cores):
     assert len(template_length) == 1, "Multiple data lengths in template"
     template_length = template_length.pop()
 
+    channel_length = {tr.stats.npts for st in streams for tr in st}
+    if len(channel_length) > 1:
+        Logger.debug("Multiple lengths of stream found, using the longest")
+    channel_length = sorted(list(channel_length))[-1]
     # pre-define stream for efficiency
     chans = {tr.id for st in streams for tr in st}.intersection(
         {tr.id for tr in template})
@@ -145,7 +145,7 @@ def _concatenate_and_correlate(streams, template, cores):
                 start_index += channel_length
                 continue
             assert len(tr) == 1, "Multiple channels found for {0}".format(chan)
-            data[i][start_index:start_index + channel_length] = tr[0].data
+            data[i][start_index:start_index + tr[0].stats.npts] = tr[0].data
             start_index += channel_length
             used_chans[j].append(UsedChannel(
                 channel=(chan.split('.')[1], chan.split('.')[-1]), used=True))
@@ -232,9 +232,10 @@ def xcorr_pick_family(family, stream, shift_len=0.2, min_cc=0.4,
     detection_ids = list(detect_streams_dict.keys())
     detect_streams = [detect_streams_dict[detection_id]
                       for detection_id in detection_ids]
-    assert len(detect_streams) > 0, "No appropriate data found, check your " \
-                                    "family and detections - make sure seed " \
-                                    "ids match"
+    if len(detect_streams) == 0:
+        Logger.warning("No appropriate data found, check your family and "
+                       "detections - make sure seed ids match")
+        return picked_dict
     if len(detect_streams) != len(family):
         Logger.warning("Not all detections have matching data. "
                        "Proceeding anyway. HINT: Make sure SEED IDs match")
@@ -386,7 +387,7 @@ def _prepare_data(family, detect_data, shift_len):
 def lag_calc(detections, detect_data, template_names, templates,
              shift_len=0.2, min_cc=0.4, horizontal_chans=['E', 'N', '1', '2'],
              vertical_chans=['Z'], cores=1, interpolate=False,
-             plot=False):
+             plot=False, plotdir=None):
     """
     Cross-correlation derived picking of seismic events.
 
@@ -434,6 +435,8 @@ def lag_calc(detections, detect_data, template_names, templates,
     :type plot: bool
     :param plot:
         To generate a plot for every detection or not, defaults to False
+    :param plotdir:
+        Path to plotting folder, plots will be output here.
 
     :returns:
         Catalog of events with picks.  No origin information is included.
@@ -513,7 +516,7 @@ def lag_calc(detections, detect_data, template_names, templates,
                 family=family, stream=detect_data,
                 min_cc=min_cc, horizontal_chans=horizontal_chans,
                 vertical_chans=vertical_chans, interpolate=interpolate,
-                cores=cores, shift_len=shift_len, plot=plot)
+                cores=cores, shift_len=shift_len, plot=plot, plotdir=plotdir)
             initial_cat.update(template_dict)
     # Order the catalogue to match the input
     output_cat = Catalog()

eqcorrscan/core/match_filter/detection.py

@@ -255,7 +255,10 @@ def _calculate_event(self, template=None, template_st=None,
                 template_prepick = template.prepick
             else:
                 template_prepick = 0
-            template_picks = template.event.picks
+            try:
+                template_picks = template.event.picks
+            except AttributeError:
+                template_picks = []
         else:
             template_prepick = 0
             template_picks = []
@@ -298,7 +301,8 @@ def _calculate_event(self, template=None, template_st=None,
                     except IndexError:
                         Logger.error("No pick for trace")
                 ev.picks.append(new_pick)
-        if estimate_origin and template is not None:
+        if estimate_origin and template is not None\
+                and template.event is not None:
             try:
                 template_origin = (template.event.preferred_origin() or
                                    template.event.origins[0])

eqcorrscan/core/match_filter/family.py

@@ -544,7 +544,7 @@ def lag_calc(self, stream, pre_processed, shift_len=0.2, min_cc=0.4,
         :param plot:
             To generate a plot for every detection or not, defaults to False.
         :type plotdir: str
-    	:param plotdir:
+        :param plotdir:
             The path to save plots to. If `plotdir=None` (default) then the
             figure will be shown on screen.
         :type parallel: bool
@@ -657,10 +657,19 @@ def relative_magnitudes(self, stream, pre_processed, process_cores=1,
             event = detection.event
             if event is None:
                 continue
-            corr_dict = {
-                p.waveform_id.get_seed_string():
-                    float(p.comments[0].text.split("=")[-1])
-                for p in event.picks}
+            corr_dict = {}
+            for pick in event.picks:
+                if len(pick.comments) == 0:
+                    continue
+                try:
+                    corr = float(pick.comments[0].text.split("=")[-1])
+                except IndexError:
+                    continue
+                corr_dict.update({pick.waveform_id.get_seed_string(): corr})
+            if len(corr_dict) == 0:
+                Logger.info("Correlations not run for {0}".format(
+                    detection.id))
+                continue
             template = self.template
             try:
                 t_mag = (

eqcorrscan/core/match_filter/matched_filter.py

@@ -99,7 +99,7 @@ def _group_detect(templates, stream, threshold, threshold_type, trig_int,
     :param plot:
         Turn plotting on or off.
     :type plotdir: str
-	:param plotdir:
+    :param plotdir:
         The path to save plots to. If `plotdir=None` (default) then the
         figure will be shown on screen.
     :type group_size: int
@@ -219,7 +219,8 @@ def _group_detect(templates, stream, threshold, threshold_type, trig_int,
                 xcorr_func=xcorr_func, concurrency=concurrency,
                 threshold=threshold, threshold_type=threshold_type,
                 trig_int=trig_int, plot=plot, plotdir=plotdir, cores=cores,
-                full_peaks=full_peaks, peak_cores=process_cores, **kwargs)
+                full_peaks=full_peaks, peak_cores=process_cores,
+                **kwargs)
             for template in template_group:
                 family = Family(template=template, detections=[])
                 for detection in detections:
@@ -412,11 +413,15 @@ def match_filter(template_names, template_list, st, threshold,
         Check arguments, defaults to True, but if running in bulk, and you are
         certain of your arguments, then set to False.
     :type full_peaks: bool
-    :param full_peaks: See `eqcorrscan.core.findpeaks.find_peaks2_short`.
+    :param full_peaks: See
+        :func: `eqcorrscan.utils.findpeaks.find_peaks2_short`
     :type peak_cores: int
     :param peak_cores:
         Number of processes to use for parallel peak-finding (if different to
         `cores`).
+    :type spike_test: bool
+    :param spike_test: If set True, raise error when there is a spike in data.
+        defaults to True.
 
     .. Note::
         When using the "fftw" correlation backend the length of the fft
@@ -594,6 +599,7 @@ def match_filter(template_names, template_list, st, threshold,
                     raise MatchFilterError(
                         'Template contains masked array, split first')
     if spike_test:
+        Logger.info("Checking for spikes in data")
         _spike_test(st)
     if cores is not None:
         parallel = True
@@ -606,6 +612,7 @@ def match_filter(template_names, template_list, st, threshold,
     templates = [t.copy() for t in template_list]
     _template_names = template_names.copy()  # This can just be a shallow copy
 
+    Logger.info("Reshaping templates")
     stream, templates, _template_names = _prep_data_for_correlation(
         stream=stream, templates=templates, template_names=_template_names)
     if len(templates) == 0:

eqcorrscan/core/match_filter/party.py

@@ -790,7 +790,7 @@ def lag_calc(self, stream, pre_processed, shift_len=0.2, min_cc=0.4,
         :param plot:
             To generate a plot for every detection or not, defaults to False
         :type plotdir: str
-    	:param plotdir:
+        :param plotdir:
             The path to save plots to. If `plotdir=None` (default) then the
             figure will be shown on screen.
         :type parallel: bool
@@ -838,7 +838,8 @@ def lag_calc(self, stream, pre_processed, shift_len=0.2, min_cc=0.4,
         """
         process_cores = process_cores or cores
         template_groups = group_templates(
-            [_f.template for _f in self.families])
+            [_f.template for _f in self.families
+             if len(_f) > 0])  # Fix for #341
         catalog = Catalog()
         for template_group in template_groups:
             family = [_f for _f in self.families
@@ -949,10 +950,13 @@ def get_catalog(self):
 
     def min_chans(self, min_chans):
         """
-        Remove detections with fewer channels used than min_chans
+        Remove detections using min_chans or fewer channels.
 
         :type min_chans: int
-        :param min_chans: Minimum number of channels to allow a detection.
+        :param min_chans:
+            Detections using more than this number of channels are maintained.
+            Note that this is a strict `if detection.no_chans > min_chans:`
+            rather than >=. Maintained for backwards compatability.
         :return: Party
 
         .. Note:: Works in place on Party.

eqcorrscan/core/match_filter/template.py

@@ -75,7 +75,7 @@ class Template(object):
     def __init__(self, name=None, st=None, lowcut=None, highcut=None,
                  samp_rate=None, filt_order=None, process_length=None,
                  prepick=None, event=None):
-        name_regex = re.compile(r"^[a-z_0-9]+$")
+        name_regex = re.compile(r"^[-a-z_0-9]+$")
         if name is not None and not re.match(name_regex, name):
             raise ValueError("Invalid name: '%s' - Must satisfy the regex "
                              "'%s'." % (name, name_regex.pattern))
@@ -398,7 +398,7 @@ def detect(self, stream, threshold, threshold_type, trig_int,
         :type plot: bool
         :param plot: Turn plotting on or off.
         :type plotdir: str
-    	:param plotdir:
+        :param plotdir:
             The path to save plots to. If `plotdir=None` (default) then the
             figure will be shown on screen.
         :type pre_processed: bool
@@ -570,7 +570,7 @@ def construct(self, method, name, lowcut, highcut, samp_rate, filt_order,
         :type plot: bool
         :param plot: Plot templates or not.
         :type plotdir: str
-    	:param plotdir:
+        :param plotdir:
             The path to save plots to. If `plotdir=None` (default) then the
             figure will be shown on screen.
         :type min_snr: float

eqcorrscan/core/match_filter/tribe.py

@@ -294,7 +294,8 @@ def write(self, filename, compress=True, catalog_format="QUAKEML"):
             if t.event is not None:
                 # Check that the name in the comment matches the template name
                 for comment in t.event.comments:
-                    if comment.text.startswith("eqcorrscan_template_"):
+                    if comment.text and comment.text.startswith(
+                            "eqcorrscan_template_"):
                         comment.text = "eqcorrscan_template_{0}".format(t.name)
                 tribe_cat.append(t.event)
         if len(tribe_cat) > 0:
@@ -448,7 +449,7 @@ def detect(self, stream, threshold, threshold_type, trig_int, plot=False,
         :type plot: bool
         :param plot: Turn plotting on or off.
         :type plotdir: str
-    	:param plotdir:
+        :param plotdir:
             The path to save plots to. If `plotdir=None` (default) then the
             figure will be shown on screen.
         :type daylong: bool
@@ -632,7 +633,7 @@ def client_detect(self, client, starttime, endtime, threshold,
         :type plot: bool
         :param plot: Turn plotting on or off.
         :type plotdir: str
-    	:param plotdir:
+        :param plotdir:
             The path to save plots to. If `plotdir=None` (default) then the
             figure will be shown on screen.
         :type min_gap: float
@@ -940,7 +941,7 @@ def construct(self, method, lowcut, highcut, samp_rate, filt_order,
         :type plot: bool
         :param plot: Plot templates or not.
         :type plotdir: str
-    	:param plotdir:
+        :param plotdir:
             The path to save plots to. If `plotdir=None` (default) then the
             figure will be shown on screen.
         :type min_snr: float