Merge remote-tracking branch 'upstream/master'

minidump-stackwalk/Makefile

@@ -30,16 +30,23 @@ EXTRA_OBJS := \
 
 VPATH += $(JSON_SRCDIR)
 
+OS := $(shell uname -s)
+ifeq ($(OS),Linux)
 CURL_CFLAGS := $(shell pkg-config libcurl --cflags)
 CURL_LIBS := $(shell pkg-config libcurl --libs)
+# Don't -Werror everywhere, some compilers are too picky.
+WERROR := -Werror
+else
+CURL_LIBS := -lcurl
+endif
 
 CXXFLAGS += \
   -I$(BREAKPAD_SRCDIR) \
   -I$(JSON_INCLUDEDIR) \
   -D__STDC_FORMAT_MACROS=1 \
   -std=gnu++0x \
   -Wno-format \
-  -Werror \
+  $(WERROR) \
   $(CURL_CFLAGS) \
   $(NULL)
 LIBS := $(CURL_LIBS)

socorro/external/rabbitmq/connection_context.py

@@ -334,4 +334,3 @@ def force_reconnect(self, name=None):
             name = self.config.executor_identity()
         if name in self.pool:
             del self.pool[name]
-

socorro/processor/symbol_cache_manager.py

@@ -245,3 +245,9 @@ def _get_existing_files(self, path):
     # -------------------------------------------------------------------------
     def close(self):
         self._notifier.stop()
+
+
+# =============================================================================
+class NoOpCacheManager(RequiredConfig):
+    def __init__(self, *args, **kwargs):
+        pass

tools/submit_crash_to_rabbitmq.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+
+"""
+Script to put crash IDs into RabbitMQ from the command line.
+This is useful when you have a local processor running without a
+collector or crashmover.
+
+To run it, pass one more more crash IDs like this::
+
+
+    $ ./tools/submit_crash_to_rabbitmq.py e9a624b0-fbcc-4e95-a0a8-d2a152160907
+
+"""
+
+import logging
+
+from configman import configuration, Namespace
+from socorrolib.lib.util import DotDict
+from socorrolib.lib.converters import change_default
+from socorro.external.rabbitmq.connection_context import ConnectionContext
+from socorro.external.rabbitmq.crashstorage import RabbitMQCrashStorage
+from socorro.database.transaction_executor import TransactionExecutor
+
+logger = logging.getLogger(__file__)
+
+
+class SingleCrashMQCrashStorage(RabbitMQCrashStorage):
+    required_config = Namespace()
+    required_config.routing_key = change_default(
+        RabbitMQCrashStorage,
+        'routing_key',
+        'socorro.normal'
+    )
+    required_config.rabbitmq_class = change_default(
+        RabbitMQCrashStorage,
+        'rabbitmq_class',
+        ConnectionContext,
+    )
+    required_config.transaction_executor_class = change_default(
+        RabbitMQCrashStorage,
+        'transaction_executor_class',
+        TransactionExecutor
+    )
+
+    def submit(self, crash_ids):
+        if not isinstance(crash_ids, (list, tuple)):
+            crash_ids = [crash_ids]
+        success = bool(crash_ids)
+        for crash_id in crash_ids:
+            if not self.save_raw_crash(
+                DotDict({'legacy_processing': 0}),
+                [],
+                crash_id
+            ):
+                success = False
+        return success
+
+
+def run(*crash_ids):
+
+    definition_source = Namespace()
+    definition_source.namespace('queuing')
+    definition_source.queuing.add_option(
+        'rabbitmq_reprocessing_class',
+        default=SingleCrashMQCrashStorage,
+    )
+    config_dict = {
+        'resource': {
+            'rabbitmq': {
+                'host': 'localhost',
+                'port': '5672',
+                'virtual_host': '/'
+            }
+        },
+        'secrets': {
+            'rabbitmq': {
+                'rabbitmq_password': 'guest',
+                'rabbitmq_user': 'guest'
+            }
+        }
+    }
+    config = configuration(
+        definition_source=definition_source,
+        values_source_list=[config_dict],
+    )
+    config.queuing.logger = logger
+    config.logger = logger
+    storage = SingleCrashMQCrashStorage(config=config['queuing'])
+    for crash_id in crash_ids:
+        print storage.submit(crash_id)
+    return 0
+
+
+if __name__ == '__main__':
+    import sys
+    args = sys.argv[1:]
+    sys.exit(run(*args))

webapp-django/crashstats/crashstats/templatetags/jinja_helpers.py

@@ -15,6 +15,7 @@
 from django.utils.safestring import mark_safe
 
 from crashstats import scrubber
+from crashstats.crashstats.utils import parse_isodate
 
 
 @library.filter
@@ -94,10 +95,9 @@ def timestamp_to_date(
 def time_tag(dt, format='%a, %b %d %H:%M %Z', future=False):
     if not isinstance(dt, (datetime.date, datetime.datetime)):
         try:
-            dt = isodate.parse_datetime(dt)
+            dt = parse_isodate(dt)
         except isodate.ISO8601Error:
             return dt
-
     return jinja2.Markup(
         '<time datetime="{}" class="{}">{}</time>'
         .format(
@@ -113,7 +113,7 @@ def human_readable_iso_date(dt):
     """ Python datetime to a human readable ISO datetime. """
     if not isinstance(dt, (datetime.date, datetime.datetime)):
         try:
-            dt = isodate.parse_datetime(dt)
+            dt = parse_isodate(dt)
         except isodate.ISO8601Error:
             # Because we're paranoid, we don't want to fail
             # the whole template rendering just because one date

webapp-django/crashstats/crashstats/tests/test_jinja_helpers.py

@@ -71,6 +71,16 @@ def test_time_tag_invalid_date(self):
         output = time_tag('junk')
         eq_(output, 'junk')
 
+    def test_parse_with_unicode_with_timezone(self):
+        # See https://bugzilla.mozilla.org/show_bug.cgi?id=1300921
+        date = u'2016-09-07T00:38:42.630775+00:00'
+        output = time_tag(date)
+
+        eq_(output, '<time datetime="{}" class="ago">{}</time>'.format(
+            '2016-09-07T00:38:42.630775+00:00',
+            'Wed, Sep 07 00:38 +00:00'
+        ))
+
 
 class TestRecursiveStateFilter(TestCase):
 

webapp-django/crashstats/crashstats/utils.py

@@ -32,14 +32,15 @@ def unixtime(value, millis=False, format='%Y-%m-%d'):
         return epoch_seconds
 
 
-def parse_isodate(ds, format_string="%b %d %Y %H:%M:%S"):
+def parse_isodate(ds):
     """
-    parses iso8601 date string and returns a truncated
-    string representation suitable for display on the status page
+    return a datetime object from a date string
     """
-    if not ds:
-        return ""
-    return isodate.parse_datetime(ds).strftime(format_string)
+    if isinstance(ds, unicode):
+        # isodate struggles to convert unicode strings with
+        # its parse_datetime() if the input string is unicode.
+        ds = ds.encode('ascii')
+    return isodate.parse_datetime(ds)
 
 
 def daterange(start_date, end_date, format='%Y-%m-%d'):

webapp-django/crashstats/crashstats/views.py

@@ -9,7 +9,6 @@
 from operator import itemgetter
 from io import BytesIO
 
-import isodate
 import isoweek
 
 from django import http
@@ -1779,7 +1778,7 @@ def report_list(request, partial=None, default_context=None):
 
             counts[(product, os_name, version)] += 1
 
-            report['date_processed'] = isodate.parse_datetime(
+            report['date_processed'] = utils.parse_isodate(
                 report['date_processed']
             ).strftime('%b %d, %Y %H:%M')
 
@@ -1794,9 +1793,7 @@ def report_list(request, partial=None, default_context=None):
                         )
                     else:
                         # old style, middleware returned a formatted string
-                        install_time = isodate.parse_datetime(
-                            install_time
-                        )
+                        install_time = utils.parse_isodate(install_time)
                 # put it back into the report
                 report['install_time'] = install_time.strftime(
                     '%Y-%m-%d %H:%M:%S'

webapp-django/crashstats/signature/views.py

@@ -2,8 +2,6 @@
 import functools
 import math
 
-import isodate
-
 from django import http
 from django.conf import settings
 from django.core.urlresolvers import reverse
@@ -433,17 +431,17 @@ def signature_graph_data(request, params, channel):
             # Set the earliest given start date as the start date
             if date.startswith('>'):
                 if date.startswith('>='):
-                    d = isodate.parse_date(date.lstrip('>='))
+                    d = utils.parse_isodate(date.lstrip('>='))
                 else:
-                    d = isodate.parse_date(date.lstrip('>')) + one_day
+                    d = utils.parse_isodate(date.lstrip('>')) + one_day
                 if not start_date or d < start_date:
                     start_date = d
             # Set the latest given end date as the end date
             elif date.startswith('<'):
                 if date.startswith('<='):
-                    d = isodate.parse_date(date.lstrip('<='))
+                    d = utils.parse_isodate(date.lstrip('<='))
                 else:
-                    d = isodate.parse_date(date.lstrip('<')) - one_day
+                    d = utils.parse_isodate(date.lstrip('<')) - one_day
                 if not end_date or d > end_date:
                     end_date = d
 

webapp-django/crashstats/supersearch/form_fields.py

@@ -5,6 +5,8 @@
 from django import forms
 from django.utils.timezone import utc
 
+from crashstats.crashstats.utils import parse_isodate
+
 
 OPERATORS = (
     '__true__', '__null__', '$', '~', '^', '@', '=', '<=', '>=', '<', '>',
@@ -146,7 +148,7 @@ class IsoDateTimeField(forms.DateTimeField):
     def to_python(self, value):
         if value:
             try:
-                return isodate.parse_datetime(value).replace(tzinfo=utc)
+                return parse_isodate(value).replace(tzinfo=utc)
             except (ValueError, isodate.isoerror.ISO8601Error):
                 # let the super method deal with that
                 pass