Rewrite of the REST Webhook interface to run with celery in parallel …

…and save the return codes of the webhooks. Also it retries 3 times if it fails.

odk_viewer/models/parsed_instance.py

@@ -6,7 +6,7 @@
 from dateutil import parser
 from bson import json_util
 from django.conf import settings
-from django.db import models
+from django.db import models, transaction
 from django.db.models.signals import post_save, pre_delete
 from restservice.utils import call_service
 from stats.tasks import stat_log
@@ -294,12 +294,15 @@ def _remove_from_mongo(sender, **kwargs):
 
 pre_delete.connect(_remove_from_mongo, sender=ParsedInstance)
 
-
 def rest_service_form_submission(sender, **kwargs):
-    parsed_instance = kwargs.get('instance')
-    created = kwargs.get('created')
-    if created:
-        call_service(parsed_instance)
+
+    # So we can be sure that all data is saved 
+    # and we can find it once we pass the task
+    # on to celery.
+    transaction.commit()
+
+    if kwargs.get('created'):
+        call_service(kwargs.get('instance'))
 
 
 post_save.connect(rest_service_form_submission, sender=ParsedInstance)

odk_viewer/pandas_mongo_bridge.py

@@ -70,7 +70,7 @@ class AbstractDataFrameBuilder(object):
     IGNORED_COLUMNS = [XFORM_ID_STRING, STATUS, ID, ATTACHMENTS, GEOLOCATION,
                        BAMBOO_DATASET_ID, DELETEDAT]
     # fields NOT within the form def that we want to include
-    ADDITIONAL_COLUMNS = [UUID, SUBMISSION_TIME]
+    ADDITIONAL_COLUMNS = [UUID, SUBMISSION_TIME, "webhooks"]
 
     """
     Group functionality used by any DataFrameBuilder i.e. XLS, CSV and KML
@@ -463,7 +463,7 @@ def _reindex(cls, key, value, ordered_columns, parent_prefix = None):
         d = {}
 
         # check for lists
-        if type(value) is list and len(value) > 0:
+        if type(value) is list and len(value) > 0 and key != "webhooks" :
             for index, item in enumerate(value):
                 # start at 1
                 index += 1
@@ -480,7 +480,7 @@ def _reindex(cls, key, value, ordered_columns, parent_prefix = None):
                         # re-create xpath the split on /
                         xpaths = "/".join(xpaths).split("/")
                         new_prefix = xpaths[:-1]
-                        if type(nested_val) is list:
+                        if type(nested_val) is list and nested_key != "webhooks":
                             # if nested_value is a list, rinse and repeat
                             d.update(cls._reindex(nested_key, nested_val,
                                 ordered_columns, new_prefix))
@@ -590,7 +590,7 @@ def export_to(self, file_or_path, data_frame_max_size=30000):
 
         # add extra columns
         columns += [col for col in self.ADDITIONAL_COLUMNS]
-
+        
         header = True
         if hasattr(file_or_path, 'read'):
             csv_file = file_or_path

odk_viewer/tasks.py

@@ -56,7 +56,6 @@ def _create_export(xform, export_type):
         result = create_kml_export.apply_async(
             (), arguments, countdown=10)
     elif export_type == Export.JSON_EXPORT:
-        print arguments
         result = create_json_export.apply_async(
                 (), arguments, countdown=10)
     else:

restservice/migrations/0003_auto__add_restserviceanswer.py

@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'RestServiceAnswer'
+        db.create_table(u'restservice_restserviceanswer', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('service', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['restservice.RestService'])),
+            ('instance', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['odk_viewer.ParsedInstance'])),
+            ('returnCode', self.gf('django.db.models.fields.CharField')(max_length=4)),
+            ('returnText', self.gf('django.db.models.fields.TextField')()),
+            ('date', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
+        ))
+        db.send_create_signal(u'restservice', ['RestServiceAnswer'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'RestServiceAnswer'
+        db.delete_table(u'restservice_restserviceanswer')
+
+
+    models = {
+        u'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        u'auth.permission': {
+            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        u'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        u'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'odk_logger.instance': {
+            'Meta': {'object_name': 'Instance'},
+            'date': ('django.db.models.fields.DateField', [], {'null': 'True'}),
+            'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'deleted_at': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_deleted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'start_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'status': ('django.db.models.fields.CharField', [], {'default': "u'submitted_via_web'", 'max_length': '20'}),
+            'survey_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['odk_logger.SurveyType']"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'surveys'", 'null': 'True', 'to': u"orm['auth.User']"}),
+            'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '249'}),
+            'xform': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'surveys'", 'null': 'True', 'to': "orm['odk_logger.XForm']"}),
+            'xml': ('django.db.models.fields.TextField', [], {})
+        },
+        'odk_logger.surveytype': {
+            'Meta': {'object_name': 'SurveyType'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'slug': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'})
+        },
+        'odk_logger.xform': {
+            'Meta': {'ordering': "('id_string',)", 'unique_together': "(('user', 'id_string'), ('user', 'sms_id_string'))", 'object_name': 'XForm'},
+            'allows_sms': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'bamboo_dataset': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '60'}),
+            'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'default': "u''", 'null': 'True'}),
+            'encrypted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'form_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'has_start_time': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'id_string': ('django.db.models.fields.SlugField', [], {'max_length': '100'}),
+            'is_crowd_form': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'json': ('django.db.models.fields.TextField', [], {'default': "u''"}),
+            'last_submission_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'num_of_submissions': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'shared': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'shared_data': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'sms_id_string': ('django.db.models.fields.SlugField', [], {'default': "''", 'max_length': '50'}),
+            'surveys_with_geopoints': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'xforms'", 'null': 'True', 'to': u"orm['auth.User']"}),
+            'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '32'}),
+            'xls': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}),
+            'xml': ('django.db.models.fields.TextField', [], {})
+        },
+        'odk_viewer.parsedinstance': {
+            'Meta': {'object_name': 'ParsedInstance'},
+            'end_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'instance': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'parsed_instance'", 'unique': 'True', 'to': "orm['odk_logger.Instance']"}),
+            'lat': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+            'lng': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+            'start_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'})
+        },
+        'restservice.restservice': {
+            'Meta': {'unique_together': "(('service_url', 'xform', 'name'),)", 'object_name': 'RestService'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'service_url': ('django.db.models.fields.URLField', [], {'max_length': '200'}),
+            'xform': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['odk_logger.XForm']"})
+        },
+        u'restservice.restserviceanswer': {
+            'Meta': {'object_name': 'RestServiceAnswer'},
+            'date': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'instance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['odk_viewer.ParsedInstance']"}),
+            'returnCode': ('django.db.models.fields.CharField', [], {'max_length': '4'}),
+            'returnText': ('django.db.models.fields.TextField', [], {}),
+            'service': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['restservice.RestService']"})
+        },
+        u'taggit.tag': {
+            'Meta': {'object_name': 'Tag'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}),
+            'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100'})
+        },
+        u'taggit.taggeditem': {
+            'Meta': {'object_name': 'TaggedItem'},
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_tagged_items'", 'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'object_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}),
+            'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_items'", 'to': u"orm['taggit.Tag']"})
+        }
+    }
+
+    complete_apps = ['restservice']