From 9cf2d0ddc1e138e1aed665137b5d9213ee47716a Mon Sep 17 00:00:00 2001 From: Didi Hoffmann Date: Mon, 4 Aug 2014 10:50:42 +0100 Subject: [PATCH] Rewrite of the REST Webhook interface to run with celery in parallel and save the return codes of the webhooks. Also it retries 3 times if it fails. --- odk_viewer/models/parsed_instance.py | 15 +- odk_viewer/pandas_mongo_bridge.py | 8 +- odk_viewer/tasks.py | 1 - restservice/RestServiceInterface.py | 3 - .../0003_auto__add_restserviceanswer.py | 152 ++++++++++++++++++ ...__add_field_restserviceanswer_iteration.py | 147 +++++++++++++++++ restservice/models.py | 20 ++- restservice/services/__init__.py | 1 - restservice/services/bamboo.py | 39 ----- restservice/services/f2dhis2.py | 14 -- restservice/services/generic_json.py | 19 --- restservice/services/generic_xml.py | 14 -- restservice/tasks.py | 131 +++++++++++++++ restservice/tests/test_restservice.py | 7 +- restservice/utils.py | 12 +- utils/export_tools.py | 34 +++- 16 files changed, 494 insertions(+), 123 deletions(-) delete mode 100644 restservice/RestServiceInterface.py create mode 100644 restservice/migrations/0003_auto__add_restserviceanswer.py create mode 100644 restservice/migrations/0004_auto__add_field_restserviceanswer_iteration.py delete mode 100644 restservice/services/__init__.py delete mode 100644 restservice/services/bamboo.py delete mode 100644 restservice/services/f2dhis2.py delete mode 100644 restservice/services/generic_json.py delete mode 100644 restservice/services/generic_xml.py create mode 100644 restservice/tasks.py diff --git a/odk_viewer/models/parsed_instance.py b/odk_viewer/models/parsed_instance.py index 15c7d1040..f33a08835 100644 --- a/odk_viewer/models/parsed_instance.py +++ b/odk_viewer/models/parsed_instance.py @@ -6,7 +6,7 @@ from dateutil import parser from bson import json_util from django.conf import settings -from django.db import models +from django.db import models, transaction from django.db.models.signals import post_save, pre_delete from restservice.utils import call_service from stats.tasks import stat_log @@ -294,12 +294,15 @@ def _remove_from_mongo(sender, **kwargs): pre_delete.connect(_remove_from_mongo, sender=ParsedInstance) - def rest_service_form_submission(sender, **kwargs): - parsed_instance = kwargs.get('instance') - created = kwargs.get('created') - if created: - call_service(parsed_instance) + + # So we can be sure that all data is saved + # and we can find it once we pass the task + # on to celery. + transaction.commit() + + if kwargs.get('created'): + call_service(kwargs.get('instance')) post_save.connect(rest_service_form_submission, sender=ParsedInstance) diff --git a/odk_viewer/pandas_mongo_bridge.py b/odk_viewer/pandas_mongo_bridge.py index acd74125c..a72f5f987 100644 --- a/odk_viewer/pandas_mongo_bridge.py +++ b/odk_viewer/pandas_mongo_bridge.py @@ -70,7 +70,7 @@ class AbstractDataFrameBuilder(object): IGNORED_COLUMNS = [XFORM_ID_STRING, STATUS, ID, ATTACHMENTS, GEOLOCATION, BAMBOO_DATASET_ID, DELETEDAT] # fields NOT within the form def that we want to include - ADDITIONAL_COLUMNS = [UUID, SUBMISSION_TIME] + ADDITIONAL_COLUMNS = [UUID, SUBMISSION_TIME, "webhooks"] """ Group functionality used by any DataFrameBuilder i.e. XLS, CSV and KML @@ -463,7 +463,7 @@ def _reindex(cls, key, value, ordered_columns, parent_prefix = None): d = {} # check for lists - if type(value) is list and len(value) > 0: + if type(value) is list and len(value) > 0 and key != "webhooks" : for index, item in enumerate(value): # start at 1 index += 1 @@ -480,7 +480,7 @@ def _reindex(cls, key, value, ordered_columns, parent_prefix = None): # re-create xpath the split on / xpaths = "/".join(xpaths).split("/") new_prefix = xpaths[:-1] - if type(nested_val) is list: + if type(nested_val) is list and nested_key != "webhooks": # if nested_value is a list, rinse and repeat d.update(cls._reindex(nested_key, nested_val, ordered_columns, new_prefix)) @@ -590,7 +590,7 @@ def export_to(self, file_or_path, data_frame_max_size=30000): # add extra columns columns += [col for col in self.ADDITIONAL_COLUMNS] - + header = True if hasattr(file_or_path, 'read'): csv_file = file_or_path diff --git a/odk_viewer/tasks.py b/odk_viewer/tasks.py index 4b71d9cd8..a155a6b88 100644 --- a/odk_viewer/tasks.py +++ b/odk_viewer/tasks.py @@ -56,7 +56,6 @@ def _create_export(xform, export_type): result = create_kml_export.apply_async( (), arguments, countdown=10) elif export_type == Export.JSON_EXPORT: - print arguments result = create_json_export.apply_async( (), arguments, countdown=10) else: diff --git a/restservice/RestServiceInterface.py b/restservice/RestServiceInterface.py deleted file mode 100644 index 633a71feb..000000000 --- a/restservice/RestServiceInterface.py +++ /dev/null @@ -1,3 +0,0 @@ -class RestServiceInterface(object): - def send(self, url, data=None): - pass diff --git a/restservice/migrations/0003_auto__add_restserviceanswer.py b/restservice/migrations/0003_auto__add_restserviceanswer.py new file mode 100644 index 000000000..5f61686ce --- /dev/null +++ b/restservice/migrations/0003_auto__add_restserviceanswer.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'RestServiceAnswer' + db.create_table(u'restservice_restserviceanswer', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('service', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['restservice.RestService'])), + ('instance', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['odk_viewer.ParsedInstance'])), + ('returnCode', self.gf('django.db.models.fields.CharField')(max_length=4)), + ('returnText', self.gf('django.db.models.fields.TextField')()), + ('date', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), + )) + db.send_create_signal(u'restservice', ['RestServiceAnswer']) + + + def backwards(self, orm): + # Deleting model 'RestServiceAnswer' + db.delete_table(u'restservice_restserviceanswer') + + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'odk_logger.instance': { + 'Meta': {'object_name': 'Instance'}, + 'date': ('django.db.models.fields.DateField', [], {'null': 'True'}), + 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'deleted_at': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_deleted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'start_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "u'submitted_via_web'", 'max_length': '20'}), + 'survey_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['odk_logger.SurveyType']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'surveys'", 'null': 'True', 'to': u"orm['auth.User']"}), + 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '249'}), + 'xform': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'surveys'", 'null': 'True', 'to': "orm['odk_logger.XForm']"}), + 'xml': ('django.db.models.fields.TextField', [], {}) + }, + 'odk_logger.surveytype': { + 'Meta': {'object_name': 'SurveyType'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'slug': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}) + }, + 'odk_logger.xform': { + 'Meta': {'ordering': "('id_string',)", 'unique_together': "(('user', 'id_string'), ('user', 'sms_id_string'))", 'object_name': 'XForm'}, + 'allows_sms': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'bamboo_dataset': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '60'}), + 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'default': "u''", 'null': 'True'}), + 'encrypted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'form_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_start_time': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'id_string': ('django.db.models.fields.SlugField', [], {'max_length': '100'}), + 'is_crowd_form': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'json': ('django.db.models.fields.TextField', [], {'default': "u''"}), + 'last_submission_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'num_of_submissions': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'shared': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'shared_data': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'sms_id_string': ('django.db.models.fields.SlugField', [], {'default': "''", 'max_length': '50'}), + 'surveys_with_geopoints': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'xforms'", 'null': 'True', 'to': u"orm['auth.User']"}), + 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '32'}), + 'xls': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}), + 'xml': ('django.db.models.fields.TextField', [], {}) + }, + 'odk_viewer.parsedinstance': { + 'Meta': {'object_name': 'ParsedInstance'}, + 'end_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'parsed_instance'", 'unique': 'True', 'to': "orm['odk_logger.Instance']"}), + 'lat': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'lng': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'start_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}) + }, + 'restservice.restservice': { + 'Meta': {'unique_together': "(('service_url', 'xform', 'name'),)", 'object_name': 'RestService'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'service_url': ('django.db.models.fields.URLField', [], {'max_length': '200'}), + 'xform': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['odk_logger.XForm']"}) + }, + u'restservice.restserviceanswer': { + 'Meta': {'object_name': 'RestServiceAnswer'}, + 'date': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['odk_viewer.ParsedInstance']"}), + 'returnCode': ('django.db.models.fields.CharField', [], {'max_length': '4'}), + 'returnText': ('django.db.models.fields.TextField', [], {}), + 'service': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['restservice.RestService']"}) + }, + u'taggit.tag': { + 'Meta': {'object_name': 'Tag'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100'}) + }, + u'taggit.taggeditem': { + 'Meta': {'object_name': 'TaggedItem'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_tagged_items'", 'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_items'", 'to': u"orm['taggit.Tag']"}) + } + } + + complete_apps = ['restservice'] \ No newline at end of file diff --git a/restservice/migrations/0004_auto__add_field_restserviceanswer_iteration.py b/restservice/migrations/0004_auto__add_field_restserviceanswer_iteration.py new file mode 100644 index 000000000..209310a2b --- /dev/null +++ b/restservice/migrations/0004_auto__add_field_restserviceanswer_iteration.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'RestServiceAnswer.iteration' + db.add_column(u'restservice_restserviceanswer', 'iteration', + self.gf('django.db.models.fields.PositiveIntegerField')(default=0), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'RestServiceAnswer.iteration' + db.delete_column(u'restservice_restserviceanswer', 'iteration') + + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'odk_logger.instance': { + 'Meta': {'object_name': 'Instance'}, + 'date': ('django.db.models.fields.DateField', [], {'null': 'True'}), + 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'deleted_at': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_deleted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'start_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "u'submitted_via_web'", 'max_length': '20'}), + 'survey_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['odk_logger.SurveyType']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'surveys'", 'null': 'True', 'to': u"orm['auth.User']"}), + 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '249'}), + 'xform': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'surveys'", 'null': 'True', 'to': "orm['odk_logger.XForm']"}), + 'xml': ('django.db.models.fields.TextField', [], {}) + }, + 'odk_logger.surveytype': { + 'Meta': {'object_name': 'SurveyType'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'slug': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}) + }, + 'odk_logger.xform': { + 'Meta': {'ordering': "('id_string',)", 'unique_together': "(('user', 'id_string'), ('user', 'sms_id_string'))", 'object_name': 'XForm'}, + 'allows_sms': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'bamboo_dataset': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '60'}), + 'date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'date_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'default': "u''", 'null': 'True'}), + 'encrypted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'form_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_start_time': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'id_string': ('django.db.models.fields.SlugField', [], {'max_length': '100'}), + 'is_crowd_form': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'json': ('django.db.models.fields.TextField', [], {'default': "u''"}), + 'last_submission_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'num_of_submissions': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'shared': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'shared_data': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'sms_id_string': ('django.db.models.fields.SlugField', [], {'default': "''", 'max_length': '50'}), + 'surveys_with_geopoints': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'xforms'", 'null': 'True', 'to': u"orm['auth.User']"}), + 'uuid': ('django.db.models.fields.CharField', [], {'default': "u''", 'max_length': '32'}), + 'xls': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}), + 'xml': ('django.db.models.fields.TextField', [], {}) + }, + 'odk_viewer.parsedinstance': { + 'Meta': {'object_name': 'ParsedInstance'}, + 'end_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'parsed_instance'", 'unique': 'True', 'to': "orm['odk_logger.Instance']"}), + 'lat': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'lng': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'start_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}) + }, + 'restservice.restservice': { + 'Meta': {'unique_together': "(('service_url', 'xform', 'name'),)", 'object_name': 'RestService'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'service_url': ('django.db.models.fields.URLField', [], {'max_length': '200'}), + 'xform': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['odk_logger.XForm']"}) + }, + u'restservice.restserviceanswer': { + 'Meta': {'object_name': 'RestServiceAnswer'}, + 'date': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'instance': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['odk_viewer.ParsedInstance']"}), + 'iteration': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'returnCode': ('django.db.models.fields.CharField', [], {'max_length': '4'}), + 'returnText': ('django.db.models.fields.TextField', [], {}), + 'service': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['restservice.RestService']"}) + }, + u'taggit.tag': { + 'Meta': {'object_name': 'Tag'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100'}) + }, + u'taggit.taggeditem': { + 'Meta': {'object_name': 'TaggedItem'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_tagged_items'", 'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_items'", 'to': u"orm['taggit.Tag']"}) + } + } + + complete_apps = ['restservice'] \ No newline at end of file diff --git a/restservice/models.py b/restservice/models.py index 6c2c9860c..b9f1487a9 100644 --- a/restservice/models.py +++ b/restservice/models.py @@ -18,11 +18,21 @@ def __unicode__(self): return u"%s:%s - %s" % (self.xform, self.long_name, self.service_url) def get_service_definition(self): - m = __import__(''.join(['restservice.services.', self.name]), - globals(), locals(), ['ServiceDefinition']) - return m.ServiceDefinition + m = __import__(''.join(['restservice.tasks']), globals(), \ + locals(),[self.name]) + a = getattr(m,self.name) + return a @property def long_name(self): - sv = self.get_service_definition() - return sv.verbose_name + return [i for i in SERVICE_CHOICES if i[0]==self.name][0][1] + + + +class RestServiceAnswer(models.Model): + service = models.ForeignKey(RestService) + instance = models.ForeignKey("odk_viewer.ParsedInstance") + returnCode = models.CharField(max_length=4) + returnText = models.TextField() + iteration = models.PositiveIntegerField(default=0) + date = models.DateTimeField(auto_now=True) diff --git a/restservice/services/__init__.py b/restservice/services/__init__.py deleted file mode 100644 index 032b71c0a..000000000 --- a/restservice/services/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ('f2dhis2','generic_json','generic_xml','bamboo') diff --git a/restservice/services/bamboo.py b/restservice/services/bamboo.py deleted file mode 100644 index c52612e3e..000000000 --- a/restservice/services/bamboo.py +++ /dev/null @@ -1,39 +0,0 @@ - -from pybamboo.dataset import Dataset -from pybamboo.connection import Connection - -from restservice.RestServiceInterface import RestServiceInterface -from utils.bamboo import get_new_bamboo_dataset, get_bamboo_url - - -class ServiceDefinition(RestServiceInterface): - id = u'bamboo' - verbose_name = u'bamboo POST' - - def send(self, url, parsed_instance): - - xform = parsed_instance.instance.xform - rows = [parsed_instance.to_dict_for_mongo()] - - # prefix meta columns names for bamboo - prefix = (u'%(id_string)s_%(id)s' - % {'id_string': xform.id_string, 'id': xform.id}) - - for row in rows: - for col, value in row.items(): - if col.startswith('_') or col.startswith('meta_') \ - or col.startswith('meta/'): - new_col = (u'%(prefix)s%(col)s' - % {'prefix': prefix, 'col': col}) - row.update({new_col: value}) - del(row[col]) - - # create dataset on bamboo first (including current submission) - if not xform.bamboo_dataset: - dataset_id = get_new_bamboo_dataset(xform, force_last=True) - xform.bamboo_dataset = dataset_id - xform.save() - else: - dataset = Dataset(connection=Connection(url=get_bamboo_url(xform)), - dataset_id=xform.bamboo_dataset) - dataset.update_data(rows=rows) diff --git a/restservice/services/f2dhis2.py b/restservice/services/f2dhis2.py deleted file mode 100644 index 24a181b1f..000000000 --- a/restservice/services/f2dhis2.py +++ /dev/null @@ -1,14 +0,0 @@ -import httplib2 -from restservice.RestServiceInterface import RestServiceInterface - - -class ServiceDefinition(RestServiceInterface): - id = u'f2dhis2' - verbose_name = u'Formhub to DHIS2' - - def send(self, url, parsed_instance): - instance = parsed_instance.instance - info = {"id_string": instance.xform.id_string, "uuid": instance.uuid} - valid_url = url % info - http = httplib2.Http() - resp, content = http.request(valid_url, 'GET') diff --git a/restservice/services/generic_json.py b/restservice/services/generic_json.py deleted file mode 100644 index 063e13dbc..000000000 --- a/restservice/services/generic_json.py +++ /dev/null @@ -1,19 +0,0 @@ -import httplib2 -import json - -from odk_viewer.models import ParsedInstance -from restservice.RestServiceInterface import RestServiceInterface - - -class ServiceDefinition(RestServiceInterface): - id = u'json' - verbose_name = u'JSON POST' - - def send(self, url, parsed_instance): - post_data = json.dumps(parsed_instance.to_dict_for_mongo()) - headers = {"Content-Type": "application/json"} - http = httplib2.Http() - resp, content = http.request(uri=url, method='POST', - headers=headers, - body=post_data) - diff --git a/restservice/services/generic_xml.py b/restservice/services/generic_xml.py deleted file mode 100644 index 58084974c..000000000 --- a/restservice/services/generic_xml.py +++ /dev/null @@ -1,14 +0,0 @@ -import httplib2 -from restservice.RestServiceInterface import RestServiceInterface - - -class ServiceDefinition(RestServiceInterface): - id = u'xml' - verbose_name = u'XML POST' - - def send(self, url, parsed_instance): - instance = parsed_instance.instance - headers = {"Content-Type": "application/xml"} - http = httplib2.Http() - resp, content = http.request( - url, method="POST", body=instance.xml, headers=headers) diff --git a/restservice/tasks.py b/restservice/tasks.py new file mode 100644 index 000000000..b4823667f --- /dev/null +++ b/restservice/tasks.py @@ -0,0 +1,131 @@ +import httplib2 +import json +from celery import task +from odk_viewer.models import ParsedInstance +from restservice.models import RestServiceAnswer +from httplib2 import RelativeURIError, ServerNotFoundError +import time + +from django.conf import settings + +#If we ever want to use it uncomment +#from utils.bamboo import get_new_bamboo_dataset, get_bamboo_url + + +def get_network_data(method, url, headers, post_data, parsed_instance, services): + http = httplib2.Http() + + for i in range(3): + try: + + if method == "GET": + requ, content = http.request(url, 'GET') + elif method == "POST": + requ, content = http.request(uri=url, method=method, + headers=headers, + body=post_data) + except RelativeURIError: + status = "500" + content = "RelativeURIError" + except ServerNotFoundError: + status = "500" + content = "ServerNotFoundError" + except Exception as e: + status = "500" + content = unicode( type(e) ) + else: + status = requ["status"] + + response = RestServiceAnswer() + response.service = services + response.instance = parsed_instance + response.returnCode = status + response.returnText = content + response.iteration = i + response.save() + + #Also add it to Mongo + record = {} + record["url"] = unicode(url) + record["status"] = unicode(status) + record["content"] = unicode(content) + record["try"] = i + record["timestamp"] = unicode(response.date) + + mongo_instance = settings.MONGO_DB.instances + + mongo_instance.update({"_id":parsed_instance.pk}, { '$push': { "webhooks" :record } }) + + #If the status is 200 everything is fine + if status == '200': + break + + #So we give the service some time to recover, if that is the problem + time.sleep(60) + + + + +@task() +def generic_json(url, parsed_instance_pk, services): + + parsed_instance = ParsedInstance.objects.get(pk=parsed_instance_pk) + + post_data = json.dumps(parsed_instance.to_dict_for_mongo()) + headers = {"Content-Type": "application/json"} + + return get_network_data("POST", url, headers, post_data, parsed_instance, services) + + + +@task() +def generic_xml(url, parsed_instance_pk, services): + + parsed_instance = ParsedInstance.objects.get(pk=parsed_instance_pk) + + instance = parsed_instance.instance + headers = {"Content-Type": "application/xml"} + + return get_network_data("POST", url, headers, instance.xml, parsed_instance, services) + +@task() +def f2dhis2(url, parsed_instance_pk, services): + parsed_instance = ParsedInstance.objects.get(pk=parsed_instance_pk) + + instance = parsed_instance.instance + info = {"id_string": instance.xform.id_string, "uuid": instance.uuid} + valid_url = url % info + + return get_network_data("GET", valid_url, None, None, parsed_instance, services) + +# For now we don't need this. +# But should work as a dropin replacement +# def bamboo(self, url, parsed_instance): +# +# xform = parsed_instance.instance.xform +# rows = [parsed_instance.to_dict_for_mongo()] +# +# # prefix meta columns names for bamboo +# prefix = (u'%(id_string)s_%(id)s' +# % {'id_string': xform.id_string, 'id': xform.id}) +# +# for row in rows: +# for col, value in row.items(): +# if col.startswith('_') or col.startswith('meta_') \ +# or col.startswith('meta/'): +# new_col = (u'%(prefix)s%(col)s' +# % {'prefix': prefix, 'col': col}) +# row.update({new_col: value}) +# del(row[col]) +# +# # create dataset on bamboo first (including current submission) +# if not xform.bamboo_dataset: +# dataset_id = get_new_bamboo_dataset(xform, force_last=True) +# xform.bamboo_dataset = dataset_id +# xform.save() +# else: +# dataset = Dataset(connection=Connection(url=get_bamboo_url(xform)), +# dataset_id=xform.bamboo_dataset) +# dataset.update_data(rows=rows) +# +# return diff --git a/restservice/tests/test_restservice.py b/restservice/tests/test_restservice.py index 8bc6174d0..08ff19dda 100644 --- a/restservice/tests/test_restservice.py +++ b/restservice/tests/test_restservice.py @@ -10,9 +10,9 @@ from main.tests.test_base import MainTestCase from odk_logger.models.xform import XForm from restservice.views import add_service, delete_service -from restservice.RestServiceInterface import RestServiceInterface from restservice.models import RestService from django.utils.unittest.case import skip +from restservice.tasks import f2dhis2 #from nose import SkipTest @@ -57,8 +57,9 @@ def test_create_rest_service(self): def test_service_definition(self): self._create_rest_service() - sv = self.restservice.get_service_definition()() - self.assertEqual(isinstance(sv, RestServiceInterface), True) + sv = self.restservice.get_service_definition() + self.assertTrue(hasattr(sv, '__call__')) + self.assertEqual(sv.__name__, "f2dhis2") def test_add_service(self): self._add_rest_service(self.service_url, self.service_name) diff --git a/restservice/utils.py b/restservice/utils.py index 44e4b0cce..201fb853d 100644 --- a/restservice/utils.py +++ b/restservice/utils.py @@ -1,16 +1,10 @@ from restservice.models import RestService - def call_service(parsed_instance): # lookup service instance = parsed_instance.instance services = RestService.objects.filter(xform=instance.xform) - # call service send with url and data parameters + # call service send with url and instance id for sv in services: - # TODO: Queue service - try: - service = sv.get_service_definition()() - service.send(sv.service_url, parsed_instance) - except: - # TODO: Handle gracefully | requeue/resend - pass + service = sv.get_service_definition() + service.delay(sv.service_url, parsed_instance.pk, sv) diff --git a/utils/export_tools.py b/utils/export_tools.py index 6ae605483..8f51741dd 100644 --- a/utils/export_tools.py +++ b/utils/export_tools.py @@ -25,6 +25,7 @@ from odk_viewer.models.parsed_instance import _is_invalid_for_mongo,\ _encode_for_mongo, dict_for_mongo, _decode_from_mongo import logging +from restservice.models import RestServiceAnswer log = logging.getLogger('utils.export_tools') @@ -111,7 +112,7 @@ def dict_to_joined_export(data, index, indices, name): # TODO: test for _geolocation and attachment lists if isinstance(data, dict): for key, val in data.iteritems(): - if isinstance(val, list): + if isinstance(val, list) and key != "webhooks": output[key] = [] for child in val: if key not in indices: @@ -144,7 +145,7 @@ class ExportBuilder(object): BAMBOO_DATASET_ID, DELETEDAT] # fields we export but are not within the form's structure EXTRA_FIELDS = [ID, UUID, SUBMISSION_TIME, INDEX, PARENT_TABLE_NAME, - PARENT_INDEX] + PARENT_INDEX, "webhooks"] SPLIT_SELECT_MULTIPLES = True # column group delimiters @@ -377,11 +378,30 @@ def to_json_export(self, path, data, *args): form_id_string = args[1] pis = ParsedInstance.objects.filter(instance__user__username=username, instance__xform__id_string=form_id_string) - json_dump = [x.to_dict() for x in pis] + + #Add the return code of the webhooks + json_dump =[] + for x in pis: + dic = x.to_dict() + webhooks = [] + for i in RestServiceAnswer.objects.filter(instance=x): + record = {} + record["url"] = i.service.service_url + record["status"] = i.returnCode + record["content"] = i.returnText + record["try"] = i.iteration + record["timestamp"] = unicode(i.date) + webhooks.append(record) + + if len(webhooks) > 0: + dic["webhooks"] = webhooks + + json_dump.append(dic) + with open(path, 'w') as outfile: json.dump(json_dump, outfile) - def to_zipped_csv(self, path, data, *args): + def to_zipped_csv(self, path, data, username, id_string, filter_query): def encode_if_str(row, key): val = row.get(key) if isinstance(val, basestring): @@ -409,18 +429,21 @@ def write_row(row, csv_writer, fields): index = 1 indices = {} survey_name = self.survey.name + for d in data: # decode mongo section names joined_export = dict_to_joined_export(d, index, indices, survey_name) output = ExportBuilder.decode_mongo_encoded_section_names( joined_export) + # attach meta fields (index, parent_index, parent_table) # output has keys for every section if survey_name not in output: output[survey_name] = {} output[survey_name][INDEX] = index output[survey_name][PARENT_INDEX] = -1 + for section in self.sections: # get data for this section and write to csv section_name = section['name'] @@ -431,7 +454,8 @@ def write_row(row, csv_writer, fields): csv_writer = csv_def['csv_writer'] # section name might not exist within the output, e.g. data was # not provided for said repeat - write test to check this - row = output.get(section_name, None) + row = output[section_name] + if type(row) == dict: write_row( self.pre_process_row(row, section),