Log a diff of the ES document and serialized enrollment (#3657)

* Log a diff of the ES document and serialized enrollment

* Revert upgrade of psycopg2

requirements.in

@@ -2,6 +2,7 @@ Django==1.10.5
 Pillow==3.4.2
 boto==2.39.0
 celery==4.0.2
+deepdiff==3.3.0
 dj-database-url==0.3.0
 dj-static==0.0.6
 django-cors-headers==2.1.0
@@ -21,6 +22,7 @@ gnupg==2.2.0
 html5lib==0.999999999
 ipython
 jsonfield==1.0.3
+jsonpatch==1.16
 newrelic
 open-discussions-client==0.2.2
 phonenumbers==8.3.2

requirements.txt

@@ -15,6 +15,7 @@ cffi==1.10.0              # via bcrypt, cryptography, pynacl
 contextlib2==0.5.5        # via raven
 cryptography==2.0.2       # via paramiko
 decorator==4.1.2          # via ipython, traitlets
+deepdiff==3.3.0
 defusedxml==0.5.0         # via python3-openid, social-auth-core
 dj-database-url==0.3.0
 dj-static==0.0.6
@@ -43,6 +44,8 @@ ipython-genutils==0.2.0   # via traitlets
 ipython==6.1.0
 jedi==0.10.2              # via ipython
 jsonfield==1.0.3
+jsonpatch==1.16
+jsonpointer==1.12         # via jsonpatch
 kombu==4.1.0              # via celery, django-server-status
 newrelic==2.88.1.73
 oauthlib==2.0.2           # via requests-oauthlib, social-auth-core

search/api.py

@@ -1,11 +1,15 @@
 """
 Functions for executing ES searches
 """
+import json
+import logging
+
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.db.models import Q as Query
 from elasticsearch.exceptions import NotFoundError
 from elasticsearch_dsl import Search, Q
+from jsonpatch import make_patch
 
 from courses.models import Program
 from dashboard.models import ProgramEnrollment
@@ -27,6 +31,9 @@
 DEFAULT_ES_LOOP_PAGE_SIZE = 100
 
 
+log = logging.getLogger(__name__)
+
+
 def execute_search(search_obj):
     """
     Executes a search against ES after checking the connection
@@ -300,4 +307,14 @@ def document_needs_updating(enrollment):
         return True
     serialized_enrollment = serialize_program_enrolled_user(enrollment)
     del serialized_enrollment['_id']
-    return serialized_enrollment != document['_source']
+    source = document['_source']
+
+    if serialized_enrollment != source:
+        # Convert OrderedDict to dict
+        reserialized_enrollment = json.loads(json.dumps(serialized_enrollment))
+
+        diff = make_patch(source, reserialized_enrollment).patch
+        serialized_diff = json.dumps(diff, indent="    ")
+        log.info("Difference found for enrollment %s: %s", enrollment, serialized_diff)
+        return True
+    return False