Added Elasticsearch index, indexing for Profile and related models (#706

)

README.md

@@ -168,20 +168,32 @@ updates, you can run this without the ``--install`` flag: ``./webpack_dev_server
 **DEBUGGING NOTE:** If you see an error related to node-sass when you run this script, try running
 ``npm rebuild node-sass``
 
+#### 3) Build the containers
+Run this command:
 
-#### 3) Run the container
+    docker-compose build
 
-For first-time container start-up, start it with a full build:
+You will also need to run this command whenever ``requirements.txt`` or ``test_requirements.txt`` change.
 
-    docker-compose up --build
+#### 4) Create an Elasticsearch index
+To do this, run this command:
+
+    docker-compose run web ./manage.py recreate_index
+
+This is required to initialize the Elasticsearch index and mappings. This command should only need
+to be run when the Elasticsearch container is first created. It may also be run afterwards to clear and recreate
+existing indexes, and to reindex relevant documents.
+
+#### 5) Run the container
+
+Start Django, PostgreSQL, and other related services:
+
+    docker-compose up
 
 In another terminal tab, navigate the the MicroMasters directory
 and add a superuser in the now-running Docker container:
 
-    docker-compose run web python3 manage.py createsuperuser
-
-Starting the container after this can be done without the ``--build``
-param: ``docker-compose up``
+    docker-compose run web ./manage.py createsuperuser
 
 You should now be able to do the following:
 
@@ -196,7 +208,7 @@ You should now be able to do the following:
 The CMS can be found at `/cms/`. Use the CMS to manage the content of the program pages and, by extension, the home 
 page.  
 
-#### Adding a new MicroMaters program
+#### Adding a new MicroMasters program
 
 1. Login to the cms with an admin account. If you don't have one, you can use the superuser account created earlier.
 
@@ -226,7 +238,7 @@ skip to step 5.
 As shown above, manage commands can be executed on the Docker-contained
 MicroMasters app. For example, you can run a Python shell with the following command:
 
-    docker-compose run web python3 manage.py shell
+    docker-compose run web ./manage.py shell
 
 Tests should be run in the Docker container, not the host machine. They can be run with the following commands:
 

backends/pipeline_api_test.py

@@ -4,7 +4,6 @@
 
 from urllib.parse import urljoin
 
-from django.test import TestCase
 import mock
 
 from backends import pipeline_api, edxorg
@@ -13,10 +12,11 @@
 from profiles.models import Profile
 from profiles.factories import UserFactory
 from profiles.util import split_name
+from search.base import ESTestCase
 
 
 # pylint: disable=no-self-use
-class EdxPipelineApiTest(TestCase):
+class EdxPipelineApiTest(ESTestCase):
     """
     Test class for APIs run during the Python Social Auth
     authentication pipeline.
@@ -52,6 +52,7 @@ def setUp(self):
         """
         Set up class
         """
+        super(EdxPipelineApiTest, self).setUp()
         self.user = UserFactory()
         self.user.social_auth.create(
             provider='not_edx',
@@ -213,7 +214,7 @@ def test_preferred_language(self, mocked_get_json):
             assert self.user_profile.edx_language_proficiencies == proficiencies
 
 
-class LinkedInPipelineTests(TestCase):
+class LinkedInPipelineTests(ESTestCase):
     """Tests for the LinkedIn pipline entry"""
 
     def test_saves_linkedin_response(self):

backends/utils_test.py

@@ -8,17 +8,16 @@
 )
 
 import pytz
-from django.test import TestCase
 from requests.exceptions import HTTPError
 
 from backends import utils
 from backends.edxorg import EdxOrgOAuth2
 from profiles.factories import UserFactory
-
+from search.base import ESTestCase
 # pylint: disable=protected-access
 
 
-class RefreshTest(TestCase):
+class RefreshTest(ESTestCase):
     """Class to test refresh token"""
 
     @classmethod

courses/management/commands/gen_fake_data_test.py

@@ -2,7 +2,6 @@
 gen_fake_data test
 """
 
-from django.test import TestCase
 import mock
 
 from courses.management.commands.gen_fake_data import Command
@@ -11,9 +10,10 @@
     CourseRun,
     Program,
 )
+from search.base import ESTestCase
 
 
-class GenFakeDataTest(TestCase):
+class GenFakeDataTest(ESTestCase):
     """
     gen_fake_data tests
     """

courses/models_test.py

@@ -5,17 +5,17 @@
 from datetime import datetime, timedelta
 
 import pytz
-from django.test import TestCase
 
 from courses.factories import (
     ProgramFactory,
     CourseFactory,
     CourseRunFactory,
 )
 from courses.models import CourseRun
+from search.base import ESTestCase
 
 
-class ProgramTests(TestCase):
+class ProgramTests(ESTestCase):
     """Tests for Program model"""
 
     def test_to_string(self):
@@ -24,16 +24,16 @@ def test_to_string(self):
         assert "{}".format(prog) == "Title"
 
 
-class CourseModelsMixin(TestCase):
+class CourseModelTests(ESTestCase):
     """Mixin for Course models"""
 
     @classmethod
     def setUpTestData(cls):
-        super(CourseModelsMixin, cls).setUpTestData()
+        super(CourseModelTests, cls).setUpTestData()
         cls.course = CourseFactory.create(title="Title")
 
     def setUp(self):
-        super(CourseModelsMixin, self).setUp()
+        super(CourseModelTests, self).setUp()
         self.now = datetime.now(pytz.utc)
 
     def create_run(self, course=None, start=None, end=None,
@@ -51,7 +51,7 @@ def create_run(self, course=None, start=None, end=None,
         )
 
 
-class CourseTests(CourseModelsMixin):
+class CourseTests(CourseModelTests):
     """Tests for Course model"""
 
     def test_to_string(self):
@@ -244,7 +244,7 @@ def test_get_next_run_enroll_none_course_future(self):
         assert next_run.pk == course_run.pk
 
 
-class CourseRunTests(CourseModelsMixin):
+class CourseRunTests(CourseModelTests):
     """Tests for Course Run model"""
 
     def test_to_string(self):

courses/serializers_test.py

@@ -2,13 +2,12 @@
 Tests for serializers
 """
 
-from django.test import TestCase
-
 from courses.factories import CourseRunFactory
 from courses.serializers import CourseRunSerializer
+from search.base import ESTestCase
 
 
-class CourseRunSerializerTests(TestCase):
+class CourseRunSerializerTests(ESTestCase):
     """
     Tests for CourseRunSerializer
     """

courses/views_tests.py

@@ -1,12 +1,12 @@
 """Tests for the API"""
 # pylint: disable=no-self-use
 from django.core.urlresolvers import reverse
-from django.test import TestCase
 
 from .factories import ProgramFactory, CourseFactory
+from search.base import ESTestCase
 
 
-class ProgramTests(TestCase):
+class ProgramTests(ESTestCase):
     """Tests for the Program API"""
     def test_lists_live_programs(self):
         """Live programs should show up"""
@@ -26,7 +26,7 @@ def test_doesnt_list_unlive_programs(self):
         assert len(resp.json) == 0
 
 
-class CourseTests(TestCase):
+class CourseTests(ESTestCase):
     """Tests for the Course API"""
     def test_list_course_if_program_live(self):
         """

dashboard/api_test.py

@@ -8,7 +8,6 @@
 
 import pytz
 from django.core.exceptions import ImproperlyConfigured
-from django.test import TestCase
 
 from edx_api.certificates.models import Certificate, Certificates
 from edx_api.enrollments.models import Enrollments
@@ -20,9 +19,10 @@
 )
 from dashboard import api
 from profiles.factories import UserFactory
+from search.base import ESTestCase
 
 
-class StatusTest(TestCase):
+class StatusTest(ESTestCase):
     """
     Tests for the different status classes
     """
@@ -93,12 +93,12 @@ def test_course_format_conditional_fields_get(self):
         assert len(api.CourseFormatConditionalFields.get_assoc_field(api.CourseStatus.OFFERED)) == 2
 
 
-class CourseMixin(TestCase):
+class CourseTests(ESTestCase):
     """Base class for APIs tests"""
 
     @classmethod
     def setUpTestData(cls):
-        super(CourseMixin, cls).setUpTestData()
+        super(CourseTests, cls).setUpTestData()
         cls.course = CourseFactory.create(title="Title")
 
         with open(os.path.join(os.path.dirname(__file__),
@@ -109,7 +109,7 @@ def setUpTestData(cls):
             [Certificate(cert_json) for cert_json in cls.certificates_json])
 
     def setUp(self):
-        super(CourseMixin, self).setUp()
+        super(CourseTests, self).setUp()
         self.now = datetime.now(pytz.utc)
 
     def create_run(self, course=None, start=None, end=None,
@@ -132,7 +132,7 @@ def create_run(self, course=None, start=None, end=None,
         return run
 
 
-class FormatRunTest(CourseMixin):
+class FormatRunTest(CourseTests):
     """Tests for the format_courserun_for_dashboard function"""
 
     def test_format_run_no_run(self):
@@ -244,7 +244,7 @@ def test_format_run_certificate(self):
         )
 
 
-class CourseRunTest(CourseMixin):
+class CourseRunTest(CourseTests):
     """Tests for get_status_for_courserun"""
 
     @classmethod
@@ -392,7 +392,7 @@ def test_not_verified_not_passed(self):
                 self.enrollments.get_enrollment_for_course("course-v1:MITx+8.MechCX+2014_T1"))
 
 
-class InfoCourseTest(CourseMixin):
+class InfoCourseTest(CourseTests):
     """Tests for get_info_for_course"""
 
     @classmethod
@@ -749,7 +749,7 @@ def mocked_get_status_for_courserun(run, enrollments):  # pylint: disable=unused
         mock_format.assert_called_once_with(run1, api.CourseStatus.OFFERED, position=1)
 
 
-class InfoProgramTest(TestCase):
+class InfoProgramTest(ESTestCase):
     """Tests for get_info_for_program"""
     @classmethod
     def setUpTestData(cls):

docker-compose.yml

@@ -32,11 +32,13 @@ web:
     MICROMASTERS_USE_WEBPACK_DEV_SERVER: 'True'
     MICROMASTERS_SECURE_SSL_REDIRECT: 'False'
     MICROMASTERS_DB_DISABLE_SSL: 'True'
+    ELASTICSEARCH_URL: elastic:9200
   env_file: .env
   ports:
     - "8079:8079"
   links:
     - db
+    - elastic
 
 watch:
   build: .
@@ -68,6 +70,7 @@ celery:
     DATABASE_URL: postgres://postgres@db:5432/postgres
     BROKER_URL: redis://redis:6379/4
     CELERY_RESULT_BACKEND: redis://redis:6379/4
+    ELASTICSEARCH_URL: elastic:9200
   links:
     - db
     - elastic

micromasters/settings.py

@@ -107,6 +107,7 @@ def get_var(name, default):
     'backends',
     'profiles',
     'dashboard',
+    'search',
 )
 
 MIDDLEWARE_CLASSES = (
@@ -409,3 +410,7 @@ def get_var(name, default):
 CELERY_ALWAYS_EAGER = get_var("CELERY_ALWAYS_EAGER", True)
 CELERY_EAGER_PROPAGATES_EXCEPTIONS = get_var(
     "CELERY_EAGER_PROPAGATES_EXCEPTIONS", True)
+
+# Elasticsearch
+ELASTICSEARCH_URL = get_var("ELASTICSEARCH_URL", None)
+ELASTICSEARCH_INDEX = get_var('ELASTICSEARCH_INDEX', 'micromasters')

micromasters/tests/test_settings.py

@@ -10,15 +10,15 @@
 from django.conf import settings
 from django.core import mail
 from django.core.exceptions import ImproperlyConfigured
-from django.test import TestCase
 import mock
 import semantic_version
 import yaml
 
 from micromasters.settings import load_fallback, get_var
+from search.base import ESTestCase
 
 
-class TestSettings(TestCase):
+class TestSettings(ESTestCase):
     """Validate that settings work as expected."""
 
     def reload_settings(self):

profiles/api_test.py

@@ -5,16 +5,16 @@
 from mock import Mock
 
 from django.db.models.signals import post_save
-from django.test import TestCase
 from factory.django import mute_signals
 from testfixtures import LogCapture
 
 from backends.edxorg import EdxOrgOAuth2
 from profiles.api import get_social_username
 from profiles.factories import ProfileFactory
+from search.base import ESTestCase
 
 
-class SocialTests(TestCase):
+class SocialTests(ESTestCase):
     """
     Tests for profile functions
     """
@@ -23,6 +23,7 @@ def setUp(self):
         """
         Create a user with a default social auth
         """
+        super(SocialTests, self).setUp()
 
         with mute_signals(post_save):
             profile = ProfileFactory.create(