Modernize ORA, commit 1 of 2

Contains installation and testing changes for the django 1.11 upgrade
preparedness initiative. Splits tests out into seperate shards on travis.

.travis.yml

@@ -1,43 +1,31 @@
 addons:
   apt:
     packages:
-    - aspell
-    - aspell-en
-    - g++
-    - gcc
-    - git
-    - gfortran
-    - libblas3gf
-    - libblas-dev
-    - liblapack3gf
-    - liblapack-dev
-    - libatlas-base-dev
-    - libfontconfig1
-    - libmysqlclient-dev
-    - libxml2-dev
-    - libxslt1-dev
     - nodejs
-    - python2.7
-    - python2.7-dev
-    - python-pip
-    - python-software-properties
 language: python
 sudo: false
 python:
   - "2.7"
+env:
+  - TOXENV=django18
+  - TOXENV=django111
+matrix:
+  include:
+    - python: 2.7
+      env: TOXENV=quality
+    - python: 2.7
+      env: TOXENV=js
+before_install:
+  - "pip install -U pip"
+cache:
+    directories:
+        - $HOME/.cache/pip
 install:
-  - "pip install coveralls"
-  - "make install"
-before_script:
-  - "export DISPLAY=:99.0"
-  - "sh -e /etc/init.d/xvfb start"
+  - "pip install tox"
 script:
-  - "make verify-generated-files"
-  - "make test"
-  - "python manage.py makemessages -l eo"
+  - tox
 branches:
     only:
       - master
-      - ora-staff-grading
 after_success:
   coveralls

MANIFEST.in

@@ -1,13 +1,13 @@
 include LICENSE
 include AUTHORS
 include README.rst
+include setup.cfg
 include openassessment/xblock/static/css/*.css
 include openassessment/xblock/static/css/lib/backgrid/*.css
 include openassessment/xblock/static/js/openassessment*.min.js
 include openassessment/xblock/static/js/lib/backgrid/*.js
+include requirements/*.txt
 recursive-include openassessment/xblock/static/js/src *.js
 recursive-include openassessment/templates *.html *.underscore
 recursive-include openassessment/locale *.po
 recursive-include openassessment/locale *.mo
-global-exclude */test*
-global-exclude */test?/*

Makefile

@@ -1,33 +1,29 @@
-all: install test
-
-.PHONY: install test
-
-# not used by travis
-install-system:
-	sudo apt-get update -qq
-	sudo xargs -a apt-packages.txt apt-get install -qq --fix-missing
-
-# not used by travis
-install-node:
-	sudo apt-get install -qq nodejs
-
-install-wheels:
-	./scripts/install-wheels.sh
-
-
+##################
+# Install commands
+##################
 install-python:
-	./scripts/install-python.sh
+	pip install -r requirements/django.txt
+	pip install -r requirements/base.txt --only-binary=lxml,libsass
 
 install-js:
 	npm install
 
-install-nltk-data:
-	./scripts/download-nltk-data.sh
+install-test:
+	pip install -r requirements/test.txt
 
+install: install-python install-js install-test javascript sass
 
+##############################
+# Generate js/css output files
+##############################
 STATIC_JS = openassessment/xblock/static/js
 STATIC_CSS = openassessment/xblock/static/css
 
+update-npm-requirements:
+	npm update --silent
+	cp ./node_modules/backgrid/lib/backgrid*.js $(STATIC_JS)/lib/backgrid/
+	cp ./node_modules/backgrid/lib/backgrid*.css $(STATIC_CSS)/lib/backgrid/
+
 javascript: update-npm-requirements
 	node_modules/.bin/uglifyjs $(STATIC_JS)/src/oa_shared.js $(STATIC_JS)/src/*.js $(STATIC_JS)/src/lms/*.js $(STATIC_JS)/lib/backgrid/backgrid.min.js -c warnings=false > "$(STATIC_JS)/openassessment-lms.min.js"
 	node_modules/.bin/uglifyjs $(STATIC_JS)/src/oa_shared.js $(STATIC_JS)/src/*.js $(STATIC_JS)/src/studio/*.js $(STATIC_JS)/lib/backgrid/backgrid.min.js -c warnings=false > "$(STATIC_JS)/openassessment-studio.min.js"
@@ -38,27 +34,17 @@ sass:
 verify-generated-files:
 	@git diff --quiet || (echo 'Modifications exist locally! Run `make javascript` or `make sass` to update bundled files.'; exit 1)
 
-install-test:
-	pip install -q -r requirements/test.txt
-
-install-sys-requirements: install-system install-node
-	npm config set loglevel warn
-
-install-dev:
-	pip install -q -r requirements/dev.txt
-
-install: install-wheels install-python install-js install-nltk-data install-test install-dev javascript sass
-
+################
+#Tests and checks
+################
 quality:
 	./node_modules/.bin/jshint $(STATIC_JS)/src -c .jshintrc --verbose
 	./node_modules/jscs/bin/jscs $(STATIC_JS)/src --verbose
 	./scripts/run-pep8.sh
 	./scripts/run-pylint.sh
 
-test: quality test-python test-js
-
 test-python:
-	./scripts/test-python.sh
+	coverage run manage.py test openassessment
 
 render-templates:
 	./scripts/render-templates.sh
@@ -69,15 +55,13 @@ test-js: render-templates
 test-js-debug: render-templates
 	./scripts/js-debugger.sh
 
-test-sandbox: test-acceptance test-a11y
+test: quality test-python test-js
 
+# acceptance and a11y tests require a functioning sandbox, and do not run on travis
 test-acceptance:
 	./scripts/test-acceptance.sh tests
 
 test-a11y:
 	./scripts/test-acceptance.sh accessibility
 
-update-npm-requirements:
-	npm update --silent
-	cp ./node_modules/backgrid/lib/backgrid*.js $(STATIC_JS)/lib/backgrid/
-	cp ./node_modules/backgrid/lib/backgrid*.css $(STATIC_CSS)/lib/backgrid/
+test-sandbox: test-acceptance test-a11y

README.rst

@@ -7,7 +7,7 @@ Open Response Assessment |build-status| |coverage-status|
 Installation, Tests, and other Developer Tasks
 ==============================================
 
-EdX engineers follow the `guides on our wiki <https://openedx.atlassian.net/wiki/display/EDUCATOR/ORA+FAQ>`_.
+EdX engineers follow the `guides on our wiki <https://openedx.atlassian.net/wiki/spaces/EDUCATOR/pages/9765004/ORA+Developer+Guide>`_.
 
 License
 =======

manage.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
-import sys
 import os
+import sys
 
 if __name__ == "__main__":
 

pylintrc

@@ -55,7 +55,7 @@
 [MASTER]
 ignore = 
 persistent = yes
-load-plugins = edx_lint.pylint,pylint_django,pylint_celery
+load-plugins = edx_lint.pylint,pylint_django
 
 [MESSAGES CONTROL]
 disable = 

requirements/base.txt

@@ -1,31 +1,22 @@
-# Include the requirements we're caching as "wheel" archives
-# to speed up the test builds.
--r wheels.txt
-
 # edX Internal Requirements
-git+https://github.com/edx/[email protected]#egg=XBlock==0.4.12
+edx-submissions>=2.0.6,<3.0.0
+git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3
+git+https://github.com/edx/[email protected]#egg=XBlock==1.0.1
+git+https://github.com/edx/[email protected]#egg=xblock-sdk==0.1.4
 
 # Third Party Requirements
-boto>=2.32.1,<3.0.0
-python-swiftclient==3.1.0
-celery==3.1.18
-defusedxml==0.4.1
-django<1.9a0 # Resolves known bug on gemnasium. See TNL-6266
-django-extensions==1.5.9
-django-model-utils==2.3.1
-git+https://github.com/edx/django-rest-framework.git@1ceda7c086fddffd1c440cc86856441bbf0bd9cb#egg=djangorestframework==3.6.3
-dogapi==1.2.1
-edx-submissions==2.0.6
-jsonfield==1.0.3
-lazy==1.1
-libsass==0.10.0
-loremipsum==1.0.5
-path.py==8.2.1
-python-dateutil==2.1
-python-memcached==1.48
+boto3>=1.4.4,<2.0.0
+python-swiftclient>=3.1.0,<4.0.0
+defusedxml>=0.4.1,<1.0.0
+django-model-utils>=2.3.1,<3.0.0
+django-nose>=1.4.1,<2.0.0
+dogapi>=1.2.1,<2.0.0
+jsonfield>=1.0.3,<2.0.0
+lazy>=1.1,<2.0
+libsass>=0.10.0,<1.0.0
+loremipsum>=1.0.5,<2.0.0
+lxml>=3.4.4,<4.0.0
+path.py>=8.2.1,<9.0.0
+python-dateutil>=2.1,<3.0
 pytz
-voluptuous==0.8.5
-
-# AI grading
-git+https://github.com/edx/ease.git@release-2015-07-14#egg=ease==0.1.3
-git+https://github.com/edx/[email protected]#egg=nltk==2.0.6
+voluptuous>=0.10.5,<1.0.0

requirements/django.txt

@@ -0,0 +1 @@
+django>=1.8,<2.0

requirements/quality.txt

@@ -0,0 +1,4 @@
+pep8==1.7.0
+
+git+https://github.com/edx/[email protected]#egg=edx_lint==0.5.4
+git+https://github.com/edx/[email protected]#egg=i18n_tools==0.3.9

requirements/test-acceptance.txt

@@ -1,5 +1,5 @@
 bok-choy==0.5.3
 ddt==1.0.0
 nose==1.3.7
-pyinstrument==0.13.1
-selenium==2.53.1
+pyinstrument==0.13.2
+selenium==2.53.1 # when upgrading to 3.x, will need to ensure geckodriver is installed

requirements/test.txt

@@ -1,14 +1,12 @@
--r base.txt
--r test-acceptance.txt
+-r quality.txt
 
-coverage==4.0.2
-django-nose==1.4.1
-freezegun==0.1.11
-mock==1.0.1
-moto==0.3.1
-pep8==1.7.0
+coverage==4.4.1
+ddt==1.0.0
 factory_boy==2.8.1
+freezegun==0.3.9
+mock==2.0.0
+moto==1.0.1
+nose==1.3.7
+tox==2.7.0
 
-git+https://github.com/edx/[email protected]#egg=django-pyfs==1.0.3
-git+https://github.com/edx/i18n-tools.git@56f048af9b6868613c14aeae760548834c495011#egg=i18n_tools
-git+https://github.com/edx/[email protected]#egg=edx_lint==0.4.1
+git+https://github.com/edx/[email protected]#egg=django-pyfs==1.0.6

scripts/render-templates.sh

@@ -3,5 +3,5 @@
 cd `dirname $BASH_SOURCE` && cd ..
 
 echo "Generating HTML fixtures for JavaScript tests..."
-export DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-"settings.test_with_coverage"}
+export DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE:-"settings.base"}
 ./scripts/render_templates.py openassessment/xblock/static/js/fixtures/templates.json

scripts/render_templates.py

@@ -100,7 +100,7 @@ def render_templates(root_dir, template_json):
         template = get_template(template_dict['template'])
         context = parse_dates(template_dict['context'])
         print "Rendering template: {}".format(template_dict['template'])
-        rendered = template.render(Context(context))
+        rendered = template.render(context)
         output_path = os.path.join(root_dir, template_dict['output'])
 
         try:

scripts/run-pep8.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-MAX_PEP8_VIOLATIONS=106
+MAX_PEP8_VIOLATIONS=60
 
 mkdir -p test/logs
 PEP8_VIOLATIONS=test/logs/pep8.txt

scripts/run-pylint.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-MAX_PYLINT_VIOLATIONS=438
+MAX_PYLINT_VIOLATIONS=400
 
 mkdir -p test/logs
 PYLINT_VIOLATIONS=test/logs/pylint.txt