From 735161a8ce3fa94b253fa62679ea2082f106d96a Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Tue, 10 Mar 2015 17:55:03 -0400
Subject: [PATCH] Optimize asset pipeline for CSS and Javascript Create
 deterministic jasvascript loading Minify all JS into single file with
 ``build.py -p`` Minify all CSS into single file with ``build.py -p``

---
 .gitignore                  |   2 +
 README.rst                  |   3 +
 Vagrantfile                 |   5 +-
 html_app/build.py           |  92 +++++++++------
 javascript_requirements.yml | 219 ++++++++++++++++++++++++++++++++++++
 package.json                |  18 +++
 provision/ansible-dev.yml   |   6 +
 provision/vagrant_env       |   1 +
 requirements.txt            |   2 +
 9 files changed, 314 insertions(+), 34 deletions(-)
 create mode 100644 README.rst
 create mode 100644 javascript_requirements.yml
 create mode 100644 package.json

diff --git a/.gitignore b/.gitignore
index 8dc7c274..ce7aa6e6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -68,3 +68,5 @@ static/
 gen/
 
 settings.yml
+
+node_modules/
\ No newline at end of file
diff --git a/README.rst b/README.rst
new file mode 100644
index 00000000..7e6fdf38
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,3 @@
+StarCellBio
+===========
+
diff --git a/Vagrantfile b/Vagrantfile
index 9bfa1ab0..7633caec 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -26,7 +26,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # using a specific IP.
   config.vm.network "private_network", ip: "192.168.33.200"
   config.vm.network :forwarded_port, guest: 8100, host: 8100
-
+  config.vm.provider "virtualbox" do |vb|
+    vb.customize ["modifyvm", :id, "--memory", "1024"]
+  end
   # Create a public network, which generally matched to bridged network.
   # Bridged networks make the machine appear as another physical device on
   # your network.
@@ -51,7 +53,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   #   vb.gui = true
   #
   #   # Use VBoxManage to customize the VM. For example to change memory:
-  #   vb.customize ["modifyvm", :id, "--memory", "1024"]
   # end
   #
   # View the documentation for the provider you're using for more
diff --git a/html_app/build.py b/html_app/build.py
index 96a5927b..16c5c76b 100755
--- a/html_app/build.py
+++ b/html_app/build.py
@@ -5,28 +5,36 @@
 Usage:
   build.py
   build.py watch
+  build.py -p | --prod
   build.py -h | --help
 
+
 Options:
+  -p --prod       Do production javascript/css pipelining
   -h --help       Show this help screen.
 
 """
 
 import logging
 import os
+import sh
+from subprocess import call
 import time
 
 from docopt import docopt
-from subprocess import call
+from ordereddict import OrderedDict  # For 2.6 compatibility
 from watchdog.observers import Observer
 from watchdog.events import RegexMatchingEventHandler
-
+import yaml
 
 ROOT = os.environ['PROJECT_HOME'] + '/html_app/'
+JAVASCRIPT_REQUIREMENTS = 'javascript_requirements.yml'
 
+prod_minify = False
 global_update_index = True
 js = dict()
 css = dict()
+
 TIME = '?_=' + str(int(time.time()))
 
 CSS_PREFIX = '<link type="text/css" href="/static/'
@@ -69,40 +77,62 @@
 """
 
 
-def index_html():
-    global css, js
-    if js.has_key('js/jquery-1.7.2.min.js'):
-        js.pop("js/jquery-1.7.2.min.js")
-    if js.has_key("js/tinymce.min.js"):
-        js.pop("js/tinymce.min.js")
-    if js.has_key("js/jquery.tinymce.min.js"):
-        js.pop("js/jquery.tinymce.min.js")
-    if js.has_key("starcellbio.app.js"):
-        js.pop("starcellbio.app.js")
-    if js.has_key("swipe/Gruntfile.js"):
-        js.pop("swipe/Gruntfile.js")
-    if js.has_key("js/soyutils.js"):
-        js.pop("js/soyutils.js")
-    if js.has_key("master_model.js"):
-        js.pop("master_model.js")
+def minify_all_js(js):
+    """Concatenates and then minifies each file passed in"""
+    all_file_packed = os.path.join(ROOT, 'gen', 'all.packed.js')
+    dynamic_scripts = []
+    uglify_params = []
+    for file_name in js:
+        path = os.path.join(ROOT, file_name)
+        if os.path.isfile(path):
+            uglify_params.append(path)
+        else:
+            dynamic_scripts.append(file_name)
+    sh.uglifyjs(
+        *uglify_params, o=all_file_packed
+    )
+    dynamic_scripts.append(os.path.join(
+        'gen',
+        os.path.basename(all_file_packed)
+    ))
+    return dynamic_scripts
 
+
+def minify_all_css(css):
+    """Concatenate and minify all css"""
+    all_file_packed = os.path.join(ROOT, 'gen', 'all.packed.css')
+    css_paths = [os.path.join(ROOT, x) for x in css]
+    sh.cleancss(o=all_file_packed, *css_paths)
+    return [os.path.join('gen', os.path.basename(all_file_packed))]
+
+
+def index_html():
+    # Concatenate all the CSS includes
+    css_list = css.keys()
+    if prod_minify:
+        css_list = minify_all_css(css_list)
     css_join = (
         CSS_PREFIX +
-        (TIME + CSS_SUFFIX + CSS_PREFIX).join(css.keys()) +
+        (TIME + CSS_SUFFIX + CSS_PREFIX).join(css_list) +
         TIME + CSS_SUFFIX
     )
+    # Concatenate all the js files
+    # Load yaml file of javascript deps
+    with open(os.path.join(
+            os.environ['PROJECT_HOME'],
+            JAVASCRIPT_REQUIREMENTS
+    )) as yaml_file:
+        js = yaml.load(yaml_file)['javascripts']
+    # Remove duplicates
+    js = list(OrderedDict.fromkeys(js))
+    # If production, then minify everything
+    if prod_minify:
+        js = minify_all_js(js)
     js_join = (
         JS_PREFIX +
-        (TIME + JS_SUFFIX + JS_PREFIX).join(js.keys()) +
+        (TIME + JS_SUFFIX + JS_PREFIX).join(js) +
         JS_SUFFIX
     )
-    js_join = JS_PREFIX + "../scb/get_user.js" + JS_SUFFIX + js_join
-    js_join = JS_PREFIX + "starcellbio.app.js" + JS_SUFFIX + js_join
-    js_join = JS_PREFIX + "js/soyutils.js" + JS_SUFFIX + js_join
-    js_join = JS_PREFIX + "js/tinymce.min.js" + JS_SUFFIX + js_join
-    js_join = JS_PREFIX + "js/jquery.tinymce.min.js" + JS_SUFFIX + js_join
-    js_join = JS_PREFIX + "js/jquery-1.7.2.min.js" + JS_SUFFIX + js_join
-    js_join = js_join + JS_PREFIX + "master_model.js" + JS_SUFFIX
 
     return HTML_PREFIX + css_join + js_join + HTML_SUFFIX
 
@@ -125,12 +155,11 @@ def processor(path):
 
     """
 
-    global global_update_index, css, js
+    global global_update_index
     update_index = False
     path = path.replace("//", "/")
     url = path.replace(ROOT, "")
     if path.endswith(".js"):
-        js[url] = 1
         update_index = True
     if path.endswith(".css"):
         css[url] = 1
@@ -161,8 +190,6 @@ def processor(path):
             "-o", outfile
         ])
         print "compile gss {} to {} ".format(infile, outfile)
-    if path.endswith(".touch_index"):
-        update_index_html()
     if update_index:
         global_update_index = True
         os.utime("{}.touch_index".format(ROOT), None)
@@ -239,7 +266,8 @@ def watch(source):
         format='[%(asctime)s] %(levelname)s: %(message)s',
         datefmt='%Y-%m-%d %H:%M:%S'
     )
-
+    if args['--prod']:
+        prod_minify = True
     if args['watch']:
         watch(".")
     else:
diff --git a/javascript_requirements.yml b/javascript_requirements.yml
new file mode 100644
index 00000000..96ff075d
--- /dev/null
+++ b/javascript_requirements.yml
@@ -0,0 +1,219 @@
+---
+# Javascript dependencies (including soy files) in sorted order of dependency
+# If you need to add a new script, please add it here.  A prefix of `/static/`
+# is assumed when pathing these. For soy files, add a .js extension
+
+javascripts:
+  - js/jquery-1.7.2.min.js
+  - js/jquery.tinymce.min.js
+  - js/tinymce.min.js
+  - js/soyutils.js
+  - starcellbio.app.js
+  - ../scb/get_user.js
+  - js/plugins/fullscreen/plugin.min.js
+  - instructor/gen/instructor_facs_page2.soy.js
+  - js/jquery-ui.js
+  - ui/WesternBlotGelView.js
+  - ui_old/WorkspaceView.js
+  - js/plugins/autosave/plugin.min.js
+  - ui/gen/facs.soy.js
+  - scb_ex/gen/ex1_text.soy.js
+  - assignments/sutd_intro_biology/gen/instructions.soy.js
+  - assignments/mit706s14/gen/instructions.soy.js
+  - ui/gen/western_blot_gel.soy.js
+  - instructor/gen/instructor_course_setup.soy.js
+  - instructor/gen/instructor_microscopy_page2.soy.js
+  - mit706s14ps2/AddMultipleDialog.js
+  - instructor/gen/instructor_facs_page1.soy.js
+  - js/jquery.history.js
+  - scb_ex/gen/addmutlpledialog.soy.js
+  - model/NotebookElement.js
+  - js/plugins/link/plugin.min.js
+  - model/Assignments.js
+  - ui/gen/userguide.soy.js
+  - assignments/microscopy_usability/gen/instructions.soy.js
+  - scb_ex/gen/addmultipledialog2.soy.js
+  - model/WesternBlot.js
+  - instructor/gen/instructor_assignment_setup.soy.js
+  - ui/gen/western_blot.soy.js
+  - model/Facs.js
+  - components/WesternBlotCanvas.js
+  - ui_old/gen/experiment_edit.soy.js
+  - instructor/InstructorExperimentSetupPage1View.js
+  - assignments/sutd_intro_biology_ps2/gen/instructions.soy.js
+  - instructor/gen/instructor_experiment_setup_page1.soy.js
+  - ui/HomepageView.js
+  - test/gen/test.soy.js
+  - js/jquery.simulate.drag-sortable.js
+  - js/plugins/charmap/plugin.min.js
+  - instructor/gen/instructor_homepage.soy.js
+  - model/Drugs.js
+  - js/plugins/insertdatetime/plugin.min.js
+  - model/Microscopy.js
+  - js/plugins/template/plugin.min.js
+  - assignments/mit706s14ps2/gen/instructions.soy.js
+  - ui_old/WesternBlotView.js
+  - js/slides.min.jquery.js
+  - instructor/InstructorExperimentSetupPage3View.js
+  - gen/microscopy_test.soy.js
+  - ui/gen/microscopy.soy.js
+  - model/FacsLanesPreparation.js
+  - js/plugins/searchreplace/plugin.min.js
+  - js/plugins/pagebreak/plugin.min.js
+  - instructor/InstructorCourseSetupView.js
+  - ug2/ug.js
+  - ui/gen/experiment_setup.soy.js
+  - model/LysateKind.js
+  - instructor/InstructorDashboardView.js
+  - js/raphael.js
+  - ui_old/gen/dashboard.soy.js
+  - ug2/parse_wiki.js
+  - utils/Utilities.js
+  - ui_old/FacsOneView.js
+  - js/plugins/tabfocus/plugin.min.js
+  - assignments/sutd_intro_biology_ps2/AddMultipleDialog.js
+  - js/jqdialog.js
+  - instructor/gen/instructor_western_blot_page1.soy.js
+  - js/second-raphael-svg-filter.js
+  - js/jquery.ba-bbq.min.js
+  - ui/NotebookView.js
+  - model/LensMap.js
+  - ui/gen/contact.soy.js
+  - model/FacsLanes.js
+  - js/plugins/emoticons/plugin.min.js
+  - js/plugins/hr/plugin.min.js
+  - ui/ExperimentSetupView.js
+  - js/plugins/anchor/plugin.min.js
+  - ui_old/FacsView.js
+  - js/plugins/legacyoutput/plugin.min.js
+  - js/plugins/visualchars/plugin.min.js
+  - components/FACSModelFactory.js
+  - instructor/gen/instructor_microscopy_page3.soy.js
+  - ui/MicroscopyView.js
+  - ui_old/Sidebar.js
+  - components/WesternBlotModelFactory.js
+  - swipe/swipe.js
+  - model/WesternBlotGel.js
+  - model/Template.js
+  - model/Experiment.js
+  - ui/gen/auth.soy.js
+  - decusability/AddMultipleDialog.js
+  - model/WesternBlotExposure.js
+  - ui/WesternBlotView.js
+  - instructor/gen/instructor_common.soy.js
+  - js/plugins/autoresize/plugin.min.js
+  - microscopy_usability/AddMultipleDialog.js
+  - instructor/InstructorMicroscopyPage3View.js
+  - instructor/InstructorWesternBlotPage4View.js
+  - js/jquery.cookie.js
+  - model/CellTreatment.js
+  - js/jquery.multiselect.js
+  - ui/gen/homepage.soy.js
+  - js/plugins/print/plugin.min.js
+  - scb_ex/gen/ex3_text.soy.js
+  - instructor/InstructorSelectTechniqueView.js
+  - js/plugins/directionality/plugin.min.js
+  - instructor/InstructorWesternBlotPage1View.js
+  - microscopy_usability/gen/addmultipledialog.soy.js
+  - js/plugins/importcss/plugin.min.js
+  - js/plugins/nonbreaking/plugin.min.js
+  - assignments/sutd_intro_biology_ps2/gen/addmultipledialog.soy.js
+  - instructor/gen/instructor_select_technique.soy.js
+  - js/plugins/table/plugin.min.js
+  - js/plugins/image/plugin.min.js
+  - model/WesternBlotLanes.js
+  - js/plugins/bbcode/plugin.min.js
+  - instructor/InstructorFacsPage2View.js
+  - js/plugins/media/plugin.min.js
+  - coin-slider/coin-slider.min.js
+  - components/ModelFactory.js
+  - assignments/tufts/gen/instructions.soy.js
+  - ui/SelectTechniqueView.js
+  - ui/gen/select_technique.soy.js
+  - assignments/decusability/gen/instructions.soy.js
+  - ui_old/WesternBlotGelView.js
+  - js/plugins/fullpage/plugin.min.js
+  - instructor/InstructorFacsPage1View.js
+  - ui_old/DashboardView.js
+  - instructor/gen/instructor_western_blot_page3.soy.js
+  - ui_old/gen/westernblotgel.soy.js
+  - instructor/InstructorFrame.js
+  - assignments/mit706s13/gen/instructions.soy.js
+  - instructor/gen/instructor_experiment_setup_page2.soy.js
+  - js/plugins/visualblocks/plugin.min.js
+  - ui_old/gen/workspace.soy.js
+  - instructor/InstructorWesternBlotPage2View.js
+  - instructor/gen/instructor_western_blot_page2.soy.js
+  - js/plugins/paste/plugin.min.js
+  - instructor/InstructorWesternBlotPage3View.js
+  - instructor/InstructorMicroscopyPage2View.js
+  - mit706s14/gen/addmultipledialog.soy.js
+  - instructor/gen/instructor_western_blot_page5.soy.js
+  - js/plugins/preview/plugin.min.js
+  - model/NotebookSection.js
+  - js/swipe.js
+  - scb_ex/AddMultipleDialog.js
+  - model/MicroscopyLanes.js
+  - instructor/gen/instructor_experiment_setup_page3.soy.js
+  - mit706s13/AddMultipleDialog.js
+  - instructor/InstructorExperimentSetupPage2View.js
+  - ui_old/gen/facs.soy.js
+  - mit706s13/gen/addmultipledialog.soy.js
+  - model.js
+  - js/plugins/lists/plugin.min.js
+  - js/plugins/spellchecker/plugin.min.js
+  - js/underscore.js
+  - instructor/gen/instructor_experiment_setup_page4.soy.js
+  - js/plugins/advlist/plugin.min.js
+  - js/plugins/layer/plugin.min.js
+  - js/jstat-1.0.0.js
+  - scb_ex/gen/ex2_text.soy.js
+  - ui_old/MakingLysatesView.js
+  - ui_old/gen/common.soy.js
+  - ui/AssignmentsView.js
+  - ui_old/gen/making_lysates.soy.js
+  - model/Treatment.js
+  - model/Notebook.js
+  - js/plugins/code/plugin.min.js
+  - model/MakingLysates.js
+  - js/jq.pubsub.js
+  - js/plugins/example/plugin.min.js
+  - instructor/InstructorAssignmentSetupView.js
+  - js/plugins/noneditable/plugin.min.js
+  - ui/gen/notebook.soy.js
+  - ui_old/gen/experiment.soy.js
+  - instructor/gen/instructor_dashboard.soy.js
+  - instructor/InstructorExperimentSetupPage4View.js
+  - js/jquery.flot.js
+  - ui_old/gen/western_blots.soy.js
+  - js/plugins/contextmenu/plugin.min.js
+  - utils/CSFRFix.js
+  - model/ModelHelpers.js
+  - ui/gen/assignments.soy.js
+  - model/CollectionSchedule.js
+  - js/jquery.layout-1.2.0.js
+  - gen/usability_test.soy.js
+  - ui/MainFrame.js
+  - ui/gen/common.soy.js
+  - instructor/gen/instructor_western_blot_page4.soy.js
+  - ui/FacsView.js
+  - js/plugins/wordcount/plugin.min.js
+  - ref_lib/rl.js
+  - components/MicroscopyModelFactory.js
+  - instructor/InstructorMicroscopyPage1View.js
+  - js/themes/modern/theme.min.js
+  - instructor/gen/instructor_microscopy_page1.soy.js
+  - ui/gen/experiment_design.soy.js
+  - instructor/InstructorWesternBlotPage5View.js
+  - ui_old/ExperimentView.js
+  - coin-slider/coin-slider.js
+  - decusability/gen/addmultipledialog.soy.js
+  - js/plugins/example_dependency/plugin.min.js
+  - ui_old/gen/experiment_setup.soy.js
+  - js/plugins/autolink/plugin.min.js
+  - js/plugins/save/plugin.min.js
+  - mit706s14/AddMultipleDialog.js
+  - mit706s14ps2/gen/addmultipledialog.soy.js
+  - js/plugins/textcolor/plugin.min.js
+  - ui/ExperimentDesignView.js
+  - master_model.js
diff --git a/package.json b/package.json
new file mode 100644
index 00000000..1ee428f3
--- /dev/null
+++ b/package.json
@@ -0,0 +1,18 @@
+{
+  "name": "starcellbio",
+  "version": "0.0.0",
+  "description": "StarCellBio",
+  "dependencies": {
+    "clean-css": "~3.1.5",
+    "uglify-js": "~2.4.16"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/starteam/starcellbio_html"
+  },
+  "author": "",
+  "license": "BSD-2-Clause",
+  "bugs": {
+    "url": "https://github.com/starteam/starcellbio_html/issues"
+  }
+}
diff --git a/provision/ansible-dev.yml b/provision/ansible-dev.yml
index 96d20000..8534f93d 100644
--- a/provision/ansible-dev.yml
+++ b/provision/ansible-dev.yml
@@ -12,6 +12,8 @@
       - libncurses5-dev
       - python-mysqldb
       - openjdk-6-jre
+      - npm
+      - nodejs-legacy
     pip_packages:
       - virtualenvwrapper
     venv_name: starcellbio
@@ -40,6 +42,9 @@
     - name: Install requirements
       pip: requirements='/vagrant/requirements.txt' virtualenv={{ venv_path }}
 
+    - name: Install node packages
+      npm: path=/vagrant
+
     - name: Create database
       sudo: yes
       mysql_db: name={{ db_name }}
@@ -76,6 +81,7 @@
         {{ venv_path }}/bin/python build.py
       environment:
         PROJECT_HOME: "/vagrant"
+        PATH: "/vagrant/node_modules/.bin:{{ ansible_env.PATH }}"
         
 
     - name: Drop profile
diff --git a/provision/vagrant_env b/provision/vagrant_env
index 0d2b04cf..87cf8cd3 100644
--- a/provision/vagrant_env
+++ b/provision/vagrant_env
@@ -2,3 +2,4 @@ source virtualenvwrapper.sh
 workon starcellbio
 cd /vagrant
 export PROJECT_HOME=/vagrant
+export PATH=$PATH:/vagrant/node_modules/.bin
diff --git a/requirements.txt b/requirements.txt
index a657780c..556b47ee 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -25,3 +25,5 @@ urwid==1.1.1
 wsgiref==0.1.2
 watchdog>=0.8.3
 docopt>=0.6.2
+ordereddict>=1.1
+sh>=1.11