From c52ed36da36c432dfbe572fc035baf0813769613 Mon Sep 17 00:00:00 2001 From: Julien Maupetit Date: Thu, 30 Aug 2018 10:39:54 +0200 Subject: [PATCH] =?UTF-8?q?=E2=AC=86=EF=B8=8F(edxapp)=20upgrade=20to=20haw?= =?UTF-8?q?thorn?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As hawthorn as been released and fun-platform's docker image updated consequently, we've upgraded the default edxapp release to hawthorn.1 Notable changes: * loading an sql dump to decrease database migrations execution time is no longer required as they have been drastically optimized * front-end webpack build now generates a 'webpack-stats.json' file that is required to run the CMS and the LMS. As it is generated during the build to the STATIC_ROOT, it cannot be collected. We were obliged to copy it "by hand" in the collectstatics jobs. This is a temporary hack that should not be required (we've declared an issue to edx-platform for that). --- .circleci/config.yml | 2 +- .../templates/cms/job_01_collectstatic.yml.j2 | 60 +++++++++++-------- .../templates/lms/job_02_collectstatic.yml.j2 | 59 ++++++++++-------- .../mysql/job_00_load_sql_dump.yml.j2 | 42 ------------- apps/edxapp/vars/all/main.yml | 2 +- group_vars/env_type/development.yml | 2 +- 6 files changed, 74 insertions(+), 93 deletions(-) delete mode 100644 apps/edxapp/templates/mysql/job_00_load_sql_dump.yml.j2 diff --git a/.circleci/config.yml b/.circleci/config.yml index 9522bff29..d55dc5b91 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -222,7 +222,7 @@ jobs: name: Test the "edxapp" application bootstrapping (with redis enabled) command: | bin/ci-bootstrap "edxapp,redis" - bin/ci-test-service "cms" "Welcome to Open edX Studio" + bin/ci-test-service "cms" "Welcome to Your Platform Studio" bin/ci-test-service "lms" "It works! This is the default homepage for this Open edX instance." # FIXME: we have deactivated plugins test coverage as the container user is diff --git a/apps/edxapp/templates/cms/job_01_collectstatic.yml.j2 b/apps/edxapp/templates/cms/job_01_collectstatic.yml.j2 index 68c153ad2..848d3e989 100644 --- a/apps/edxapp/templates/cms/job_01_collectstatic.yml.j2 +++ b/apps/edxapp/templates/cms/job_01_collectstatic.yml.j2 @@ -21,17 +21,17 @@ spec: deployment_stamp: "{{ deployment_stamp }}" spec: containers: - - name: edxapp-cms-collectstatic-{{ job_stamp }} - image: "{{ edxapp_image_name }}:{{ edxapp_image_tag }}" - env: - - name: DJANGO_SETTINGS_MODULE - value: cms.envs.fun.docker_run - command: ["python", "manage.py", "cms", "collectstatic", "--noinput"] - volumeMounts: - - mountPath: /config - name: edxapp-config - - mountPath: /edx/app/edxapp/staticfiles - name: edxapp-v-static + - name: edxapp-cms-collectstatic-{{ job_stamp }} + image: "{{ edxapp_image_name }}:{{ edxapp_image_tag }}" + env: + - name: DJANGO_SETTINGS_MODULE + value: cms.envs.fun.docker_run + command: ["python", "manage.py", "cms", "collectstatic", "--noinput"] + volumeMounts: + - mountPath: /config + name: edxapp-config + - mountPath: /edx/app/edxapp/staticfiles + name: edxapp-v-static initContainers: # This initContainer has nothing mounted on its "/config" directory. We # copy the content of its "/config" directory (fun-platform default @@ -44,6 +44,14 @@ spec: # with all project settings that will be used to run the service variant # container. Please refer to the documentation to better understand our # settings generation mecanism. + # + # FIXME: the last command of this initContainer aims at copying a + # webpack-stats.json file generated during the build in the STATIC_ROOT. + # As it cannot be collected and it is required to run this service, we + # are constrained to copy it from the image's STATIC_ROOT to the statics + # PVC. We've declared an issue in the edx-platform's bug tracker to + # better understand the role of this file and its location: + # https://openedx.atlassian.net/browse/OPEN-2411 - name: init-create-config image: "{{ edxapp_image_name }}:{{ edxapp_image_tag }}" imagePullPolicy: IfNotPresent @@ -52,7 +60,9 @@ spec: - "-c" - cp -r /config/* /tmp/config && cp -rL /tmp/configmap-cms/* /tmp/config/cms/ && - cp /tmp/secret/credentials.vault.yml /tmp/config/cms/secrets.yml + cp /tmp/secret/credentials.vault.yml /tmp/config/cms/secrets.yml && + mkdir -p /tmp/staticfiles/studio && + cp /edx/app/edxapp/staticfiles/studio/webpack-stats.json /tmp/staticfiles/studio/ volumeMounts: - mountPath: /tmp/config name: edxapp-config @@ -60,17 +70,19 @@ spec: name: edxapp-configmap-cms - mountPath: /tmp/secret name: edxapp-secret + - mountPath: /tmp/staticfiles + name: edxapp-v-static volumes: - - name: edxapp-configmap-cms - configMap: - defaultMode: 420 - name: edxapp-cms-{{ deployment_stamp }} - - name: edxapp-config - emptyDir: {} # volume that lives as long as the pod lives - - name: edxapp-secret - secret: - secretName: edxapp-{{ secret_id }} - - name: edxapp-v-static - persistentVolumeClaim: - claimName: edxapp-pvc-static + - name: edxapp-configmap-cms + configMap: + defaultMode: 420 + name: edxapp-cms-{{ deployment_stamp }} + - name: edxapp-config + emptyDir: {} # volume that lives as long as the pod lives + - name: edxapp-secret + secret: + secretName: edxapp-{{ secret_id }} + - name: edxapp-v-static + persistentVolumeClaim: + claimName: edxapp-pvc-static restartPolicy: Never diff --git a/apps/edxapp/templates/lms/job_02_collectstatic.yml.j2 b/apps/edxapp/templates/lms/job_02_collectstatic.yml.j2 index fc604e78f..159c77b04 100644 --- a/apps/edxapp/templates/lms/job_02_collectstatic.yml.j2 +++ b/apps/edxapp/templates/lms/job_02_collectstatic.yml.j2 @@ -21,17 +21,17 @@ spec: deployment_stamp: "{{ deployment_stamp }}" spec: containers: - - name: edxapp-lms-collectstatic-{{ job_stamp }} - image: "{{ edxapp_image_name }}:{{ edxapp_image_tag }}" - env: - - name: DJANGO_SETTINGS_MODULE - value: lms.envs.fun.docker_run - command: ["python", "manage.py", "lms", "collectstatic", "--noinput"] - volumeMounts: - - mountPath: /config - name: edxapp-config - - mountPath: /edx/app/edxapp/staticfiles - name: edxapp-v-static + - name: edxapp-lms-collectstatic-{{ job_stamp }} + image: "{{ edxapp_image_name }}:{{ edxapp_image_tag }}" + env: + - name: DJANGO_SETTINGS_MODULE + value: lms.envs.fun.docker_run + command: ["python", "manage.py", "lms", "collectstatic", "--noinput"] + volumeMounts: + - mountPath: /config + name: edxapp-config + - mountPath: /edx/app/edxapp/staticfiles + name: edxapp-v-static initContainers: # This initContainer has nothing mounted on its "/config" directory. We # copy the content of its "/config" directory (fun-platform default @@ -44,6 +44,14 @@ spec: # with all project settings that will be used to run the service variant # container. Please refer to the documentation to better understand our # settings generation mecanism. + # + # FIXME: the last command of this initContainer aims at copying a + # webpack-stats.json file generated during the build in the STATIC_ROOT. + # As it cannot be collected and it is required to run this service, we + # are constrained to copy it from the image's STATIC_ROOT to the statics + # PVC. We've declared an issue in the edx-platform's bug tracker to + # better understand the role of this file and its location: + # https://openedx.atlassian.net/browse/OPEN-2411 - name: init-create-config image: "{{ edxapp_image_name }}:{{ edxapp_image_tag }}" imagePullPolicy: IfNotPresent @@ -52,7 +60,8 @@ spec: - "-c" - cp -r /config/* /tmp/config && cp -rL /tmp/configmap-lms/* /tmp/config/lms/ && - cp /tmp/secret/credentials.vault.yml /tmp/config/lms/secrets.yml + cp /tmp/secret/credentials.vault.yml /tmp/config/lms/secrets.yml && + cp /edx/app/edxapp/staticfiles/webpack-stats.json /tmp/staticfiles/ volumeMounts: - mountPath: /tmp/config name: edxapp-config @@ -60,17 +69,19 @@ spec: name: edxapp-configmap-lms - mountPath: /tmp/secret name: edxapp-secret + - mountPath: /tmp/staticfiles + name: edxapp-v-static volumes: - - name: edxapp-configmap-lms - configMap: - defaultMode: 420 - name: edxapp-lms-{{ deployment_stamp }} - - name: edxapp-config - emptyDir: {} # volume that lives as long as the pod lives - - name: edxapp-secret - secret: - secretName: edxapp-{{ secret_id }} - - name: edxapp-v-static - persistentVolumeClaim: - claimName: edxapp-pvc-static + - name: edxapp-configmap-lms + configMap: + defaultMode: 420 + name: edxapp-lms-{{ deployment_stamp }} + - name: edxapp-config + emptyDir: {} # volume that lives as long as the pod lives + - name: edxapp-secret + secret: + secretName: edxapp-{{ secret_id }} + - name: edxapp-v-static + persistentVolumeClaim: + claimName: edxapp-pvc-static restartPolicy: Never diff --git a/apps/edxapp/templates/mysql/job_00_load_sql_dump.yml.j2 b/apps/edxapp/templates/mysql/job_00_load_sql_dump.yml.j2 deleted file mode 100644 index ddcf401a5..000000000 --- a/apps/edxapp/templates/mysql/job_00_load_sql_dump.yml.j2 +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: edxapp-mysql-load-sql-dump-{{ job_stamp }} - namespace: "{{ project_name }}" - labels: - app: edxapp - service: mysql - version: "{{ edxapp_image_tag }}" - job_stamp: "{{ job_stamp }}" - deployment_stamp: "{{ deployment_stamp }}" -spec: - template: - metadata: - name: mysql-load-sql-dump-{{ job_stamp }} - labels: - app: edxapp - service: mysql - version: "{{ edxapp_image_tag }}" - job_stamp: "{{ job_stamp }}" - deployment_stamp: "{{ deployment_stamp }}" - spec: - containers: - - name: mysql-load-sql-dump-{{ job_stamp }} - image: "{{ edxapp_mysql_image_name }}:{{ edxapp_mysql_image_tag }}" - env: - - name: MYSQL_DATABASE - value: {{ edxapp_mysql_database }} - envFrom: - - secretRef: - name: edxapp-{{ secret_id }} - command: - - "bash" - - "-c" - - cd /tmp && - curl -sL -o edx-database.sql "{{ edxapp_sql_dump_url }}" && - mysql - -u ${MYSQL_USER} - -h ${MYSQL_SERVICE_HOST} - --password=${MYSQL_PASSWORD} - ${MYSQL_DATABASE} < edx-database.sql - restartPolicy: Never diff --git a/apps/edxapp/vars/all/main.yml b/apps/edxapp/vars/all/main.yml index 764a6b9bc..e1251863d 100644 --- a/apps/edxapp/vars/all/main.yml +++ b/apps/edxapp/vars/all/main.yml @@ -6,7 +6,7 @@ edxapp_lms_host: "lms.{{ project_name}}.{{ domain_name }}" # -- edxapp (cms/lms) edxapp_image_name: "fundocker/edxapp" -edxapp_image_tag: "ginkgo.1-1.0.6" +edxapp_image_tag: "hawthorn.1-1.0.0" edxapp_django_port: 8000 edxapp_sql_dump_url: "https://gist.github.com/jmaupetit/1f9d270d7d2106774fd94ba89a51ab78/raw/b0004f2825623d03de58710bf936db175e96bc90/edx-database-ginko.sql" diff --git a/group_vars/env_type/development.yml b/group_vars/env_type/development.yml index eee20a386..8e734b119 100644 --- a/group_vars/env_type/development.yml +++ b/group_vars/env_type/development.yml @@ -2,7 +2,7 @@ domain_name: "{{ lookup('env', 'OPENSHIFT_DOMAIN') }}.nip.io" # Use development images in the development environment -edxapp_image_tag: "ginkgo.1-1.0.6-dev" +edxapp_image_tag: "hawthorn.1-1.0.0-dev" richie_image_tag: "0.1.0-alpha.3-alpine-dev" apps: