From b72b681ddcbf63f67330cf1a09f713ec8abc63d1 Mon Sep 17 00:00:00 2001 From: Ray Pulspiher Date: Tue, 30 Apr 2024 09:52:42 -0700 Subject: [PATCH 1/4] Update Canvas migration so that it will fully work - moved alter table before later scripts that use the new field. --- docker_build_files/ope-canvas/Dockerfile | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docker_build_files/ope-canvas/Dockerfile b/docker_build_files/ope-canvas/Dockerfile index 3cb70ae0..649c425c 100644 --- a/docker_build_files/ope-canvas/Dockerfile +++ b/docker_build_files/ope-canvas/Dockerfile @@ -36,7 +36,8 @@ ENV BUNDLER_VERSION 2.2.33 #ENV BUNDLER_VERSION 2.2.17 -ENV CANVAS_RELEASE prod +#ENV CANVAS_RELEASE prod +ENV CANVAS_RELEASE release/2022-07-06.145 #ENV CANVAS_RELEASE release/2022-05-25.135 #ENV CANVAS_RELEASE release/2022-01-19.127 #ENV CANVAS_RELEASE release/2021-12-15.42 @@ -158,8 +159,8 @@ RUN cd ${APP_HOME} \ && git config --global --add safe.directory ${APP_HOME} \ && git init \ && git remote add origin https://github.com/instructure/canvas-lms \ - #&& git fetch origin --no-tags --depth 1 refs/tags/${CANVAS_RELEASE}:refs/tags/${CANVAS_RELEASE} \ - && git fetch --no-tags --depth 1 origin ${CANVAS_RELEASE} \ + && git fetch origin --no-tags --depth 1 refs/tags/${CANVAS_RELEASE}:refs/tags/${CANVAS_RELEASE} \ + #&& git fetch --no-tags --depth 1 origin ${CANVAS_RELEASE} \ && git checkout ${CANVAS_RELEASE} @@ -193,7 +194,7 @@ RUN cd build/vendor \ # INSTALL QTIMigrationTool -RUN mkdir ${APP_HOME}/vendor \ +RUN #mkdir ${APP_HOME}/vendor \ && cd ${APP_HOME}/vendor \ && git clone https://github.com/instructure/QTIMigrationTool.git QTIMigrationTool \ && chmod +x QTIMigrationTool/migrate.py @@ -376,6 +377,10 @@ RUN echo "Settings file permissions..." \ && dos2unix /usr/src/*.sh +# Fix migration error where singleton column added too late +RUN mv /usr/src/app/db/migrate/20210812210129_add_singleton_column.rb \ + /usr/src/app/db/migrate/20111111214311_add_singleton_column.rb + USER root From 2f98396e5dc1fbac08a5d272c3eb3309ea1a7cfa Mon Sep 17 00:00:00 2001 From: Ray Pulspiher Date: Wed, 8 May 2024 18:14:33 -0700 Subject: [PATCH 2/4] Adjust helper scripts to detect if we need to use the new docker compose command instead of the older docker-compose command. To use new format, uninstall the docker-compose command (apt remove docker-compose or pip uninstall docker-compose) --- docker_build_files/down.sh | 11 ++- .../fix_role_overrides_migration_error.sh | 12 ++- docker_build_files/flush_redis_keys.sh | 10 ++- docker_build_files/recompile_canvas_assets.sh | 11 ++- docker_build_files/up.sh | 76 ++++++++++--------- 5 files changed, 82 insertions(+), 38 deletions(-) diff --git a/docker_build_files/down.sh b/docker_build_files/down.sh index 58e33ccc..a4a90ae3 100755 --- a/docker_build_files/down.sh +++ b/docker_build_files/down.sh @@ -1,4 +1,13 @@ #!/bin/sh -docker-compose down --remove-orphans +compose=`which docker-compose` +if [ -z "$compose" ]; then + compose="docker compose" +fi; +echo "Using Compose: $compose" + + + +#docker-compose down --remove-orphans +$compose down --remove-orphans diff --git a/docker_build_files/fix_role_overrides_migration_error.sh b/docker_build_files/fix_role_overrides_migration_error.sh index 2d6b623f..5e7704a5 100755 --- a/docker_build_files/fix_role_overrides_migration_error.sh +++ b/docker_build_files/fix_role_overrides_migration_error.sh @@ -4,6 +4,16 @@ # https://github.com/instructure/canvas-lms/issues/1806 # Then run this command to drop the index on that table, then the migration can finish. -docker-compose exec ope-postgresql bash -c "psql -U postgres -d canvas_production -c 'drop index index_role_overrides_on_context_role_permission;'" + +compose=`which docker-compose` + +if [ -z "$compose" ]; then + compose="docker compose" +fi; +echo "Using Compose: $compose" + + + +$compose exec ope-postgresql bash -c "psql -U postgres -d canvas_production -c 'drop index index_role_overrides_on_context_role_permission;'" diff --git a/docker_build_files/flush_redis_keys.sh b/docker_build_files/flush_redis_keys.sh index acbcf685..b45bc10e 100755 --- a/docker_build_files/flush_redis_keys.sh +++ b/docker_build_files/flush_redis_keys.sh @@ -1,6 +1,14 @@ #!/bin/bash -docker-compose exec ope-redis sh -c "redis-cli flushdb" +compose=`which docker-compose` + +if [ -z "$compose" ]; then + compose="docker compose" +fi; +echo "Using Compose: $compose" + + +$compose exec ope-redis sh -c "redis-cli flushdb" # flushall clears ALL dbs. diff --git a/docker_build_files/recompile_canvas_assets.sh b/docker_build_files/recompile_canvas_assets.sh index ffe4e84c..089d7aca 100755 --- a/docker_build_files/recompile_canvas_assets.sh +++ b/docker_build_files/recompile_canvas_assets.sh @@ -1,5 +1,14 @@ #!/bin/bash -docker-compose exec ope-canvas bash -c "cd /usr/src/app; bundle exec rake canvas:compile_assets" + +compose=`which docker-compose` + +if [ -z "$compose" ]; then + compose="docker compose" +fi; +echo "Using Compose: $compose" + + +$compose exec ope-canvas bash -c "cd /usr/src/app; bundle exec rake canvas:compile_assets" diff --git a/docker_build_files/up.sh b/docker_build_files/up.sh index 633db90d..57a56644 100755 --- a/docker_build_files/up.sh +++ b/docker_build_files/up.sh @@ -4,16 +4,21 @@ PY3=`which python3` PY2=`which python` +compose=`which docker-compose` PY=$PY3 if [ -z "$PY" ]; then - echo Switching to py2 + #echo Switching to py2 PY=$PY2 fi; echo "Using Python: $PY" +if [ -z "$compose" ]; then + compose="docker compose" +fi; +echo "Using Compose: $compose" SCRIPT=$(readlink -f "$0") BASEDIR=$(dirname "$SCRIPT") @@ -21,37 +26,37 @@ ROOTDIR=$(dirname "$BASEDIR") cd "$BASEDIR" -# Only run this stuff if fog is enabled +# Only run this stuff if fog is enabled and running in old opensuse OS if [ -f ope-fog/.enabled ]; then - - systemctl disable rpcbind - systemctl stop rpcbind - - # change to force update/checkout after adjusting gitattribute linefeeds - - # Need these loaded for tftp server to work - echo "Ensuring kernel modules are loaded..." - modprobe nf_conntrack_tftp - echo "nf_conntrack_tftp" > /etc/modules-load.d/nf_conntrack_tftp.conf - modprobe nf_nat_tftp - echo "nf_nat_tftp" > /etc/modules-load.d/nf_nat_tftp.conf - modprobe nf_conntrack_ftp - echo "nf_conntrack_ftp" > /etc/modules-load.d/nf_conntrack_ftp.conf - modprobe nf_conntrack_netbios_ns - echo "nf_conntrack_netbios_ns" > /etc/modules-load.d/nf_conntrack_netbios_ns.conf - modprobe nfs - echo "nfs" > /etc/modules-load.d/nfs.conf - modprobe nfsd - echo "nfsd" > /etc/modules-load.d/nfsd.conf - modprobe ipip - echo "ipip" > /etc/modules-load.d/ipip.conf - - - # Add some rules to track tftp traffic - #WLAN_IF=eth0 - #iptables -A INPUT -i $WLAN_IF -p udp -m state --state ESTABLISHED,RELATED -j ACCEPT - #iptables -A INPUT -i $WLAN_IF -p udp --dport 69 -m state --state NEW -j ACCEPT - + if /usr/bin/grep -q "SUSE" "/etc/os-release"; then + systemctl disable rpcbind + systemctl stop rpcbind + + # change to force update/checkout after adjusting gitattribute linefeeds + + # Need these loaded for tftp server to work + echo "Ensuring kernel modules are loaded..." + modprobe nf_conntrack_tftp + echo "nf_conntrack_tftp" > /etc/modules-load.d/nf_conntrack_tftp.conf + modprobe nf_nat_tftp + echo "nf_nat_tftp" > /etc/modules-load.d/nf_nat_tftp.conf + modprobe nf_conntrack_ftp + echo "nf_conntrack_ftp" > /etc/modules-load.d/nf_conntrack_ftp.conf + modprobe nf_conntrack_netbios_ns + echo "nf_conntrack_netbios_ns" > /etc/modules-load.d/nf_conntrack_netbios_ns.conf + modprobe nfs + echo "nfs" > /etc/modules-load.d/nfs.conf + modprobe nfsd + echo "nfsd" > /etc/modules-load.d/nfsd.conf + modprobe ipip + echo "ipip" > /etc/modules-load.d/ipip.conf + + + # Add some rules to track tftp traffic + #WLAN_IF=eth0 + #iptables -A INPUT -i $WLAN_IF -p udp -m state --state ESTABLISHED,RELATED -j ACCEPT + #iptables -A INPUT -i $WLAN_IF -p udp --dport 69 -m state --state NEW -j ACCEPT + fi fi @@ -64,17 +69,20 @@ rebuild_param="" if [ "$build_flag" = "auto" ]; then rebuild_param="auto" fi -echo "Rebuilding docker compose..." +#echo "Rebuilding docker compose..." $PY $ROOTDIR/build_tools/rebuild_compose.py "$rebuild_param" if [ "$build_flag" = "b" ]; then echo "Building docker containers..." - docker-compose build + #docker-compose build + $compose build fi echo "Bringing up containers..." # Bring up without rebuilding (in case container is out of date it will still start) -docker-compose up -d --no-build --remove-orphans +#docker-compose up -d --no-build --remove-orphans +$compose up -d --no-build --remove-orphans + #echo "Bringing up bridge for fog..." From dc48bf3cf8aaa84e3c8e3c06efef072e9dffbb01 Mon Sep 17 00:00:00 2001 From: Ray Pulspiher Date: Wed, 8 May 2024 18:16:17 -0700 Subject: [PATCH 3/4] Update gateway - fix issue with canvas rce upstream server not working in different circumstances and causing nginx errors --- docker_build_files/ope-gateway/Dockerfile | 4 ++-- docker_build_files/ope-gateway/nginx.tmpl | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docker_build_files/ope-gateway/Dockerfile b/docker_build_files/ope-gateway/Dockerfile index 6d9d9464..73d9a97e 100644 --- a/docker_build_files/ope-gateway/Dockerfile +++ b/docker_build_files/ope-gateway/Dockerfile @@ -9,14 +9,14 @@ LABEL rebuild=8 LABEL maintainer="Ray Pulsipher " LABEL description="Web Gateway for Open Prison Education - source files at https://github.com/operepo" LABEL vendor="openprisoneducation.com" -LABEL version="0.8" +LABEL version="0.9" ARG VERSION=dev RUN apk add nano openssl -LABEL rebuild=139 +LABEL rebuild=141 # public_certs moved under /usr/share/nginx/html # Make public_certs folder so we can make it easy to grab crt file #RUN mkdir -p /public_certs \ diff --git a/docker_build_files/ope-gateway/nginx.tmpl b/docker_build_files/ope-gateway/nginx.tmpl index 6963d79b..a8e6a865 100644 --- a/docker_build_files/ope-gateway/nginx.tmpl +++ b/docker_build_files/ope-gateway/nginx.tmpl @@ -11,18 +11,19 @@ {{ if .Address }} {{/* If we got the containers from swarm and this container's port is published to host, use host IP:PORT */}} {{ if and .Container.Node.ID .Address.HostPort }} - # {{ .Container.Node.Name }}/{{ .Container.Name }} + # {{ .Container.Node.Name }}/{{ .Container.Name }} . server {{ .Container.Node.Address.IP }}:{{ .Address.HostPort }} max_fails=0; {{/* If there is no swarm node or the port is not published on host, use container's IP:PORT */}} {{ else if .Network }} - # {{ .Container.Name }} + # {{ .Container.Name }} .. server {{ .Network.IP }}:{{ .Address.Port }} max_fails=0; {{ end }} {{ else if .Network }} - # {{ .Container.Name }} + # {{ .Container.Name }} ... {{ if .Network.IP }} server {{ .Network.IP }} max_fails=0 down; {{ else }} + # .... server 127.0.0.1 max_fails=0 down; {{ end }} {{ end }} @@ -172,12 +173,12 @@ server { } {{ end }} -{{ if ne $canvas_host "" }} -upstream ope-canvas-rce { - # RCE Server - server ope-canvas-rce:3010; -} -{{ end }} +#{{ if ne $canvas_host "" }} +#upstream ope-canvas-rce { +# # RCE Server +# server ope-canvas-rce:3010; +#} +#{{ end }} {{ range $host, $containers := groupByMulti $ "Env.VIRTUAL_HOST" "," }} From f27771231ac01e10f2f66aed70542a0a59beb895 Mon Sep 17 00:00:00 2001 From: Ray Pulspiher Date: Wed, 8 May 2024 18:19:15 -0700 Subject: [PATCH 4/4] Rebuild canvas release/2022-07-06.145 and matching rce apps. --- docker_build_files/ope-canvas-rce/Dockerfile | 13 +++++++------ docker_build_files/ope-canvas/Dockerfile | 1 + 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docker_build_files/ope-canvas-rce/Dockerfile b/docker_build_files/ope-canvas-rce/Dockerfile index 28840c34..f59ae37d 100644 --- a/docker_build_files/ope-canvas-rce/Dockerfile +++ b/docker_build_files/ope-canvas-rce/Dockerfile @@ -4,7 +4,7 @@ FROM instructure/node-passenger:12 LABEL maintainer="Ray Pulsipher " LABEL description="Canvas LMS by Instructure - Image for Open Prison Education" LABEL vendor="openprisoneducation.com" -LABEL version="1.2" +LABEL version="1.3" USER root @@ -16,15 +16,16 @@ LABEL rebuild=7 # Pull the RCE server ENV RCE_APP_HOME /usr/src/rce/ WORKDIR $RCE_APP_HOME -ENV RCE_RELEASE master +#ENV RCE_RELEASE master +ENV RCE_RELEASE v1.21 RUN mkdir -p ${RCE_APP_HOME} \ && cd ${RCE_APP_HOME} \ && git init \ && git remote add origin https://github.com/instructure/canvas-rce-api \ - && git fetch origin master --no-tags --depth 1 \ - && git checkout master \ - #&& git fetch origin --no-tags --depth 1 refs/tags/${RCE_RELEASE}:refs/tags/${RCE_RELEASE} \ - #&& git checkout ${RCE_RELEASE} \ + #&& git fetch origin master --no-tags --depth 1 \ + #&& git checkout master \ + && git fetch origin --no-tags --depth 1 refs/tags/${RCE_RELEASE}:refs/tags/${RCE_RELEASE} \ + && git checkout ${RCE_RELEASE} \ && npm install --production \ && echo Checkout done. diff --git a/docker_build_files/ope-canvas/Dockerfile b/docker_build_files/ope-canvas/Dockerfile index 649c425c..07016574 100644 --- a/docker_build_files/ope-canvas/Dockerfile +++ b/docker_build_files/ope-canvas/Dockerfile @@ -378,6 +378,7 @@ RUN echo "Settings file permissions..." \ # Fix migration error where singleton column added too late +# https://groups.google.com/g/canvas-lms-users/c/pk6pzDb0-Gw RUN mv /usr/src/app/db/migrate/20210812210129_add_singleton_column.rb \ /usr/src/app/db/migrate/20111111214311_add_singleton_column.rb