Merge pull request #1620 from ucfopen/issue/1616-run-first-script-upd…

…ates-part-two First-time setup and dev docker compose config overhaul
ucfopen · Nov 26, 2024 · 353f7f9 · 353f7f9
2 parents 28da569 + a7dabad
commit 353f7f9
Show file tree

Hide file tree

Showing 17 changed files with 1,000 additions and 362 deletions.
diff --git a/.env b/.env
@@ -1,19 +1,25 @@
-# =========================
+# =================================
 # CONFIGURE MATERIA
-# 1. Copy `.env` to `.env.local` (make changes in .env.local)
-# 2. Set every `<MUST_SET>` value
-# 3. Comb through for any other settings you may wish to change
-# 4. Visit Materia in browser, check logs in fuel/app/logs for debugging
 #
-# DETAILED INSTRUCTIONS
-# !!! View the Materia README for options, descriptions & permitted values !!!
+# This .env file is not referenced by the default development & nondev docker stack! Use docker/.env.local instead.
+# It's recommended to use this .env file as a template for production.
+# Your docker compose file(s) will need to be updated to reference your final .env and its relative path in the env_file: directive of each service.
+#
+# DETAILED INSTRUCTIONS FOR PRODUCTION USE
+#
+# 1. If coming from a nondev instance, copy relevant values from docker/.env.local
+# 2. Set every `<MUST_SET>` value
+# 3. Review remaining optional settings
+# 4. Update your docker compose file(s) to point to the final .env file and location. Services should use the `env_file:` directive to include env vars as required
+# 5. Remove any dev `environment:` references in compose file(s) if they are present
+# 6. Start docker containers (or stop, rm, and restart if already running)
+# 7. Visit Materia in browser, check logs in fuel/app/logs for debugging
 #
-# Only `BOOL_` options become boolean values, and ONLY `true` evaluates to true
+# Refer to docker/README.md for more information about environment configs
 #
-# Docker/Dev notes:
-# docker/.env.local is used instead of .env.local
+# !! NOTE: Only `BOOL_` options become boolean values, and ONLY `true` evaluates to true
 
-# GENERAL ===================
+# GENERAL ==========================
 
 FUEL_ENV=production
 #DATABASE_URL=<MUST_SET>
@@ -24,43 +30,46 @@ BOOL_SEND_EMAILS=false
 #FUEL_ALWAYS_LOAD_MODULES=""
 #GOOGLE_ANALYTICS_ID=xxx
 
-# LOGGING ===================
+# LOGGING ==========================
 
 #FUEL_LOG_THRESHOLD=300
 #LOG_HANDLER=DEFAULT
 
-# ASSETS ===================
+# ASSETS ===========================
 
-#URLS_STATIC=
-#URLS_ENGINES=
+#URLS_STATIC= 								# Set if static files and assets come from a different domain. While used in dev, not required for prod
+#URLS_ENGINES= 								# same as above
 #BOOL_ADMIN_UPLOADER_ENABLE=true
-ASSET_STORAGE_DRIVER=file # file | s3 | db (db not recommended)
+ASSET_STORAGE_DRIVER=file					# file | s3 | db (db not recommended)
 
-# AWS S3 ===================
+# AWS S3 ===========================
+# Note: the <MUST_SET> options here are only required if S3 storage driver is enabled
 
-# ASSET_STORAGE_S3_REGION=us-east-1
-# ASSET_STORAGE_S3_BASEPATH=media
+# ASSET_STORAGE_S3_CREDENTIAL_PROVIDER=		# env | imds
 # ASSET_STORAGE_S3_BUCKET=<MUST_SET>
-# ASSET_STORAGE_S3_ENDPOINT= # endpoint not required for S3 on AWS
-# AWS_ACCESS_KEY_ID=<MUST_SET>
-# AWS_SECRET_ACCESS_KEY=<MUST_SET>
-# AWS_SESSION_TOKEN=<MUST_SET> # STS token for s3 development
+# ASSET_STORAGE_S3_ENDPOINT=				# not required for S3 on AWS
+# ASSET_STORAGE_S3_KEY=<MUST_SET>
+# ASSET_STORAGE_S3_SECRET=<MUST_SET>
+# ASSET_STORAGE_S3_REGION=
+# ASSET_STORAGE_S3_BUCKET=
+# ASSET_STORAGE_S3_BASEPATH=
+# AWS_SESSION_TOKEN=<MUST_SET> 				# STS token for s3 development. Not required if using imds
 
-# SESSION & CACHE ===================
+# SESSION & CACHE ==================
 
 #MEMCACHED_HOST=localhost
 #MEMCACHED_PORT=11211
-#CACHE_DRIVER=file
-#SESSION_DRIVER=file
+#CACHE_DRIVER=file							# memcached | file
+#SESSION_DRIVER=file						# memcached | file | db
 SESSION_EXPIRATION=21600
 
-# THEME ===================
+# THEME ============================
 
 #THEME_ACTIVE=default
 # relative to /fuel/packages/
 #THEME_PACKAGE=materia-theme-ucf
 
-# AUTH ===================
+# AUTH ============================
 
 #AUTH_DRIVERS=Materiaauth
 #AUTH_SALT=<MUST_SET>
@@ -72,14 +81,14 @@ SESSION_EXPIRATION=21600
 #USER_INSTRUCTOR_PASSWORD
 #USER_STUDENT_PASSWORD
 
-# CRYPTO ===================
+# CRYPTO ==========================
 
 #CRYPTO_KEY=<MUST_SET>
 #CRYPTO_IV=<MUST_SET>
 #CRYPTO_HMAC=<MUST_SET>
 #CIPHER_KEY=<MUST_SET>
 
-# LTI ===================
+# LTI ============================
 
 #BOOL_LTI_RESTRICT_LOGINS_TO_LAUNCHES=false
 #LTI_GUID=
@@ -94,17 +103,18 @@ LTI_KEY="materia-production-lti-key"
 #BOOL_LTI_GRACEFUL_CONFIG_FALLBACK=true
 #BOOL_LTI_LOG_FOR_DEBUGGING=false
 
-# Question Generation ===
+# QUESTION GENERATION ===========
+# Note: <MUST_SET> options are only required if GENERATION_ENABLED is true
 
 #GENERATION_ENABLED=true
 #GENERATION_ALLOW_IMAGES=false
-#GENERATION_API_PROVIDER=<MUST_SET>
-#GENERATION_API_ENDPOINT=
-#GENERATION_API_KEY=
-#GENERATION_API_VERSION=
-#GENERATION_API_MODEL=
-#GENERATION_LOG_STATS=true
+#GENERATION_API_PROVIDER=<MUST_SET>			# openai | azure_openai
+#GENERATION_API_KEY=<MUST_SET>
+#GENERATION_API_ENDPOINT=					# required for azure
+#GENERATION_API_VERSION=					# required for azure
+#GENERATION_API_MODEL=						# required for openai
+#GENERATION_LOG_STATS=true					# optional. stats are logged to debug threshold.
 
-# webserver settings =======
+# WEBSERVER =====================
 
-#IS_SERVER_HTTPS=true
+#IS_SERVER_HTTPS=true						# defaults to true, only set to false if required
diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml
@@ -8,7 +8,7 @@ jobs:
       uses: actions/checkout@v2
 
     - name: Run Tests
-      run: cd docker && ./run_tests_ci.sh
+      run: cd docker && touch .env.local && ./run_tests_ci.sh
 
   docker_images_and_release_zip:
     needs: ci_tests
@@ -39,7 +39,8 @@ jobs:
       - name: Build App and Webserver Images
         run: |
           cd docker
-          docker compose build --no-cache webserver app fakes3
+          docker compose build --no-cache webserver app
+          docker compose -f docker-compose.yml -f docker-compose.development.yml build --no-cache fakes3
 
       - name: Push Dev App and Webserver Images
         if: ${{ startsWith(github.ref, 'refs/tags/v') && (contains(github.ref, '-alpha') || contains(github.ref, '-rc')) }}

diff --git a/.gitignore b/.gitignore
@@ -14,6 +14,7 @@ nbproject/
 
 # custom config
 .env.local
+.env.local.mysql
 
 # hidden files created by Windows
 Thumbs.db
@@ -123,3 +124,9 @@ public/dist/*
 !public/dist/package.json
 !public/dist/path.js
 !public/dist/README.md
+
+public/js
+public/css
+
+# docker override is now untracked as of v10.3.0
+docker/docker-compose.override.yml
diff --git a/README.md b/README.md
@@ -26,37 +26,54 @@ Materia is configured to use Docker containers in production environments, orche
 
 ### Docker Deployment
 
-We publish production ready docker and nginx containers in the [Materia Docker repository](https://github.com/orgs/ucfopen/packages/container/package/materia).  For more info on using Docker in Production, read the [Materia Docker Readme](docker/README.md)
+Refer to the [Materia Docker Readme](docker/README.md) for a full breakdown of Materia's docker configuration and deployment in both development and production.
 
 ### Configuration
 
-Visit the [Server Variables](https://ucfopen.github.io/Materia-Docs/admin/server-variables.html) page on our docs site for information about configuration through environment variables.
+Materia uses environment variables to facilitate application and webserver configuration. These are typically sourced from `.env` or `.env.local` files on the host machine that are volume mounted into their associated containers.
 
-## Development
+Visit docker README as well as the [Server Variables](https://ucfopen.github.io/Materia-Docs/admin/server-variables.html) page on our docs site for information about configuration through environment variables. The [root env file](.env) serves as a configuration template for production instances of Materia.
 
-Code contributors should review the [CONTRIBUTING](CONTRIBUTING.md) document before proceeding.
+## Setup
+
+Materia provides a pair of setup scripts for out-of-the-box deployment based on two different use cases:
+
+1. `run_first_for_dev.sh` sets up and configures your local instance of Materia for development. This includes additional volume mounts for project files and makes use of additional containers for mysql, s3, and memcached.
+2. `run_first_for_nondev.sh` is ideal for users who just want to explore Materia locally and potentially transition to a production instance. The script dynamically configures the override compose file based on selections you make in the script.
 
-### Local Dev with Docker
+> [!NOTE]
+> `yq` is required for the nondev script. This may come preinstalled in some OS distributions but you should ensure the correct version is installed. Consult the [yq installation guide](https://github.com/mikefarah/yq?tab=readme-ov-file#install) for more info.
 
-Get started with a local dev server:
+In either case, first-time setup involves the following:
 
 ```
 git clone https://github.com/ucfopen/Materia.git
-
 cd Materia/docker
+```
 
-./run_first.sh
+Followed by either:
+```
+./run_first_for_dev.sh
+```
+For local development or 
+
+```
+./run_first_for_nondev.sh
 ```
+For creating a local instance where development is not desired.
 
-The `run_first.sh` script only has to be run once for initial setup. Afterwards, your local copy will persist in a docker volume unless you explicitly use `docker compose down` or delete the volume manually.
+The `run_first` scripts only have to be run once for initial setup. Afterwards, your local copy will persist in a docker volume unless you explicitly use `docker compose down` or delete the volumes manually.
 
-Use `docker compose up` to run your local instance. The compose process must persist to keep the application alive. Materia is configured to run at `https://127.0.0.1` by default.
+Use `docker compose up` to run your local instance. The compose process must persist to keep the application alive. Materia is configured to run at `https://localhost` by default.
 
-In a separate terminal window, run `yarn dev` to enable the webpack dev server and live reloading while making changes to JS and CSS assets.
+Note that Materia uses a self-signed certificate to facilitate https traffic locally. Your browser may require security exceptions for your application on ports `443` and `8008` (if setup for local development).
 
-Note that Materia uses a self-signed certificate to facilitate https traffic locally. Your browser may require security exceptions for both `127.0.0.1:443` and `127.0.0.1:8008`.
+> [!NOTE]
+> If local development of static assets (JS and CSS) is desired, run `yarn dev` in a separate terminal window to enable the webpack dev server and live reloading. This requires node and yarn to be installed on the host machine.
 
-More info about Materia Docker can be found in the [Materia Docker Readme](docker/README.md)
+## Transitioning to Production
+
+More information about creating a production-capable Materia instance can be found in the [Materia Docker Readme](docker/README.md).
 
 ### Creating additional users
 
@@ -66,20 +83,14 @@ See the wiki page for [Creating a local user](https://github.com/ucfopen/Materia
 
 Tests run in the docker environment to maintain consistency. View the `run_tests_*.sh` scripts in the docker directory for options.
 
-#### Running A Single Test Group
-
-Inspect the actual test command in `/.run_tests.sh` for guidance, but as of the time of writing this, you can run a subset of the tests in the docker environment to save time.
-
-The following command will run just the **Oauth** tests rather quickly:
-
-```
-./run_tests.sh --group=Oauth
-```
-
 ### Git Hooks
 
 There is a pre-commit hook available to ensure your code follows our linting standards. Check out the comments contained inside the hook files (in the githooks directory) to install it, you may need a few dependencies installed to get linting working.
 
+## Contributing
+
+Code contributors should review the [CONTRIBUTING](CONTRIBUTING.md) document before proceeding.
+
 ## Authentication
 
 Materia supports two forms of authentication:
@@ -109,3 +120,33 @@ To use an actual S3 bucket for local dev:
 1. Set `DEV_ONLY_FAKES3_DISABLED` environment variable in `docker/.env` to `true`
 2. Set `ASSET_STORAGE_S3_BUCKET` to your bucket name
 3. Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` in `.env.local`. (Tip: You can run `aws configure export-credentials --profile YOUR_PROFILE_NAME --format env-no-export` to get these)
+
+> [!NOTE]
+> Note that `fakes3` asset storage is disabled when `FUEL_ENV` is set to `production`.
+
+## Widget Management
+
+A default list of widgets will be installed as part of the first-time setup process, but the widget ecosystem has many more! Peruse the [Materia Widget Gallery](https://ucfopen.github.io/materia-widget-gallery/) to view additional widgets to install.
+
+### Installing Widgets
+
+Widgets can be installed in one of two ways:
+
+1. A user with the `super_user` role can visit the Widget Admin panel by navigating to `your.materia.url/admin/widget` or by selecting the orange "Admin" button at the top. Select a `.wigt` file from the file upload dialog to install it.
+2. Widgets can be installed from the cli on the application container once running:
+
+```
+$ docker exec -it <container name or id> sh
+$ wget url/for/materia/widget.wigt
+$ php oil r widget:install widget.wigt
+```
+
+The `.wigt` files do not need to be retained once a widget is installed.
+
+## Theming
+
+Theme overrides are facilitated through FuelPHP packages, installed via composer. Materia ships with `Materia-Theme-UCF` by default.
+
+Review the [repository README](https://github.com/ucfopen/Materia-Theme-UCF) for `Materia-Theme-UCF` for a breakdown of using theme overrides in Materia 10.x and later.
+
+
diff --git a/docker/.env b/docker/.env
@@ -1,11 +1,31 @@
-## docker/.env contains environment variables used by Materia during local development
-## we do not recommend making edits directly to this file. Instead, make a .env.local in the same directory (docker/) and override the values below as desired.
+## This env file contains environment variable configurations for development and nondev (but NOT production!)
+## Instead of editing these values directly, use .env.local for overrides
+
+## In a production environment, a final .env file should not include any of the dev defaults for unique keys:
+## AUTH_SALT
+## AUTH_SIMPLEAUTH_SALT
+## CIPHER_KEY
+## LTI_KEY
+## LTI_SECRET
+
+## use the .env file in the root directory as a template for a production .env. Your compose file(s) will need to be updated accordingly.
+
+# fuel environment
+# the run_first_for_nondev script will automatically override this to production
+FUEL_ENV=development
+
+# general configurations
+SYSTEM_EMAIL=[email protected]
+
+# theming
+THEME_PACKAGE=materia-theme-ucf
 
 # database settings
 MYSQL_ROOT_PASSWORD=drRoots
 MYSQL_USER=materia
 MYSQL_PASSWORD=odin
 MYSQL_DATABASE=materia
+DATABASE_URL=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@mysql/${MYSQL_DATABASE}
 
 # passwords/hashes/keys
 DEV_ONLY_USER_PASSWORD=kogneato
@@ -14,10 +34,34 @@ DEV_ONLY_AUTH_SALT=111b776e5f862058e2e075b640b3de5fb601d0ac57639c733a2d10edffd2a
 DEV_ONLY_AUTH_SIMPLEAUTH_SALT=33e0d379060e3877d634632853c10a70dff9710b751e5af00a0f637884df417e
 DEV_ONLY_SECRET_CIPHER_KEY=e0beaea1704555ae3c75650703bb106fac24b8967c77a667124fbe745c3346ed
 
-# s3-specific asset storage values
+# auth
+AUTH_DRIVERS=Materiaauth
+
+# caching
+CACHE_DRIVER=memcached
+MEMCACHED_HOST=memcached
 
-# overrides default value in the base .env (which isn't loaded into dev environment)
+# logging
+FUEL_LOG_THRESHOLD=100
+LOG_HANDLER=STDOUT
+BOOL_LTI_LOG_FOR_DEBUGGING=true
+
+# lti config
+LTI_KEY=materia-lti-key
+LTI_SECRET=materia-lti-secret
+
+# session
+SESSION_DRIVER=memcached
+
+# urls
+URLS_ENGINES=https://localhost:8008/widget/
+URLS_STATIC=https://localhost:8008/
+
+# asset storage
 ASSET_STORAGE_DRIVER=s3
+
+# s3-specific asset storage values
+
 # provider must be one of the following: env | imds
 ASSET_STORAGE_S3_CREDENTIAL_PROVIDER=env
 ASSET_STORAGE_S3_BUCKET=fake_bucket