Merge pull request #225 from metabrainz/schema-change-2022-q2

Follow database schema change 2022 Q2

README.md

@@ -44,7 +44,7 @@ This repo contains everything needed to run a musicbrainz slave server with sear
 
 * CPU: 16 threads (or 2 without indexed search)
 * RAM: 16 GB (or 4 without indexed search)
-* Disk Space: 150 GB (or 60 without indexed search)
+* Disk Space: 200 GB (or 70 without indexed search)
             + system disk usage
 
 ### Required software
@@ -81,13 +81,13 @@ If you use [UFW](https://help.ubuntu.com/community/UFW) to manage your firewall:
 
 ## Components version
 
-* Current MB Branch: [v-2022-04-18](build/musicbrainz/Dockerfile#L53)
-* Current DB_SCHEMA_SEQUENCE: [26](build/musicbrainz/DBDefs.pm#L112)
+* Current MB Branch: [v-2022-05-16-schema-change](build/musicbrainz/Dockerfile#L53)
+* Current DB_SCHEMA_SEQUENCE: [27](build/musicbrainz/Dockerfile#L129)
 * Postgres Version: [12](docker-compose.yml)
   (can be changed by setting the environment variable `POSTGRES_VERSION`)
 * MB Solr search server: [3.4.2](docker-compose.yml#L88)
   (can be changed by setting the environment variable `MB_SOLR_VERSION`)
-* Search Index Rebuilder: [2.1.1](build/sir/Dockerfile#L37)
+* Search Index Rebuilder: [3.0.1](build/sir/Dockerfile#L37)
 
 ## Installation
 
@@ -575,7 +575,6 @@ admin/purge-message-queues
 sudo docker-compose run --rm search load-search-indexes.sh --force
 sudo docker-compose run --rm musicbrainz recreatedb.sh
 sudo docker-compose up -d
-admin/setup-amqp-triggers clean
 admin/setup-amqp-triggers install
 admin/configure add replication-cron
 sudo docker-compose up -d

admin/setup-amqp-triggers

@@ -26,6 +26,44 @@ fi
 LOCAL_TRIGGERS_DIR="$(dirname "${BASH_SOURCE[0]}")/.setup-amqp-triggers-sql"
 REMOTE_TRIGGERS_DIR=/tmp/indexer-sql
 
+generate_sql_scripts() {
+  $DOCKER_COMPOSE_CMD exec indexer python -m sir triggers
+  $DOCKER_COMPOSE_CMD exec indexer ./GenerateDropSql.pl
+}
+
+copy_sql_scripts() {
+  remove_temp_sql_scripts_from_host
+  remove_temp_sql_scripts_from_container
+
+  indexer_container_id="$($DOCKER_COMPOSE_CMD ps -q indexer)"
+  $DOCKER_CMD cp "$indexer_container_id":/code/sql "$LOCAL_TRIGGERS_DIR"
+  if [ "$(stat -c %U "$LOCAL_TRIGGERS_DIR")" != "$(id -un)" ]
+  then
+    sudo chown -R "$(id -un):$(id -gn)" "$LOCAL_TRIGGERS_DIR"
+  fi
+
+  musicbrainz_container_id="$($DOCKER_COMPOSE_CMD ps -q musicbrainz)"
+  $DOCKER_CMD cp "$LOCAL_TRIGGERS_DIR" "$musicbrainz_container_id":"$REMOTE_TRIGGERS_DIR"
+
+  remove_temp_sql_scripts_from_host
+}
+
+remove_temp_sql_scripts_from_host() {
+  if [ -e "$LOCAL_TRIGGERS_DIR" ]
+  then
+    if [ "$(stat -c %U "$LOCAL_TRIGGERS_DIR")" != "$(id -un)" ]
+    then
+      sudo rm -fr"${1:+v}" "$LOCAL_TRIGGERS_DIR"
+    else
+      rm -fr"${1:+v}" "$LOCAL_TRIGGERS_DIR"
+    fi
+  fi
+}
+
+remove_temp_sql_scripts_from_container() {
+  $DOCKER_COMPOSE_CMD exec musicbrainz rm -fr"${1:+v}" "$REMOTE_TRIGGERS_DIR"
+}
+
 case "$1" in
   clean     )
     if ! $DOCKER_COMPOSE_CMD ps musicbrainz | grep -qw 'running\|Up'
@@ -38,74 +76,48 @@ case "$1" in
 
     echo "Cleaning indexer triggers install/uninstall scripts up ..."
 
-    $DOCKER_COMPOSE_CMD exec musicbrainz rm -frv "$REMOTE_TRIGGERS_DIR"
-    rm -frv "$LOCAL_TRIGGERS_DIR"
+    remove_temp_sql_scripts_from_host -v
+    remove_temp_sql_scripts_from_container -v
     ;;
   install   )
-    if ! $DOCKER_COMPOSE_CMD ps indexer | grep -qw 'running\|Up'
-    then
-      echo >&2 "$SCRIPT_NAME: cannot install: " \
-               "the Docker Compose service 'indexer' is not up"
-      echo >&2 "Try '$DOCKER_COMPOSE_CMD up -d indexer' from '$MB_DOCKER_ROOT'"
-      exit 69 # EX_UNAVAILABLE
-    fi
-
-    if [ -e "$LOCAL_TRIGGERS_DIR" ]
-    then
-      echo >&2 "$SCRIPT_NAME: cannot install: file '$LOCAL_TRIGGERS_DIR' exists"
-      echo >&2 "Either $SCRIPT_NAME is already running or stopped before its time."
-      echo >&2 "Remove this file if you are sure the script is not still running:"
-      echo >&2 "    rm -frv $(printf %q "$LOCAL_TRIGGERS_DIR")"
-      exit 70 # EX_SOFTWARE
-    fi
-
-    if $DOCKER_COMPOSE_CMD exec musicbrainz test -e "$REMOTE_TRIGGERS_DIR"
-    then
-      echo >&2 "$SCRIPT_NAME: cannot install: file '$REMOTE_TRIGGERS_DIR' exists in 'musicbrainz' docker compose service"
-      echo >&2 "Either $SCRIPT_NAME is already running or stopped before its time."
-      echo >&2 "Revert partial installation if you are sure the script is not still running:"
-      echo >&2 "    $SCRIPT_NAME uninstall"
-      exit 70 # EX_SOFTWARE
-    fi
+    for service in indexer musicbrainz
+    do
+      if ! $DOCKER_COMPOSE_CMD ps "$service" | grep -qw 'running\|Up'
+      then
+        echo >&2 "$SCRIPT_NAME: cannot install: " \
+                 "the Docker Compose service '$service' is not up"
+        echo >&2 "Try '$DOCKER_COMPOSE_CMD up -d $service' from '$MB_DOCKER_ROOT'"
+        exit 69 # EX_UNAVAILABLE
+      fi
+    done
+
+    generate_sql_scripts
+    copy_sql_scripts
 
     echo "Installing indexer triggers ..."
 
-    $DOCKER_COMPOSE_CMD exec indexer python -m sir triggers
-
-    indexer_container_id="$($DOCKER_COMPOSE_CMD ps -q indexer)"
-    $DOCKER_CMD cp "$indexer_container_id":/code/sql "$LOCAL_TRIGGERS_DIR"
-    if [ "$(stat -c %U "$LOCAL_TRIGGERS_DIR")" != "$(id -un)" ]
-    then
-      sudo chown -R "$(id -un):$(id -gn)" "$LOCAL_TRIGGERS_DIR"
-    fi
-
-    musicbrainz_container_id="$($DOCKER_COMPOSE_CMD ps -q musicbrainz)"
-    $DOCKER_CMD cp "$LOCAL_TRIGGERS_DIR" "$musicbrainz_container_id":"$REMOTE_TRIGGERS_DIR"
-
     $DOCKER_COMPOSE_CMD exec musicbrainz indexer-triggers.sh "$REMOTE_TRIGGERS_DIR" create
-
-    rm -fr "$LOCAL_TRIGGERS_DIR"
     ;;
   uninstall )
-    if ! $DOCKER_COMPOSE_CMD ps musicbrainz | grep -qw 'running\|Up'
-    then
-      echo >&2 "$SCRIPT_NAME: cannot uninstall: " \
-               "the Docker Compose service 'musicbrainz' is not up"
-      echo >&2 "Try '$DOCKER_COMPOSE_CMD up -d musicbrainz' from '$MB_DOCKER_ROOT'"
-      exit 69 # EX_UNAVAILABLE
-    fi
-
-    if $DOCKER_COMPOSE_CMD exec musicbrainz test ! -e "$REMOTE_TRIGGERS_DIR"
-    then
-      echo >&2 "$SCRIPT_NAME: cannot uninstall: file '$REMOTE_TRIGGERS_DIR' does not exist"
-      exit 70 # EX_SOFTWARE
-    fi
+    for service in indexer musicbrainz
+    do
+      if ! $DOCKER_COMPOSE_CMD ps "$service" | grep -qw 'running\|Up'
+      then
+        echo >&2 "$SCRIPT_NAME: cannot uninstall: " \
+                 "the Docker Compose service '$service' is not up"
+        echo >&2 "Try '$DOCKER_COMPOSE_CMD up -d $service' from '$MB_DOCKER_ROOT'"
+        exit 69 # EX_UNAVAILABLE
+      fi
+    done
+
+    generate_sql_scripts
+    copy_sql_scripts
 
     echo "Uninstalling indexer triggers ..."
 
     $DOCKER_COMPOSE_CMD exec musicbrainz indexer-triggers.sh "$REMOTE_TRIGGERS_DIR" drop
 
-    $DOCKER_COMPOSE_CMD exec musicbrainz rm -fr "$REMOTE_TRIGGERS_DIR"
+    remove_temp_sql_scripts_from_container
     ;;
   help|-h|--help)
     echo "$HELP"

build/musicbrainz-dev/DBDefs.pm

@@ -122,7 +122,7 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases(
 # replication_control.current_schema_sequence.
 # This is required, there is no default in order to prevent it changing without
 # manual intervention.
-sub DB_SCHEMA_SEQUENCE { 26 }
+sub DB_SCHEMA_SEQUENCE { 27 }
 
 # What type of server is this?
 # * RT_MASTER - This is a master replication server.  Changes are allowed, and

build/musicbrainz/Dockerfile

@@ -50,7 +50,7 @@ RUN mkdir -p /usr/local/share/keyrings && \
         zlib1g-dev \
     && rm -rf /var/lib/apt/lists/*
 
-ARG MUSICBRAINZ_SERVER_VERSION=v-2022-04-18
+ARG MUSICBRAINZ_SERVER_VERSION=v-2022-05-16-schema-change
 LABEL org.metabrainz.musicbrainz-server.version="${MUSICBRAINZ_SERVER_VERSION}"
 RUN git clone --depth=1 --branch $MUSICBRAINZ_SERVER_VERSION https://github.com/metabrainz/musicbrainz-server.git musicbrainz-server
 
@@ -126,7 +126,7 @@ ARG POSTGRES_PASSWORD=doesntmatteraslongasyoudontcompiletests
 ENV BASH_ENV=/noninteractive.bash_env \
     MUSICBRAINZ_BASE_FTP_URL=ftp://ftp.eu.metabrainz.org/pub/musicbrainz \
     MUSICBRAINZ_CATALYST_DEBUG=0 \
-    MUSICBRAINZ_DB_SCHEMA_SEQUENCE=26 \
+    MUSICBRAINZ_DB_SCHEMA_SEQUENCE=27 \
     MUSICBRAINZ_DEVELOPMENT_SERVER=0 \
     MUSICBRAINZ_SEARCH_SERVER=search:8983/solr \
     MUSICBRAINZ_SERVER_PROCESSES=10 \

build/musicbrainz/scripts/indexer-triggers.sh

@@ -20,7 +20,6 @@ case "$2" in
   create)
     admin/psql < "$INDEXER_SQL_DIR/CreateFunctions.sql"
     admin/psql < "$INDEXER_SQL_DIR/CreateTriggers.sql"
-    admin/GenerateSQLScripts.pl "$INDEXER_SQL_DIR/"
     ;;
   drop  )
     admin/psql < "$INDEXER_SQL_DIR/DropTriggers.sql"

build/musicbrainz/scripts/upgrade-db-schema.sh

@@ -2,6 +2,6 @@
 
 set -e -u
 
-export MUSICBRAINZ_DB_SCHEMA_SEQUENCE=25
+export MUSICBRAINZ_DB_SCHEMA_SEQUENCE=26
 
 dockerize -wait tcp://db:5432 -timeout 60s ./upgrade.sh

build/sir-dev/Dockerfile

@@ -1,5 +1,5 @@
 ARG PYTHON_VERSION=2.7
-ARG BASE_IMAGE_DATE=20210115
+ARG BASE_IMAGE_DATE=20220421
 FROM metabrainz/python:$PYTHON_VERSION-$BASE_IMAGE_DATE
 
 ARG PYTHON_VERSION

build/sir/Dockerfile

@@ -1,5 +1,5 @@
 ARG PYTHON_VERSION=2.7
-ARG BASE_IMAGE_DATE=20210115
+ARG BASE_IMAGE_DATE=20220421
 FROM metabrainz/python:$PYTHON_VERSION-$BASE_IMAGE_DATE
 
 ARG PYTHON_VERSION
@@ -34,7 +34,7 @@ RUN apt-get update \
 # Installing sir #
 ##################
 
-ARG SIR_VERSION=2.1.1
+ARG SIR_VERSION=3.0.1
 LABEL org.metabrainz.sir.version="${SIR_VERSION}"
 
 # hadolint ignore=DL3003