Add an --initial-sql flag to InitDb.pl; allow configuring the REPLICATION_TYPE for create_test_db.sh

.circleci/config.yml

@@ -8,7 +8,7 @@ branch_filters: &job_filters
 executors:
   musicbrainz-tests:
     docker:
-      - image: metabrainz/musicbrainz-tests:v-2024-05-23
+      - image: metabrainz/musicbrainz-tests:v-2024-05-27
         user: root
     working_directory: /home/musicbrainz/musicbrainz-server
 

admin/InitDb.pl

@@ -6,10 +6,12 @@
 use English;
 use FindBin;
 use lib "$FindBin::Bin/../lib";
+use String::ShellQuote qw( shell_quote );
 
 use DBDefs;
 use MusicBrainz::Server::Constants qw( @FULL_SCHEMA_LIST );
 use MusicBrainz::Server::Replication qw( :replication_type );
+use MusicBrainz::Script::Utils qw( find_files );
 
 use aliased 'MusicBrainz::Server::DatabaseConnectionFactory' => 'Databases';
 
@@ -23,6 +25,7 @@
 my $fCreateDB;
 my $fInstallExtension;
 my $fExtensionSchema;
+my $initial_sql = 'InsertDefaultRows.sql';
 my $tmp_dir;
 
 use Getopt::Long;
@@ -57,11 +60,15 @@ sub RequireMinimumPostgreSQLVersion
 
 sub RunSQLScript
 {
-    my ($db, $file, $startmessage, $path) = @_;
-    $startmessage ||= "Running $file";
+    my ($db, $file, $startmessage) = @_;
+    $startmessage ||= "Running $file ...";
     print localtime() . " : $startmessage ($file)\n" unless $fQuiet;
 
-    $path //= $sqldir;
+    my $quoted_file_path = do {
+        my ($file_path) = find_files($file, $file, $sqldir,
+                                     DBDefs->MB_SERVER_ROOT);
+        shell_quote($file_path);
+    };
 
     my $opts = $db->shell_args;
     my $echo = ($fEcho ? '-e' : '');
@@ -83,8 +90,8 @@ sub RunSQLScript
         $ENV{'PGOPTIONS'} .= ' -c client_min_messages=WARNING';
     }
 
-    print "$psql $quiet $echo -f $path/$file $opts 2>&1 $stdout |\n" if $fVerbose;
-    open(PIPE, "$psql $quiet $echo -f $path/$file $opts 2>&1 $stdout |")
+    print "$psql $quiet $echo -f $quoted_file_path $opts 2>&1 $stdout |\n" if $fVerbose;
+    open(PIPE, "$psql $quiet $echo -f $quoted_file_path $opts 2>&1 $stdout |")
         or die "exec '$psql': $OS_ERROR";
     while (<PIPE>)
     {
@@ -115,7 +122,7 @@ sub InstallExtension
 
     chomp($sharedir);
 
-    RunSQLScript($db, "$sharedir/contrib/$ext", "Installing $ext extension ...", '');
+    RunSQLScript($db, "$sharedir/contrib/$ext", "Installing $ext extension ...");
 }
 
 sub CreateReplicationFunction
@@ -265,8 +272,8 @@ sub CreateRelations
         push @opts, '--database', $databaseName;
         system($^X, "$FindBin::Bin/MBImport.pl", @opts, @$import);
         die "\nFailed to import dataset.\n" if ($CHILD_ERROR >> 8);
-    } else {
-        RunSQLScript($DB, 'InsertDefaultRows.sql', 'Adding default rows ...');
+    } elsif ($initial_sql) {
+        RunSQLScript($DB, $initial_sql);
     }
 
     RunSQLScript($DB, 'CreatePrimaryKeys.sql', 'Creating primary keys ...');
@@ -473,6 +480,8 @@ sub Usage
 
      --install-extension Install a postgres extension module
      --extension-schema  Which schema to install the extension module into.
+     --initial-sql       SQL file to initialize the database with.
+                         (default: admin/sql/InsertDefaultRows.sql)
 
 After the import option, you may specify one or more MusicBrainz data dump
 files for importing into the database. Once this script runs to completion
@@ -506,6 +515,7 @@ sub Usage
     'extension-schema=s'  => \$fExtensionSchema,
     'tmp-dir=s'           => \$tmp_dir,
     'reptype=s'           => \$REPTYPE,
+    'initial-sql=s'       => \$initial_sql,
 ) or exit 2;
 
 my $DB = Databases->get($databaseName);

admin/MBImport.pl

@@ -10,6 +10,7 @@
 use Getopt::Long;
 use DBDefs;
 use Sql;
+use MusicBrainz::Script::Utils qw( find_mbdump_file );
 use MusicBrainz::Server::Replication qw( :replication_type );
 use MusicBrainz::Server::Constants qw( @FULL_TABLE_LIST );
 
@@ -328,7 +329,7 @@ sub empty
 sub ImportAllTables
 {
     for my $table (@$import_tables) {
-        my $file = (find_file($table))[0];
+        my $file = find_mbdump_file($table, @ARGV);
         $file or print("No data file found for '$table', skipping\n"), next;
         $imported_tables{$table} = 1;
 
@@ -365,27 +366,11 @@ sub ImportAllTables
     return 1;
 }
 
-sub find_file
-{
-    my $table = shift;
-    my @r;
-
-    for my $arg (@ARGV)
-    {
-        use File::Basename;
-        push(@r, $arg), next if -f $arg and basename($arg) eq $table;
-        push(@r, "$arg/$table"), next if -f "$arg/$table";
-        push(@r, "$arg/mbdump/$table"), next if -f "$arg/mbdump/$table";
-    }
-
-    @r;
-}
-
 sub read_all_and_check
 {
     my $file = shift;
 
-    my @files = find_file($file);
+    my @files = find_mbdump_file($file, @ARGV);
     my %contents;
     my %uniq;
 

admin/replication/ImportReplicationChanges

@@ -9,6 +9,7 @@ use FindBin;
 use lib "$FindBin::Bin/../../lib";
 
 use Getopt::Long;
+use MusicBrainz::Script::Utils qw( find_mbdump_file );
 use MusicBrainz::Server::Context;
 use DBDefs;
 use Sql;
@@ -194,7 +195,7 @@ sub ImportReplicationTables
 
     for my $table (('dbmirror_pending', 'dbmirror_pendingdata'))
     {
-        my $file = find_file($table);
+        my $file = find_mbdump_file($table, @ARGV);
         $file or print("No data file found for '$table', skipping\n"), die;
 
         if (not empty($table))
@@ -217,7 +218,7 @@ sub ImportReplicationTables
 
 sub ImportDBMirror2ReplicationTables {
     for my $table (qw( pending_data pending_keys pending_ts )) {
-        my $file = find_file($table);
+        my $file = find_mbdump_file($table, @ARGV);
 
         if (!$file) {
             # See NOTE-NOPK-1 in LoadReplicationChanges.
@@ -236,21 +237,6 @@ sub ImportDBMirror2ReplicationTables {
     return 1;
 }
 
-sub find_file
-{
-    my $table = shift;
-
-    for my $arg (@ARGV)
-    {
-        use File::Basename;
-        return $arg if -f $arg and basename($arg) eq $table;
-        return "$arg/$table" if -f "$arg/$table";
-        return "$arg/mbdump/$table" if -f "$arg/mbdump/$table";
-    }
-
-    undef;
-}
-
 {
     my @tmpdirs;
 

docker/musicbrainz-tests/artwork-indexer.service

@@ -2,4 +2,4 @@
 
 cd /home/musicbrainz/artwork-indexer
 
-exec sudo -E -H -u musicbrainz bash -c '. venv/bin/activate; exec python3 indexer.py --max-wait=1'
+exec sudo -E -H -u musicbrainz bash -c '. venv/bin/activate; exec python3 indexer.py --config=config.selenium.ini --max-wait=1'

docker/musicbrainz-tests/run_circleci_tests.sh

@@ -20,7 +20,7 @@ sudo -E -H -u musicbrainz cp docker/musicbrainz-tests/DBDefs.pm lib/
 
 sv_start_if_down postgresql redis
 
-sudo -E -H -u musicbrainz carton exec -- ./script/create_test_db.sh
+REPLICATION_TYPE=1 sudo -E -H -u musicbrainz carton exec -- ./script/create_test_db.sh
 
 sudo -E -H -u musicbrainz make -C po test_source all_quiet deploy
 

docker/musicbrainz-tests/run_selenium_tests.sh

@@ -39,17 +39,11 @@ rabbitmqctl add_user sir sir
 rabbitmqctl add_vhost /sir-test
 rabbitmqctl set_permissions -p /sir-test sir '.*' '.*' '.*'
 
-# Install the sir triggers into musicbrainz_selenium.
+# Generate the sir extensions and triggers, which is required before
+# invoking create_selenium_db.sh.
 export SIR_DIR=/home/musicbrainz/sir
 cd "$SIR_DIR"
 sudo -E -H -u musicbrainz sh -c '. venv/bin/activate; python -m sir amqp_setup; python -m sir extension; python -m sir triggers --broker-id=1'
-psql -U postgres -f sql/CreateExtension.sql musicbrainz_selenium
-psql -U musicbrainz -f sql/CreateFunctions.sql musicbrainz_selenium
-psql -U musicbrainz -f sql/CreateTriggers.sql musicbrainz_selenium
-
-# Install the artwork_indexer schema into musicbrainz_selenium.
-cd /home/musicbrainz/artwork-indexer
-sudo -E -H -u musicbrainz sh -c '. venv/bin/activate; python indexer.py --setup-schema'
 
 cd /home/musicbrainz/musicbrainz-server
 

docker/templates/Dockerfile.tests.m4

@@ -167,7 +167,7 @@ COPY --chown=postgres:postgres \
 RUN sudo -E -H -u postgres touch \
     $PGDATA/pg_ident.conf
 
-COPY docker/musicbrainz-tests/artwork-indexer-config.ini artwork-indexer/config.ini
+COPY docker/musicbrainz-tests/artwork-indexer-config.ini artwork-indexer/config.selenium.ini
 COPY docker/musicbrainz-tests/artwork-redirect-config.ini artwork-redirect/config.ini
 COPY docker/musicbrainz-tests/sir-config.ini sir/config.ini
 

lib/MusicBrainz/Script/Utils.pm

@@ -3,18 +3,64 @@ use strict;
 use warnings;
 
 use English;
+use List::AllUtils qw( uniq );
 
 use feature 'state';
 
 use base 'Exporter';
 
 our @EXPORT_OK = qw(
+    find_files
+    find_mbdump_file
     get_primary_keys
     get_foreign_keys
     log
     retry
 );
 
+=sub find_files
+
+Looks for files named C<$file> in C<@search_paths>. The given paths may
+contain a direct reference to the file, or directories which will be checked
+instead.
+
+Returns an array of found files (in the specified search order).
+
+=cut
+
+sub find_files {
+    my ($file, @search_paths) = @_;
+
+    return uniq(grep { -f } map {
+        my $search_path = $_;
+        (
+            ($search_path =~ m/\Q$file\E$/ ? $search_path : ()),
+            "$search_path/$file"
+        )
+    } @search_paths);
+}
+
+=sub find_mbdump_file
+
+Looks for an mbdump file named C<$table> in C<@search_paths>. The semantics
+are the same as for C<find_files>, except:
+
+ 1. The file is additionally searched for under an 'mbdump' sub-directory in
+    each search path.
+ 2. The first matching file is returned in scalar context.
+
+=cut
+
+sub find_mbdump_file {
+    my ($table, @search_paths) = @_;
+
+    my @result = find_files($table, map {
+        ($_, "$_/mbdump")
+    } @search_paths);
+
+    return wantarray ? @result : $result[0];
+}
+
 =sub get_foreign_keys
 
 Get a list of foreign key columns for (C<$schema>, C<$table>).

script/create_selenium_db.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset -o pipefail
+
+MB_SERVER_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../" && pwd)
+cd "$MB_SERVER_ROOT"
+
+./script/create_test_db.sh SELENIUM
+
+./admin/psql SELENIUM < ./t/sql/selenium.sql
+
+if [[ $# -gt 0 ]]; then
+    EXTRA_SQL="$1"
+    if [[ -f "$EXTRA_SQL" ]]; then
+        ./admin/psql SELENIUM < "$EXTRA_SQL"
+    fi
+fi
+
+DROP_SQL=$(cat <<'SQL'
+\set ON_ERROR_STOP 1
+DROP EXTENSION IF EXISTS amqp CASCADE;
+DROP SCHEMA IF EXISTS artwork_indexer CASCADE;
+SQL
+)
+echo "$DROP_SQL" | ./admin/psql --system SELENIUM
+
+SIR_DIR="${SIR_DIR:="$MB_SERVER_ROOT"/../sir}"
+
+if [ -d "$SIR_DIR" ]; then
+    ./admin/psql --system SELENIUM < "$SIR_DIR"/sql/CreateExtension.sql
+    ./admin/psql SELENIUM < "$SIR_DIR"/sql/CreateFunctions.sql
+    ./admin/psql SELENIUM < "$SIR_DIR"/sql/CreateTriggers.sql
+fi
+
+ARTWORK_INDEXER_DIR="${ARTWORK_INDEXER_DIR:="$MB_SERVER_ROOT"/../artwork-indexer}"
+
+if [ -d "$ARTWORK_INDEXER_DIR" ]; then
+    pushd "$ARTWORK_INDEXER_DIR"
+    VENV_DIR="$([ -d .venv ] && echo .venv || echo venv)"
+    . "$VENV_DIR"/bin/activate
+    python indexer.py --config=config.selenium.ini --setup-schema
+    popd
+fi