Add github workflow with database tests of all schema files

.github/workflows/test.yaml

@@ -0,0 +1,217 @@
+name: Test database creation and TAP_SCHEMA upload
+on: [push]
+
+# TODO: Make scripts for complex steps and put into /scripts
+# TODO: Put common setup in into another workflow
+
+jobs:
+
+  postgresql:
+
+    runs-on: ubuntu-latest
+
+    env:
+      PGHOST: localhost
+      PGPORT: 5432
+      PGUSER: rubin
+      PGPASSWORD: rubin
+      PGDATABASE: sdm_schemas_test
+
+    services:
+      postgres:
+        image: postgres:13
+        env:
+          POSTGRES_USER: rubin
+          POSTGRES_PASSWORD: rubin
+          POSTGRES_DB: sdm_schemas_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip uv
+          uv pip install --system -r requirements.txt
+
+      - name: Set engine URL
+        run: echo "FELIS_ENGINE_URL=postgresql+psycopg2://rubin:rubin@localhost:5432/sdm_schemas_test" >> $GITHUB_ENV
+
+      # Create databases from YAML files. Drop existing databases since some of
+      # the schema names are duplicated and also ignore constraints since many
+      # schemas are missing required indices for FK constraints.
+      - name: Create databases
+        run: |
+          for file in yml/*.yaml; do
+            echo "Creating database from $file..."
+            felis --log-level ERROR create --drop --ignore-constraints "$file"
+            echo "Done creating database from $file"
+          done
+
+      - name: Create SQL files
+        run: |
+          for yaml_file in yml/*.yaml; do
+            echo "Creating SQL from $yaml_file..."
+            felis --log-level ERROR create --ignore-constraints --engine-url=postgresql:// --output-file "${yaml_file}.sql" $yaml_file
+          done
+
+      # TODO: Load SQL files
+      # db_name=$(grep -m 1 '^name:' ${yaml_file} | awk '{print $2}' | tr -d '"')
+      # echo "Creating schema..."
+      # psql -c "CREATE SCHEMA ${db_name};"
+      # echo "Executing SQL..."
+      # psql < "${sql_file}"
+      # echo "Dropping schema..."
+      # psql -c "DROP SCHEMA ${db_name} CASCADE;"
+      # echo "Done creating SQL from $yaml_file"
+
+      - name: Initialize TAP_SCHEMA database
+        run: |
+          felis init-tap $FELIS_ENGINE_URL
+
+      # Upload to TAP_SCHEMA database. Clear the tables after each upload to
+      # avoid conflicts between schemas.
+      - name: Upload to TAP_SCHEMA database
+        run: |
+          set +e
+          for yaml_file in yml/*.yaml; do
+            filename=$(basename "yaml_file" .yaml)
+            echo "Uploading to TAP_SCHEMA from $yaml_file..."
+            felis --log-level ERROR load-tap "$yaml_file"
+            echo "Done uploading to TAP_SCHEMA from $yaml_file"
+            psql -q -c "TRUNCATE TABLE columns, key_columns, keys, schemas, tables;"
+          done
+          set -e
+
+  mysql:
+
+    runs-on: ubuntu-latest
+
+    services:
+      # TODO: Use MariaDB instead of MySQL
+      mysql:
+        image: mysql:8
+        env:
+          MYSQL_ROOT_PASSWORD: root
+          MYSQL_USER: rubin
+          MYSQL_PASSWORD: rubin
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd "mysqladmin ping --silent"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+
+      - name: Check out repo
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip uv
+          uv pip install --system -r requirements.txt
+
+      - name: Grant privileges to rubin MySQL user
+        run: |
+          mysql -h 127.0.0.1 -P 3306 -u root -proot -e "GRANT ALL PRIVILEGES ON *.* TO 'rubin'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" > /dev/null 2>&1
+
+      - name: Set MySQL credentials
+        run: |
+          cat <<EOF > ~/.my.cnf
+          [client]
+          user=rubin
+          password=rubin
+          host=127.0.0.1
+          port=3306
+          EOF
+          chmod 600 ~/.my.cnf
+
+      - name: Set FELIS_ENGINE_URL
+        run: echo "FELIS_ENGINE_URL=mysql+pymysql://rubin:rubin@localhost:3306" >> $GITHUB_ENV
+
+      # Create databases from YAML files. Drop existing databases since some of
+      # the schema names are duplicated and also ignore constraints since many
+      # schemas are missing required indices for FK constraints.
+      #
+      # Capture output from felis create command and print the first 20 lines
+      # when there are errors. Some extremely long exception messages can occur
+      # here and need to be truncated for now.
+      - name: Create databases
+        run: |
+          set +e
+          for yaml_file in yml/*.yaml; do
+            filename=$(basename "$yaml_file")
+            echo "Creating database from $yaml_file..."
+            output=$(felis --log-level ERROR create --drop --ignore-constraints "$file" 2>&1)
+            echo "$output" | awk 'NR<=20 {print} NR==21 {print "... [output truncated]"; exit}'
+            db_name=$(grep -m 1 '^name:' ${yaml_file} | awk '{print $2}' | tr -d '"')
+            mysql -e "DROP DATABASE ${db_name};"
+            echo "Done creating database from $yaml_file"
+          done
+          set -e
+
+      - name: Create and test load SQL files
+        run: |
+          set +e
+          for yaml_file in yml/*.yaml; do
+            sql_file="${db_name}.sql"
+            echo "Creating SQL from $yaml_file..."
+            felis --log-level ERROR create --ignore-constraints --engine-url=mysql+pymysql:// --output-file "${sql_file}" "$yaml_file"
+          done
+          set -e
+
+      # TODO: Load SQL files
+      # db_name=$(grep -m 1 '^name:' ${yaml_file} | awk '{print $2}' | tr -d '"')
+      # echo "Creating db..."
+      # mysql -e "CREATE DATABASE ${db_name};"
+      # echo "Executing SQL..."
+      # mysql < "${sql_file}"
+      # echo "Dropping db..."
+      # mysql -e "DROP DATABASE ${db_name};"
+      # echo "Done creating SQL from $yaml_file"
+
+      - name: Initialize TAP_SCHEMA database
+        run: |
+          mysql -e "CREATE DATABASE TAP_SCHEMA;"
+          felis init-tap ${FELIS_ENGINE_URL}/TAP_SCHEMA
+
+        # Upload to TAP_SCHEMA database. Clear the tables after each upload to
+        # avoid conflicts between schemas.
+      - name: Upload to TAP_SCHEMA database
+        run: |
+          set +e
+          for yaml_file in yml/*.yaml; do
+            filename=$(basename "$yaml_file")
+            echo "Uploading to TAP_SCHEMA from $yaml_file..."
+            felis --log-level ERROR load-tap --engine-url ${FELIS_ENGINE_URL}/TAP_SCHEMA "$yaml_file"
+            mysql -DTAP_SCHEMA -e "
+              DELETE FROM \`columns\`;
+              DELETE FROM \`key_columns\`;
+              DELETE FROM \`keys\`;
+              DELETE FROM \`schemas\`;
+              DELETE FROM \`tables\`;
+            "
+            echo "Done uploading to TAP_SCHEMA from $yaml_file"
+          done
+          set -e