diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 00000000..34f71350 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,217 @@ +name: Test database creation and TAP_SCHEMA upload +on: [push] + +# TODO: Make scripts for complex steps and put into /scripts +# TODO: Put common setup in into another workflow + +jobs: + + postgresql: + + runs-on: ubuntu-latest + + env: + PGHOST: localhost + PGPORT: 5432 + PGUSER: rubin + PGPASSWORD: rubin + PGDATABASE: sdm_schemas_test + + services: + postgres: + image: postgres:13 + env: + POSTGRES_USER: rubin + POSTGRES_PASSWORD: rubin + POSTGRES_DB: sdm_schemas_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Check out repo + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + + - name: Set engine URL + run: echo "FELIS_ENGINE_URL=postgresql+psycopg2://rubin:rubin@localhost:5432/sdm_schemas_test" >> $GITHUB_ENV + + # Create databases from YAML files. Drop existing databases since some of + # the schema names are duplicated and also ignore constraints since many + # schemas are missing required indices for FK constraints. + - name: Create databases + run: | + for file in yml/*.yaml; do + echo "Creating database from $file..." + felis --log-level ERROR create --drop --ignore-constraints "$file" + echo "Done creating database from $file" + done + + - name: Create SQL files + run: | + for yaml_file in yml/*.yaml; do + echo "Creating SQL from $yaml_file..." + felis --log-level ERROR create --ignore-constraints --engine-url=postgresql:// --output-file "${yaml_file}.sql" $yaml_file + done + + # TODO: Load SQL files + # db_name=$(grep -m 1 '^name:' ${yaml_file} | awk '{print $2}' | tr -d '"') + # echo "Creating schema..." + # psql -c "CREATE SCHEMA ${db_name};" + # echo "Executing SQL..." + # psql < "${sql_file}" + # echo "Dropping schema..." + # psql -c "DROP SCHEMA ${db_name} CASCADE;" + # echo "Done creating SQL from $yaml_file" + + - name: Initialize TAP_SCHEMA database + run: | + felis init-tap $FELIS_ENGINE_URL + + # Upload to TAP_SCHEMA database. Clear the tables after each upload to + # avoid conflicts between schemas. + - name: Upload to TAP_SCHEMA database + run: | + set +e + for yaml_file in yml/*.yaml; do + filename=$(basename "yaml_file" .yaml) + echo "Uploading to TAP_SCHEMA from $yaml_file..." + felis --log-level ERROR load-tap "$yaml_file" + echo "Done uploading to TAP_SCHEMA from $yaml_file" + psql -q -c "TRUNCATE TABLE columns, key_columns, keys, schemas, tables;" + done + set -e + + mysql: + + runs-on: ubuntu-latest + + services: + # TODO: Use MariaDB instead of MySQL + mysql: + image: mysql:8 + env: + MYSQL_ROOT_PASSWORD: root + MYSQL_USER: rubin + MYSQL_PASSWORD: rubin + ports: + - 3306:3306 + options: >- + --health-cmd "mysqladmin ping --silent" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + + - name: Check out repo + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + + - name: Grant privileges to rubin MySQL user + run: | + mysql -h 127.0.0.1 -P 3306 -u root -proot -e "GRANT ALL PRIVILEGES ON *.* TO 'rubin'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" > /dev/null 2>&1 + + - name: Set MySQL credentials + run: | + cat < ~/.my.cnf + [client] + user=rubin + password=rubin + host=127.0.0.1 + port=3306 + EOF + chmod 600 ~/.my.cnf + + - name: Set FELIS_ENGINE_URL + run: echo "FELIS_ENGINE_URL=mysql+pymysql://rubin:rubin@localhost:3306" >> $GITHUB_ENV + + # Create databases from YAML files. Drop existing databases since some of + # the schema names are duplicated and also ignore constraints since many + # schemas are missing required indices for FK constraints. + # + # Capture output from felis create command and print the first 20 lines + # when there are errors. Some extremely long exception messages can occur + # here and need to be truncated for now. + - name: Create databases + run: | + set +e + for yaml_file in yml/*.yaml; do + filename=$(basename "$yaml_file") + echo "Creating database from $yaml_file..." + output=$(felis --log-level ERROR create --drop --ignore-constraints "$file" 2>&1) + echo "$output" | awk 'NR<=20 {print} NR==21 {print "... [output truncated]"; exit}' + db_name=$(grep -m 1 '^name:' ${yaml_file} | awk '{print $2}' | tr -d '"') + mysql -e "DROP DATABASE ${db_name};" + echo "Done creating database from $yaml_file" + done + set -e + + - name: Create and test load SQL files + run: | + set +e + for yaml_file in yml/*.yaml; do + sql_file="${db_name}.sql" + echo "Creating SQL from $yaml_file..." + felis --log-level ERROR create --ignore-constraints --engine-url=mysql+pymysql:// --output-file "${sql_file}" "$yaml_file" + done + set -e + + # TODO: Load SQL files + # db_name=$(grep -m 1 '^name:' ${yaml_file} | awk '{print $2}' | tr -d '"') + # echo "Creating db..." + # mysql -e "CREATE DATABASE ${db_name};" + # echo "Executing SQL..." + # mysql < "${sql_file}" + # echo "Dropping db..." + # mysql -e "DROP DATABASE ${db_name};" + # echo "Done creating SQL from $yaml_file" + + - name: Initialize TAP_SCHEMA database + run: | + mysql -e "CREATE DATABASE TAP_SCHEMA;" + felis init-tap ${FELIS_ENGINE_URL}/TAP_SCHEMA + + # Upload to TAP_SCHEMA database. Clear the tables after each upload to + # avoid conflicts between schemas. + - name: Upload to TAP_SCHEMA database + run: | + set +e + for yaml_file in yml/*.yaml; do + filename=$(basename "$yaml_file") + echo "Uploading to TAP_SCHEMA from $yaml_file..." + felis --log-level ERROR load-tap --engine-url ${FELIS_ENGINE_URL}/TAP_SCHEMA "$yaml_file" + mysql -DTAP_SCHEMA -e " + DELETE FROM \`columns\`; + DELETE FROM \`key_columns\`; + DELETE FROM \`keys\`; + DELETE FROM \`schemas\`; + DELETE FROM \`tables\`; + " + echo "Done uploading to TAP_SCHEMA from $yaml_file" + done + set -e