diff --git a/.github/workflows/test_postgresql.yaml b/.github/workflows/test_postgresql.yaml new file mode 100644 index 00000000..a7629d33 --- /dev/null +++ b/.github/workflows/test_postgresql.yaml @@ -0,0 +1,104 @@ +name: Test PostgreSQL database creation and TAP_SCHEMA upload +on: + push: + pull_request: + +jobs: + + test: + + runs-on: ubuntu-latest + + env: + PGHOST: localhost + PGPORT: 5432 + PGUSER: rubin + PGPASSWORD: rubin + PGDATABASE: sdm_schemas_test + + services: + postgres: + image: postgres:13 + env: + POSTGRES_USER: rubin + POSTGRES_PASSWORD: rubin + POSTGRES_DB: sdm_schemas_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + + - name: Check out repo + uses: actions/checkout@v4 + + # - name: Setup CI environment + # uses: .github/actions/setup + + - name: Check out repo + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip uv + uv pip install --system -r requirements.txt + + - name: Set engine URL + run: echo "FELIS_ENGINE_URL=postgresql+psycopg2://rubin:rubin@localhost:5432/sdm_schemas_test" >> $GITHUB_ENV + + # Create databases from YAML files. Drop existing databases since some of + # the schema names are duplicated and also ignore constraints since many + # schemas are missing required indices for FK constraints. + - name: Create databases + run: | + for file in yml/*.yaml; do + echo "Creating database from $file..." + felis --log-level ERROR create --drop --ignore-constraints "$file" + echo "Done creating database from $file" + done + + - name: Create SQL files + run: | + mkdir sql + for yaml_file in yml/*.yaml; do + echo "Creating SQL from $yaml_file..." + felis --log-level ERROR create --ignore-constraints --engine-url=postgresql:// --output-file sql/"${yaml_file}.sql" $yaml_file + done + + - name: Load SQL files + run: | + for yaml_file in yml/*.yaml; do + db_name=$(grep -m 1 '^name:' ${yaml_file} | awk '{print $2}' | tr -d '"') + psql -c "CREATE SCHEMA ${db_name};" + psql < "sql/${sql_file}" + psql -c "DROP SCHEMA ${db_name} CASCADE;" + echo "Done creating SQL from $yaml_file" + done + + - name: Initialize TAP_SCHEMA database + run: | + felis init-tap $FELIS_ENGINE_URL + + # Upload to TAP_SCHEMA database. Clear the tables after each upload to + # avoid conflicts between schemas. + - name: Upload to TAP_SCHEMA database + run: | + set +e + for yaml_file in yml/*.yaml; do + filename=$(basename "yaml_file" .yaml) + echo "Uploading to TAP_SCHEMA from $yaml_file..." + felis --log-level ERROR load-tap "$yaml_file" + echo "Done uploading to TAP_SCHEMA from $yaml_file" + psql -q -c "TRUNCATE TABLE columns, key_columns, keys, schemas, tables;" + done + set -e