diff --git a/README.md b/README.md index 7d7d9255a..d54367e14 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,14 @@ When you have started the development code of the Management System, the first i Both users actually have the same privileges by default, but the first one logging in can see the _System Dashboard_ and becomes the _System Admin_. +**Local Development with DB : Helper commands** \ +`yarn dev-ms-db` : starts the docker container and setups the default db \ +`yarn dev-ms-db-create`: create a new branch specific database \ +`yarn dev-ms-db-use `: switch between dbs, also creates a branch db if not created already \ +`yarn dev-ms-db-delete <--all | --branch >` : delete db \ +`yarn dev-ms-db-migrate`: create new prisma migration based on changes made to `schema.prisma` file \ +`yarn dev-ms-db-deploy` : deploys the migrations to db + # Contributions If you are interested in developing PROCEED further, we are very open for help and project contributions. Regularly there are on-boarding development workshops and, if you are interested, we have weekly video calls with all developers. diff --git a/package.json b/package.json index 236f9a79a..be7b22ae1 100644 --- a/package.json +++ b/package.json @@ -38,12 +38,12 @@ }, "scripts": { "dev": "ts-node src/engine/native/node/index.ts --trace-warnings", - "dev-ms-db": "cd src/management-system-v2 && docker compose -f docker-compose-dev.yml up -d && ./db-helper.sh --init", - "dev-ms-db-create": "cd src/management-system-v2 && ./db-helper.sh --new", + "dev-ms-db": "cd src/management-system-v2 && docker compose -f docker-compose-dev.yml up -d && node db-helper.js --init", + "dev-ms-db-create": "cd src/management-system-v2 && node db-helper.js --new", "dev-ms-db-deploy": "cd src/management-system-v2 && yarn prisma migrate deploy", "dev-ms-db-migrate": "cd src/management-system-v2 && yarn prisma migrate dev", - "dev-ms-db-use": "cd src/management-system-v2 && ./db-helper.sh --use", - "dev-ms-db-delete": "cd src/management-system-v2 && ./db-helper.sh --delete", + "dev-ms-db-use": "cd src/management-system-v2 && node db-helper.js --use", + "dev-ms-db-delete": "cd src/management-system-v2 && node db-helper.js --delete", "dev-ms": "cd src/management-system-v2 && yarn dev", "dev-ms-old": "cd src/management-system && yarn web:dev", "dev-ms-old-iam": "cd src/management-system && yarn web:dev-iam", diff --git a/src/management-system-v2/db-helper.js b/src/management-system-v2/db-helper.js new file mode 100644 index 000000000..40c5ac794 --- /dev/null +++ b/src/management-system-v2/db-helper.js @@ -0,0 +1,292 @@ +const { execSync } = require('child_process'); +const fs = require('fs'); + +// Configurable parameters +const CONFIG = { + containerName: 'postgres_database_proceed', + postgresUser: 'proceed', + postgresPassword: 'proceed', + defaultDb: 'proceed_db', + dbHost: 'localhost', + dbPort: '5433', + defaultSchema: 'public', + envFile: './.env', + checkInterval: 2000, +}; + +// Parse command-line arguments +const args = process.argv.slice(2); +const options = parseArguments(args); + +(async function main() { + validateEnvironment(); + + // Wait for the container to be healthy (if initializing) + if (options.init) { + await waitForContainerHealth(); + } + + // Determine database name + const branchName = getBranchName(options.branch); + const dbName = options.init ? determineDatabase(branchName) : getBranchDatabaseName(branchName); + + // Handle options + if (options.delete) { + handleDatabaseDeletion(dbName, options); + return; + } + + if (options.createNew) { + handleNewDatabaseCreation(dbName); + } + + if (options.changeDb) { + handleDatabaseSwitch(dbName, options.changeOption); + } + + // Apply the schema (if initializing or creating a new database) + if (options.init || options.createNew) { + applyDatabaseSchema(dbName); + } +})(); + +// ---------------- Helper Functions ---------------- + +function parseArguments(args) { + const options = { + init: false, + createNew: false, + delete: false, + deleteAll: false, + deleteBranch: '', + changeDb: false, + changeOption: '', + branch: '', + }; + + for (let i = 0; i < args.length; i++) { + switch (args[i]) { + case '--init': + options.init = true; // creates the default db + break; + case '--new': + options.createNew = true; // creates new branch specific db + break; + case '--delete': + options.delete = true; // delete db + break; + case '--all': + options.deleteAll = true; // deletes all branch specific dbs except default one + break; + case '--branch': + options.deleteBranch = args[++i]; // delete only a branch db : --branch + break; + case '--use': + options.changeDb = true; // switch between dbs, also creates new branch db if not present + options.changeOption = args[++i]; + break; + default: + options.branch = args[i]; + } + } + + return options; +} + +function validateEnvironment() { + try { + execSync('docker --version'); + execSync(`docker exec ${CONFIG.containerName} psql --version`); + } catch (error) { + console.error('Docker or PostgreSQL is not configured correctly.'); + process.exit(1); + } +} + +async function waitForContainerHealth() { + console.log(`Waiting for container "${CONFIG.containerName}" to be healthy...`); + while (!isContainerHealthy()) { + console.log('Still waiting...'); + await delay(CONFIG.checkInterval); + } + console.log(`Container "${CONFIG.containerName}" is healthy!`); +} + +function isContainerHealthy() { + try { + const status = execSync( + `docker inspect -f "{{.State.Health.Status}}" ${CONFIG.containerName}`, + { + encoding: 'utf-8', + }, + ).trim(); + return status === 'healthy'; + } catch (error) { + console.error(`Error checking container health: ${error.message}`); + return false; + } +} + +function determineDatabase(branchName) { + const branchDbName = getBranchDatabaseName(branchName); + + if (checkDatabaseExists(branchDbName)) { + console.log(`Using existing branch-specific database: ${branchDbName}`); + updateEnvFile(branchDbName); + return branchDbName; + } + + if (checkDatabaseExists(CONFIG.defaultDb)) { + console.log(`Using existing default database: ${CONFIG.defaultDb}`); + updateEnvFile(CONFIG.defaultDb); + return CONFIG.defaultDb; + } + + console.log(`No databases found. Creating default database: ${CONFIG.defaultDb}`); + createDatabase(CONFIG.defaultDb); + updateEnvFile(CONFIG.defaultDb); + return CONFIG.defaultDb; +} + +function handleDatabaseDeletion(dbName, options) { + if (options.deleteAll) { + deleteAllBranchDatabases(); + } else if (options.deleteBranch) { + deleteDatabase(getBranchDatabaseName(options.deleteBranch)); + } else { + deleteDatabase(dbName); + } +} + +async function deleteAllBranchDatabases() { + console.log('Deleting all branch-specific databases...'); + const databases = listDatabases().filter( + (db) => db.startsWith('proceed_db_') && db !== CONFIG.defaultDb, + ); + await Promise.all(databases.map(deleteDatabase)); +} + +function handleNewDatabaseCreation(dbName) { + if (checkDatabaseExists(dbName)) { + console.log(`Database ${dbName} already exists.`); + } else { + console.log(`Creating new database: ${dbName}`); + createDatabase(dbName); + updateEnvFile(dbName); + } +} + +function handleDatabaseSwitch(dbName, option) { + const targetDb = option === 'default' ? CONFIG.defaultDb : dbName; + if (!checkDatabaseExists(targetDb)) { + console.log(`Database ${targetDb} does not exist. Creating it...`); + createDatabase(targetDb); + options.createNew = true; + } + console.log(`Switching to database: ${targetDb}`); + updateEnvFile(targetDb); +} + +function applyDatabaseSchema(dbName) { + console.log(`Applying schema to database: ${dbName}`); + const schemaPath = './prisma/migrations'; + if (!fs.existsSync(schemaPath) || !fs.readdirSync(schemaPath).length) { + console.error('No migration files found. Skipping schema application.'); + return; + } + try { + execSync('yarn prisma migrate deploy', { stdio: 'inherit' }); + console.log('Schema applied successfully.'); + } catch (error) { + console.error(`Failed to apply schema: ${error.message}`); + process.exit(1); + } +} + +// Utility Functions +function getBranchName(branch) { + if (branch) return sanitizeBranchName(branch); + + try { + return sanitizeBranchName(execSync('git rev-parse --abbrev-ref HEAD').toString().trim()); + } catch (error) { + console.error('Failed to determine current branch name.'); + process.exit(1); + } +} + +function sanitizeBranchName(name) { + return name.replace(/[-\/]/g, '_'); +} + +function getBranchDatabaseName(branchName) { + return `proceed_db_${sanitizeBranchName(branchName)}`; +} + +function checkDatabaseExists(dbName) { + try { + const result = execSync( + `docker exec -e PGPASSWORD="${CONFIG.postgresPassword}" -i "${CONFIG.containerName}" psql -U "${CONFIG.postgresUser}" -d "${CONFIG.defaultDb}" -tAc "SELECT 1 FROM pg_database WHERE datname='${dbName}';"`, + ) + .toString() + .trim(); + return result === '1'; + } catch { + return false; + } +} + +function createDatabase(dbName) { + try { + execSync( + `docker exec -e PGPASSWORD="${CONFIG.postgresPassword}" -i "${CONFIG.containerName}" psql -U "${CONFIG.postgresUser}" -d "postgres" -c "CREATE DATABASE ${dbName};"`, + ); + console.log(`Database ${dbName} created successfully.`); + } catch (error) { + console.error(`Failed to create database: ${error.message}`); + process.exit(1); + } +} + +function deleteDatabase(dbName) { + try { + execSync( + `docker exec -e PGPASSWORD="${CONFIG.postgresPassword}" -i "${CONFIG.containerName}" psql -U "${CONFIG.postgresUser}" -d "${CONFIG.defaultDb}" -c "DROP DATABASE ${dbName};"`, + ); + console.log(`Database ${dbName} deleted successfully.`); + } catch (error) { + console.error(`Failed to delete database: ${error.message}`); + } +} + +function listDatabases() { + try { + const result = execSync( + `docker exec -e PGPASSWORD="${CONFIG.postgresPassword}" -i "${CONFIG.containerName}" psql -U "${CONFIG.postgresUser}" -d "${CONFIG.defaultDb}" -tAc "SELECT datname FROM pg_database;"`, + ) + .toString() + .trim(); + return result.split('\n').filter((db) => db); + } catch { + return []; + } +} + +function updateEnvFile(dbName) { + const databaseUrl = `postgresql://${CONFIG.postgresUser}:${CONFIG.postgresPassword}@${CONFIG.dbHost}:${CONFIG.dbPort}/${dbName}?schema=${CONFIG.defaultSchema}`; + if (fs.existsSync(CONFIG.envFile)) { + let content = fs.readFileSync(CONFIG.envFile, 'utf-8'); + const regex = /^DATABASE_URL=.*$/m; + content = regex.test(content) + ? content.replace(regex, `DATABASE_URL=${databaseUrl}`) + : `${content}\nDATABASE_URL=${databaseUrl}`; + fs.writeFileSync(CONFIG.envFile, content); + } else { + fs.writeFileSync(CONFIG.envFile, `DATABASE_URL=${databaseUrl}\n`); + } + console.log(`Updated DATABASE_URL in ${CONFIG.envFile}`); +} + +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/management-system-v2/db-helper.sh b/src/management-system-v2/db-helper.sh deleted file mode 100755 index 91cbc4a24..000000000 --- a/src/management-system-v2/db-helper.sh +++ /dev/null @@ -1,269 +0,0 @@ -#!/bin/bash - -# Configurable parameters -CONTAINER_NAME="postgres_database_proceed" # Replace with your Docker container name -POSTGRES_USER="proceed" # PostgreSQL user -POSTGRES_PASSWORD="proceed" # PostgreSQL password -DEFAULT_DB="proceed_db" # The default database name when --default is used -ENV_FILE="./.env" -DB_HOST="localhost" -DB_PORT="5432" -DEFAULT_SCHEMA="public" - -# Parse command-line arguments -INIT=false -CREATE_NEW_DB=false -DELETE_DB=false -DELETE_ALL=false -DELETE_BRANCH="" -CHANGE_DB=false -CHANGE_OPTION="" - -while [[ "$#" -gt 0 ]]; do - case $1 in - --init) - INIT=true - ;; - --new) - CREATE_NEW_DB=true - ;; - --delete) - DELETE_DB=true - ;; - --branch) - DELETE_BRANCH="$2" - shift # Shift past the branch name argument - ;; - --all) - DELETE_ALL=true - ;; - --use) - CHANGE_DB=true - ;; - branch|default) - CHANGE_OPTION="$1" - ;; - --*) # Handle any unknown options - echo "Unknown option: $1" - exit 1 - ;; - *) # Handle non-option arguments (like branch names) - BRANCH_NAME="$1" - ;; - esac - shift # Shift to the next argument -done - -# Function to update the DATABASE_URL in .env file -update_env_file() { - local db_name=$1 - local env_file=$2 - - # Generate the new DATABASE_URL in the required format - DATABASE_URL="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$DB_HOST:$DB_PORT/$db_name?schema=$DEFAULT_SCHEMA" - - # Replace DATABASE_URL in the .env file - if [ -f "$env_file" ]; then - echo "Updating DATABASE_URL in $env_file" - - # Check if the DATABASE_URL already exists in the file - if grep -q "^DATABASE_URL=" "$env_file"; then - # If DATABASE_URL exists, update it - sed -i '' "s|^DATABASE_URL=.*|DATABASE_URL=$DATABASE_URL|" "$env_file" - else - # If DATABASE_URL does not exist, append it to the file - echo "DATABASE_URL=$DATABASE_URL" >> "$env_file" - fi - - if [ $? -eq 0 ]; then - echo "DATABASE_URL successfully updated in $env_file" - else - echo "Failed to update DATABASE_URL in $env_file" - exit 1 - fi - else - echo "$env_file not found! Creating new .env file..." - echo "DATABASE_URL=$DATABASE_URL" > "$env_file" - if [ $? -eq 0 ]; then - echo "Created new $env_file with DATABASE_URL" - else - echo "Failed to create $env_file" - exit 1 - fi - fi -} - -# Get the current branch name (only if not provided) -if [ -z "$BRANCH_NAME" ]; then - BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) -fi - -# Replace hyphens & slash with underscores in the branch name -BRANCH_NAME_SAFE=$(echo "$BRANCH_NAME" | tr '-' '_' | tr '/' '_') - -# Determine which database to use -if [ "$DELETE_DB" == false ]; then - if [ "$INIT" == true ]; then - # Check if branch-specific database exists - BRANCH_DB_NAME="proceed_db_${BRANCH_NAME_SAFE}" - BRANCH_DB_EXISTS=$(docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" -i "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$DEFAULT_DB" -tAc "SELECT 1 FROM pg_database WHERE datname='$BRANCH_DB_NAME';") - - # Check if default database exists - DEFAULT_DB_EXISTS=$(docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" -i "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "postgres" -tAc "SELECT 1 FROM pg_database WHERE datname='$DEFAULT_DB';") - - if [ "$BRANCH_DB_EXISTS" == "1" ]; then - DB_NAME=$BRANCH_DB_NAME - echo "Using existing branch-specific database: $DB_NAME" - elif [ "$DEFAULT_DB_EXISTS" == "1" ]; then - DB_NAME=$DEFAULT_DB - echo "Branch database not found. Using default database: $DB_NAME" - else - DB_NAME=$DEFAULT_DB - echo "Neither branch nor default database exists. Creating default database: $DB_NAME" - docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" -i "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "postgres" -c "CREATE DATABASE $DB_NAME;" - if [ $? -ne 0 ]; then - echo "Failed to create default database: $DB_NAME" - exit 1 - fi - echo "Default database $DB_NAME created successfully." - fi - # Update .env file for init - update_env_file "$DB_NAME" "$ENV_FILE" - else - DB_NAME="proceed_db_${BRANCH_NAME_SAFE}" - echo "Using branch-specific database: $DB_NAME" - fi -fi - -# Check if the Docker container is running -if ! docker ps | grep -q "$CONTAINER_NAME"; then - echo "Starting Docker container: $CONTAINER_NAME" - docker start "$CONTAINER_NAME" - if [ $? -ne 0 ]; then - echo "Failed to start Docker container: $CONTAINER_NAME" - exit 1 - fi -fi - -# Handle --delete option: Delete the specified database or all branch-specific databases -if [ "$DELETE_DB" == true ]; then - if [ "$DELETE_ALL" == true ]; then - echo "Deleting all branch-specific databases except the default one..." - - # Get a list of all databases and delete the ones that match the branch pattern - DATABASES=$(docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" -i "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$DEFAULT_DB" -tAc "SELECT datname FROM pg_database WHERE datname LIKE 'proceed_db_%';") - - for DB in $DATABASES; do - if [ "$DB" != "$DEFAULT_DB" ]; then - echo "Dropping database: $DB" - docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" -i "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$DEFAULT_DB" -c "DROP DATABASE $DB;" - if [ $? -eq 0 ]; then - echo "Database $DB dropped successfully." - else - echo "Failed to drop the database: $DB." - fi - fi - done - exit 0 - elif [ -n "$DELETE_BRANCH" ]; then - # Handle --delete --branch - BRANCH_NAME_SAFE=$(echo "$DELETE_BRANCH" | tr '-' '_' | tr '/' '_') - DB_NAME="proceed_db_${BRANCH_NAME_SAFE}" - - echo "Checking if the database $DB_NAME exists for deletion..." - DB_EXISTS=$(docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" -i "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$DEFAULT_DB" -tAc "SELECT 1 FROM pg_database WHERE datname='$DB_NAME';") - - if [ "$DB_EXISTS" == "1" ]; then - echo "Dropping the database: $DB_NAME" - docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" -i "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$DEFAULT_DB" -c "DROP DATABASE $DB_NAME;" - if [ $? -eq 0 ]; then - echo "Database $DB_NAME dropped successfully." - else - echo "Failed to drop the database: $DB_NAME." - exit 1 - fi - else - echo "Database $DB_NAME does not exist. Nothing to delete." - fi - exit 0 - else - # Handle default deletion - if [ "$INIT" == false ]; then - echo "Checking if the database $DB_NAME exists for deletion..." - DB_EXISTS=$(docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" -i "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$DEFAULT_DB" -tAc "SELECT 1 FROM pg_database WHERE datname='$DB_NAME';") - - if [ "$DB_EXISTS" == "1" ]; then - echo "Dropping the database: $DB_NAME" - docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" -i "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$DEFAULT_DB" -c "DROP DATABASE $DB_NAME;" - if [ $? -eq 0 ]; then - echo "Database $DB_NAME dropped successfully." - else - echo "Failed to drop the database: $DB_NAME." - exit 1 - fi - else - echo "Database $DB_NAME does not exist. Nothing to delete." - fi - else - echo "Cannot delete the default database." - exit 1 - fi - fi - exit 0 -fi - -# Handle --new option: Create a new database if it doesn't exist -if [ "$CREATE_NEW_DB" == true ]; then - if [ "$INIT" == false ]; then - echo "Checking if the database $DB_NAME exists..." - DB_EXISTS=$(docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" -i "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$DEFAULT_DB" -tAc "SELECT 1 FROM pg_database WHERE datname='$DB_NAME';") - - if [ "$DB_EXISTS" == "1" ]; then - echo "Database $DB_NAME already exists. Reusing it." - else - # Create a new database for the branch if it doesn't exist - echo "Creating a new database for branch: $BRANCH_NAME_SAFE" - docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" -i "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$DEFAULT_DB" -c "CREATE DATABASE $DB_NAME;" - - if [ $? -ne 0 ]; then - echo "Failed to create database: $DB_NAME" - exit 1 - fi - echo "Database $DB_NAME created." - fi - # Update .env file for new database - update_env_file "$DB_NAME" "$ENV_FILE" - else - echo "Using the default database. Skipping branch-specific database creation." - fi -fi - -# Handle --use option: Update the DATABASE_URL in the .env file with the default or branch database -if [ "$CHANGE_DB" == true ]; then - if [ "$CHANGE_OPTION" == "default" ]; then - DB_NAME=$DEFAULT_DB - echo "Switching to default database: $DB_NAME" - elif [ "$CHANGE_OPTION" = "branch" ]; then - DB_NAME="proceed_db_${BRANCH_NAME_SAFE}" - echo "Switching to branch-specific database: $DB_NAME" - else - echo "Unknown option for --use: $CHANGE_OPTION. Use 'default' or 'branch'." - exit 1 - fi - - # Update .env file for database change - update_env_file "$DB_NAME" "$ENV_FILE" -fi - -# Apply the schema to the selected database (only on --new and --init) -if [ "$CREATE_NEW_DB" == true ] || [ "$INIT" == true ]; then - echo "Applying the schema using Prisma..." - yarn prisma migrate deploy - - if [ $? -eq 0 ]; then - echo "Schema applied successfully to $DB_NAME." - else - echo "Failed to apply the schema to $DB_NAME." - exit 1 - fi -fi diff --git a/src/management-system-v2/docker-compose-dev.yml b/src/management-system-v2/docker-compose-dev.yml index 415ebc0b0..580a59d8f 100644 --- a/src/management-system-v2/docker-compose-dev.yml +++ b/src/management-system-v2/docker-compose-dev.yml @@ -2,12 +2,17 @@ services: postgres: image: postgres:latest container_name: postgres_database_proceed + healthcheck: + test: ['CMD', 'pg_isready', '-U', 'postgres'] + interval: 10s + timeout: 5s + retries: 5 environment: POSTGRES_DB: proceed_db POSTGRES_USER: proceed POSTGRES_PASSWORD: proceed ports: - - '5432:5432' + - '5433:5432' volumes: - postgres_data:/var/lib/postgresql/data diff --git a/src/management-system-v2/lib/data/file-manager-facade.ts b/src/management-system-v2/lib/data/file-manager-facade.ts index 1d0af0214..d1da62476 100644 --- a/src/management-system-v2/lib/data/file-manager-facade.ts +++ b/src/management-system-v2/lib/data/file-manager-facade.ts @@ -326,7 +326,7 @@ export async function updateArtifactProcessReference( processId: string, status: boolean, ) { - if (DEPLOYMENT_ENV !== 'cloud') return; + //if (DEPLOYMENT_ENV !== 'cloud') return; const artifact = await db.artifact.findUnique({ where: { fileName }, diff --git a/src/management-system-v2/prisma/schema.prisma b/src/management-system-v2/prisma/schema.prisma index ea6d4c5dd..fd6531431 100644 --- a/src/management-system-v2/prisma/schema.prisma +++ b/src/management-system-v2/prisma/schema.prisma @@ -93,7 +93,7 @@ model Artifact { artifactType String // user-task, script, image, etc. processReferences ArtifactProcessReference[] // Relation to track references versionReferences ArtifactVersionReference[] - refCounter Int @default(1) + refCounter Int @default(0) @@map("artifact") }