feat: add pgbouncer (#114)

* feat: Add option to set up a pgbouncer server that can manage traffic to the
actual database

* chore: increase timeout value to 90 minutes for deployment action

* deps: update to pyyaml 6.0.2 to fix build on ubuntu-latest

.github/workflows/build.yaml

@@ -26,7 +26,7 @@ jobs:
           cache: "npm"
 
       - name: Install Dependencies
-        run: npm ci
+        run: npm run install:all
 
       - name: Compile project
         run: npm run build

.github/workflows/deploy.yaml

@@ -7,7 +7,7 @@ jobs:
   build_package_and_deploy:
     name: Build, package and deploy
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 90
     env:
       AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION_DEPLOY }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_DEPLOY }}
@@ -21,8 +21,8 @@ jobs:
           node-version: 18
           cache: "npm"
 
-      - name: Install Dependencies
-        run: npm ci
+      - name: Install All Dependencies
+        run: npm run install:all
 
       - name: Compile project
         run: npm run build

.gitignore

@@ -11,3 +11,4 @@ __pycache__
 tests/*.egg*
 tests/*venv*
 tests/__pycache__
+integration_tests/cdk/cdk.out

integration_tests/cdk/app.py

@@ -79,20 +79,25 @@ def __init__(
             ),
             allocated_storage=app_config.db_allocated_storage,
             instance_type=aws_ec2.InstanceType(app_config.db_instance_type),
+            add_pgbouncer=True,
             removal_policy=RemovalPolicy.DESTROY,
         )
 
-        pgstac_db.db.connections.allow_default_port_from_any_ipv4()
+        assert pgstac_db.security_group
+
+        pgstac_db.security_group.add_ingress_rule(
+            aws_ec2.Peer.any_ipv4(), aws_ec2.Port.tcp(5432)
+        )
 
         PgStacApiLambda(
             self,
             "pgstac-api",
+            db=pgstac_db.connection_target,
+            db_secret=pgstac_db.pgstac_secret,
             api_env={
                 "NAME": app_config.build_service_name("STAC API"),
                 "description": f"{app_config.stage} STAC API",
             },
-            db=pgstac_db.db,
-            db_secret=pgstac_db.pgstac_secret,
         )
 
         TitilerPgstacApiLambda(
@@ -102,7 +107,7 @@ def __init__(
                 "NAME": app_config.build_service_name("titiler pgSTAC API"),
                 "description": f"{app_config.stage} titiler pgstac API",
             },
-            db=pgstac_db.db,
+            db=pgstac_db.connection_target,
             db_secret=pgstac_db.pgstac_secret,
             buckets=[],
             lambda_function_options={
@@ -113,7 +118,7 @@ def __init__(
         TiPgApiLambda(
             self,
             "tipg-api",
-            db=pgstac_db.db,
+            db=pgstac_db.connection_target,
             db_secret=pgstac_db.pgstac_secret,
             api_env={
                 "NAME": app_config.build_service_name("tipg API"),

integration_tests/cdk/requirements.txt

@@ -3,5 +3,5 @@ constructs==10.3.0
 pydantic==2.0.2
 pydantic-settings==2.0.1
 python-dotenv==1.0.0
-pyyaml==6.0
+pyyaml==6.0.2
 types-PyYAML==6.0.12.10

lib/database/PgBouncer.ts

@@ -0,0 +1,251 @@
+import {
+  aws_ec2 as ec2,
+  aws_iam as iam,
+  aws_lambda as lambda,
+  aws_secretsmanager as secretsmanager,
+  CustomResource,
+  Stack,
+} from "aws-cdk-lib";
+import { Construct } from "constructs";
+
+import * as fs from "fs";
+import * as path from "path";
+
+// used to populate pgbouncer config:
+// see https://www.pgbouncer.org/config.html for details
+export interface PgBouncerConfigProps {
+  poolMode?: "transaction" | "session" | "statement";
+  maxClientConn?: number;
+  defaultPoolSize?: number;
+  minPoolSize?: number;
+  reservePoolSize?: number;
+  reservePoolTimeout?: number;
+  maxDbConnections?: number;
+  maxUserConnections?: number;
+}
+
+export interface PgBouncerProps {
+  /**
+   * Name for the pgbouncer instance
+   */
+  instanceName: string;
+
+  /**
+   * VPC to deploy PgBouncer into
+   */
+  vpc: ec2.IVpc;
+
+  /**
+   * The RDS instance to connect to
+   */
+  database: {
+    connections: ec2.Connections;
+    secret: secretsmanager.ISecret;
+  };
+
+  /**
+   * Maximum connections setting for the database.
+   * PgBouncer will use 10 fewer than this value.
+   */
+  dbMaxConnections: number;
+
+  /**
+   * Whether to deploy in public subnet
+   * @default false
+   */
+  usePublicSubnet?: boolean;
+
+  /**
+   * Instance type for PgBouncer
+   * @default t3.micro
+   */
+  instanceType?: ec2.InstanceType;
+
+  /**
+   * PgBouncer configuration options
+   */
+  pgBouncerConfig?: PgBouncerConfigProps;
+}
+
+export class PgBouncer extends Construct {
+  public readonly instance: ec2.Instance;
+  public readonly pgbouncerSecret: secretsmanager.Secret;
+  public readonly securityGroup: ec2.SecurityGroup;
+
+  // The max_connections parameter in PgBouncer determines the maximum number of
+  // connections to open on the actual database instance. We want that number to
+  // be slightly smaller than the actual max_connections value on the RDS instance
+  // so we perform this calculation.
+
+  private getDefaultConfig(
+    dbMaxConnections: number
+  ): Required<PgBouncerConfigProps> {
+    // maxDbConnections (and maxUserConnections) are the only settings that need
+    // to be responsive to the database size/max_connections setting
+    return {
+      poolMode: "transaction",
+      maxClientConn: 1000,
+      defaultPoolSize: 5,
+      minPoolSize: 0,
+      reservePoolSize: 5,
+      reservePoolTimeout: 5,
+      maxDbConnections: dbMaxConnections - 10,
+      maxUserConnections: dbMaxConnections - 10,
+    };
+  }
+
+  constructor(scope: Construct, id: string, props: PgBouncerProps) {
+    super(scope, id);
+
+    // Set defaults for optional props
+    const defaultInstanceType = ec2.InstanceType.of(
+      ec2.InstanceClass.T3,
+      ec2.InstanceSize.MICRO
+    );
+
+    const instanceType = props.instanceType ?? defaultInstanceType;
+    const defaultConfig = this.getDefaultConfig(props.dbMaxConnections);
+
+    // Merge provided config with defaults
+    const pgBouncerConfig: Required<PgBouncerConfigProps> = {
+      ...defaultConfig,
+      ...props.pgBouncerConfig,
+    };
+
+    // Create role for PgBouncer instance to enable writing to CloudWatch
+    const role = new iam.Role(this, "InstanceRole", {
+      description:
+        "pgbouncer instance role with Systems Manager + CloudWatch permissions",
+      assumedBy: new iam.ServicePrincipal("ec2.amazonaws.com"),
+      managedPolicies: [
+        iam.ManagedPolicy.fromAwsManagedPolicyName(
+          "AmazonSSMManagedInstanceCore"
+        ),
+        iam.ManagedPolicy.fromAwsManagedPolicyName(
+          "CloudWatchAgentServerPolicy"
+        ),
+      ],
+    });
+
+    // Add policy to allow reading RDS credentials from Secrets Manager
+    role.addToPolicy(
+      new iam.PolicyStatement({
+        actions: ["secretsmanager:GetSecretValue"],
+        resources: [props.database.secret.secretArn],
+      })
+    );
+
+    // Create a security group and allow connections from the Lambda IP ranges for this region
+    this.securityGroup = new ec2.SecurityGroup(this, "PgBouncerSecurityGroup", {
+      vpc: props.vpc,
+      description: "Security group for PgBouncer instance",
+      allowAllOutbound: true,
+    });
+
+    // Create PgBouncer instance
+    this.instance = new ec2.Instance(this, "Instance", {
+      vpc: props.vpc,
+      vpcSubnets: {
+        subnetType: props.usePublicSubnet
+          ? ec2.SubnetType.PUBLIC
+          : ec2.SubnetType.PRIVATE_WITH_EGRESS,
+      },
+      securityGroup: this.securityGroup,
+      instanceType,
+      instanceName: props.instanceName,
+      machineImage: ec2.MachineImage.fromSsmParameter(
+        "/aws/service/canonical/ubuntu/server/jammy/stable/current/amd64/hvm/ebs-gp2/ami-id",
+        { os: ec2.OperatingSystemType.LINUX }
+      ),
+      role,
+      blockDevices: [
+        {
+          deviceName: "/dev/xvda",
+          volume: ec2.BlockDeviceVolume.ebs(20, {
+            volumeType: ec2.EbsDeviceVolumeType.GP3,
+            encrypted: true,
+            deleteOnTermination: true,
+          }),
+        },
+      ],
+      userData: this.loadUserDataScript(pgBouncerConfig, props.database),
+      userDataCausesReplacement: true,
+      associatePublicIpAddress: props.usePublicSubnet,
+    });
+
+    // Allow PgBouncer to connect to RDS
+    props.database.connections.allowFrom(
+      this.instance,
+      ec2.Port.tcp(5432),
+      "Allow PgBouncer to connect to RDS"
+    );
+
+    // Create a new secret for pgbouncer connection credentials
+    this.pgbouncerSecret = new secretsmanager.Secret(this, "PgBouncerSecret", {
+      description: `Connection information for PgBouncer instance ${props.instanceName}`,
+      generateSecretString: {
+        generateStringKey: "dummy",
+        secretStringTemplate: "{}",
+      },
+    });
+
+    // Grant the role permission to read the new secret
+    this.pgbouncerSecret.grantRead(role);
+
+    // Update pgbouncerSecret to contain pgstacSecret values but with new value for host
+    const secretUpdaterFn = new lambda.Function(this, "SecretUpdaterFunction", {
+      runtime: lambda.Runtime.NODEJS_20_X,
+      handler: "index.handler",
+      code: lambda.Code.fromAsset(
+        path.join(__dirname, "lambda/pgbouncer-secret-updater")
+      ),
+      environment: {
+        SOURCE_SECRET_ARN: props.database.secret.secretArn,
+        TARGET_SECRET_ARN: this.pgbouncerSecret.secretArn,
+      },
+    });
+
+    props.database.secret.grantRead(secretUpdaterFn);
+    this.pgbouncerSecret.grantWrite(secretUpdaterFn);
+
+    new CustomResource(this, "pgbouncerSecretBootstrapper", {
+      serviceToken: secretUpdaterFn.functionArn,
+      properties: {
+        instanceIp: props.usePublicSubnet
+          ? this.instance.instancePublicIp
+          : this.instance.instancePrivateIp,
+      },
+    });
+  }
+
+  private loadUserDataScript(
+    pgBouncerConfig: Required<NonNullable<PgBouncerProps["pgBouncerConfig"]>>,
+    database: { secret: secretsmanager.ISecret }
+  ): ec2.UserData {
+    const userDataScript = ec2.UserData.forLinux();
+
+    // Set environment variables with configuration parameters
+    userDataScript.addCommands(
+      'export SECRET_ARN="' + database.secret.secretArn + '"',
+      'export REGION="' + Stack.of(this).region + '"',
+      'export POOL_MODE="' + pgBouncerConfig.poolMode + '"',
+      'export MAX_CLIENT_CONN="' + pgBouncerConfig.maxClientConn + '"',
+      'export DEFAULT_POOL_SIZE="' + pgBouncerConfig.defaultPoolSize + '"',
+      'export MIN_POOL_SIZE="' + pgBouncerConfig.minPoolSize + '"',
+      'export RESERVE_POOL_SIZE="' + pgBouncerConfig.reservePoolSize + '"',
+      'export RESERVE_POOL_TIMEOUT="' +
+        pgBouncerConfig.reservePoolTimeout +
+        '"',
+      'export MAX_DB_CONNECTIONS="' + pgBouncerConfig.maxDbConnections + '"',
+      'export MAX_USER_CONNECTIONS="' + pgBouncerConfig.maxUserConnections + '"'
+    );
+
+    // Load the startup script
+    const scriptPath = path.join(__dirname, "./pgbouncer-setup.sh");
+    let script = fs.readFileSync(scriptPath, "utf8");
+
+    userDataScript.addCommands(script);
+
+    return userDataScript;
+  }
+}