From 7d5950be94e6c925211a60ff60372d7714cb4971 Mon Sep 17 00:00:00 2001 From: bsinghvi Date: Wed, 1 May 2024 18:55:16 -0400 Subject: [PATCH 1/2] Passing in Docs Cache endpoint as environment variable --- .github/workflows/diff-fdr-dev.yml | 7 +++++ .../fdr-deploy/scripts/elasticache-stack.ts | 5 ++- .../fdr-deploy/scripts/fdr-deploy-stack.ts | 31 +++++++++---------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/.github/workflows/diff-fdr-dev.yml b/.github/workflows/diff-fdr-dev.yml index c94a01e57d..b8b942c369 100644 --- a/.github/workflows/diff-fdr-dev.yml +++ b/.github/workflows/diff-fdr-dev.yml @@ -1,6 +1,13 @@ name: Diff FDR Dev on: + pull_request: + branches: + - "*" + paths: + - "servers/fdr-deploy/scripts/**" + - ".github/workflows/diff-fdr-dev.yml" + - "pnpm-lock.yaml" push: branches: - "*" diff --git a/servers/fdr-deploy/scripts/elasticache-stack.ts b/servers/fdr-deploy/scripts/elasticache-stack.ts index a4fdd082f6..706039eeac 100644 --- a/servers/fdr-deploy/scripts/elasticache-stack.ts +++ b/servers/fdr-deploy/scripts/elasticache-stack.ts @@ -1,5 +1,5 @@ import { EnvironmentType } from "@fern-fern/fern-cloud-sdk/api"; -import { Environment, Stack, StackProps, Token } from "aws-cdk-lib"; +import { CfnOutput, Environment, Stack, StackProps, Token } from "aws-cdk-lib"; import { IVpc, Peer, Port, SecurityGroup } from "aws-cdk-lib/aws-ec2"; import { CfnReplicationGroup, CfnSubnetGroup } from "aws-cdk-lib/aws-elasticache"; import { Construct } from "constructs"; @@ -72,5 +72,8 @@ export class ElastiCacheStack extends Stack { Port.tcp(Token.asNumber(this.redisEndpointPort)), "Redis Port Ingress rule", ); + + new CfnOutput(this, `${props.cacheName}Host`, { value: this.redisEndpointAddress }); + new CfnOutput(this, `${props.cacheName}Port`, { value: this.redisEndpointPort }); } } diff --git a/servers/fdr-deploy/scripts/fdr-deploy-stack.ts b/servers/fdr-deploy/scripts/fdr-deploy-stack.ts index 29683c6843..2c81228f90 100644 --- a/servers/fdr-deploy/scripts/fdr-deploy-stack.ts +++ b/servers/fdr-deploy/scripts/fdr-deploy-stack.ts @@ -1,5 +1,5 @@ import { EnvironmentInfo, EnvironmentType } from "@fern-fern/fern-cloud-sdk/api"; -import { CfnOutput, Duration, RemovalPolicy, Stack, StackProps } from "aws-cdk-lib"; +import { Duration, RemovalPolicy, Stack, StackProps } from "aws-cdk-lib"; import { Certificate } from "aws-cdk-lib/aws-certificatemanager"; import { Alarm } from "aws-cdk-lib/aws-cloudwatch"; import * as actions from "aws-cdk-lib/aws-cloudwatch-actions"; @@ -21,8 +21,6 @@ const CONTAINER_NAME = "fern-definition-registry"; const SERVICE_NAME = "fdr"; export class FdrDeployStack extends Stack { - private readonly fernDocsCacheEndpoint: string; - constructor( scope: Construct, id: string, @@ -79,6 +77,18 @@ export class FdrDeployStack extends Stack { versioned: true, }); + const fernDocsCache = new ElastiCacheStack(this, "FernDocsCache", { + cacheName: "FernDocsCache", + IVpc: vpc, + numCacheShards: 1, + numCacheReplicasPerShard: environmentType === EnvironmentType.Prod ? 2 : undefined, + clusterMode: "enabled", + cacheNodeType: "cache.r7g.large", + envType: environmentType, + env: props?.env, + ingressSecurityGroup: fdrSg, + }); + const cloudmapNamespaceName = environmentInfo.cloudMapNamespaceInfo.namespaceName; const cloudMapNamespace = PrivateDnsNamespace.fromPrivateDnsNamespaceAttributes(this, "private-cloudmap", { namespaceArn: environmentInfo.cloudMapNamespaceInfo.namespaceArn, @@ -111,6 +121,7 @@ export class FdrDeployStack extends Stack { ALGOLIA_SEARCH_API_KEY: getEnvironmentVariableOrThrow("ALGOLIA_SEARCH_API_KEY"), SLACK_TOKEN: getEnvironmentVariableOrThrow("FERNIE_SLACK_APP_TOKEN"), LOG_LEVEL: getLogLevel(environmentType), + DOCS_CACHE_ENDPOINT: `${fernDocsCache.redisEndpointAddress}:${fernDocsCache.redisEndpointPort}`, ENABLE_CUSTOMER_NOTIFICATIONS: (environmentType === "PROD").toString(), }, containerName: CONTAINER_NAME, @@ -197,20 +208,6 @@ export class FdrDeployStack extends Stack { evaluationPeriods: 5, }); lb500CountAlarm.addAlarmAction(new actions.SnsAction(snsTopic)); - - const fernDocsCache = new ElastiCacheStack(this, "FernDocsElastiCache", { - cacheName: "FernDocsElastiCache", - IVpc: vpc, - numCacheShards: 1, - numCacheReplicasPerShard: environmentType === EnvironmentType.Prod ? 2 : undefined, - clusterMode: "enabled", - cacheNodeType: "cache.r7g.large", - envType: environmentType, - env: props?.env, - }); - - this.fernDocsCacheEndpoint = `${fernDocsCache.redisEndpointAddress}:${fernDocsCache.redisEndpointPort}`; - new CfnOutput(this, "FernDocsCacheEndpoint", { value: this.fernDocsCacheEndpoint }); } } From a35a31a13ee4096d9c61e7b9ee9fb104c9d35de3 Mon Sep 17 00:00:00 2001 From: bsinghvi Date: Wed, 1 May 2024 19:54:14 -0400 Subject: [PATCH 2/2] Refactoring cache --- servers/fdr-deploy/bin/fdr-deploy.ts | 30 ++++---- .../fdr-deploy/scripts/elasticache-stack.ts | 3 +- .../fdr-deploy/scripts/fdr-deploy-stack.ts | 77 +++++++++++++++++-- 3 files changed, 87 insertions(+), 23 deletions(-) diff --git a/servers/fdr-deploy/bin/fdr-deploy.ts b/servers/fdr-deploy/bin/fdr-deploy.ts index c5373ff602..2bc519d48b 100644 --- a/servers/fdr-deploy/bin/fdr-deploy.ts +++ b/servers/fdr-deploy/bin/fdr-deploy.ts @@ -17,24 +17,20 @@ async function main() { if (environmentInfo == null) { throw new Error(`No info for environment ${environmentType}`); } - switch (environmentType) { - case EnvironmentType.Dev: - case EnvironmentType.Dev2: - case EnvironmentType.Prod: - new FdrDeployStack( - app, - `fdr-${environmentType.toLowerCase()}`, - version, - environmentType, - environmentInfo, - { - env: { account: "985111089818", region: "us-east-1" }, - }, - ); - break; - default: - return; + if (!(environmentType in EnvironmentType)) { + return; } + + new FdrDeployStack( + app, + `fdr-${environmentType.toLowerCase()}`, + version, + environmentType as EnvironmentType, + environmentInfo, + { + env: { account: "985111089818", region: "us-east-1" }, + }, + ); } } diff --git a/servers/fdr-deploy/scripts/elasticache-stack.ts b/servers/fdr-deploy/scripts/elasticache-stack.ts index 706039eeac..f82287c64f 100644 --- a/servers/fdr-deploy/scripts/elasticache-stack.ts +++ b/servers/fdr-deploy/scripts/elasticache-stack.ts @@ -61,6 +61,7 @@ export class ElastiCacheStack extends Stack { this.replicationGroup.cfnOptions.updatePolicy = { useOnlineResharding: true, }; + this.replicationGroup; this.replicationGroup.addDependency(this.subnetGroup); @@ -74,6 +75,6 @@ export class ElastiCacheStack extends Stack { ); new CfnOutput(this, `${props.cacheName}Host`, { value: this.redisEndpointAddress }); - new CfnOutput(this, `${props.cacheName}Port`, { value: this.redisEndpointPort }); + new CfnOutput(this, `${props.cacheName}Po`, { value: this.redisEndpointPort }); } } diff --git a/servers/fdr-deploy/scripts/fdr-deploy-stack.ts b/servers/fdr-deploy/scripts/fdr-deploy-stack.ts index 2c81228f90..2eb0a499d7 100644 --- a/servers/fdr-deploy/scripts/fdr-deploy-stack.ts +++ b/servers/fdr-deploy/scripts/fdr-deploy-stack.ts @@ -1,11 +1,12 @@ import { EnvironmentInfo, EnvironmentType } from "@fern-fern/fern-cloud-sdk/api"; -import { Duration, RemovalPolicy, Stack, StackProps } from "aws-cdk-lib"; +import { CfnOutput, Duration, Environment, RemovalPolicy, Stack, StackProps, Token } from "aws-cdk-lib"; import { Certificate } from "aws-cdk-lib/aws-certificatemanager"; import { Alarm } from "aws-cdk-lib/aws-cloudwatch"; import * as actions from "aws-cdk-lib/aws-cloudwatch-actions"; -import { Peer, Port, SecurityGroup, Vpc } from "aws-cdk-lib/aws-ec2"; +import { IVpc, Peer, Port, SecurityGroup, Vpc } from "aws-cdk-lib/aws-ec2"; import { Cluster, ContainerImage, LogDriver, Volume } from "aws-cdk-lib/aws-ecs"; import { ApplicationLoadBalancedFargateService } from "aws-cdk-lib/aws-ecs-patterns"; +import { CfnReplicationGroup, CfnSubnetGroup } from "aws-cdk-lib/aws-elasticache"; import { ApplicationProtocol, HttpCodeTarget } from "aws-cdk-lib/aws-elasticloadbalancingv2"; import { LogGroup } from "aws-cdk-lib/aws-logs"; import { ARecord, HostedZone, RecordTarget } from "aws-cdk-lib/aws-route53"; @@ -15,11 +16,22 @@ import { PrivateDnsNamespace } from "aws-cdk-lib/aws-servicediscovery"; import * as sns from "aws-cdk-lib/aws-sns"; import { EmailSubscription } from "aws-cdk-lib/aws-sns-subscriptions"; import { Construct } from "constructs"; -import { ElastiCacheStack } from "./elasticache-stack"; const CONTAINER_NAME = "fern-definition-registry"; const SERVICE_NAME = "fdr"; +interface ElastiCacheProps { + readonly cacheName: string; + readonly IVpc: IVpc; + readonly numCacheShards: number; + readonly numCacheReplicasPerShard: number | undefined; + readonly clusterMode: "enabled" | "disabled"; + readonly cacheNodeType: string; + readonly envType: EnvironmentType; + readonly env?: Environment; + readonly ingressSecurityGroup?: SecurityGroup; +} + export class FdrDeployStack extends Stack { constructor( scope: Construct, @@ -77,7 +89,7 @@ export class FdrDeployStack extends Stack { versioned: true, }); - const fernDocsCache = new ElastiCacheStack(this, "FernDocsCache", { + const fernDocsCacheEndpoint = this.constructElastiCacheInstance(scope, { cacheName: "FernDocsCache", IVpc: vpc, numCacheShards: 1, @@ -121,7 +133,7 @@ export class FdrDeployStack extends Stack { ALGOLIA_SEARCH_API_KEY: getEnvironmentVariableOrThrow("ALGOLIA_SEARCH_API_KEY"), SLACK_TOKEN: getEnvironmentVariableOrThrow("FERNIE_SLACK_APP_TOKEN"), LOG_LEVEL: getLogLevel(environmentType), - DOCS_CACHE_ENDPOINT: `${fernDocsCache.redisEndpointAddress}:${fernDocsCache.redisEndpointPort}`, + DOCS_CACHE_ENDPOINT: fernDocsCacheEndpoint, ENABLE_CUSTOMER_NOTIFICATIONS: (environmentType === "PROD").toString(), }, containerName: CONTAINER_NAME, @@ -209,6 +221,61 @@ export class FdrDeployStack extends Stack { }); lb500CountAlarm.addAlarmAction(new actions.SnsAction(snsTopic)); } + + private constructElastiCacheInstance(scope: Construct, props: ElastiCacheProps): string { + const envPrefix = props.envType + "-"; + + const cacheSecurityGroupName = envPrefix + props.cacheName + "SecurityGroup"; + const cacheSecurityGroup = new SecurityGroup(scope, cacheSecurityGroupName, { + vpc: props.IVpc, + allowAllOutbound: true, + description: `${cacheSecurityGroupName} CDK`, + }); + + const cacheSubnetGroupName = envPrefix + props.cacheName + "SubnetGroup"; + const cacheSubnetGroup = new CfnSubnetGroup(this, cacheSubnetGroupName, { + description: `${cacheSubnetGroupName} CDK`, + cacheSubnetGroupName, + subnetIds: props.IVpc.publicSubnets.map(({ subnetId }) => subnetId), + }); + + const cacheReplicationGroupName = envPrefix + props.cacheName + "ReplicationGroup"; + const cacheReplicationGroup = new CfnReplicationGroup(this, cacheReplicationGroupName, { + replicationGroupId: cacheReplicationGroupName, + replicationGroupDescription: `Replication Group for the ${cacheReplicationGroupName} ElastiCache stack`, + automaticFailoverEnabled: true, + autoMinorVersionUpgrade: true, + engine: "redis", + engineVersion: "7.0", + cacheParameterGroupName: "default.redis7.cluster.on", + cacheNodeType: props.cacheNodeType, + numNodeGroups: props.numCacheShards, + replicasPerNodeGroup: props.numCacheReplicasPerShard, + clusterMode: props.clusterMode, + cacheSubnetGroupName: cacheSubnetGroup.ref, + securityGroupIds: [cacheSecurityGroup.securityGroupId], + }); + + cacheReplicationGroup.cfnOptions.updatePolicy = { + useOnlineResharding: true, + }; + + cacheReplicationGroup.addDependency(cacheSubnetGroup); + + const cacheEndpointAddress = cacheReplicationGroup.attrConfigurationEndPointAddress; + const cacheEndpointPort = cacheReplicationGroup.attrConfigurationEndPointPort; + + cacheSecurityGroup.addIngressRule( + props.ingressSecurityGroup || Peer.anyIpv4(), + Port.tcp(Token.asNumber(cacheEndpointPort)), + "Redis Port Ingress rule", + ); + + new CfnOutput(this, `${props.cacheName}Host`, { value: cacheEndpointAddress }); + new CfnOutput(this, `${props.cacheName}Port`, { value: cacheEndpointPort }); + + return `${cacheEndpointAddress}:${cacheEndpointPort}`; + } } function getServiceDomainName(environmentType: EnvironmentType, environmentInfo: EnvironmentInfo) {