diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 82fa63db52..0f8990dafb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2877,6 +2877,9 @@ importers: servers/fdr-deploy/getdocs-lambda: dependencies: + '@fern-api/venus-api-sdk': + specifier: ^0.10.1-5-ged06d22 + version: 0.10.1-5-ged06d22 pg: specifier: ^8.11.3 version: 8.13.1 diff --git a/servers/fdr-deploy/getdocs-lambda/cdk/getLambdaStack.ts b/servers/fdr-deploy/getdocs-lambda/cdk/getLambdaStack.ts index 1a708c4bdf..f47b40d45b 100644 --- a/servers/fdr-deploy/getdocs-lambda/cdk/getLambdaStack.ts +++ b/servers/fdr-deploy/getdocs-lambda/cdk/getLambdaStack.ts @@ -9,8 +9,11 @@ import * as path from "path"; interface GetDocsLambdaProps { vpc: IVpc; environmentType: EnvironmentType; - databaseName: string; - rdsProxyEndpoint: string; // Add this property + rdsProxyEndpoint: string; + redisEndpoint: string; + redisSecurityGroupID: string; + cacheSecurityGroupID: string; + venusURL: string; } export class GetDocsLambda extends Construct { @@ -26,6 +29,16 @@ export class GetDocsLambda extends Construct { "imported-rds-proxy-sg", rdsProxySecurityGroupId ); + const redisSecurityGroup = SecurityGroup.fromSecurityGroupId( + this, + "imported-rds-proxy-sg", + props.redisSecurityGroupID + ); + const cacheSecurityGroup = SecurityGroup.fromSecurityGroupId( + this, + "imported-cache-sg", + props.cacheSecurityGroupID + ); // Create the Lambda function this.lambdaFunction = new lambda.Function(this, "get-docs-lambda", { @@ -33,11 +46,16 @@ export class GetDocsLambda extends Construct { handler: "index.handler", code: lambda.Code.fromAsset(path.join(__dirname, "../dist")), vpc: props.vpc, - securityGroups: [rdsProxySecurityGroup], + securityGroups: [ + rdsProxySecurityGroup, + redisSecurityGroup, + cacheSecurityGroup, + ], environment: { RDS_PROXY_ENDPOINT: props.rdsProxyEndpoint, - DATABASE_NAME: props.databaseName, + REDIS_ENDPOINT: props.redisEndpoint, NODE_ENV: props.environmentType.toLowerCase(), + VENUS_URL: props.venusURL, }, timeout: Duration.seconds(30), memorySize: 256, diff --git a/servers/fdr-deploy/getdocs-lambda/package.json b/servers/fdr-deploy/getdocs-lambda/package.json index 2e547d5924..90f5c81201 100644 --- a/servers/fdr-deploy/getdocs-lambda/package.json +++ b/servers/fdr-deploy/getdocs-lambda/package.json @@ -9,7 +9,9 @@ "watch": "tsc -w" }, "dependencies": { - "pg": "^8.11.3" + "@fern-api/venus-api-sdk": "^0.10.1-5-ged06d22", + "pg": "^8.11.3", + "redis": "^4.13.0" }, "devDependencies": { "@types/aws-lambda": "^8.10.131", diff --git a/servers/fdr-deploy/getdocs-lambda/src/ParsedBaseUrl.ts b/servers/fdr-deploy/getdocs-lambda/src/ParsedBaseUrl.ts new file mode 100644 index 0000000000..7e2e923d74 --- /dev/null +++ b/servers/fdr-deploy/getdocs-lambda/src/ParsedBaseUrl.ts @@ -0,0 +1,49 @@ +const HAS_HTTPS_REGEX = /^https?:\/\//i; + +export class ParsedBaseUrl { + public readonly hostname: string; + public readonly path: string | undefined; + + private constructor({ + hostname, + path, + }: { + hostname: string; + path: string | undefined; + }) { + this.hostname = hostname; + this.path = path; + } + + public getFullUrl(): string { + if (this.path == null) { + return this.hostname; + } + return `${this.hostname}${this.path}`; + } + + public toURL(): URL { + return new URL(`https://${this.getFullUrl()}`); + } + + public static parse(url: string): ParsedBaseUrl { + try { + let urlWithHttpsPrefix = url; + if (!HAS_HTTPS_REGEX.test(url)) { + urlWithHttpsPrefix = "https://" + url; + } + const parsedURL = new URL(urlWithHttpsPrefix); + return new ParsedBaseUrl({ + hostname: parsedURL.hostname, + path: + parsedURL.pathname === "/" || parsedURL.pathname === "" + ? undefined + : parsedURL.pathname, + }); + } catch (e) { + throw new Error( + `Failed to parse URL: ${url}. The error was ${(e as Error)?.message}` + ); + } + } +} diff --git a/servers/fdr-deploy/getdocs-lambda/src/index.ts b/servers/fdr-deploy/getdocs-lambda/src/index.ts index 2a7683350b..227c4507c0 100644 --- a/servers/fdr-deploy/getdocs-lambda/src/index.ts +++ b/servers/fdr-deploy/getdocs-lambda/src/index.ts @@ -1,59 +1,168 @@ /* eslint-disable turbo/no-undeclared-env-vars */ +import { FernVenusApi, FernVenusApiClient } from "@fern-api/venus-api-sdk"; import { APIGatewayProxyHandler } from "aws-lambda"; import { Client } from "pg"; +import { createClient } from "redis"; +import { ParsedBaseUrl } from "./ParsedBaseUrl"; import { DocumentData, GetDocsEvent } from "./types"; -export const handler: APIGatewayProxyHandler = async (event) => { - const client = new Client({ - host: process.env.RDS_PROXY_ENDPOINT, - database: process.env.DATABASE_NAME, - port: 5432, - ssl: { - rejectUnauthorized: false, // Configure based on your security requirements - }, +// copied from https://github.com/fern-api/fern-platform/blob/main/servers/fdr/src/services/auth/AuthService.ts#L96 +function getVenusClient({ token }: { token?: string }): FernVenusApiClient { + return new FernVenusApiClient({ + environment: process.env.venusUrl || "", + token, }); +} - try { - const { docId } = JSON.parse(event.body || "{}") as GetDocsEvent; +const BEARER_REGEX = /^bearer\s+/i; +export function getTokenFromAuthHeader(authHeader: string) { + return authHeader.replace(BEARER_REGEX, ""); +} - if (!docId) { - return { - statusCode: 400, - body: JSON.stringify({ error: "docId is required" }), - }; - } +async function checkUserBelongsToOrg({ + authHeader, + orgId, +}: { + authHeader: string | undefined; + orgId: string; +}): Promise { + if (authHeader == null) { + throw new Error("Authorization header was not specified"); + } + const token = getTokenFromAuthHeader(authHeader); + const venus = getVenusClient({ token }); + const response = await venus.organization.isMember( + FernVenusApi.OrganizationId(orgId) + ); + if (!response.ok) { + throw new Error("Failed to resolve user's organizations"); + } + const belongsToOrg = response.body; + if (!belongsToOrg) { + throw new Error("User does not belong to organization"); + } +} - await client.connect(); +export const handler: APIGatewayProxyHandler = async (event) => { + const { url } = JSON.parse(event.body || "") as GetDocsEvent; - const query = ` - SELECT id, content, version, created_at, updated_at - FROM documents - WHERE id = $1 - `; + if (!url) { + return { + statusCode: 400, + body: JSON.stringify({ error: "url is required" }), + }; + } - const result = await client.query(query, [docId]); + const parsedUrl = ParsedBaseUrl.parse(url); - if (result.rows.length === 0) { - return { - statusCode: 404, - body: JSON.stringify({ error: "Document not found" }), - }; + let client; + + const authHeader = event.headers.Authorization; + if (authHeader) { + try { + await checkUserBelongsToOrg({ authHeader, orgId: "fern" }); + } catch { + throw new Error("User does not belong to fern"); } - return { - statusCode: 200, - body: JSON.stringify(result.rows[0]), - }; - } catch (error) { - console.error("Error:", error); + // first - try to get from redis cache + const redis = createClient({ + url: `redis://${process.env.REDIS_ENDPOINT}`, + }); + + await redis.connect(); + const cachedResponse = await redis.get(parsedUrl.getFullUrl()); + if (cachedResponse != null) { + console.log(`Cache HIT for ${url}`); + console.log(cachedResponse); + // const filesV2 = Object.fromEntries( + // await Promise.all( + // Object.entries(cachedResponse.dbFiles).map( + // async ([fileId, dbFileInfo]) => { + // const presignedUrl = + // await this.app.services.s3.getPresignedDocsAssetsDownloadUrl({ + // key: dbFileInfo.s3Key, + // isPrivate: + // cachedResponse.usesPublicS3 === true ? false : true, + // }); + + // switch (dbFileInfo.type) { + // case "image": { + // const { s3Key, ...image } = dbFileInfo; + // return [fileId, { ...image, url: presignedUrl }]; + // } + // default: + // return [fileId, { type: "url", url: presignedUrl }]; + // } + // } + // ) + // ) + // ); + // return { + // ...cachedResponse.response, + // definition: { + // ...cachedResponse.response.definition, + // filesV2, + // }, + // }; + } else { + console.log(`Cache MISS for ${url}`); + console.log("Connecting to RDS"); + + // second - try to get from rds db + try { + client = new Client({ + host: process.env.RDS_PROXY_ENDPOINT, + port: 5432, + // The RDS Proxy will use IAM authentication, so we don't need username/password + ssl: { + rejectUnauthorized: false, + }, + }); + + await client.connect(); + + const query = ` + SELECT url, docsDefinition + FROM Docs + WHERE url = $1 + `; + + const result = await client.query(query, [parsedUrl]); + + if (result.rows.length === 0) { + return { + statusCode: 404, + body: JSON.stringify({ error: "Document not found" }), + }; + } + + console.log("found result"); + console.log(result.rows[0]); + return { + statusCode: 200, + body: JSON.stringify(result.rows[0]), + }; + } catch (error) { + console.error("Error:", error); + return { + statusCode: 500, + body: JSON.stringify({ + error: "Internal server error", + message: error instanceof Error ? error.message : "Unknown error", + }), + }; + } finally { + if (client) { + await client.end().catch(console.error); + } + } + } return { statusCode: 500, - body: JSON.stringify({ - error: "Internal server error", - message: error instanceof Error ? error.message : "Unknown error", - }), + body: "Internal server error", }; - } finally { - await client.end(); + } else { + throw new Error("Authorization header was not specified"); } }; diff --git a/servers/fdr-deploy/getdocs-lambda/src/types.ts b/servers/fdr-deploy/getdocs-lambda/src/types.ts index ff64c57313..9ef32735de 100644 --- a/servers/fdr-deploy/getdocs-lambda/src/types.ts +++ b/servers/fdr-deploy/getdocs-lambda/src/types.ts @@ -1,5 +1,5 @@ export interface GetDocsEvent { - docId: string; + url: string; } export interface GetDocsResponse { @@ -8,9 +8,6 @@ export interface GetDocsResponse { } export interface DocumentData { - id: string; - content: string; - version: string; - created_at: Date; - updated_at: Date; + url: string; + docsDefinition: string; } diff --git a/servers/fdr-deploy/scripts/fdr-deploy-stack.ts b/servers/fdr-deploy/scripts/fdr-deploy-stack.ts index bb0e815517..c23f4da378 100644 --- a/servers/fdr-deploy/scripts/fdr-deploy-stack.ts +++ b/servers/fdr-deploy/scripts/fdr-deploy-stack.ts @@ -137,23 +137,6 @@ export class FdrDeployStack extends Stack { new EmailSubscription("support@buildwithfern.com") ); - // GetDocs-Lambda - const rdsProxyEndpoint = Fn.importValue("RDSProxyEndpoint"); - const getDocsLambda = new GetDocsLambda(this, "getdocs-lambda", { - vpc, - environmentType, - databaseName: "temp", - rdsProxyEndpoint, - }); - - const getDocsApi = new LambdaRestApi(this, "getdocs-api", { - handler: getDocsLambda.lambdaFunction, - proxy: false, - }); - - const docs = getDocsApi.root.addResource("docs"); - docs.addMethod("POST"); - const privateApiDefinitionSourceBucket = new Bucket( this, "fdr-api-definition-source-files", @@ -235,17 +218,18 @@ export class FdrDeployStack extends Stack { zone: hostedZone, }); - const fernDocsCacheEndpoint = this.constructElastiCacheInstance(this, { - cacheName: options.cacheName, - IVpc: vpc, - numCacheShards: 1, - numCacheReplicasPerShard: 0, - clusterMode: "enabled", - cacheNodeType: options.cacheNodeType, - envType: environmentType, - env: props?.env, - ingressSecurityGroup: fdrSg, - }); + const { fernDocsCacheEndpoint, redisSecurityGroupID } = + this.constructElastiCacheInstance(this, { + cacheName: options.cacheName, + IVpc: vpc, + numCacheShards: 1, + numCacheReplicasPerShard: 0, + clusterMode: "enabled", + cacheNodeType: options.cacheNodeType, + envType: environmentType, + env: props?.env, + ingressSecurityGroup: fdrSg, + }); const cloudmapNamespaceName = environmentInfo.cloudMapNamespaceInfo.namespaceName; @@ -260,6 +244,31 @@ export class FdrDeployStack extends Stack { } ); + // Set up getdocs-lambda, which gets docsDefinition from RDS + const rdsProxyEndpoint = Fn.importValue("RDSProxyEndpoint"); + const getDocsLambda = new GetDocsLambda(this, "getdocs-lambda", { + vpc, + environmentType, + rdsProxyEndpoint, + redisEndpoint: fernDocsCacheEndpoint, + redisSecurityGroupID: fdrSg.securityGroupId, + cacheSecurityGroupID: redisSecurityGroupID, + venusURL: `http://venus.${cloudmapNamespaceName}:8080/`, + }); + + const getDocsApi = new LambdaRestApi(this, "getdocs-api", { + handler: getDocsLambda.lambdaFunction, + proxy: false, + }); + + const docs = getDocsApi.root.addResource("docs"); + docs.addMethod("POST"); + + new CfnOutput(this, "GetDocsApiUrl", { + value: getDocsApi.url, + }); + // end getdocs-lambda + const fargateService = new ApplicationLoadBalancedFargateService( this, SERVICE_NAME, @@ -446,7 +455,7 @@ export class FdrDeployStack extends Stack { private constructElastiCacheInstance( scope: Construct, props: ElastiCacheProps - ): string { + ): { fernDocsCacheEndpoint: string; redisSecurityGroupID: string } { const envPrefix = props.envType + "-"; const cacheSecurityGroupName = @@ -512,7 +521,10 @@ export class FdrDeployStack extends Stack { "Redis Port Ingress rule" ); - return `${cacheEndpointAddress}:${cacheEndpointPort}`; + return { + fernDocsCacheEndpoint: `${cacheEndpointAddress}:${cacheEndpointPort}`, + redisSecurityGroupID: cacheSecurityGroup.securityGroupId, + }; } }