From 6c882e0acc36b632ff80286e72bac08734d70d72 Mon Sep 17 00:00:00 2001 From: yashkh-amzn Date: Tue, 4 Mar 2025 18:06:14 -0800 Subject: [PATCH] feat(logs): add support for fieldIndexPolicies in log group L2 Construct (#33416) ### Issue # (if applicable) https://github.com/aws/aws-cdk/issues/33366 Closes #33366 ### Reason for this change Field Indexing for CloudWatch Logs (CWL) was launched in Nov 2024. A lot of CWL customers are asking for indexing support in L2 construct. This feature will enable that property under FieldIndexPolicies as a JSON object in the LogGroup construct. ### Description of changes The change here is just populating the `fieldIndexPolicies` property of the LogGroup CFN with the list of fields provided by the user. The format of this property will be like this: ``` const fieldIndexPolicy = new FieldIndexPolicy({ fields: ['Operation', 'RequestId'], }); new LogGroup(this, 'LogGroupLambda', { dataProtectionPolicy: dataProtectionPolicy, fieldIndexPolicies: [fieldIndexPolicy], }); ``` ### Describe any new or updated permissions being added No new permissions have been added. ### Description of how you validated changes Added unit tests. Will add integ tests after getting a confirmation from the CDK team on the implementation. ### Checklist - [X] My code adheres to the [CONTRIBUTING GUIDE](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) and [DESIGN GUIDELINES](https://github.com/aws/aws-cdk/blob/main/docs/DESIGN_GUIDELINES.md) ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../aws-cdk-log-group-integ.assets.json | 4 +- .../aws-cdk-log-group-integ.template.json | 8 +++ .../integ.log-group.js.snapshot/manifest.json | 2 +- .../integ.log-group.js.snapshot/tree.json | 8 +++ .../test/aws-logs/test/integ.log-group.ts | 7 +- packages/aws-cdk-lib/aws-logs/README.md | 23 ++++++ .../aws-logs/lib/field-index-policy.ts | 34 +++++++++ packages/aws-cdk-lib/aws-logs/lib/index.ts | 1 + .../aws-cdk-lib/aws-logs/lib/log-group.ts | 15 ++++ .../aws-logs/test/loggroup.test.ts | 70 ++++++++++++++++++- 10 files changed, 167 insertions(+), 5 deletions(-) create mode 100644 packages/aws-cdk-lib/aws-logs/lib/field-index-policy.ts diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/aws-cdk-log-group-integ.assets.json b/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/aws-cdk-log-group-integ.assets.json index 1a63f09b0460b..3f39978bd9c2f 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/aws-cdk-log-group-integ.assets.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/aws-cdk-log-group-integ.assets.json @@ -1,7 +1,7 @@ { "version": "39.0.0", "files": { - "f73440b0e32a261b64b3f44b72f9e681bc775595740055ca82b47830bc9b3535": { + "67c684a3a20aaf6222eb1845b08f0d8dde625acf4e0f6a6430aa7e1f94a6017a": { "source": { "path": "aws-cdk-log-group-integ.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "f73440b0e32a261b64b3f44b72f9e681bc775595740055ca82b47830bc9b3535.json", + "objectKey": "67c684a3a20aaf6222eb1845b08f0d8dde625acf4e0f6a6430aa7e1f94a6017a.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/aws-cdk-log-group-integ.template.json b/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/aws-cdk-log-group-integ.template.json index 67ad305cbcb04..cb312fe63be3a 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/aws-cdk-log-group-integ.template.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/aws-cdk-log-group-integ.template.json @@ -112,6 +112,14 @@ ] } }, + "FieldIndexPolicies": [ + { + "Fields": [ + "Operation", + "RequestId" + ] + } + ], "RetentionInDays": 731 }, "UpdateReplacePolicy": "Retain", diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/manifest.json b/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/manifest.json index 4cdf84372be65..c0800da6610fb 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/manifest.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/manifest.json @@ -18,7 +18,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/f73440b0e32a261b64b3f44b72f9e681bc775595740055ca82b47830bc9b3535.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/67c684a3a20aaf6222eb1845b08f0d8dde625acf4e0f6a6430aa7e1f94a6017a.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/tree.json b/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/tree.json index 4f4b3c111c0da..49e8317d80b4a 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/tree.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.js.snapshot/tree.json @@ -166,6 +166,14 @@ ] } }, + "fieldIndexPolicies": [ + { + "Fields": [ + "Operation", + "RequestId" + ] + } + ], "retentionInDays": 731 } }, diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.ts b/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.ts index 162e76687dfa0..6b55c786dc0ef 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.ts +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-logs/test/integ.log-group.ts @@ -1,7 +1,7 @@ import { Bucket } from 'aws-cdk-lib/aws-s3'; import { App, Stack, StackProps } from 'aws-cdk-lib'; import { IntegTest } from '@aws-cdk/integ-tests-alpha'; -import { LogGroup, DataProtectionPolicy, DataIdentifier, CustomDataIdentifier } from 'aws-cdk-lib/aws-logs'; +import { LogGroup, DataProtectionPolicy, DataIdentifier, CustomDataIdentifier, FieldIndexPolicy } from 'aws-cdk-lib/aws-logs'; class LogGroupIntegStack extends Stack { constructor(scope: App, id: string, props?: StackProps) { @@ -19,8 +19,13 @@ class LogGroupIntegStack extends Stack { s3BucketAuditDestination: bucket, }); + const fieldIndexPolicy = new FieldIndexPolicy({ + fields: ['Operation', 'RequestId'], + }); + new LogGroup(this, 'LogGroupLambda', { dataProtectionPolicy: dataProtectionPolicy, + fieldIndexPolicies: [fieldIndexPolicy], }); } } diff --git a/packages/aws-cdk-lib/aws-logs/README.md b/packages/aws-cdk-lib/aws-logs/README.md index 07863d584b932..f5cee1f441a08 100644 --- a/packages/aws-cdk-lib/aws-logs/README.md +++ b/packages/aws-cdk-lib/aws-logs/README.md @@ -441,6 +441,29 @@ new logs.LogGroup(this, 'LogGroupLambda', { }); ``` +## Field Index Policies + +Creates or updates a field index policy for the specified log group. You can use field index policies to create field indexes on fields found in log events in the log group. Creating field indexes lowers the costs for CloudWatch Logs Insights queries that reference those field indexes, because these queries attempt to skip the processing of log events that are known to not match the indexed field. Good fields to index are fields that you often need to query for and fields that have high cardinality of values. + +For more information, see [Create field indexes to improve query performance and reduce costs](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CloudWatchLogs-Field-Indexing.html). + +Only log groups in the Standard log class support field index policies. +Currently, this array supports only one field index policy object. + +Example: + +```ts + +const fieldIndexPolicy = new logs.FieldIndexPolicy({ + fields: ['Operation', 'RequestId'], +}); + +new logs.LogGroup(this, 'LogGroup', { + logGroupName: 'cdkIntegLogGroup', + fieldIndexPolicies: [fieldIndexPolicy], +}); +``` + ## Notes Be aware that Log Group ARNs will always have the string `:*` appended to diff --git a/packages/aws-cdk-lib/aws-logs/lib/field-index-policy.ts b/packages/aws-cdk-lib/aws-logs/lib/field-index-policy.ts new file mode 100644 index 0000000000000..2df85826a866c --- /dev/null +++ b/packages/aws-cdk-lib/aws-logs/lib/field-index-policy.ts @@ -0,0 +1,34 @@ +import { Construct } from 'constructs'; + +/** + * Creates a field index policy for CloudWatch Logs log groups. + */ +export class FieldIndexPolicy { + private readonly fieldIndexPolicyProps: FieldIndexPolicyProps; + + constructor(props: FieldIndexPolicyProps) { + if (props.fields.length > 20) { + throw new Error('A maximum of 20 fields can be indexed per log group'); + } + this.fieldIndexPolicyProps = props; + } + + /** + * @internal + */ + public _bind(_scope: Construct) { + return { Fields: this.fieldIndexPolicyProps.fields }; + } +} + +/** + * Properties for creating field index policies + */ +export interface FieldIndexPolicyProps { + /** + * List of fields to index in log events. + * + * @default no fields + */ + readonly fields: string[]; +} diff --git a/packages/aws-cdk-lib/aws-logs/lib/index.ts b/packages/aws-cdk-lib/aws-logs/lib/index.ts index 71f2717cc4447..19981d33d8e7d 100644 --- a/packages/aws-cdk-lib/aws-logs/lib/index.ts +++ b/packages/aws-cdk-lib/aws-logs/lib/index.ts @@ -8,6 +8,7 @@ export * from './log-retention'; export * from './policy'; export * from './query-definition'; export * from './data-protection-policy'; +export * from './field-index-policy'; // AWS::Logs CloudFormation Resources: export * from './logs.generated'; diff --git a/packages/aws-cdk-lib/aws-logs/lib/log-group.ts b/packages/aws-cdk-lib/aws-logs/lib/log-group.ts index 9ed431de8d39a..ef1c71b477a01 100644 --- a/packages/aws-cdk-lib/aws-logs/lib/log-group.ts +++ b/packages/aws-cdk-lib/aws-logs/lib/log-group.ts @@ -1,5 +1,6 @@ import { Construct } from 'constructs'; import { DataProtectionPolicy } from './data-protection-policy'; +import { FieldIndexPolicy } from './field-index-policy'; import { LogStream } from './log-stream'; import { CfnLogGroup } from './logs.generated'; import { MetricFilter } from './metric-filter'; @@ -506,6 +507,13 @@ export interface LogGroupProps { */ readonly dataProtectionPolicy?: DataProtectionPolicy; + /** + * Field Index Policies for this log group. + * + * @default - no field index policies for this log group. + */ + readonly fieldIndexPolicies?: FieldIndexPolicy[]; + /** * How long, in days, the log contents will be retained. * @@ -630,6 +638,12 @@ export class LogGroup extends LogGroupBase { } const dataProtectionPolicy = props.dataProtectionPolicy?._bind(this); + const fieldIndexPolicies: any[] = []; + if (props.fieldIndexPolicies) { + props.fieldIndexPolicies.forEach((fieldIndexPolicy) => { + fieldIndexPolicies.push(fieldIndexPolicy._bind(this)); + }); + } const resource = new CfnLogGroup(this, 'Resource', { kmsKeyId: props.encryptionKey?.keyArn, @@ -643,6 +657,7 @@ export class LogGroup extends LogGroupBase { Statement: dataProtectionPolicy?.statement, Configuration: dataProtectionPolicy?.configuration, } : undefined, + ...(props.fieldIndexPolicies && { fieldIndexPolicies: fieldIndexPolicies }), }); resource.applyRemovalPolicy(props.removalPolicy); diff --git a/packages/aws-cdk-lib/aws-logs/test/loggroup.test.ts b/packages/aws-cdk-lib/aws-logs/test/loggroup.test.ts index c12a7570253bd..722c5ba301597 100644 --- a/packages/aws-cdk-lib/aws-logs/test/loggroup.test.ts +++ b/packages/aws-cdk-lib/aws-logs/test/loggroup.test.ts @@ -4,7 +4,7 @@ import * as iam from '../../aws-iam'; import * as kms from '../../aws-kms'; import { Bucket } from '../../aws-s3'; import { App, CfnParameter, Fn, RemovalPolicy, Stack } from '../../core'; -import { LogGroup, RetentionDays, LogGroupClass, DataProtectionPolicy, DataIdentifier, CustomDataIdentifier, ILogGroup, ILogSubscriptionDestination, FilterPattern } from '../lib'; +import { LogGroup, RetentionDays, LogGroupClass, DataProtectionPolicy, DataIdentifier, CustomDataIdentifier, ILogGroup, ILogSubscriptionDestination, FilterPattern, FieldIndexPolicy } from '../lib'; describe('log group', () => { test('set kms key when provided', () => { @@ -921,6 +921,66 @@ describe('log group', () => { }); }); +test('set field index policy with four fields indexed', () => { + // GIVEN + const stack = new Stack(); + + const fieldIndexPolicy = new FieldIndexPolicy({ + fields: ['Operation', 'RequestId', 'timestamp', 'message'], + }); + + // WHEN + const logGroupName = 'test-field-index-log-group'; + new LogGroup(stack, 'LogGroup', { + logGroupName: logGroupName, + fieldIndexPolicies: [fieldIndexPolicy], + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Logs::LogGroup', { + LogGroupName: logGroupName, + FieldIndexPolicies: [{ + Fields: [ + 'Operation', + 'RequestId', + 'timestamp', + 'message', + ], + }], + }); +}); + +test('set more than 20 field indexes in a field index policy', () => { + let message; + try { + // GIVEN + const stack = new Stack(); + const fieldIndexPolicy = new FieldIndexPolicy({ + fields: createMoreThan20FieldIndexes(), + }); + + // WHEN + const logGroupName = 'test-field-multiple-field-index-policies'; + new LogGroup(stack, 'LogGroup', { + logGroupName: logGroupName, + fieldIndexPolicies: [fieldIndexPolicy], + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Logs::LogGroup', { + LogGroupName: logGroupName, + FieldIndexPolicies: [{ + Fields: ['abc'], + }], + }); + } catch (e) { + message = (e as Error).message; + } + + expect(message).toBeDefined(); + expect(message).toEqual('A maximum of 20 fields can be indexed per log group'); +}); + describe('subscription filter', () => { test('add subscription filter with custom name', () => { // GIVEN @@ -953,6 +1013,14 @@ function dataDrivenTests(cases: string[], body: (suffix: string) => void): void } } +function createMoreThan20FieldIndexes(): string[] { + let arr: string[] = []; + for (let i = 0; i < 23; i++) { + arr.push('abc' + i.toString()); + } + return arr; +} + class FakeDestination implements ILogSubscriptionDestination { public bind(_scope: Construct, _sourceLogGroup: ILogGroup) { return {