From a5c586f048eee417e430777604b1a75e5cac154f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Sobo=C5=84?= Date: Thu, 19 Dec 2024 09:27:02 +0100 Subject: [PATCH] [GCP][CDR] Add actor.entity.id and target.entity.id fields to audit logs (#11983) --- packages/gcp/changelog.yml | 5 + .../pipeline/test-audit.log-expected.json | 258 +++++++++++++++++- ...cemanager-googleapis-com.log-expected.json | 15 + ...t-compute-googleapis-com.log-expected.json | 15 + ...edentials-googleapis-com.log-expected.json | 14 + .../pipeline/test-sdh-3695.log-expected.json | 16 +- .../elasticsearch/ingest_pipeline/default.yml | 108 ++++++++ .../data_stream/audit/fields/base-fields.yml | 8 + packages/gcp/docs/README.md | 2 + packages/gcp/docs/audit.md | 2 + packages/gcp/manifest.yml | 2 +- 11 files changed, 439 insertions(+), 6 deletions(-) diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml index 8987791bf8a..01796e2460c 100644 --- a/packages/gcp/changelog.yml +++ b/packages/gcp/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.40.0" + changes: + - description: Add `actor.entity.id` and `target.entity.id` fields to audit logs. + type: enhancement + link: https://github.com/elastic/integrations/pull/11983 - version: "2.39.0" changes: - description: Add `related.entity` field to audit logs. diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json index dc2cb2a31f5..71d4e32a594 100644 --- a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json @@ -53,6 +53,13 @@ "level": "INFO", "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, + "actor": { + "entity": { + "id": [ + "xxx@xxx.xxx" + ] + } + }, "related": { "ip": [ "192.168.1.1" @@ -73,7 +80,14 @@ }, "tags": [ "preserve_original_event" - ] + ], + "target": { + "entity": { + "id": [ + "projects/elastic-beats" + ] + } + } }, { "@timestamp": "2019-12-19T00:45:51.228Z", @@ -137,6 +151,13 @@ "level": "INFO", "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, + "actor": { + "entity": { + "id": [ + "xxx@xxx.xxx" + ] + } + }, "related": { "ip": [ "192.168.1.1" @@ -158,6 +179,13 @@ "tags": [ "preserve_original_event" ], + "target": { + "entity": { + "id": [ + "projects/elastic-beats/global/machineTypes" + ] + } + }, "user_agent": { "device": { "name": "Mac" @@ -246,6 +274,13 @@ "level": "INFO", "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, + "actor": { + "entity": { + "id": [ + "xxx@xxx.xxx" + ] + } + }, "related": { "ip": [ "192.168.1.1" @@ -267,6 +302,13 @@ "tags": [ "preserve_original_event" ], + "target": { + "entity": { + "id": [ + "projects/elastic-beats/global/instances" + ] + } + }, "user_agent": { "device": { "name": "Mac" @@ -342,6 +384,13 @@ "level": "INFO", "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, + "actor": { + "entity": { + "id": [ + "xxx@xxx.xxx" + ] + } + }, "related": { "ip": [ "192.168.1.1" @@ -363,6 +412,13 @@ "tags": [ "preserve_original_event" ], + "target": { + "entity": { + "id": [ + "projects/elastic-beats/global/instances" + ] + } + }, "user_agent": { "device": { "name": "Mac" @@ -485,6 +541,13 @@ }, "type": "kubernetes" }, + "actor": { + "entity": { + "id": [ + "system:serviceaccount:cert-manager:cert-manager-webhook" + ] + } + }, "related": { "ip": [ "10.11.12.13" @@ -609,6 +672,13 @@ "level": "NOTICE", "logger": "projects/foo/logs/cloudaudit.googleapis.com%2Factivity" }, + "actor": { + "entity": { + "id": [ + "user@mycompany.com" + ] + } + }, "related": { "ip": [ "67.43.156.13" @@ -642,6 +712,13 @@ "tags": [ "preserve_original_event" ], + "target": { + "entity": { + "id": [ + "projects/foo/global/images/windows-server-2016-v20200805" + ] + } + }, "user_agent": { "device": { "name": "Mac" @@ -704,6 +781,13 @@ "level": "NOTICE", "logger": "projects/foo/logs/cloudaudit.googleapis.com%2Factivity" }, + "actor": { + "entity": { + "id": [ + "user@mycompany.com" + ] + } + }, "related": { "ip": [ "67.43.156.13" @@ -737,6 +821,13 @@ "tags": [ "preserve_original_event" ], + "target": { + "entity": { + "id": [ + "projects/foo/zones/us-central1-a/instances/win10-test" + ] + } + }, "user_agent": { "device": { "name": "Mac" @@ -811,6 +902,13 @@ }, "type": "kubernetes" }, + "actor": { + "entity": { + "id": [ + "xxx@xxx.xxx" + ] + } + }, "related": { "ip": [ "192.168.1.1" @@ -900,6 +998,13 @@ }, "type": "kubernetes" }, + "actor": { + "entity": { + "id": [ + "xxx@xxx.xxx" + ] + } + }, "related": { "ip": [ "192.168.1.1" @@ -986,6 +1091,13 @@ }, "type": "kubernetes" }, + "actor": { + "entity": { + "id": [ + "system:anonymous" + ] + } + }, "related": { "ip": [ "127.0.0.1" @@ -1070,6 +1182,13 @@ }, "type": "kubernetes" }, + "actor": { + "entity": { + "id": [ + "system:serviceaccount:kube-system:generic-garbage-collector" + ] + } + }, "related": { "ip": [ "::1" @@ -1157,6 +1276,14 @@ "level": "INFO", "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, + "actor": { + "entity": { + "id": [ + "sub", + "xxx@xxx.xxx" + ] + } + }, "related": { "user": [ "xxx@xxx.xxx" @@ -1174,6 +1301,13 @@ "tags": [ "preserve_original_event" ], + "target": { + "entity": { + "id": [ + "projects/project" + ] + } + }, "user_agent": { "device": { "name": "Other" @@ -1300,6 +1434,13 @@ }, "type": "kubernetes" }, + "actor": { + "entity": { + "id": [ + "xxx@xxx.xxx" + ] + } + }, "related": { "entity": [], "ip": [ @@ -1691,6 +1832,13 @@ }, "type": "kubernetes" }, + "actor": { + "entity": { + "id": [ + "system:addon-manager" + ] + } + }, "related": { "entity": [], "ip": [ @@ -1783,6 +1931,14 @@ }, "type": "kubernetes" }, + "actor": { + "entity": { + "id": [ + "serviceAccount:service-xxxx@developer.gserviceaccount.com", + "service-xxxx@developer.gserviceaccount.com" + ] + } + }, "related": { "entity": [ "serviceAccount:service-xxxx@developer.gserviceaccount.com" @@ -1803,6 +1959,13 @@ "tags": [ "preserve_original_event" ], + "target": { + "entity": { + "id": [ + "projects/elastic-product/zones/us-central1-a/clusters/demo-elastic-co" + ] + } + }, "user_agent": { "device": { "name": "Other" @@ -1865,6 +2028,13 @@ "level": "INFO", "logger": "projects/elastic/logs/cloudaudit.googleapis.com%2Fdata_access" }, + "actor": { + "entity": { + "id": [ + "xxx-compute@developer.gserviceaccount.com" + ] + } + }, "related": { "entity": [ "projects/_/buckets/dataflow-staging-us-central1-xxx/objects/staging/jfxrt-xxx.jar", @@ -1880,6 +2050,13 @@ "tags": [ "preserve_original_event" ], + "target": { + "entity": { + "id": [ + "projects/_/buckets/dataflow-staging-us-central1-xxx/objects/staging/jfxrt-xxx.jar" + ] + } + }, "user_agent": { "device": { "name": "Other" @@ -1952,6 +2129,14 @@ }, "type": "kubernetes" }, + "actor": { + "entity": { + "id": [ + "serviceAccount:servoce-xxxx@developer.gserviceaccount.com", + "servoce-xxxx@developer.gserviceaccount.com" + ] + } + }, "related": { "entity": [ "serviceAccount:servoce-xxxx@developer.gserviceaccount.com" @@ -1972,6 +2157,13 @@ "tags": [ "preserve_original_event" ], + "target": { + "entity": { + "id": [ + "projects/elastic-product/zones/us-central1-a/clusters/demo-elastic-co" + ] + } + }, "user_agent": { "device": { "name": "Other" @@ -2038,6 +2230,14 @@ "level": "INFO", "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, + "actor": { + "entity": { + "id": [ + "sub", + "xxx@xxx.xxx" + ] + } + }, "related": { "entity": [ "projects/project", @@ -2055,6 +2255,13 @@ "tags": [ "preserve_original_event" ], + "target": { + "entity": { + "id": [ + "projects/project" + ] + } + }, "user_agent": { "device": { "name": "Other" @@ -2112,6 +2319,13 @@ "level": "INFO", "logger": "projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fsystem_event" }, + "actor": { + "entity": { + "id": [ + "system@google.com" + ] + } + }, "related": { "entity": [ "projects/elastic-siem/zones/us-central1-c/instances/sep-perf-debian-11-155", @@ -2126,7 +2340,14 @@ }, "tags": [ "preserve_original_event" - ] + ], + "target": { + "entity": { + "id": [ + "projects/elastic-siem/zones/us-central1-c/instances/sep-perf-debian-11-155" + ] + } + } }, { "@timestamp": "2021-09-13T03:10:14.801Z", @@ -2210,7 +2431,14 @@ }, "tags": [ "preserve_original_event" - ] + ], + "target": { + "entity": { + "id": [ + "projects/elastic-siem" + ] + } + } }, { "@timestamp": "2022-06-01T11:15:10.842Z", @@ -2289,6 +2517,14 @@ "level": "INFO", "logger": "projects/elastic/logs/cloudaudit.googleapis.com%2Fdata_access" }, + "actor": { + "entity": { + "id": [ + "serviceAccount:servoce-xxxx@developer.gserviceaccount.com", + "servoce-xxxx@developer.gserviceaccount.com" + ] + } + }, "orchestrator": { "cluster": { "name": "demo-elastic-co" @@ -2315,6 +2551,13 @@ "tags": [ "preserve_original_event" ], + "target": { + "entity": { + "id": [ + "projects/elastic-product/zones/us-central1-a/clusters/demo-elastic-co" + ] + } + }, "user_agent": { "device": { "name": "Other" @@ -2382,7 +2625,14 @@ }, "tags": [ "preserve_original_event" - ] + ], + "target": { + "entity": { + "id": [ + "projects/elastic-siem/zones/us-central1-c/clusters/endpoint-gke-cluster" + ] + } + } } ] } diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log-expected.json index 2987836a147..b0d0a2f8996 100644 --- a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log-expected.json +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log-expected.json @@ -139,6 +139,14 @@ "level": "NOTICE", "logger": "projects/project-id/logs/cloudaudit.googleapis.com%2Factivity" }, + "actor": { + "entity": { + "id": [ + "serviceAccount:madeupprincipal@project-id.iam.gserviceaccount.com", + "made-up-ci-account@project-id.iam.gserviceaccount.com" + ] + } + }, "related": { "entity": [ "projects/project-id/roles/x", @@ -182,6 +190,13 @@ "tags": [ "preserve_original_event" ], + "target": { + "entity": { + "id": [ + "projects/project-id" + ] + } + }, "user_agent": { "device": { "name": "Other" diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log-expected.json index 2d874cc183e..392169743ae 100644 --- a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log-expected.json +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log-expected.json @@ -206,6 +206,14 @@ "level": "NOTICE", "logger": "projects/project-id/logs/cloudaudit.googleapis.com%2Factivity" }, + "actor": { + "entity": { + "id": [ + "serviceAccount:project-id@cloudservices.gserviceaccount.com", + "project-id@cloudservices.gserviceaccount.com" + ] + } + }, "related": { "entity": [ "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/x-logs-network", @@ -243,6 +251,13 @@ "tags": [ "preserve_original_event" ], + "target": { + "entity": { + "id": [ + "projects/project-id/zones/us-central1-a/instances/x-logs" + ] + } + }, "user_agent": { "device": { "name": "Other" diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log-expected.json index 3c1d622e611..b672c0a8151 100644 --- a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log-expected.json +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log-expected.json @@ -61,6 +61,13 @@ "level": "INFO", "logger": "projects/project-id/logs/cloudaudit.googleapis.com%2Fdata_access" }, + "actor": { + "entity": { + "id": [ + "principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/..." + ] + } + }, "related": { "entity": [ "projects/-/serviceAccounts/made-up-ci-account@project-id.iam.gserviceaccount.com", @@ -92,6 +99,13 @@ "tags": [ "preserve_original_event" ], + "target": { + "entity": { + "id": [ + "projects/-/serviceAccounts/somenumber" + ] + } + }, "user_agent": { "device": { "name": "Other" diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-sdh-3695.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-sdh-3695.log-expected.json index 1e08be1feda..c14765499de 100644 --- a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-sdh-3695.log-expected.json +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-sdh-3695.log-expected.json @@ -39,6 +39,13 @@ "level": "INFO", "logger": "organizations/123456789098/logs/cloudaudit.googleapis.com%2Fdata_access" }, + "actor": { + "entity": { + "id": [ + "joel.miller@contoso.com" + ] + } + }, "related": { "entity": [ "organizations/123456789098", @@ -53,7 +60,14 @@ }, "tags": [ "preserve_original_event" - ] + ], + "target": { + "entity": { + "id": [ + "organizations/123456789098" + ] + } + } } ] } diff --git a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 5db73156fd8..cc8aa28cbdd 100644 --- a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -194,6 +194,114 @@ processors: ctx.related = ctx.related ?: [:]; ctx.related.entity = entities; +## +# Extract origin, target, and related entity IDs +## + - append: + field: actor.entity.id + value: "{{{json.protoPayload.authenticationInfo.principalSubject}}}" + if: ctx.json?.protoPayload?.authenticationInfo?.principalSubject != null + - append: + field: actor.entity.id + value: "{{{json.protoPayload.authenticationInfo.principalEmail}}}" + if: ctx.json?.protoPayload?.authenticationInfo?.principalEmail != null + - append: + field: target.entity.id + value: "{{{json.protoPayload.resourceName}}}" + if: ctx.json?.resource?.type != 'k8s_cluster' && ctx.json?.protoPayload?.resourceName != null + - append: + field: target.entity.id + value: "{{{json.resource.disk_id}}}" + if: ctx.json?.resouce?.disk_id != null + - script: + description: Appends any relevant entity to `related.entity` for all events + lang: painless + on_failure: + - set: + description: Adds error reason to the document + field: error.message + value: "{{{ _ingest.on_failure_message }}}" + source: | + void addValue(Set entities, def value) { + if (value != null && value != "") { + entities.add(value); + } + } + + boolean isKubernetes = false; + if (ctx.json?.resource?.type != null) { + String typ = ctx.json.resource.type; + isKubernetes = (typ == "k8s_cluster" || typ == "gke_cluster" || typ == "kubernetes"); + } + + // Using tree set to ensure a sorting is kept (testing purposes) + TreeSet entities = new TreeSet(); + + addValue(entities, ctx.json?.protoPayload?.request?.parent); + if (!isKubernetes) { + addValue(entities, ctx.json?.protoPayload?.resourceName); + addValue(entities, ctx.json?.protoPayload?.response?.user); + } + + HashMap authInfo = ctx.json?.protoPayload?.authenticationInfo ?: new HashMap(); + if (!isKubernetes) { + addValue(entities, authInfo.principalEmail); + } + addValue(entities, authInfo.principalSubject); + addValue(entities, authInfo.serviceAccountKeyName); + if (authInfo.serviceAccountDelegationInfo instanceof List) { + for (def i: authInfo.serviceAccountDelegationInfo) { + addValue(entities, i.principalSubject); + addValue(entities, i.firstPartyPrincipal?.principalEmail); + addValue(entities, i.thirdPartyPrincipal?.principalEmail); + } + } + + String serviceName = ctx.json?.protoPayload?.serviceName ?: ''; + if (serviceName == "compute.googleapis.com") { + if (ctx.json?.protoPayload?.request?.networkInterfaces instanceof List) { + for (def e: ctx.json.protoPayload.request.networkInterfaces) { + addValue(entities, e.network); + } + } + if (ctx.json?.protoPayload?.request?.serviceAccounts instanceof List) { + for (def e: ctx.json.protoPayload.request.serviceAccounts) { + addValue(entities, e.email); + } + } + if (ctx.json?.protoPayload?.request?.disks instanceof List) { + for (def e: ctx.json.protoPayload.request.disks) { + addValue(entities, e.source); + } + } + } else if (serviceName == "cloudresourcemanager.googleapis.com") { + if (ctx.json?.protoPayload?.request?.policy?.bindings instanceof List) { + for (def e: ctx.json.protoPayload.request.policy.bindings) { + addValue(entities, e.role); + for (def m: e.members) { + addValue(entities, m); + } + } + } + if (ctx.json?.protoPayload?.response?.bindings instanceof List) { + for (def e: ctx.json.protoPayload.response.bindings) { + addValue(entities, e.role); + for (def m: e.members) { + addValue(entities, m); + } + } + } + } else if (serviceName == "iamcredentials.googleapis.com") { + if (ctx.json?.protoPayload?.metadata?.identityDelegationChain instanceof List) { + for (def e: ctx.json.protoPayload.metadata.identityDelegationChain) { + addValue(entities, e); + } + } + } + + ctx.related = ctx.related ?: [:]; + ctx.related.entity = entities; + ## # AuthenticationInfo # .protoPayload.authenticationInfo diff --git a/packages/gcp/data_stream/audit/fields/base-fields.yml b/packages/gcp/data_stream/audit/fields/base-fields.yml index 529f2bfaeee..145bb85e959 100644 --- a/packages/gcp/data_stream/audit/fields/base-fields.yml +++ b/packages/gcp/data_stream/audit/fields/base-fields.yml @@ -18,6 +18,10 @@ type: constant_keyword description: Event dataset value: gcp.audit +- name: actor.entity.id + description: | + ID or multiple IDs of the entity performing the action described by the event. + type: keyword - name: related.entity description: | A collection of all entity identifiers associated with the document. @@ -25,3 +29,7 @@ Example identifiers include (but not limited to) cloud resource IDs, email addresses, and hostnames. type: keyword +- name: target.entity.id + description: | + ID or multiple IDs of the entity targeted by the action described by the event. + type: keyword diff --git a/packages/gcp/docs/README.md b/packages/gcp/docs/README.md index 040dea358f3..5c36a22a04c 100644 --- a/packages/gcp/docs/README.md +++ b/packages/gcp/docs/README.md @@ -224,6 +224,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | +| actor.entity.id | ID or multiple IDs of the entity performing the action described by the event. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | @@ -289,6 +290,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | input.type | Input type | keyword | | log.offset | Log offset | long | | related.entity | A collection of all entity identifiers associated with the document. If the document contains multiple entities, identifiers for each will be included. Example identifiers include (but not limited to) cloud resource IDs, email addresses, and hostnames. | keyword | +| target.entity.id | ID or multiple IDs of the entity targeted by the action described by the event. | keyword | An example event for `audit` looks as following: diff --git a/packages/gcp/docs/audit.md b/packages/gcp/docs/audit.md index c2d8be56e80..6475998f0b3 100644 --- a/packages/gcp/docs/audit.md +++ b/packages/gcp/docs/audit.md @@ -13,6 +13,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | +| actor.entity.id | ID or multiple IDs of the entity performing the action described by the event. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | @@ -78,6 +79,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | input.type | Input type | keyword | | log.offset | Log offset | long | | related.entity | A collection of all entity identifiers associated with the document. If the document contains multiple entities, identifiers for each will be included. Example identifiers include (but not limited to) cloud resource IDs, email addresses, and hostnames. | keyword | +| target.entity.id | ID or multiple IDs of the entity targeted by the action described by the event. | keyword | An example event for `audit` looks as following: diff --git a/packages/gcp/manifest.yml b/packages/gcp/manifest.yml index 539c0e69fdb..ac90ee00680 100644 --- a/packages/gcp/manifest.yml +++ b/packages/gcp/manifest.yml @@ -1,6 +1,6 @@ name: gcp title: Google Cloud Platform -version: "2.39.0" +version: "2.40.0" description: Collect logs and metrics from Google Cloud Platform with Elastic Agent. type: integration icons: