diff --git a/osi/blueprint.yaml b/osi/blueprint.yaml new file mode 100644 index 0000000..5e82dfd --- /dev/null +++ b/osi/blueprint.yaml @@ -0,0 +1,51 @@ +version: "2" +ubi-pipeline: + source: + http: + path: "/ubi" + processor: + - date: + from_time_received: true + destination: "@timestamp" + route: + - ubi-events: '/type == "event"' + - ubi-queries: '/type == "query"' + sink: + - opensearch: + hosts: ["https://${aws_opensearch_domain.opensearch_ubi.endpoint}"] + index: "ubi_events" + aws: + sts_role_arn: "${aws_iam_role.ubi.arn}" + region: "${data.aws_region.current.name}" + routes: [ubi-events] + - s3: + aws: + sts_role_arn: "${aws_iam_role.ubi.arn}" + region: "${data.aws_region.current.name}" + bucket: "${aws_s3_bucket.ubi_queries_events_bucket.id}" + object_key: + path_prefix: ubi_events/ + threshold: + event_collect_timeout: "60s" + codec: + ndjson: + routes: [ubi-events] + - opensearch: + hosts: ["https://${aws_opensearch_domain.opensearch_ubi.endpoint}"] + index: "ubi_queries" + aws: + sts_role_arn: "${aws_iam_role.ubi.arn}" + region: "${data.aws_region.current.name}" + routes: [ubi-queries] + - s3: + aws: + sts_role_arn: "${aws_iam_role.ubi.arn}" + region: "${data.aws_region.current.name}" + bucket: "${aws_s3_bucket.ubi_queries_events_bucket.id}" + object_key: + path_prefix: ubi_queries/ + threshold: + event_collect_timeout: "60s" + codec: + ndjson: + routes: [ubi-queries] diff --git a/osi/delete-indexes.sh b/osi/delete-indexes.sh new file mode 100755 index 0000000..573eed8 --- /dev/null +++ b/osi/delete-indexes.sh @@ -0,0 +1,12 @@ +#!/bin/bash -e + +. stack.properties + +OPENSEARCH_ENDPOINT=`terraform output "opensearch_domain_endpoint" | jq -r .` + +awscurl \ + "https://${OPENSEARCH_ENDPOINT}/ubi_events,ubi_queries" \ + -X DELETE \ + --region ${AWS_REGION} \ + --service es \ + --profile ${AWS_PROFILE} diff --git a/osi/get-indexed-events.sh b/osi/get-indexed-events.sh new file mode 100755 index 0000000..629e2b7 --- /dev/null +++ b/osi/get-indexed-events.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e + +. stack.properties + +OPENSEARCH_ENDPOINT=`terraform output "opensearch_domain_endpoint" | jq -r .` + +awscurl \ + "https://${OPENSEARCH_ENDPOINT}/_cat/indices" \ + -X GET \ + --region ${AWS_REGION} \ + --service es \ + --profile ${AWS_PROFILE} + +awscurl \ + "https://${OPENSEARCH_ENDPOINT}/ubi_events/_search" \ + -X GET \ + --region ${AWS_REGION} \ + --service es \ + --profile ${AWS_PROFILE} | jq diff --git a/osi/get-indexed-queries.sh b/osi/get-indexed-queries.sh new file mode 100755 index 0000000..a8d3be0 --- /dev/null +++ b/osi/get-indexed-queries.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e + +. stack.properties + +OPENSEARCH_ENDPOINT=`terraform output "opensearch_domain_endpoint" | jq -r .` + +awscurl \ + "https://${OPENSEARCH_ENDPOINT}/_cat/indices" \ + -X GET \ + --region ${AWS_REGION} \ + --service es \ + --profile ${AWS_PROFILE} + +awscurl \ + "https://${OPENSEARCH_ENDPOINT}/ubi_queries/_search" \ + -X GET \ + --region ${AWS_REGION} \ + --service es \ + --profile ${AWS_PROFILE} | jq diff --git a/osi/main.tf b/osi/main.tf new file mode 100644 index 0000000..9955587 --- /dev/null +++ b/osi/main.tf @@ -0,0 +1,151 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.36" + } + } + + required_version = ">= 1.2.0" +} + +provider "aws" { + region = "us-east-1" + profile = "mtnfog" +} + +data "aws_region" "current" {} +data "aws_caller_identity" "current" {} + +locals { + account_id = data.aws_caller_identity.current.account_id +} + +output "ingest_endpoint_url" { + value = tolist(aws_osis_pipeline.ubi_events_pipeline.ingest_endpoint_urls)[0] +} + +resource "aws_iam_role" "ubi" { + name = "ubiosisrole" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "osis-pipelines.amazonaws.com" + } + } + ] + }) +} + +data "aws_iam_policy_document" "access_policy" { + statement { + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["${aws_iam_role.ubi.arn}"] + } + + actions = ["es:*"] + } +} + +resource "aws_opensearch_domain" "opensearch_ubi" { + + domain_name = "osi-ubi-domain" + engine_version = "OpenSearch_2.17" + + cluster_config { + instance_type = "t3.small.search" + } + + encrypt_at_rest { + enabled = true + } + + domain_endpoint_options { + enforce_https = true + tls_security_policy = "Policy-Min-TLS-1-2-2019-07" + } + + node_to_node_encryption { + enabled = true + } + + ebs_options { + ebs_enabled = true + volume_size = 10 + } + + #access_policies = data.aws_iam_policy_document.access_policy.json +} + +resource "aws_iam_policy" "ubi" { + name = "osis_role_policy" + description = "Policy for OSIS pipeline role" + policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Action = ["es:DescribeDomain"] + Effect = "Allow" + Resource = "arn:aws:es:${data.aws_region.current.name}:${local.account_id}:domain/*" + }, + { + Action = ["es:ESHttp*"] + Effect = "Allow" + Resource = "arn:aws:es:${data.aws_region.current.name}:${local.account_id}:domain/osi-ubi-domain/*" + }, + { + Action = ["s3:PutObject"] + Effect = "Allow" + Resource = "arn:aws:s3:::${aws_s3_bucket.ubi_queries_events_bucket.id}/*" + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "ubi" { + role = aws_iam_role.ubi.name + policy_arn = aws_iam_policy.ubi.arn +} + +resource "aws_cloudwatch_log_group" "ubi" { + name = "/aws/vendedlogs/OpenSearchIngestion/ubi-pipeline" + retention_in_days = 14 + tags = { + Name = "UBI OSIS Pipeline Log Group" + } +} + +resource "aws_s3_bucket" "ubi_queries_events_bucket" { + bucket = "ubi-queries-events-sink" +} + +resource "aws_osis_pipeline" "ubi_events_pipeline" { + pipeline_name = "ubi-pipeline" + pipeline_configuration_body = file("blueprint.yaml") + max_units = 1 + min_units = 1 + log_publishing_options { + is_logging_enabled = true + cloudwatch_log_destination { + log_group = aws_cloudwatch_log_group.ubi.name + } + } + tags = { + Name = "UBI OpenSearch Ingestion Pipeline for UBI" + } +} + +output "opensearch_domain_endpoint" { + value = aws_opensearch_domain.opensearch_ubi.endpoint +} + +output "opensearch_ingest_pipeline_endpoint" { + value = aws_osis_pipeline.ubi_events_pipeline.ingest_endpoint_urls +} \ No newline at end of file diff --git a/osi/requirements.txt b/osi/requirements.txt new file mode 100644 index 0000000..7929f08 --- /dev/null +++ b/osi/requirements.txt @@ -0,0 +1 @@ +awscurl==0.36 \ No newline at end of file diff --git a/osi/send-event.sh b/osi/send-event.sh new file mode 100755 index 0000000..fa001eb --- /dev/null +++ b/osi/send-event.sh @@ -0,0 +1,14 @@ +#!/bin/bash -e + +. stack.properties + +OSIS_PIPELINE_ENDPOINT_URL=`terraform output -json "opensearch_ingest_pipeline_endpoint" | jq -r .[0]` + +awscurl \ + --service osis \ + --region ${AWS_REGION} \ + --profile ${AWS_PROFILE} \ + -X POST \ + -H "Content-Type: application/json" \ + -d '[{"type": "event", "action_name": "click", "query_id": "99999999-4455-6677-8899-aabbccddeeff", "event_attributes": {"position": {"ordinal": 1}, "object": {"object_id": "1234", "object_id_field": "ean", "user_id": "abc"}}}]' \ + https://${OSIS_PIPELINE_ENDPOINT_URL}/ubi diff --git a/osi/send-query.sh b/osi/send-query.sh new file mode 100755 index 0000000..6a91434 --- /dev/null +++ b/osi/send-query.sh @@ -0,0 +1,14 @@ +#!/bin/bash -e + +. stack.properties + +OSIS_PIPELINE_ENDPOINT_URL=`terraform output -json "opensearch_ingest_pipeline_endpoint" | jq -r .[0]` + +awscurl \ + --service osis \ + --region ${AWS_REGION} \ + --profile ${AWS_PROFILE} \ + -X POST \ + -H "Content-Type: application/json" \ + -d '[{"type": "query", "user_query": "computer", "query_id": "00112233-4455-6677-8899-aabbccddeeff"}]' \ + https://${OSIS_PIPELINE_ENDPOINT_URL}/ubi diff --git a/osi/stack.properties b/osi/stack.properties new file mode 100644 index 0000000..a797cad --- /dev/null +++ b/osi/stack.properties @@ -0,0 +1,5 @@ +# The name of the AWS profile. +AWS_PROFILE=mtnfog + +# The AWS region. +AWS_REGION=us-east-1