diff --git a/.github/workflows/user-list-search.yml b/.github/workflows/user-list-search.yml index 7183b1bf0..a4473ea77 100644 --- a/.github/workflows/user-list-search.yml +++ b/.github/workflows/user-list-search.yml @@ -78,19 +78,6 @@ jobs: codedeploy-group-name-pattern: UserListSearch-{0}-EventHandler function-name-pattern: UserListSearch-{0}-EventHandler secrets: inherit - - kinesis-lambda: - uses: ./.github/workflows/reuse-build-and-push-lambda.yml - needs: [infrastructure] - with: - scope: user-list-search-kinesis-to-sqs - sentry-project: user-list-search - s3-bucket-pattern: pocket-userlistsearch-{0}-kinesis-consumer - s3-key: kinesis-${{ github.sha }}.zip - codedeploy-app-name-pattern: UserListSearch-{0}-UnifiedEventsConsumer - codedeploy-group-name-pattern: UserListSearch-{0}-UnifiedEventsConsumer - function-name-pattern: UserListSearch-{0}-UnifiedEventsConsumer - secrets: inherit item-update-lambda: uses: ./.github/workflows/reuse-build-and-push-lambda.yml @@ -189,7 +176,6 @@ jobs: needs: - api - events-lambda - - kinesis-lambda - item-update-lambda - item-delete-lambda - item-update-backfill-lambda diff --git a/infrastructure/user-list-search/kinesis_consumer.tf b/infrastructure/user-list-search/kinesis_consumer.tf deleted file mode 100644 index 73d40c10c..000000000 --- a/infrastructure/user-list-search/kinesis_consumer.tf +++ /dev/null @@ -1,180 +0,0 @@ -locals { - ue_function_name = "${local.prefix}-UnifiedEventsConsumer" -} - -resource "aws_lambda_function" "unified_events_consumer" { - function_name = local.ue_function_name - filename = data.archive_file.lambda_zip.output_path #Dummy lambda that just logs the event. - role = aws_iam_role.lambda_role.arn - runtime = "nodejs20.x" - handler = "index.handler" - source_code_hash = data.archive_file.lambda_zip.output_base64sha256 #Dummy lambda that just logs the event. - # depends_on = [aws_cloudwatch_log_group.unified_events_consumer] - timeout = 300 - environment { - variables = local.lambda_env - } - tags = local.tags - publish = true # We need to publish an initial version - memory_size = 256 - lifecycle { - ignore_changes = [ - environment["GIT_SHA"], - filename, - source_code_hash - ] - } - - tracing_config { - mode = "Active" - } -} - -resource "aws_cloudwatch_log_group" "unified_events_consumer" { - name = "/aws/lambda/${local.ue_function_name}" - retention_in_days = 14 -} - -resource "aws_lambda_alias" "unified_events_consumer" { - function_name = aws_lambda_function.unified_events_consumer.function_name - function_version = split(":", aws_lambda_function.unified_events_consumer.qualified_arn)[7] - name = "DEPLOYED" - lifecycle { - ignore_changes = [ - //ignore so that code deploy can change this app - function_version - ] - } -} - -resource "aws_lambda_event_source_mapping" "kinesis_consumer" { - event_source_arn = data.aws_kinesis_stream.unified.arn - function_name = aws_lambda_alias.unified_events_consumer.arn #We set the function to our alias - starting_position = "LATEST" - batch_size = 10000 - maximum_record_age_in_seconds = 60 - enabled = true - lifecycle { - ignore_changes = [enabled] - } -} - -resource "aws_iam_role" "lambda_role" { - name = "${local.prefix}-LambdaExecutionRole" - tags = local.tags - assume_role_policy = data.aws_iam_policy_document.lambda_assume.json -} - -resource "aws_iam_role_policy_attachment" "lambda_role_xray_write" { - role = aws_iam_role.lambda_role.name - policy_arn = data.aws_iam_policy.aws_xray_write_only_access.arn -} - -resource "aws_iam_role_policy" "lambda_execution_policy" { - name = "${local.prefix}-KinesisEventAccessPolicy" - role = aws_iam_role.lambda_role.id - policy = data.aws_iam_policy_document.lambda_execution_policy.json -} - -data "aws_iam_policy_document" "lambda_assume" { - version = "2012-10-17" - - statement { - effect = "Allow" - actions = [ - "sts:AssumeRole" - ] - - principals { - identifiers = [ - "lambda.amazonaws.com" - ] - - type = "Service" - } - } -} - -data "aws_iam_policy_document" "lambda_execution_policy" { - version = "2012-10-17" - - statement { - effect = "Allow" - actions = [ - "sqs:SendMessage", - "sqs:SendMessageBatch", - ] - resources = [ - aws_sqs_queue.user_items_update.arn, - aws_sqs_queue.user_list_import.arn, - aws_sqs_queue.user_items_delete.arn - ] - } - - statement { - effect = "Allow" - actions = [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents", - "logs:DescribeLogStreams" - ] - resources = [ - "arn:aws:logs:*:*:*" - ] - } - - statement { - effect = "Allow" - actions = [ - "kinesis:ListStreams", - "kinesis:ListShards", - "kinesis:DescribeLimits", - "kinesis:ListStreamConsumers" - ] - resources = [ - "*" - ] - } - - statement { - effect = "Allow" - actions = [ - "kinesis:SubscribeToShard", - "kinesis:Describe*", - "kinesis:Get*", - "kinesis:ListTagsForStream" - ] - resources = [ - data.aws_kinesis_stream.unified.arn - ] - } - - statement { - effect = "Allow" - actions = [ - "ssm:GetParameter*" - ] - - resources = [ - "arn:aws:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:parameter/${local.name}/${local.env}", - "arn:aws:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:parameter/${local.name}/${local.env}/*" - ] - } - - statement { - effect = "Allow" - actions = [ - "secretsmanager:GetSecretValue", - "kms:Decrypt" - ] - - resources = [ - "arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:${local.name}/${local.env}", - "arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:${local.name}/${local.env}/*", - "arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:${local.name}/Default", - "arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:${local.name}/Default/*", - data.aws_kms_alias.secrets_manager.target_key_arn - ] - } -} diff --git a/infrastructure/user-list-search/kinesis_consumer_codedeploy.tf b/infrastructure/user-list-search/kinesis_consumer_codedeploy.tf deleted file mode 100644 index f96341898..000000000 --- a/infrastructure/user-list-search/kinesis_consumer_codedeploy.tf +++ /dev/null @@ -1,64 +0,0 @@ -resource "aws_codedeploy_app" "lambda_unified_events_consumer" { - compute_platform = "Lambda" - name = "${local.prefix}-UnifiedEventsConsumer" -} - -resource "aws_iam_role" "lambda_codedeploy_role" { - name = "${local.prefix}-LambdaCodeDeployRole" - assume_role_policy = data.aws_iam_policy_document.codedeploy_assume_role.json -} - -resource "aws_iam_role_policy_attachment" "lambda_codedeploy_role" { - policy_arn = "arn:aws:iam::aws:policy/service-role/AWSCodeDeployRoleForLambda" - #Depending on the service there are different types. - role = aws_iam_role.lambda_codedeploy_role.name -} - -resource "aws_codedeploy_deployment_group" "lambda_unified_events_consumer" { - app_name = aws_codedeploy_app.lambda_unified_events_consumer.name - deployment_config_name = "CodeDeployDefault.LambdaAllAtOnce" - deployment_group_name = "${local.prefix}-UnifiedEventsConsumer" - service_role_arn = aws_iam_role.lambda_codedeploy_role.arn - - deployment_style { - deployment_type = "BLUE_GREEN" - deployment_option = "WITH_TRAFFIC_CONTROL" - } - - auto_rollback_configuration { - enabled = true - events = [ - "DEPLOYMENT_FAILURE" - ] - } -} - -resource "aws_codestarnotifications_notification_rule" "lambda_unified_events_consumer_notifications" { - detail_type = "BASIC" - event_type_ids = [ - "codedeploy-application-deployment-failed", - ] - - name = aws_codedeploy_app.lambda_unified_events_consumer.name - resource = "arn:aws:codedeploy:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:application:${aws_codedeploy_app.lambda_unified_events_consumer.name}" - - target { - address = data.aws_sns_topic.backend-deploy-topic.arn - } -} - -resource "aws_s3_bucket" "lambda_unified_events_consumer_code_bucket" { - bucket = "pocket-${lower(local.prefix)}-kinesis-consumer" - tags = local.tags -} - -resource "aws_s3_bucket_acl" "lambda_unified_events_consumer_code_bucket" { - acl = "private" - bucket = aws_s3_bucket.lambda_unified_events_consumer_code_bucket.id -} - -resource "aws_s3_bucket_public_access_block" "lambda_unified_events_consumer_code_bucket" { - bucket = aws_s3_bucket.lambda_unified_events_consumer_code_bucket.id - block_public_acls = true - block_public_policy = true -} diff --git a/infrastructure/user-list-search/lambda_bucket.tf b/infrastructure/user-list-search/lambda_bucket.tf new file mode 100644 index 000000000..f63e8e7c2 --- /dev/null +++ b/infrastructure/user-list-search/lambda_bucket.tf @@ -0,0 +1,16 @@ +# NOTE: The bucket is called kinesis-consumer, because that was the old name of the lambdas deployed from here. +resource "aws_s3_bucket" "lambda_unified_events_consumer_code_bucket" { + bucket = "pocket-${lower(local.prefix)}-kinesis-consumer" + tags = local.tags +} + +resource "aws_s3_bucket_acl" "lambda_unified_events_consumer_code_bucket" { + acl = "private" + bucket = aws_s3_bucket.lambda_unified_events_consumer_code_bucket.id +} + +resource "aws_s3_bucket_public_access_block" "lambda_unified_events_consumer_code_bucket" { + bucket = aws_s3_bucket.lambda_unified_events_consumer_code_bucket.id + block_public_acls = true + block_public_policy = true +} diff --git a/infrastructure/user-list-search/lambda_codedeploy.tf b/infrastructure/user-list-search/lambda_codedeploy.tf new file mode 100644 index 000000000..59d2f3047 --- /dev/null +++ b/infrastructure/user-list-search/lambda_codedeploy.tf @@ -0,0 +1,41 @@ +resource "aws_iam_role" "lambda_codedeploy_role" { + name = "${local.prefix}-LambdaCodeDeployRole" + assume_role_policy = data.aws_iam_policy_document.codedeploy_assume_role.json +} + + +resource "aws_iam_role_policy_attachment" "lambda_codedeploy_role" { + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSCodeDeployRoleForLambda" + #Depending on the service there are different types. + role = aws_iam_role.lambda_codedeploy_role.name +} + +resource "aws_iam_role" "lambda_role" { + name = "${local.prefix}-LambdaExecutionRole" + tags = local.tags + assume_role_policy = data.aws_iam_policy_document.lambda_assume.json +} + +resource "aws_iam_role_policy_attachment" "lambda_role_xray_write" { + role = aws_iam_role.lambda_role.name + policy_arn = data.aws_iam_policy.aws_xray_write_only_access.arn +} + +data "aws_iam_policy_document" "lambda_assume" { + version = "2012-10-17" + + statement { + effect = "Allow" + actions = [ + "sts:AssumeRole" + ] + + principals { + identifiers = [ + "lambda.amazonaws.com" + ] + + type = "Service" + } + } +} diff --git a/infrastructure/user-list-search/locals.tf b/infrastructure/user-list-search/locals.tf index 0f3ddaf60..d8cf000eb 100644 --- a/infrastructure/user-list-search/locals.tf +++ b/infrastructure/user-list-search/locals.tf @@ -56,6 +56,7 @@ locals { userEvents = local.workspace.sns_topic_user_events corpusEvents = local.workspace.sns_topic_corpus_events collectionEvents = local.workspace.sns_topic_collection_events + listEvents = local.workspace.sns_topic_list_events } # environment or workspace-specific local variables go here. @@ -80,6 +81,7 @@ locals { sns_topic_user_events = "PocketEventBridge-Dev-UserEvents" sns_topic_corpus_events = "PocketEventBridge-Dev-CorpusEvents" sns_topic_collection_events = "PocketEventBridge-Dev-CollectionEvents" + sns_topic_list_events = "PocketEventBridge-Dev-ListEvents" userApiUri = "https://user-list-search.getpocket.dev" otlpCollectorUrl = "https://otel-collector.getpocket.dev:443" } @@ -97,6 +99,7 @@ locals { sns_topic_user_events = "PocketEventBridge-Prod-UserEvents" sns_topic_corpus_events = "PocketEventBridge-Prod-CorpusEvents" sns_topic_collection_events = "PocketEventBridge-Prod-CollectionEvents" + sns_topic_list_events = "PocketEventBridge-Prod-ListEvents" userApiUri = "https://user-list-search.readitlater.com" otlpCollectorUrl = "https://otel-collector.readitlater.com:443" } diff --git a/infrastructure/user-list-search/metrics.tf b/infrastructure/user-list-search/metrics.tf index 5f7006395..fb024cf2f 100644 --- a/infrastructure/user-list-search/metrics.tf +++ b/infrastructure/user-list-search/metrics.tf @@ -246,25 +246,6 @@ module "dashboard_alarm" { merge(local.metrics.list_item_update_lambda.throttles, { metadata = { color = "#ff7f0e", yAxis = "right" } }), ] }, - { - x = 0.0 - y = 24.0 - width = 12.0 - height = 6.0 - properties = { - title = "Event Kinesis Consumer" - stacked = false - region = data.aws_region.current.name, - stat = "Average" - period = 60 - } - metrics = [ - local.metrics.event_consumer_lambda.duration, - local.metrics.event_consumer_lambda.errors, - merge(local.metrics.event_consumer_lambda.iterator_age, { metadata = { yAxis = "right" } }) - ] - - }, { x = 12.0 y = 24.0 diff --git a/infrastructure/user-list-search/metrics_alarm_definitions.tf b/infrastructure/user-list-search/metrics_alarm_definitions.tf index 322f93751..1f07ee3f1 100644 --- a/infrastructure/user-list-search/metrics_alarm_definitions.tf +++ b/infrastructure/user-list-search/metrics_alarm_definitions.tf @@ -118,27 +118,6 @@ locals { ok_actions = [] alarm_actions = [] } - - event_consumer_lambda_errors = { - name = "${local.prefix}-EventConsumerLambdaErrors" - description = "More than 1 error for 3 consecutive minutes" - - metrics = [ - local.metrics.event_consumer_lambda.duration, - local.metrics.event_consumer_lambda.iterator_age, - local.metrics.event_consumer_lambda.errors - ] - - threshold = 10 - operator = ">" - return_data_on_id = local.metrics.event_consumer_lambda.errors.id - // The kinesis consumer lambda that we listen on for item updates has more then 10 errors for 10 consecutive minutes - period = 60 - breaches = 10 - tags = local.tags - ok_actions = [] - alarm_actions = [] - } } # TODO: EventHandler metrics diff --git a/infrastructure/user-list-search/metrics_metric_definitions.tf b/infrastructure/user-list-search/metrics_metric_definitions.tf index 51c38ea42..44e09adac 100644 --- a/infrastructure/user-list-search/metrics_metric_definitions.tf +++ b/infrastructure/user-list-search/metrics_metric_definitions.tf @@ -70,35 +70,6 @@ locals { expression = "IF(user_list_import_queue_messages_deleted, user_list_import_queue_messages_deleted, 1)/IF(user_list_import_queue_messages_sent, user_list_import_queue_messages_sent, 1)*100", } } - event_consumer_lambda = { - duration = { - id = "event_consumer_lambda_duration" - namespace = "AWS/Lambda" - metric = "Duration" - statistic = "Sum" - dimensions = { - FunctionName = aws_lambda_function.unified_events_consumer.function_name - } - }, - errors = { - id = "event_consumer_lambda_errors" - namespace = "AWS/Lambda" - metric = "Errors" - statistic = "Sum" - dimensions = { - FunctionName = aws_lambda_function.unified_events_consumer.function_name - } - }, - iterator_age = { - id = "event_consumer_lambda_iterator_age" - namespace = "AWS/Lambda" - metric = "IteratorAge" - statistic = "Sum" - dimensions = { - FunctionName = aws_lambda_function.unified_events_consumer.function_name - } - } - } list_item_import_lambda = { invocations = { diff --git a/infrastructure/user-list-search/sqs_sns_topic_subscription.tf b/infrastructure/user-list-search/sqs_sns_topic_subscription.tf index 3110fccdd..ffc7528bc 100644 --- a/infrastructure/user-list-search/sqs_sns_topic_subscription.tf +++ b/infrastructure/user-list-search/sqs_sns_topic_subscription.tf @@ -1,5 +1,6 @@ locals { userEventsSnsTopicArn = "arn:aws:sns:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:${local.snsTopicName.userEvents}" + listEventsSnsTopicArn = "arn:aws:sns:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:${local.snsTopicName.listEvents}" corpusEventsSnsTopicArn = "arn:aws:sns:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:${local.snsTopicName.corpusEvents}" collectionEventsSnsTopicArn = "arn:aws:sns:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:${local.snsTopicName.collectionEvents}" } @@ -28,6 +29,18 @@ resource "aws_sns_topic_subscription" "user_events_sns_topic_subscription" { depends_on = [aws_sqs_queue.user_events_sns_topic_dlq, aws_sqs_queue.user_list_events] } +resource "aws_sns_topic_subscription" "list_events_sns_topic_subscription" { + topic_arn = local.listEventsSnsTopicArn + protocol = "sqs" + endpoint = aws_sqs_queue.user_list_events.arn + # The version of terraform used in this service does not support redrive_policy + # an update is required. It is not blocking but should be prioritized soon + # redrive_policy = jsonencode({ + # deadLetterTargetArn: aws_sqs_queue.user_events_sns_topic_dlq.arn + # }) + depends_on = [aws_sqs_queue.user_events_sns_topic_dlq, aws_sqs_queue.user_list_events] +} + resource "aws_sns_topic_subscription" "corpus_events_sns_topic_subscription" { topic_arn = local.corpusEventsSnsTopicArn protocol = "sqs" @@ -101,7 +114,7 @@ data "aws_iam_policy_document" "user_events_sqs_policy_document" { } condition { test = "ArnEquals" - values = [local.userEventsSnsTopicArn] + values = [local.userEventsSnsTopicArn, local.listEventsSnsTopicArn] variable = "aws:SourceArn" } } @@ -198,7 +211,7 @@ data "aws_iam_policy_document" "user_events_sns_topic_dlq_policy_document" { } condition { test = "ArnEquals" - values = [local.userEventsSnsTopicArn] + values = [local.userEventsSnsTopicArn, local.listEventsSnsTopicArn] variable = "aws:SourceArn" } } diff --git a/lambdas/user-list-search-events/config.ts b/lambdas/user-list-search-events/config.ts deleted file mode 100644 index 95a5ad530..000000000 --- a/lambdas/user-list-search-events/config.ts +++ /dev/null @@ -1,9 +0,0 @@ -export const config = { - endpoint: process.env.USER_LIST_SEARCH_URI || 'http://localhost:4000', - accountDeletePath: '/batchDelete', - sentry: { - dsn: process.env.SENTRY_DSN || '', - release: process.env.GIT_SHA || '', - environment: process.env.NODE_ENV || 'development', - }, -}; diff --git a/lambdas/user-list-search-events/package.json b/lambdas/user-list-search-events/package.json index c965bab33..f6f8f789f 100644 --- a/lambdas/user-list-search-events/package.json +++ b/lambdas/user-list-search-events/package.json @@ -15,8 +15,10 @@ "test": "jest \"\\.spec\\.ts\" --runInBand --forceExit" }, "dependencies": { + "@aws-sdk/client-sqs": "3.716.0", "@pocket-tools/event-bridge": "workspace:*", "@pocket-tools/ts-logger": "workspace:*", + "@pocket-tools/types": "workspace:*", "@sentry/aws-serverless": "8.47.0", "tslib": "2.8.0" }, diff --git a/lambdas/user-list-search-events/src/accountDeleteHandler.spec.ts b/lambdas/user-list-search-events/src/accountDeleteHandler.spec.ts new file mode 100644 index 000000000..eff342220 --- /dev/null +++ b/lambdas/user-list-search-events/src/accountDeleteHandler.spec.ts @@ -0,0 +1,48 @@ +import { config } from './config'; +import nock from 'nock'; +import { SQSRecord } from 'aws-lambda'; +import { __handler } from './index'; +import { serverLogger } from '@pocket-tools/ts-logger'; +import { PocketEventType } from '@pocket-tools/event-bridge'; + +describe('accountDelete handler', () => { + let serverLoggerSpy: jest.SpyInstance; + beforeEach(() => { + serverLoggerSpy = jest.spyOn(serverLogger, 'error'); + nock(config.endpoint) + .post(config.accountDeletePath) + .reply(400, { errors: ['this is an error'] }); + }); + it('return a failed message if response is not ok', async () => { + const record = { + messageId: '123', + body: JSON.stringify({ + Message: JSON.stringify({ + 'detail-type': PocketEventType.ACCOUNT_DELETION, + source: 'user-event', + account: '123456789012', + id: '1234567890', + region: 'us-east-2', + time: '2022-09-10T17:29:22Z', + version: '0', + detail: { + eventType: 'account-deletion', + version: '12', + timestamp: 123456789, + userId: 1, + email: 'test@me.com', + apiId: '1', + isPremium: 'true', + }, + }), + }), + }; + + const messages = await __handler({ Records: [record] as SQSRecord[] }); + + expect(messages.batchItemFailures).toHaveLength(1); + expect(messages.batchItemFailures[0].itemIdentifier).toEqual('123'); + expect(nock.isDone()).toBeTruthy(); + expect(serverLoggerSpy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/lambdas/user-list-search-events/src/accountDeleteHandler.ts b/lambdas/user-list-search-events/src/accountDeleteHandler.ts new file mode 100644 index 000000000..996177ce3 --- /dev/null +++ b/lambdas/user-list-search-events/src/accountDeleteHandler.ts @@ -0,0 +1,54 @@ +import { config } from './config'; +import { AccountDelete, IncomingBaseEvent } from '@pocket-tools/event-bridge'; +import { PocketEventRecord } from './handlerMap'; +import * as Sentry from '@sentry/aws-serverless'; +import { serverLogger } from '@pocket-tools/ts-logger'; + +type AccountDeleteEvent = Exclude & { + pocketEvent: AccountDelete & IncomingBaseEvent; +}; + +/** + * Given an account delete event, call the batchDelete endpoint on the + * user-list-search to delete all indexes associated with the user. + * @param record SQSRecord containing forwarded event from eventbridge + * @throws Error if response is not ok + */ +export async function accountDeleteHandler( + event: AccountDeleteEvent[], +): Promise { + const promises = event.map((e) => { + return proccessAccountDeleteEvent(e); + }); + + const failedEventIds = await Promise.all(promises); + return failedEventIds.filter((id) => id !== null) as string[]; +} + +async function proccessAccountDeleteEvent( + event: AccountDeleteEvent, +): Promise { + const postBody = { + userId: event.pocketEvent.detail.userId, + }; + if (event.pocketEvent.detail.traceId !== null) { + postBody['traceId'] = event.pocketEvent.detail.traceId; + } + const res = await fetch(config.endpoint + config.accountDeletePath, { + method: 'post', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(postBody), + }); + if (!res.ok) { + const data = (await res.json()) as any; + const error = new Error( + `batchDelete - ${res.status}\n${JSON.stringify(data.errors)}`, + ); + Sentry.captureException(error); + serverLogger.error(error); + return event.messageId; + } + return null; +} diff --git a/lambdas/user-list-search-kinesis-to-sqs/src/config/index.ts b/lambdas/user-list-search-events/src/config/index.ts similarity index 90% rename from lambdas/user-list-search-kinesis-to-sqs/src/config/index.ts rename to lambdas/user-list-search-events/src/config/index.ts index 5a9952182..afe58bb29 100644 --- a/lambdas/user-list-search-kinesis-to-sqs/src/config/index.ts +++ b/lambdas/user-list-search-events/src/config/index.ts @@ -5,10 +5,11 @@ const localAwsEndpoint = : undefined; export const config = { + endpoint: process.env.USER_LIST_SEARCH_URI || 'http://localhost:4000', + accountDeletePath: '/batchDelete', aws: { region: process.env.AWS_REGION || 'us-east-1', sqs: { - waitTimeSeconds: 20, endpoint: localAwsEndpoint, userItemsUpdateUrl: process.env.SQS_USER_ITEMS_UPDATE_URL || diff --git a/lambdas/user-list-search-events/src/handlerMap.ts b/lambdas/user-list-search-events/src/handlerMap.ts new file mode 100644 index 000000000..13e8073e4 --- /dev/null +++ b/lambdas/user-list-search-events/src/handlerMap.ts @@ -0,0 +1,33 @@ +import { + IncomingBaseEvent, + PocketEvent, + PocketEventType, +} from '@pocket-tools/event-bridge'; +import { accountDeleteHandler } from './accountDeleteHandler'; +import { premiumPurchaseHandler } from './premiumPurchaseHandler'; +import { itemUpdateHandler } from './itemUpdateHandler'; +import { itemDeleteHandler } from './itemDeleteHandler'; + +export interface PocketEventRecord { + messageId: string; + pocketEvent: PocketEvent & IncomingBaseEvent; +} + +// Mapping of detail-type (via event bridge message) +// to function that should be invoked to process the message +export const handlerMap: { + [key: string]: (event: PocketEventRecord[]) => Promise; +} = { + [PocketEventType.ACCOUNT_DELETION]: accountDeleteHandler, + [PocketEventType.PREMIUM_PURCHASE]: premiumPurchaseHandler, + [PocketEventType.ADD_ITEM]: itemUpdateHandler, + [PocketEventType.ARCHIVE_ITEM]: itemUpdateHandler, + [PocketEventType.UNARCHIVE_ITEM]: itemUpdateHandler, + [PocketEventType.DELETE_ITEM]: itemDeleteHandler, + [PocketEventType.ADD_TAGS]: itemUpdateHandler, + [PocketEventType.REMOVE_TAGS]: itemUpdateHandler, + [PocketEventType.REPLACE_TAGS]: itemUpdateHandler, + [PocketEventType.CLEAR_TAGS]: itemUpdateHandler, + [PocketEventType.FAVORITE_ITEM]: itemUpdateHandler, + [PocketEventType.UNFAVORITE_ITEM]: itemUpdateHandler, +}; diff --git a/lambdas/user-list-search-events/src/index.ts b/lambdas/user-list-search-events/src/index.ts new file mode 100644 index 000000000..6a5fbade3 --- /dev/null +++ b/lambdas/user-list-search-events/src/index.ts @@ -0,0 +1,71 @@ +import { config } from './config'; +import * as Sentry from '@sentry/aws-serverless'; +Sentry.init({ + ...config.sentry, +}); +import { SQSBatchResponse, SQSEvent } from 'aws-lambda'; +import { handlerMap, PocketEventRecord } from './handlerMap'; +import { sqsLambdaEventBridgeEvent } from '@pocket-tools/event-bridge'; +import { serverLogger } from '@pocket-tools/ts-logger'; + +/** + * Processes messages originating from event bridge. The detail-type field in + * the message is used to determine which handler should be used for processing. + * @param event + * @returns + */ +export async function __handler(event: SQSEvent): Promise { + const failedRecordEventIds: string[] = []; + + // Reduce the events in the record to a map of detail-type to PocketEventRecord[] that we can then mass pass to the appropriate handler + // string is actually PocketEventType but type enums didn't work here + const parsedPocketEvents: Record = + event.Records.map((record) => { + try { + const pocketEvent = sqsLambdaEventBridgeEvent(record); + if (pocketEvent === null) { + return null; + } + return { + pocketEvent, + messageId: record.messageId, + }; + } catch (error) { + serverLogger.error('Failed to parse record', error); + failedRecordEventIds.push(record.messageId); + return null; + } + }) + .filter((record) => record !== null) + .filter((record) => + Object.keys(handlerMap).includes(record.pocketEvent['detail-type']), + ) + .reduce((acc, eventRecord: PocketEventRecord) => { + const detailType = eventRecord.pocketEvent['detail-type']; + if (!acc[detailType]) { + acc[detailType] = []; + } + acc[detailType].push(eventRecord); + return acc; + }, {}); + + // For each detail-type, call the appropriate handler and save it to a list of promises we should await + const promises: Promise[] = []; + for (const eventType of Object.keys(parsedPocketEvents)) { + promises.push(handlerMap[eventType](parsedPocketEvents[eventType])); + } + + // Await all the promises and collect any failed record event ids + const responses = await Promise.all(promises); + failedRecordEventIds.push(...responses.flat()); + + return { + batchItemFailures: failedRecordEventIds.map((id) => ({ + itemIdentifier: id, + })), + }; +} + +export const handler = Sentry.wrapHandler(__handler, { + captureTimeoutWarning: false, +}); diff --git a/lambdas/user-list-search-events/src/itemDeleteHandler.ts b/lambdas/user-list-search-events/src/itemDeleteHandler.ts new file mode 100644 index 000000000..aff3cd66f --- /dev/null +++ b/lambdas/user-list-search-events/src/itemDeleteHandler.ts @@ -0,0 +1,50 @@ +import { DeleteItem, IncomingBaseEvent } from '@pocket-tools/event-bridge'; +import { PocketEventRecord } from './handlerMap'; +import { config } from './config'; +import { sendMessage } from './sqsClient'; +import { serverLogger } from '@pocket-tools/ts-logger'; + +type DeleteItemEvent = Exclude & { + pocketEvent: DeleteItem & IncomingBaseEvent; +}; + +/** + * NOTE: Even though our SQS queue message can handle multiple user items, we are only sending one item at a time for easier error tracking. + * This is similar to the old Kinesis to SQS handler. + * + * Given an item delete event, load the item to delete to the SQS queue + * @param record Pocket Event containing forwarded event from eventbridge + */ +export async function itemDeleteHandler( + events: DeleteItemEvent[], +): Promise { + const failedIds: string[] = []; + for (const record of events) { + const savedItemId = record.pocketEvent.detail.savedItem.id + ? parseInt(record.pocketEvent.detail.savedItem.id) + : null; + if (!savedItemId) { + failedIds.push(record.messageId); + continue; + } + + try { + await sendMessage( + { + userItems: [ + { + userId: record.pocketEvent.detail.user.id, + itemIds: [savedItemId], + }, + ], + }, + config.aws.sqs.userItemsDeleteUrl, + ); + } catch (error) { + serverLogger.error('Failed to send message to Item Update SQS', error); + failedIds.push(record.messageId); + } + } + + return failedIds; +} diff --git a/lambdas/user-list-search-events/src/itemUpdateHandler.ts b/lambdas/user-list-search-events/src/itemUpdateHandler.ts new file mode 100644 index 000000000..725a43c53 --- /dev/null +++ b/lambdas/user-list-search-events/src/itemUpdateHandler.ts @@ -0,0 +1,53 @@ +import { + ListEvent as BaseEvent, + IncomingBaseEvent, +} from '@pocket-tools/event-bridge'; +import { PocketEventRecord } from './handlerMap'; +import { config } from './config'; +import { sendMessage } from './sqsClient'; +import { serverLogger } from '@pocket-tools/ts-logger'; + +type ListEvent = Exclude & { + pocketEvent: BaseEvent & IncomingBaseEvent; +}; + +/** + * NOTE: Even though our SQS queue message can handle multiple user items, we are only sending one item at a time for easier error tracking. + * This is similar to the old Kinesis to SQS handler. + * + * Given an item update event, load the item to update to the SQS queue + * @param record Pocket Event containing forwarded event from eventbridge + */ +export async function itemUpdateHandler( + events: ListEvent[], +): Promise { + const failedIds: string[] = []; + for (const record of events) { + const savedItemId = record.pocketEvent.detail.savedItem.id + ? parseInt(record.pocketEvent.detail.savedItem.id) + : null; + if (!savedItemId) { + failedIds.push(record.messageId); + continue; + } + + try { + await sendMessage( + { + userItems: [ + { + userId: record.pocketEvent.detail.user.id, + itemIds: [savedItemId], + }, + ], + }, + config.aws.sqs.userItemsUpdateUrl, + ); + } catch (error) { + serverLogger.error('Failed to send message to Item Update SQS', error); + failedIds.push(record.messageId); + } + } + + return failedIds; +} diff --git a/lambdas/user-list-search-events/src/premiumPurchaseHandler.ts b/lambdas/user-list-search-events/src/premiumPurchaseHandler.ts new file mode 100644 index 000000000..a93adb984 --- /dev/null +++ b/lambdas/user-list-search-events/src/premiumPurchaseHandler.ts @@ -0,0 +1,51 @@ +import { + PremiumPurchaseEvent as BaseEvent, + IncomingBaseEvent, +} from '@pocket-tools/event-bridge'; +import { PocketEventRecord } from './handlerMap'; +import { sendMessage } from './sqsClient'; +import { config } from './config'; +import { serverLogger } from '@pocket-tools/ts-logger'; + +type PremiumPurchaseEvent = Exclude & { + pocketEvent: BaseEvent & IncomingBaseEvent; +}; + +/** + * NOTE: Even though our SQS queue message can handle multiple users, we are only sending one user at a time for easier error tracking. + * This is similar to the old Kinesis to SQS handler. + * + * Given a premium purchase event, load the user to import into the SQS Queue + * @param record Pocket Event containing forwarded event from eventbridge + */ +export async function premiumPurchaseHandler( + events: PremiumPurchaseEvent[], +): Promise { + const failedIds: string[] = []; + for (const record of events) { + const userId = record.pocketEvent.detail.user.id + ? parseInt(record.pocketEvent.detail.user.id) + : null; + if (!userId) { + failedIds.push(record.messageId); + continue; + } + + try { + await sendMessage( + { + users: [{ userId }], + }, + config.aws.sqs.userListImportUrl, + ); + } catch (error) { + serverLogger.error( + 'Failed to send message to Premium Import SQS Queue', + error, + ); + failedIds.push(record.messageId); + } + } + + return failedIds; +} diff --git a/lambdas/user-list-search-events/src/sqsClient.ts b/lambdas/user-list-search-events/src/sqsClient.ts new file mode 100644 index 000000000..898bcfe82 --- /dev/null +++ b/lambdas/user-list-search-events/src/sqsClient.ts @@ -0,0 +1,29 @@ +import { + SendMessageCommand, + SendMessageCommandOutput, + SQSClient, +} from '@aws-sdk/client-sqs'; +import { config } from './config'; +import { UserSearchIndexSqsMessage } from '@pocket-tools/types'; + +const sqsClient = new SQSClient({ + endpoint: config.aws.sqs.endpoint, + region: config.aws.region, + maxAttempts: 3, +}); + +/** + * Send SQS message to queue + * @param data + */ +export async function sendMessage( + data: UserSearchIndexSqsMessage, + queueUrl: string, +): Promise { + const command = new SendMessageCommand({ + MessageBody: JSON.stringify(data), + QueueUrl: queueUrl, + }); + + return await sqsClient.send(command); +} diff --git a/lambdas/user-list-search-indexing/jest.config.ts b/lambdas/user-list-search-indexing/jest.config.ts index 70c768064..99ed9a1f3 100644 --- a/lambdas/user-list-search-indexing/jest.config.ts +++ b/lambdas/user-list-search-indexing/jest.config.ts @@ -3,9 +3,8 @@ import type { Config } from 'jest'; const config: Config = { preset: 'ts-jest', testEnvironment: 'node', - testMatch: ['**/?(*.)+(spec|integration).ts'], + testMatch: ['**/?(*.)+(jest|spec).[jt]s?(x)'], testPathIgnorePatterns: ['/dist/'], - displayName: 'user-list-search-kinesis-to-sqs-lambda', setupFilesAfterEnv: ['jest-extended/all'], moduleNameMapper: { "^(\\.\\/.+)\\.js$": "$1", diff --git a/lambdas/user-list-search-indexing/package.json b/lambdas/user-list-search-indexing/package.json index 1bce8345b..c94beec7d 100644 --- a/lambdas/user-list-search-indexing/package.json +++ b/lambdas/user-list-search-indexing/package.json @@ -15,6 +15,7 @@ "test": "jest \"\\.spec\\.ts\" --runInBand --forceExit" }, "dependencies": { + "@pocket-tools/types": "workspace:*", "@sentry/aws-serverless": "8.47.0", "nanoid": "3.3.8", "tslib": "2.8.0" diff --git a/lambdas/user-list-search-indexing/src/helper.spec.ts b/lambdas/user-list-search-indexing/src/helper.spec.ts index b4c805251..64e06bced 100644 --- a/lambdas/user-list-search-indexing/src/helper.spec.ts +++ b/lambdas/user-list-search-indexing/src/helper.spec.ts @@ -2,7 +2,10 @@ import { config } from './config/index.ts'; import nock from 'nock'; import { processUserImport, processUserItem } from './helper.ts'; -import { UserItemsSqsMessage, UserListImportSqsMessage } from './types.ts'; +import { + UserItemsSqsMessage, + UserListImportSqsMessage, +} from '@pocket-tools/types'; describe('Item functions', () => { describe('itemDelete', () => { diff --git a/lambdas/user-list-search-indexing/src/helper.ts b/lambdas/user-list-search-indexing/src/helper.ts index 65f7ff9d1..4ed0d4b17 100644 --- a/lambdas/user-list-search-indexing/src/helper.ts +++ b/lambdas/user-list-search-indexing/src/helper.ts @@ -1,4 +1,7 @@ -import { UserItemsSqsMessage, UserListImportSqsMessage } from './types.ts'; +import { + UserItemsSqsMessage, + UserListImportSqsMessage, +} from '@pocket-tools/types'; import { nanoid } from 'nanoid'; import { config } from './config/index.ts'; diff --git a/lambdas/user-list-search-indexing/src/itemDelete.ts b/lambdas/user-list-search-indexing/src/itemDelete.ts index e7787bfa5..4c5d1467c 100644 --- a/lambdas/user-list-search-indexing/src/itemDelete.ts +++ b/lambdas/user-list-search-indexing/src/itemDelete.ts @@ -5,7 +5,7 @@ Sentry.init({ }); import { SQSEvent, SQSRecord } from 'aws-lambda'; import { processUserItem } from './helper.ts'; -import { UserItemsSqsMessage } from './types.ts'; +import { UserItemsSqsMessage } from '@pocket-tools/types'; export const processor = async (event: SQSEvent): Promise => { return await Promise.all( diff --git a/lambdas/user-list-search-indexing/src/itemUpdate.ts b/lambdas/user-list-search-indexing/src/itemUpdate.ts index b5b5c1302..42a0cb78c 100644 --- a/lambdas/user-list-search-indexing/src/itemUpdate.ts +++ b/lambdas/user-list-search-indexing/src/itemUpdate.ts @@ -6,8 +6,7 @@ Sentry.init({ import type { SQSEvent, SQSRecord } from 'aws-lambda'; import { processUserItem } from './helper.ts'; - -import { UserItemsSqsMessage } from './types.ts'; +import { UserItemsSqsMessage } from '@pocket-tools/types'; export const processor = async (event: SQSEvent): Promise => { return await Promise.all( diff --git a/lambdas/user-list-search-indexing/src/types.ts b/lambdas/user-list-search-indexing/src/types.ts deleted file mode 100644 index 59e087ba8..000000000 --- a/lambdas/user-list-search-indexing/src/types.ts +++ /dev/null @@ -1,16 +0,0 @@ -//Dupe of types from user-list-search-sq-to-kinesis.. may be worthwile to make this a package.. - -export type UserItemsSqsMessage = { - userItems: { - userId: number; - itemIds: number[]; - }[]; -}; - -export type UserListImportSqsMessage = { - users: { - userId: number; - }[]; -}; - -export type SqsMessage = UserItemsSqsMessage | UserListImportSqsMessage; diff --git a/lambdas/user-list-search-indexing/src/userListImport.ts b/lambdas/user-list-search-indexing/src/userListImport.ts index 42a49c6de..57b2affee 100644 --- a/lambdas/user-list-search-indexing/src/userListImport.ts +++ b/lambdas/user-list-search-indexing/src/userListImport.ts @@ -3,7 +3,7 @@ import * as Sentry from '@sentry/aws-serverless'; Sentry.init({ ...config.sentry, }); -import type { UserListImportSqsMessage } from './types.ts'; +import type { UserListImportSqsMessage } from '@pocket-tools/types'; import type { SQSEvent, SQSRecord } from 'aws-lambda'; import { processUserImport } from './helper.ts'; diff --git a/lambdas/user-list-search-kinesis-to-sqs/README.md b/lambdas/user-list-search-kinesis-to-sqs/README.md deleted file mode 100644 index 2513423a8..000000000 --- a/lambdas/user-list-search-kinesis-to-sqs/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# User List Search Kinesis to SQS - -This lambda translates data from the [Pocket Unified Event Stream](https://github.com/Pocket/spec/tree/main/unified-event-stream) to various SQS queues depending on the type of list action performed to index it for premium search. - -TODO: We should deprecate this to take data from the Pocket Event Bridge and process list events for Search from that. (See User List Search Events lambda) diff --git a/lambdas/user-list-search-kinesis-to-sqs/eslint.config.mjs b/lambdas/user-list-search-kinesis-to-sqs/eslint.config.mjs deleted file mode 100644 index 00cf7f035..000000000 --- a/lambdas/user-list-search-kinesis-to-sqs/eslint.config.mjs +++ /dev/null @@ -1,3 +0,0 @@ -import lambda from '@pocket-tools/eslint-config/lambda'; -import tseslint from 'typescript-eslint'; -export default tseslint.config(...lambda); diff --git a/lambdas/user-list-search-kinesis-to-sqs/jest.config.ts b/lambdas/user-list-search-kinesis-to-sqs/jest.config.ts deleted file mode 100644 index 70c768064..000000000 --- a/lambdas/user-list-search-kinesis-to-sqs/jest.config.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { Config } from 'jest'; - -const config: Config = { - preset: 'ts-jest', - testEnvironment: 'node', - testMatch: ['**/?(*.)+(spec|integration).ts'], - testPathIgnorePatterns: ['/dist/'], - displayName: 'user-list-search-kinesis-to-sqs-lambda', - setupFilesAfterEnv: ['jest-extended/all'], - moduleNameMapper: { - "^(\\.\\/.+)\\.js$": "$1", - "^(\\..\\/.+)\\.js$": "$1" - }, -}; - -export default config; \ No newline at end of file diff --git a/lambdas/user-list-search-kinesis-to-sqs/package.json b/lambdas/user-list-search-kinesis-to-sqs/package.json deleted file mode 100644 index 1d6d0c67a..000000000 --- a/lambdas/user-list-search-kinesis-to-sqs/package.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "name": "user-list-search-kinesis-to-sqs", - "version": "1.0.0", - "description": "", - "type": "module", - "main": "dist/index.js", - "files": [ - "dist", - "package.json" - ], - "scripts": { - "build": "rm -rf dist && tsc", - "format": "eslint --fix", - "lint": "eslint --fix-dry-run", - "test": "jest \"\\.spec\\.ts\" --runInBand --forceExit" - }, - "dependencies": { - "@aws-sdk/client-sqs": "3.716.0", - "@pocket-tools/ts-logger": "workspace:*", - "@sentry/aws-serverless": "8.47.0", - "highland": "2.13.5", - "tslib": "2.8.0", - "uuid": "^10.0.0" - }, - "devDependencies": { - "@pocket-tools/eslint-config": "workspace:*", - "@types/aws-lambda": "8.10.145", - "@types/highland": "^2.12.11", - "@types/jest": "29.5.14", - "@types/node": "^22.8.2", - "jest": "29.7.0", - "jest-extended": "4.0.2", - "nock": "14.0.0-beta.11", - "ts-jest": "29.2.5", - "ts-node": "10.9.2", - "tsconfig": "workspace:*", - "typescript": "5.7.2" - } -} diff --git a/lambdas/user-list-search-kinesis-to-sqs/src/handler.spec.ts b/lambdas/user-list-search-kinesis-to-sqs/src/handler.spec.ts deleted file mode 100644 index c8359301f..000000000 --- a/lambdas/user-list-search-kinesis-to-sqs/src/handler.spec.ts +++ /dev/null @@ -1,266 +0,0 @@ -import { getHandler, KinesisEvent } from './handler.ts'; -import { SendMessageBatchCommandOutput, SQSClient } from '@aws-sdk/client-sqs'; - -const createEvent = (msg: Record): KinesisEvent => { - const data = Buffer.from(JSON.stringify(msg)).toString('base64'); - - return { - Records: [ - { - kinesis: { - data, - }, - }, - ], - }; -}; - -const sendMessageBatchSuccess: SendMessageBatchCommandOutput = { - Failed: [], - Successful: [], - $metadata: {}, -}; - -describe('kinesis', () => { - describe('handler', () => { - let handler: any; - - beforeEach(async () => { - jest - .spyOn(SQSClient.prototype, 'send') - .mockImplementation(() => Promise.resolve(sendMessageBatchSuccess)); - - handler = await getHandler(new SQSClient(), { - userListImportUrl: 'userListImportUrl', - userItemsUpdateUrl: 'userItemsUpdateUrl', - userItemsDeleteUrl: 'userItemsDeleteUrl', - }); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('handles premium subscription events', async () => { - const premiumSubscriptionCreatedEvent = createEvent({ - type: 'premium-subscription-created', - data: { - user_id: 111, - }, - }); - - expect(await handler(premiumSubscriptionCreatedEvent)).toContainAllValues( - [true, true, true, true, true, true, true, true, true, true], - ); - }); - - it('handles user list item created events', async () => { - const userListItemCreatedEvent = createEvent({ - type: 'user-list-item-created', - data: { - user_id: 111, - item_id: 222, - }, - }); - - expect(await handler(userListItemCreatedEvent)).toContainAllValues([ - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - ]); - }); - - it('handles user list item tag added events', async () => { - const userItemTagsAddedEvent = createEvent({ - type: 'user-item-tags-added', - data: { - user_id: 111, - item_id: 222, - tags: ['the', 'dude', 'abides'], - }, - }); - - expect(await handler(userItemTagsAddedEvent)).toContainAllValues([ - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - ]); - }); - - it('handles user list item tag removed events', async () => { - const userItemTagsRemovedEvent = createEvent({ - type: 'user-item-tags-removed', - data: { - user_id: 111, - item_id: 222, - tags: ['the', 'dude', 'abides'], - }, - }); - - expect(await handler(userItemTagsRemovedEvent)).toContainAllValues([ - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - ]); - }); - - it('handles user list item tag replaced events', async () => { - const userItemTagsReplacedEvent = createEvent({ - type: 'user-item-tags-replaced', - data: { - user_id: 111, - item_id: 222, - tags: ['the', 'dude', 'abides'], - }, - }); - - expect(await handler(userItemTagsReplacedEvent)).toContainAllValues([ - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - ]); - }); - - it('handles user list item archived events', async () => { - const userItemArchivedEvent = createEvent({ - type: 'user-item-archived', - data: { - user_id: 111, - item_id: 222, - }, - }); - - expect(await handler(userItemArchivedEvent)).toContainAllValues([ - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - ]); - }); - - it('handles user list item deleted events', async () => { - const userItemDeletedEvent = createEvent({ - type: 'user-item-deleted', - data: { - user_id: 111, - item_id: 222, - }, - }); - - expect(await handler(userItemDeletedEvent)).toContainAllValues([ - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - ]); - }); - it('handles user list item unarchived', async () => { - const userItemUnarchivedEvent = createEvent({ - type: 'user-item-unarchived', - data: { - user_id: 111, - item_id: 222, - }, - }); - - expect(await handler(userItemUnarchivedEvent)).toContainAllValues([ - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - ]); - }); - - it('handles user list item favorited', async () => { - const userItemFavoritedEvent = createEvent({ - type: 'user-item-favorited', - data: { - user_id: 111, - item_id: 222, - }, - }); - - expect(await handler(userItemFavoritedEvent)).toContainAllValues([ - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - ]); - }); - - it('handles user list item unfavorited', async () => { - const userItemUnfavoritedEvent = createEvent({ - type: 'user-item-unfavorited', - data: { - user_id: 111, - item_id: 222, - }, - }); - - expect(await handler(userItemUnfavoritedEvent)).toContainAllValues([ - true, - true, - true, - true, - true, - true, - true, - true, - true, - true, - ]); - }); - }); -}); diff --git a/lambdas/user-list-search-kinesis-to-sqs/src/handler.ts b/lambdas/user-list-search-kinesis-to-sqs/src/handler.ts deleted file mode 100644 index 2eb44fddf..000000000 --- a/lambdas/user-list-search-kinesis-to-sqs/src/handler.ts +++ /dev/null @@ -1,233 +0,0 @@ -import SqsWritable from './sqs/writeable.ts'; -import _ from 'highland'; -import { SQS } from '@aws-sdk/client-sqs'; -import { - SqsMessage, - UserItemsSqsMessage, - UserListImportSqsMessage, -} from './sqs/types.ts'; -import { serverLogger } from '@pocket-tools/ts-logger'; - -const MAX_JOBS_PER_MESSAGE = 1000; - -export type UnifiedEvent = { - type: string; - source: string; - version: string; - timestamp: number; - data: Record; -}; - -export type UserItemEvent = UnifiedEvent & { - data: { - user_id: number; - item_id: number; - }; -}; - -export type UserItemTagsEvent = UnifiedEvent & { - data: { - user_id: number; - item_id: number; - tags: string[]; - }; -}; - -export type PremiumSubscriptionCreatedEvent = UnifiedEvent & { - type: 'premium-subscription-created'; - data: { - user_id: number; - }; -}; - -export type PipelineError = { - type: 'error'; - error: any; -}; - -// These are all message types in the stream -export type PipelineMessage = UnifiedEvent | PipelineError; - -// TODO: Find these type in AWS source | types not available as far as I can tell -export type KinesisEvent = { - Records: KinesisRecord[]; -}; - -export type KinesisRecord = { - kinesis: KinesisData; -}; - -export type KinesisData = { - data: string; -}; - -// TODO: Catch exception -const getPayloadFromRecord = (record: KinesisRecord): string => { - return Buffer.from(record.kinesis.data, 'base64').toString('ascii'); -}; - -export const getMessageFromPayload = (payload: string): PipelineMessage => { - if (payload === '\n') { - //Something is writing a new line to the Unified Event stream, because. Reasons. - return { - type: 'error', - error: new Error('getMessageFromPayload: newline error'), - }; - } - - try { - return JSON.parse(payload); - } catch (e) { - if (e instanceof SyntaxError) { - console.error('JSON Error', { - payload, - }); - return { - type: 'error', - error: e, - }; - } else { - throw e; - } - } -}; - -const getMessagesFromRecords = ( - records: KinesisRecord[], -): PipelineMessage[] => { - return records.map((record: KinesisRecord) => { - const payload = getPayloadFromRecord(record); - return getMessageFromPayload(payload); - }); -}; - -const getUserListImportSqsMessage = ( - msgs: PremiumSubscriptionCreatedEvent[], -): UserListImportSqsMessage => { - return { - users: msgs.map((m) => { - return { - userId: m.data.user_id, - }; - }), - }; -}; - -const getUserItemsUpdateSqsMessage = ( - msgs: (UserItemEvent | UserItemTagsEvent)[], -): UserItemsSqsMessage => { - serverLogger.info('Processing messages', { - messages: JSON.stringify(msgs), - }); - return { - userItems: msgs.map((m) => ({ - userId: m.data.user_id, - itemIds: [m.data.item_id], - })), - }; -}; - -type PipelineOptions = { - queueUrl: string; - events: PipelineMessage[]; - msgType: string; - transformer: (msg: PipelineMessage[]) => SqsMessage; -}; - -export const getHandler: any = ( - sqsClient: SQS, - config: { - userListImportUrl: string; - userItemsUpdateUrl: string; - userItemsDeleteUrl: string; - }, -) => { - const createPipelinePromise = (opts: PipelineOptions): Promise => { - return new Promise((resolve, reject) => { - _(opts.events) - .filter((msg: PipelineMessage) => msg.type === opts.msgType) - .batch(MAX_JOBS_PER_MESSAGE) - .map(opts.transformer) - .map(JSON.stringify) - .pipe( - new SqsWritable({ - sqsClient, - queueUrl: opts.queueUrl, - }), - ) - .on('finish', () => resolve(true)) - .on('error', (error) => - reject( - `createPipelinePromise ${opts.msgType} failed to process: ${error}`, - ), - ); - }); - }; - - return async (event: KinesisEvent, context: any): Promise => { - const events: PipelineMessage[] = getMessagesFromRecords(event.Records); - - return await Promise.all([ - createPipelinePromise({ - events, - queueUrl: config.userListImportUrl, - msgType: 'premium-subscription-created', - transformer: getUserListImportSqsMessage, - }), - createPipelinePromise({ - events, - queueUrl: config.userItemsUpdateUrl, - msgType: 'user-list-item-created', - transformer: getUserItemsUpdateSqsMessage, - }), - createPipelinePromise({ - events, - queueUrl: config.userItemsUpdateUrl, - msgType: 'user-item-archived', - transformer: getUserItemsUpdateSqsMessage, - }), - createPipelinePromise({ - events, - queueUrl: config.userItemsUpdateUrl, - msgType: 'user-item-unarchived', - transformer: getUserItemsUpdateSqsMessage, - }), - createPipelinePromise({ - events, - queueUrl: config.userItemsDeleteUrl, - msgType: 'user-item-deleted', - transformer: getUserItemsUpdateSqsMessage, - }), - createPipelinePromise({ - events, - queueUrl: config.userItemsUpdateUrl, - msgType: 'user-item-tags-added', - transformer: getUserItemsUpdateSqsMessage, - }), - createPipelinePromise({ - events, - queueUrl: config.userItemsUpdateUrl, - msgType: 'user-item-tags-removed', - transformer: getUserItemsUpdateSqsMessage, - }), - createPipelinePromise({ - events, - queueUrl: config.userItemsUpdateUrl, - msgType: 'user-item-tags-replaced', - transformer: getUserItemsUpdateSqsMessage, - }), - createPipelinePromise({ - events, - queueUrl: config.userItemsUpdateUrl, - msgType: 'user-item-favorited', - transformer: getUserItemsUpdateSqsMessage, - }), - createPipelinePromise({ - events, - queueUrl: config.userItemsUpdateUrl, - msgType: 'user-item-unfavorited', - transformer: getUserItemsUpdateSqsMessage, - }), - ]); - }; -}; diff --git a/lambdas/user-list-search-kinesis-to-sqs/src/index.ts b/lambdas/user-list-search-kinesis-to-sqs/src/index.ts deleted file mode 100644 index 52dce89c7..000000000 --- a/lambdas/user-list-search-kinesis-to-sqs/src/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { config } from './config/index.ts'; -import * as Sentry from '@sentry/aws-serverless'; -Sentry.init({ - ...config.sentry, -}); - -import { getHandler } from './handler.ts'; -import { SQSClient } from '@aws-sdk/client-sqs'; - -export const client = new SQSClient({ - endpoint: config.aws.sqs.endpoint, - region: config.aws.region, -}); - -export const processor = getHandler(client, config.aws.sqs); - -export const handler = Sentry.wrapHandler(processor); diff --git a/lambdas/user-list-search-kinesis-to-sqs/src/sqs/readme.md b/lambdas/user-list-search-kinesis-to-sqs/src/sqs/readme.md deleted file mode 100644 index fba3bf5db..000000000 --- a/lambdas/user-list-search-kinesis-to-sqs/src/sqs/readme.md +++ /dev/null @@ -1,3 +0,0 @@ -# SQS - -Borrowed from: diff --git a/lambdas/user-list-search-kinesis-to-sqs/src/sqs/utils.ts b/lambdas/user-list-search-kinesis-to-sqs/src/sqs/utils.ts deleted file mode 100644 index c293e5b53..000000000 --- a/lambdas/user-list-search-kinesis-to-sqs/src/sqs/utils.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { v4 as uuidv4 } from 'uuid'; - -export const generateId = uuidv4; - -export class SQSBatchSendError extends Error { - details = null; - - constructor(message, details) { - super(message); - this.name = this.constructor.name; - this.details = details; - Error.captureStackTrace(this, this.constructor); - } -} diff --git a/lambdas/user-list-search-kinesis-to-sqs/src/sqs/writeable.ts b/lambdas/user-list-search-kinesis-to-sqs/src/sqs/writeable.ts deleted file mode 100644 index 49843d1b5..000000000 --- a/lambdas/user-list-search-kinesis-to-sqs/src/sqs/writeable.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { Writable } from 'stream'; -import { generateId, SQSBatchSendError } from './utils.ts'; -import { - SendMessageBatchCommand, - SendMessageBatchRequestEntry, - SQS, -} from '@aws-sdk/client-sqs'; - -type SQSWritableStreamOptions = { - sqsClient: SQS; - queueUrl: string; - sqsBatchSize?: number; -}; - -export default class SqsWritable extends Writable { - sqsClient: SQS; - queueUrl: string; - sqsBatchSize = 10; - buffer: SendMessageBatchRequestEntry[] = []; - - constructor(options: SQSWritableStreamOptions) { - super({ objectMode: true }); - this.sqsClient = options.sqsClient; - this.queueUrl = options.queueUrl; - this.sqsBatchSize = options.sqsBatchSize || 10; - } - - async send(entries: SendMessageBatchRequestEntry[]): Promise { - try { - const result = await this.sqsClient.send( - new SendMessageBatchCommand({ - Entries: entries, - QueueUrl: this.queueUrl, - }), - ); - if (result.Failed && result.Failed.length) { - throw new SQSBatchSendError('SQS Batch Send Error', result.Failed); - } - } catch (error) { - this.emit('failedEntries', entries); - const failedIds = entries.map((e) => e.Id); - this.buffer = this.buffer.filter( - (entry) => failedIds.indexOf(entry.Id) === -1, - ); - throw error; - } - } - - async _write(chunk, encoding, callback): Promise { - try { - if (typeof chunk === 'string') { - this.buffer.push({ - Id: generateId(), - MessageBody: chunk, - // hack alert: because the parser does not immediately map a resolved id to - // an item id, we are unable to retrieve any information related to that - // resolved id (e.g. content & authors). to work around this delay on the parser's - // side, we are delaying the availability of these messages here. like i - // said, hack. - DelaySeconds: 60, - }); - } else if (Buffer.isBuffer(chunk)) { - this.buffer.push({ - Id: generateId(), - MessageBody: chunk.toString(), - DelaySeconds: 60, - }); - } else { - this.buffer.push(chunk); - } - - if (this.buffer.length >= this.sqsBatchSize) { - await this.send(this.buffer); - this.buffer = []; - } - - callback(); - } catch (error) { - callback(error); - } - } - - async _final(callback): Promise { - try { - if (this.buffer.length > 0) { - await this.send(this.buffer); - this.buffer = []; - } - - return callback(); - } catch (error) { - return callback(error); - } - } -} diff --git a/lambdas/user-list-search-kinesis-to-sqs/tsconfig.json b/lambdas/user-list-search-kinesis-to-sqs/tsconfig.json deleted file mode 100644 index 8fae1b426..000000000 --- a/lambdas/user-list-search-kinesis-to-sqs/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "tsconfig/lambda.json", - "compilerOptions": { - "outDir": "dist", - }, - "exclude": ["node_modules", "jest.config.ts", "jest.setup.js"], - "include": ["./**/*.ts"], - "files": ["node_modules/jest-extended/types/index.d.ts"] -} \ No newline at end of file diff --git a/packages/backend-benchmarking/package.json b/packages/backend-benchmarking/package.json index bd13f7b05..af29f70de 100644 --- a/packages/backend-benchmarking/package.json +++ b/packages/backend-benchmarking/package.json @@ -41,9 +41,9 @@ }, "devDependencies": { "@pocket-tools/eslint-config": "workspace:*", + "@types/chance": "1.1.6", "@types/jest": "29.5.14", "@types/node": "^22.8.2", - "@types/chance": "1.1.6", "jest": "29.7.0", "ts-jest": "29.2.5", "ts-node": "10.9.2", diff --git a/packages/types/.npmignore b/packages/types/.npmignore new file mode 100644 index 000000000..51e322a5a --- /dev/null +++ b/packages/types/.npmignore @@ -0,0 +1,10 @@ +src +.github +.idea +.prettier* +eslint.config.mjs +tsconfig.json +*.spec.ts +*.integration.ts +jest.config.js +.turbo \ No newline at end of file diff --git a/packages/types/README.md b/packages/types/README.md new file mode 100644 index 000000000..af3202f5c --- /dev/null +++ b/packages/types/README.md @@ -0,0 +1,3 @@ +# Pocket Types + +A set of common types used across the Pocket application diff --git a/packages/types/eslint.config.mjs b/packages/types/eslint.config.mjs new file mode 100644 index 000000000..638ac644e --- /dev/null +++ b/packages/types/eslint.config.mjs @@ -0,0 +1,3 @@ +import packages from '@pocket-tools/eslint-config/packages'; +import tseslint from 'typescript-eslint'; +export default tseslint.config(...packages); diff --git a/packages/types/jest.config.ts b/packages/types/jest.config.ts new file mode 100644 index 000000000..b9e51506d --- /dev/null +++ b/packages/types/jest.config.ts @@ -0,0 +1,24 @@ +import type { Config } from 'jest'; + +const config: Config = { + preset: 'ts-jest/presets/default-esm', // or other ESM presets + testEnvironment: 'node', + testMatch: ['**/?(*.)+(jest|spec).[jt]s?(x)'], + testPathIgnorePatterns: ['/dist/'], + setupFilesAfterEnv: ['jest-extended/all'], + moduleNameMapper: { + "^(\\.{1,2}/.*)\\.js$": "$1" + }, + transform: { + ['^.+.tsx?$']: [ + 'ts-jest', + { + tsconfig: '/tsconfig.json', + isolatedModules: true, + useESM: true, + }, + ], + } +}; + +export default config; \ No newline at end of file diff --git a/packages/types/package.json b/packages/types/package.json new file mode 100644 index 000000000..944141455 --- /dev/null +++ b/packages/types/package.json @@ -0,0 +1,55 @@ +{ + "name": "@pocket-tools/types", + "version": "0.0.0-development", + "description": "Common types in the Pocket monorepo", + "keywords": [ + "types" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/Pocket/pocket-monorepo.git" + }, + "license": "Apache-2.0", + "author": "", + "type": "module", + "exports": { + "import": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "require": "./dist/index.cjs" + } + }, + "main": "dist/index.js", + "module": "dist/index.mjs", + "types": "dist/index.d.ts", + "files": [ + "dist", + "package.json" + ], + "scripts": { + "build": "tsup src/index.ts --format cjs,esm --dts", + "dev": "pnpm run build --watch", + "format": "eslint --fix", + "lint": "eslint --fix-dry-run" + }, + "dependencies": {}, + "devDependencies": { + "@jest/globals": "29.7.0", + "@pocket-tools/eslint-config": "workspace:*", + "@types/jest": "29.5.14", + "@types/node": "^22.8.2", + "jest": "29.7.0", + "jest-extended": "4.0.2", + "ts-jest": "29.2.5", + "ts-node": "10.9.2", + "tsconfig": "workspace:*", + "tsup": "8.3.5", + "typescript": "5.7.2" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts new file mode 100644 index 000000000..0d558c67a --- /dev/null +++ b/packages/types/src/index.ts @@ -0,0 +1 @@ +export * from './searchIndexing.ts'; diff --git a/lambdas/user-list-search-kinesis-to-sqs/src/sqs/types.ts b/packages/types/src/searchIndexing.ts similarity index 66% rename from lambdas/user-list-search-kinesis-to-sqs/src/sqs/types.ts rename to packages/types/src/searchIndexing.ts index 797e0bb60..e43319328 100644 --- a/lambdas/user-list-search-kinesis-to-sqs/src/sqs/types.ts +++ b/packages/types/src/searchIndexing.ts @@ -11,4 +11,6 @@ export type UserListImportSqsMessage = { }[]; }; -export type SqsMessage = UserItemsSqsMessage | UserListImportSqsMessage; +export type UserSearchIndexSqsMessage = + | UserItemsSqsMessage + | UserListImportSqsMessage; diff --git a/packages/types/tsconfig.json b/packages/types/tsconfig.json new file mode 100644 index 000000000..7db90efa4 --- /dev/null +++ b/packages/types/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "tsconfig/library.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "resolveJsonModule": true + }, + "exclude": ["node_modules", "dist"], + "include": ["src/**/*.ts", "src/config"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 97b113220..b5971ed88 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -36,7 +36,7 @@ importers: devDependencies: '@commitlint/cli': specifier: 19.6.0 - version: 19.6.0(@types/node@22.10.5)(typescript@5.8.0-dev.20250103) + version: 19.6.0(@types/node@22.10.5)(typescript@5.8.0-dev.20250106) '@commitlint/config-conventional': specifier: ^19.6.0 version: 19.6.0 @@ -45,7 +45,7 @@ importers: version: link:packages/eslint-config syncpack: specifier: ^13.0.0 - version: 13.0.0(typescript@5.8.0-dev.20250103) + version: 13.0.0(typescript@5.8.0-dev.20250106) tsconfig: specifier: workspace:* version: link:packages/tsconfig @@ -1710,12 +1710,18 @@ importers: lambdas/user-list-search-events: dependencies: + '@aws-sdk/client-sqs': + specifier: 3.716.0 + version: 3.716.0 '@pocket-tools/event-bridge': specifier: workspace:* version: link:../../packages/event-bridge '@pocket-tools/ts-logger': specifier: workspace:* version: link:../../packages/ts-logger + '@pocket-tools/types': + specifier: workspace:* + version: link:../../packages/types '@sentry/aws-serverless': specifier: 8.47.0 version: 8.47.0 @@ -1756,6 +1762,9 @@ importers: lambdas/user-list-search-indexing: dependencies: + '@pocket-tools/types': + specifier: workspace:* + version: link:../../packages/types '@sentry/aws-serverless': specifier: 8.47.0 version: 8.47.0 @@ -1800,64 +1809,6 @@ importers: specifier: 5.7.2 version: 5.7.2 - lambdas/user-list-search-kinesis-to-sqs: - dependencies: - '@aws-sdk/client-sqs': - specifier: 3.716.0 - version: 3.716.0 - '@pocket-tools/ts-logger': - specifier: workspace:* - version: link:../../packages/ts-logger - '@sentry/aws-serverless': - specifier: 8.47.0 - version: 8.47.0 - highland: - specifier: 2.13.5 - version: 2.13.5 - tslib: - specifier: 2.8.0 - version: 2.8.0 - uuid: - specifier: ^10.0.0 - version: 10.0.0 - devDependencies: - '@pocket-tools/eslint-config': - specifier: workspace:* - version: link:../../packages/eslint-config - '@types/aws-lambda': - specifier: 8.10.145 - version: 8.10.145 - '@types/highland': - specifier: ^2.12.11 - version: 2.13.0 - '@types/jest': - specifier: 29.5.14 - version: 29.5.14 - '@types/node': - specifier: ^22.8.2 - version: 22.10.5 - jest: - specifier: 29.7.0 - version: 29.7.0(@types/node@22.10.5)(ts-node@10.9.2(@types/node@22.10.5)(typescript@5.7.2)) - jest-extended: - specifier: 4.0.2 - version: 4.0.2(jest@29.7.0(@types/node@22.10.5)(ts-node@10.9.2(@types/node@22.10.5)(typescript@5.7.2))) - nock: - specifier: 14.0.0-beta.11 - version: 14.0.0-beta.11 - ts-jest: - specifier: 29.2.5 - version: 29.2.5(@babel/core@7.26.0)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.0))(esbuild@0.24.2)(jest@29.7.0(@types/node@22.10.5)(ts-node@10.9.2(@types/node@22.10.5)(typescript@5.7.2)))(typescript@5.7.2) - ts-node: - specifier: 10.9.2 - version: 10.9.2(@types/node@22.10.5)(typescript@5.7.2) - tsconfig: - specifier: workspace:* - version: link:../../packages/tsconfig - typescript: - specifier: 5.7.2 - version: 5.7.2 - packages/apollo-utils: dependencies: '@apollo/cache-control-types': @@ -2692,6 +2643,42 @@ importers: specifier: 5.7.2 version: 5.7.2 + packages/types: + devDependencies: + '@jest/globals': + specifier: 29.7.0 + version: 29.7.0 + '@pocket-tools/eslint-config': + specifier: workspace:* + version: link:../eslint-config + '@types/jest': + specifier: 29.5.14 + version: 29.5.14 + '@types/node': + specifier: ^22.8.2 + version: 22.10.5 + jest: + specifier: 29.7.0 + version: 29.7.0(@types/node@22.10.5)(ts-node@10.9.2(@types/node@22.10.5)(typescript@5.7.2)) + jest-extended: + specifier: 4.0.2 + version: 4.0.2(jest@29.7.0(@types/node@22.10.5)(ts-node@10.9.2(@types/node@22.10.5)(typescript@5.7.2))) + ts-jest: + specifier: 29.2.5 + version: 29.2.5(@babel/core@7.26.0)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.0))(esbuild@0.24.2)(jest@29.7.0(@types/node@22.10.5)(ts-node@10.9.2(@types/node@22.10.5)(typescript@5.7.2)))(typescript@5.7.2) + ts-node: + specifier: 10.9.2 + version: 10.9.2(@types/node@22.10.5)(typescript@5.7.2) + tsconfig: + specifier: workspace:* + version: link:../tsconfig + tsup: + specifier: 8.3.5 + version: 8.3.5(jiti@2.4.2)(postcss@8.4.49)(tsx@4.19.2)(typescript@5.7.2)(yaml@2.7.0) + typescript: + specifier: 5.7.2 + version: 5.7.2 + servers/account-data-deleter: dependencies: '@aws-sdk/client-s3': @@ -8052,9 +8039,6 @@ packages: '@types/graphql-upload@16.0.7': resolution: {integrity: sha512-7vCoxIv2pVTvV8n+miYyfkINdguWsYomAkPlOfHoM6z/qzsiBAdfRb6lNc8PvEUhe7TXaxX4+LHubejw1og1DQ==} - '@types/highland@2.13.0': - resolution: {integrity: sha512-ZB4u9dUDnBmqGtbY4t7gAiW9L5oHWfP1TknVzlucd9se+AYXhcr3/5V51ViNBj8lqmRC0xvTRSBmSPU4hkPGAQ==} - '@types/http-assert@1.5.6': resolution: {integrity: sha512-TTEwmtjgVbYAzZYWyeHPrrtWnfVkm8tQkP8P21uQifPgMRgjrow3XDEYqucuC8SKZJT7pUnhU/JymvjggxO9vw==} @@ -10615,9 +10599,6 @@ packages: resolution: {integrity: sha512-qlspKUK7IlSQv2o+5I7yhUd7TxlOG2Vr5LTa3ve2XSNVKAL/n/u/7KLvKmFNimomDIKvZFXWHv0T12mv7rT8Aw==} engines: {node: '>=8'} - highland@2.13.5: - resolution: {integrity: sha512-dn2flPapIIAa4BtkB2ahjshg8iSJtrJtdhEb9/oiOrS5HMQTR/GuhFpqJ+11YBdtnl3AwWKvbZd1Uxr8uAmA7A==} - highlight.js@10.7.3: resolution: {integrity: sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==} @@ -14354,8 +14335,8 @@ packages: engines: {node: '>=14.17'} hasBin: true - typescript@5.8.0-dev.20250103: - resolution: {integrity: sha512-PT/H6jgiDGZHb+f431Oq8uqRe1IhfFDtgoJK/IB1tevn6s5P7O59oe1uuHZKAVtiLfw55tLgOXcE9nen3VteyA==} + typescript@5.8.0-dev.20250106: + resolution: {integrity: sha512-o7wnxO5cYLyfc7QIUC5XqIhONzvtHj4pKEF3S1lxXh/3zDnAScSaSUzRKyKIRXuTXDgYuaWKhew4jrBRmVAK1Q==} engines: {node: '>=14.17'} hasBin: true @@ -16017,7 +15998,7 @@ snapshots: '@smithy/protocol-http': 4.1.8 '@smithy/signature-v4': 4.2.4 '@smithy/types': 3.7.2 - tslib: 2.8.1 + tslib: 2.8.0 '@aws-sdk/token-providers@3.714.0(@aws-sdk/client-sso-oidc@3.716.0(@aws-sdk/client-sts@3.716.0))': dependencies: @@ -16111,7 +16092,7 @@ snapshots: '@babel/generator@7.26.2': dependencies: '@babel/parser': 7.26.3 - '@babel/types': 7.26.0 + '@babel/types': 7.26.3 '@jridgewell/gen-mapping': 0.3.8 '@jridgewell/trace-mapping': 0.3.25 jsesc: 3.1.0 @@ -16682,11 +16663,11 @@ snapshots: dependencies: commander: 12.1.0 - '@commitlint/cli@19.6.0(@types/node@22.10.5)(typescript@5.8.0-dev.20250103)': + '@commitlint/cli@19.6.0(@types/node@22.10.5)(typescript@5.8.0-dev.20250106)': dependencies: '@commitlint/format': 19.5.0 '@commitlint/lint': 19.6.0 - '@commitlint/load': 19.6.1(@types/node@22.10.5)(typescript@5.8.0-dev.20250103) + '@commitlint/load': 19.6.1(@types/node@22.10.5)(typescript@5.8.0-dev.20250106) '@commitlint/read': 19.5.0 '@commitlint/types': 19.5.0 tinyexec: 0.3.2 @@ -16733,15 +16714,15 @@ snapshots: '@commitlint/rules': 19.6.0 '@commitlint/types': 19.5.0 - '@commitlint/load@19.6.1(@types/node@22.10.5)(typescript@5.8.0-dev.20250103)': + '@commitlint/load@19.6.1(@types/node@22.10.5)(typescript@5.8.0-dev.20250106)': dependencies: '@commitlint/config-validator': 19.5.0 '@commitlint/execute-rule': 19.5.0 '@commitlint/resolve-extends': 19.5.0 '@commitlint/types': 19.5.0 chalk: 5.4.1 - cosmiconfig: 9.0.0(typescript@5.8.0-dev.20250103) - cosmiconfig-typescript-loader: 6.1.0(@types/node@22.10.5)(cosmiconfig@9.0.0(typescript@5.8.0-dev.20250103))(typescript@5.8.0-dev.20250103) + cosmiconfig: 9.0.0(typescript@5.8.0-dev.20250106) + cosmiconfig-typescript-loader: 6.1.0(@types/node@22.10.5)(cosmiconfig@9.0.0(typescript@5.8.0-dev.20250106))(typescript@5.8.0-dev.20250106) lodash.isplainobject: 4.0.6 lodash.merge: 4.6.2 lodash.uniq: 4.5.0 @@ -16829,11 +16810,11 @@ snapshots: '@envelop/core@5.0.2': dependencies: '@envelop/types': 5.0.0 - tslib: 2.8.1 + tslib: 2.8.0 '@envelop/types@5.0.0': dependencies: - tslib: 2.8.1 + tslib: 2.8.0 '@esbuild/aix-ppc64@0.23.1': optional: true @@ -17423,7 +17404,7 @@ snapshots: '@repeaterjs/repeater': 3.0.6 '@whatwg-node/disposablestack': 0.0.5 graphql: 16.9.0 - tslib: 2.8.1 + tslib: 2.8.0 value-or-promise: 1.0.12 '@graphql-tools/git-loader@8.0.16(graphql@16.9.0)': @@ -17502,7 +17483,7 @@ snapshots: dependencies: '@graphql-tools/utils': 9.2.1(graphql@16.9.0) graphql: 16.9.0 - tslib: 2.8.1 + tslib: 2.8.0 '@graphql-tools/merge@9.0.16(graphql@16.9.0)': dependencies: @@ -17564,7 +17545,7 @@ snapshots: '@graphql-tools/merge': 8.4.2(graphql@16.9.0) '@graphql-tools/utils': 9.2.1(graphql@16.9.0) graphql: 16.9.0 - tslib: 2.8.1 + tslib: 2.8.0 value-or-promise: 1.0.12 '@graphql-tools/url-loader@8.0.22(@types/node@22.10.5)(encoding@0.1.13)(graphql@16.9.0)': @@ -20375,10 +20356,6 @@ snapshots: fs-capacitor: 8.0.0 graphql: 16.9.0 - '@types/highland@2.13.0': - dependencies: - '@types/node': 22.10.5 - '@types/http-assert@1.5.6': {} '@types/http-cache-semantics@4.0.4': {} @@ -20728,7 +20705,7 @@ snapshots: '@whatwg-node/disposablestack@0.0.5': dependencies: - tslib: 2.8.1 + tslib: 2.8.0 '@whatwg-node/fetch@0.10.1': dependencies: @@ -21458,7 +21435,7 @@ snapshots: chalk-template@1.1.0: dependencies: - chalk: 5.3.0 + chalk: 5.4.1 chalk@1.1.3: dependencies: @@ -21939,12 +21916,12 @@ snapshots: object-assign: 4.1.1 vary: 1.1.2 - cosmiconfig-typescript-loader@6.1.0(@types/node@22.10.5)(cosmiconfig@9.0.0(typescript@5.8.0-dev.20250103))(typescript@5.8.0-dev.20250103): + cosmiconfig-typescript-loader@6.1.0(@types/node@22.10.5)(cosmiconfig@9.0.0(typescript@5.8.0-dev.20250106))(typescript@5.8.0-dev.20250106): dependencies: '@types/node': 22.10.5 - cosmiconfig: 9.0.0(typescript@5.8.0-dev.20250103) + cosmiconfig: 9.0.0(typescript@5.8.0-dev.20250106) jiti: 2.4.2 - typescript: 5.8.0-dev.20250103 + typescript: 5.8.0-dev.20250106 cosmiconfig@8.3.6(typescript@5.7.2): dependencies: @@ -21964,14 +21941,14 @@ snapshots: optionalDependencies: typescript: 5.7.2 - cosmiconfig@9.0.0(typescript@5.8.0-dev.20250103): + cosmiconfig@9.0.0(typescript@5.8.0-dev.20250106): dependencies: env-paths: 2.2.1 import-fresh: 3.3.0 js-yaml: 4.1.0 parse-json: 5.2.0 optionalDependencies: - typescript: 5.8.0-dev.20250103 + typescript: 5.8.0-dev.20250106 cpu-features@0.0.2: dependencies: @@ -22313,7 +22290,7 @@ snapshots: dependencies: semver: 7.6.3 shelljs: 0.8.5 - typescript: 5.8.0-dev.20250103 + typescript: 5.8.0-dev.20250106 dreamopt@0.8.0: dependencies: @@ -23542,10 +23519,6 @@ snapshots: hexoid@2.0.0: {} - highland@2.13.5: - dependencies: - util-deprecate: 1.0.2 - highlight.js@10.7.3: {} hmac-drbg@1.0.1: @@ -25023,7 +24996,7 @@ snapshots: log-symbols@6.0.0: dependencies: - chalk: 5.3.0 + chalk: 5.4.1 is-unicode-supported: 1.3.0 log-update@4.0.0: @@ -25650,7 +25623,7 @@ snapshots: ora@8.0.1: dependencies: - chalk: 5.3.0 + chalk: 5.4.1 cli-cursor: 4.0.0 cli-spinners: 2.9.2 is-interactive: 2.0.0 @@ -27219,15 +27192,15 @@ snapshots: synckit@0.9.2: dependencies: '@pkgr/core': 0.1.1 - tslib: 2.8.1 + tslib: 2.8.0 - syncpack@13.0.0(typescript@5.8.0-dev.20250103): + syncpack@13.0.0(typescript@5.8.0-dev.20250106): dependencies: '@effect/schema': 0.71.1(effect@3.6.5) chalk: 5.3.0 chalk-template: 1.1.0 commander: 12.1.0 - cosmiconfig: 9.0.0(typescript@5.8.0-dev.20250103) + cosmiconfig: 9.0.0(typescript@5.8.0-dev.20250106) effect: 3.6.5 enquirer: 2.4.1 fast-check: 3.21.0 @@ -27604,7 +27577,7 @@ snapshots: typescript@5.7.2: {} - typescript@5.8.0-dev.20250103: {} + typescript@5.8.0-dev.20250106: {} ua-parser-js@1.0.40: {}