diff --git a/CHANGELOG.md b/CHANGELOG.md index caab38dd2..cd89b1b6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.94.0](https://github.com/rudderlabs/rudder-config-schema/compare/v1.93.1...v1.94.0) (2024-09-30) + + +### Features + +* add 5 min and 15 min WH sync options, controlled by LOW_GRANULAR_WH_SYNC ff ([#1703](https://github.com/rudderlabs/rudder-config-schema/issues/1703)) ([21a5015](https://github.com/rudderlabs/rudder-config-schema/commit/21a50153194c6db5da563858953ae39ed638b662)) +* adding cloud source and web source for lytics bulk ([#1685](https://github.com/rudderlabs/rudder-config-schema/issues/1685)) ([b2a433b](https://github.com/rudderlabs/rudder-config-schema/commit/b2a433b6f6515ef0818aaa0746225bc6c4f1049d)) +* bq configurable partitions ([#1630](https://github.com/rudderlabs/rudder-config-schema/issues/1630)) ([a7e21e5](https://github.com/rudderlabs/rudder-config-schema/commit/a7e21e542e41c761c66394731772703ec48bb553)) +* enable support for RN device mode in the Facebook App Events ([#1701](https://github.com/rudderlabs/rudder-config-schema/issues/1701)) ([a193c6e](https://github.com/rudderlabs/rudder-config-schema/commit/a193c6e518bf6c914a3c99656ea6d3c0aeb8d946)) +* **fb_custom_audience:** hide particular fields from ui config when VDMv2 is enabled ([#1702](https://github.com/rudderlabs/rudder-config-schema/issues/1702)) ([1791773](https://github.com/rudderlabs/rudder-config-schema/commit/1791773d94da4872d5bd1d8d048cc6f07efe7e61)) +* hide mixpanel service account settings ([#1714](https://github.com/rudderlabs/rudder-config-schema/issues/1714)) ([1f73739](https://github.com/rudderlabs/rudder-config-schema/commit/1f73739087e3e734cf83b360dee1b1c28f6d9528)) +* redshift serverless ([#1710](https://github.com/rudderlabs/rudder-config-schema/issues/1710)) ([6ac3dd6](https://github.com/rudderlabs/rudder-config-schema/commit/6ac3dd6761bf7a3b1d9ac73c4540f14092fb5c21)) + + +### Bug Fixes + +* add `device` mode enum to FB App Events in reactnative schema.json ([#1707](https://github.com/rudderlabs/rudder-config-schema/issues/1707)) ([879b966](https://github.com/rudderlabs/rudder-config-schema/commit/879b966c39df9039f3e673ffb13b78ee126cd18a)) +* fixing labels for snapchat conversion event mapping ([#1713](https://github.com/rudderlabs/rudder-config-schema/issues/1713)) ([4a00440](https://github.com/rudderlabs/rudder-config-schema/commit/4a0044014c043871b002404004e2de0895ad3421)) +* **ga4_v2:** update ui-config.jt file to reflect consent management changes ([#1717](https://github.com/rudderlabs/rudder-config-schema/issues/1717)) ([103745a](https://github.com/rudderlabs/rudder-config-schema/commit/103745a8715b4e450d809812c21a67443473e0c2)) +* legacy consent management fields data in lytics bulk upload destination ([5d4d116](https://github.com/rudderlabs/rudder-config-schema/commit/5d4d116cb7763e69b10ed2993e278b78039ca3a2)) +* webhook url regex to restrict localhost and private ips ([#1696](https://github.com/rudderlabs/rudder-config-schema/issues/1696)) ([774bd43](https://github.com/rudderlabs/rudder-config-schema/commit/774bd43e43f0576e9d6e9b5e046ec19c1c13b7e9)) +* webhook url regex to restrict private ipv6 ([#1705](https://github.com/rudderlabs/rudder-config-schema/issues/1705)) ([5b91bc7](https://github.com/rudderlabs/rudder-config-schema/commit/5b91bc7f841e830edefc2add80d1ba6b7a98d7d3)) + ### [1.93.1](https://github.com/rudderlabs/rudder-config-schema/compare/v1.93.0...v1.93.1) (2024-09-30) diff --git a/package-lock.json b/package-lock.json index ffb35cc45..9cfc0fbd8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-config-schema", - "version": "1.93.1", + "version": "1.94.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-config-schema", - "version": "1.93.1", + "version": "1.94.0", "license": "MIT", "dependencies": { "ajv": "^8.12.0", diff --git a/package.json b/package.json index 4b675d7af..19e93e80d 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-config-schema", - "version": "1.93.1", + "version": "1.94.0", "description": "", "main": "src/index.ts", "private": true, diff --git a/src/configurations/destinations/azure_datalake/schema.json b/src/configurations/destinations/azure_datalake/schema.json index 6ebea7af0..2b0ff54f3 100644 --- a/src/configurations/destinations/azure_datalake/schema.json +++ b/src/configurations/destinations/azure_datalake/schema.json @@ -34,7 +34,7 @@ }, "syncFrequency": { "type": "string", - "enum": ["30", "60", "180", "360", "720", "1440"], + "enum": ["5", "15", "30", "60", "180", "360", "720", "1440"], "default": "180" }, "syncStartAt": { diff --git a/src/configurations/destinations/azure_datalake/ui-config.json b/src/configurations/destinations/azure_datalake/ui-config.json index da7d2616e..770c38f86 100644 --- a/src/configurations/destinations/azure_datalake/ui-config.json +++ b/src/configurations/destinations/azure_datalake/ui-config.json @@ -89,6 +89,16 @@ "label": "Sync Frequency", "value": "syncFrequency", "options": [ + { + "name": "Every 5 minutes", + "value": "5", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, + { + "name": "Every 15 minutes", + "value": "15", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, { "name": "Every 30 minutes", "value": "30" diff --git a/src/configurations/destinations/azure_synapse/schema.json b/src/configurations/destinations/azure_synapse/schema.json index 66ee96019..7efa36263 100644 --- a/src/configurations/destinations/azure_synapse/schema.json +++ b/src/configurations/destinations/azure_synapse/schema.json @@ -43,7 +43,7 @@ }, "syncFrequency": { "type": "string", - "pattern": "^(30|60|180|360|720|1440)$", + "pattern": "^(5|15|30|60|180|360|720|1440)$", "default": "180" }, "syncStartAt": { diff --git a/src/configurations/destinations/azure_synapse/ui-config.json b/src/configurations/destinations/azure_synapse/ui-config.json index dfa6fe079..a54861bb5 100644 --- a/src/configurations/destinations/azure_synapse/ui-config.json +++ b/src/configurations/destinations/azure_synapse/ui-config.json @@ -88,6 +88,16 @@ "label": "Sync Frequency", "value": "syncFrequency", "options": [ + { + "name": "Every 5 minutes", + "value": "5", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, + { + "name": "Every 15 minutes", + "value": "15", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, { "name": "Every 30 minutes", "value": "30" diff --git a/src/configurations/destinations/bq/db-config.json b/src/configurations/destinations/bq/db-config.json index c9733b01a..dc87d8e71 100644 --- a/src/configurations/destinations/bq/db-config.json +++ b/src/configurations/destinations/bq/db-config.json @@ -35,7 +35,9 @@ "skipUsersTable", "jsonPaths", "underscoreDivideNumbers", - "allowUsersContextTraits" + "allowUsersContextTraits", + "partitionColumn", + "partitionType" ], "android": [ "connectionMode", diff --git a/src/configurations/destinations/bq/schema.json b/src/configurations/destinations/bq/schema.json index d33632196..ca4b66f9d 100644 --- a/src/configurations/destinations/bq/schema.json +++ b/src/configurations/destinations/bq/schema.json @@ -30,9 +30,19 @@ }, "syncFrequency": { "type": "string", - "pattern": "^(30|60|180|360|720|1440)$", + "pattern": "^(5|15|30|60|180|360|720|1440)$", "default": "180" }, + "partitionColumn": { + "type": "string", + "pattern": "^(_PARTITIONTIME|loaded_at|received_at)$", + "default": "_PARTITIONTIME" + }, + "partitionType": { + "type": "string", + "pattern": "^(hour|day)$", + "default": "day" + }, "syncStartAt": { "type": "string" }, diff --git a/src/configurations/destinations/bq/ui-config.json b/src/configurations/destinations/bq/ui-config.json index 0b5794409..41760ea6b 100644 --- a/src/configurations/destinations/bq/ui-config.json +++ b/src/configurations/destinations/bq/ui-config.json @@ -72,6 +72,16 @@ "label": "Sync Frequency", "value": "syncFrequency", "options": [ + { + "name": "Every 5 minutes", + "value": "5", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, + { + "name": "Every 15 minutes", + "value": "15", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, { "name": "Every 30 minutes", "value": "30" @@ -132,6 +142,52 @@ }, "required": false, "footerNote": "Note: Please specify time in UTC" + }, + { + "type": "singleSelect", + "immutable": true, + "label": "Partition Column", + "value": "partitionColumn", + "options": [ + { + "name": "Ingestion Time", + "value": "_PARTITIONTIME" + }, + { + "name": "Loaded At", + "value": "loaded_at" + }, + { + "name": "Received At", + "value": "received_at" + } + ], + "defaultOption": { + "name": "Ingestion Time", + "value": "_PARTITIONTIME" + }, + "required": false + }, + { + "type": "singleSelect", + "immutable": true, + "label": "Partition Type", + "value": "partitionType", + "options": [ + { + "name": "Hour", + "value": "hour" + }, + { + "name": "Day", + "value": "day" + } + ], + "defaultOption": { + "name": "Day", + "value": "day" + }, + "required": false } ] }, diff --git a/src/configurations/destinations/clickhouse/schema.json b/src/configurations/destinations/clickhouse/schema.json index c7dda6c6a..4b0c6352d 100644 --- a/src/configurations/destinations/clickhouse/schema.json +++ b/src/configurations/destinations/clickhouse/schema.json @@ -53,7 +53,7 @@ }, "syncFrequency": { "type": "string", - "pattern": "^(30|60|180|360|720|1440)$", + "pattern": "^(5|15|30|60|180|360|720|1440)$", "default": "180" }, "syncStartAt": { diff --git a/src/configurations/destinations/clickhouse/ui-config.json b/src/configurations/destinations/clickhouse/ui-config.json index bf6fc2c38..99db4f4d5 100644 --- a/src/configurations/destinations/clickhouse/ui-config.json +++ b/src/configurations/destinations/clickhouse/ui-config.json @@ -95,6 +95,16 @@ "label": "Sync Frequency", "value": "syncFrequency", "options": [ + { + "name": "Every 5 minutes", + "value": "5", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, + { + "name": "Every 15 minutes", + "value": "15", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, { "name": "Every 30 minutes", "value": "30" diff --git a/src/configurations/destinations/deltalake/schema.json b/src/configurations/destinations/deltalake/schema.json index 30495a3b8..ecb176764 100644 --- a/src/configurations/destinations/deltalake/schema.json +++ b/src/configurations/destinations/deltalake/schema.json @@ -38,7 +38,7 @@ }, "syncFrequency": { "type": "string", - "pattern": "^(30|60|180|360|720|1440)$", + "pattern": "^(5|15|30|60|180|360|720|1440)$", "default": "180" }, "syncStartAt": { diff --git a/src/configurations/destinations/deltalake/ui-config.json b/src/configurations/destinations/deltalake/ui-config.json index be27dc021..19a463c33 100644 --- a/src/configurations/destinations/deltalake/ui-config.json +++ b/src/configurations/destinations/deltalake/ui-config.json @@ -93,6 +93,16 @@ "label": "Sync Frequency", "value": "syncFrequency", "options": [ + { + "name": "Every 5 minutes", + "value": "5", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, + { + "name": "Every 15 minutes", + "value": "15", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, { "name": "Every 30 minutes", "value": "30" diff --git a/src/configurations/destinations/fb/db-config.json b/src/configurations/destinations/fb/db-config.json index 8d5942efb..28c1776a7 100644 --- a/src/configurations/destinations/fb/db-config.json +++ b/src/configurations/destinations/fb/db-config.json @@ -33,7 +33,8 @@ "cloud": ["page", "screen", "track"], "device": { "android": ["identify", "screen", "track"], - "ios": ["identify", "screen", "track"] + "ios": ["identify", "screen", "track"], + "reactnative": ["identify", "screen", "track"] } }, "supportedConnectionModes": { @@ -42,7 +43,7 @@ "web": ["cloud"], "unity": ["cloud"], "amp": ["cloud"], - "reactnative": ["cloud"], + "reactnative": ["cloud", "device"], "flutter": ["cloud"], "cordova": ["cloud"], "shopify": ["cloud"], @@ -98,6 +99,7 @@ "ketchConsentPurposes" ], "reactnative": [ + "useNativeSDK", "connectionMode", "consentManagement", "oneTrustCookieCategories", diff --git a/src/configurations/destinations/fb/schema.json b/src/configurations/destinations/fb/schema.json index e649d0852..d5789b0b2 100644 --- a/src/configurations/destinations/fb/schema.json +++ b/src/configurations/destinations/fb/schema.json @@ -631,7 +631,7 @@ }, "reactnative": { "type": "string", - "enum": ["cloud"] + "enum": ["cloud", "device"] }, "flutter": { "type": "string", diff --git a/src/configurations/destinations/ga4_v2/ui-config.json b/src/configurations/destinations/ga4_v2/ui-config.json index 351414323..5759521ef 100644 --- a/src/configurations/destinations/ga4_v2/ui-config.json +++ b/src/configurations/destinations/ga4_v2/ui-config.json @@ -18,9 +18,7 @@ "type": "nestedSelect", "label": "Select Account -> Property -> DataStream -> Protocol Secrets", "apiName": "getGA4ConfigData", - "apiDependencies": [ - "rudderAccountId" - ], + "apiDependencies": ["rudderAccountId"], "placeholder": "Select config details", "preRequisites": { "fields": [ @@ -2177,4 +2175,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/configurations/destinations/gcs_datalake/schema.json b/src/configurations/destinations/gcs_datalake/schema.json index cc0fc7a76..3c2f12f49 100644 --- a/src/configurations/destinations/gcs_datalake/schema.json +++ b/src/configurations/destinations/gcs_datalake/schema.json @@ -46,7 +46,7 @@ }, "syncFrequency": { "type": "string", - "enum": ["30", "60", "180", "360", "720", "1440"], + "enum": ["5", "15", "30", "60", "180", "360", "720", "1440"], "default": "180" }, "syncStartAt": { diff --git a/src/configurations/destinations/gcs_datalake/ui-config.json b/src/configurations/destinations/gcs_datalake/ui-config.json index 19d2a0f4f..855297cd5 100644 --- a/src/configurations/destinations/gcs_datalake/ui-config.json +++ b/src/configurations/destinations/gcs_datalake/ui-config.json @@ -96,6 +96,16 @@ "label": "Sync Frequency", "value": "syncFrequency", "options": [ + { + "name": "Every 5 minutes", + "value": "5", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, + { + "name": "Every 15 minutes", + "value": "15", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, { "name": "Every 30 minutes", "value": "30" diff --git a/src/configurations/destinations/lytics_bulk_upload/db-config.json b/src/configurations/destinations/lytics_bulk_upload/db-config.json index f9c1cae01..f99eeb183 100644 --- a/src/configurations/destinations/lytics_bulk_upload/db-config.json +++ b/src/configurations/destinations/lytics_bulk_upload/db-config.json @@ -6,14 +6,16 @@ "saveDestinationResponse": true, "includeKeys": ["oneTrustCookieCategories"], "excludeKeys": [], - "supportedSourceTypes": ["warehouse"], + "supportedSourceTypes": ["warehouse", "web", "cloud"], "supportedMessageTypes": { "cloud": ["track"] }, "syncBehaviours": ["upsert"], "disableJsonMapper": false, "supportedConnectionModes": { - "warehouse": ["cloud"] + "warehouse": ["cloud"], + "web": ["cloud"], + "cloud": ["cloud"] }, "destConfig": { "defaultConfig": [ @@ -23,7 +25,9 @@ "timestampField", "streamTraitsMapping" ], - "warehouse": ["connectionMode", "oneTrustCookieCategories", "ketchConsentPurposes"] + "cloud": ["connectionMode", "oneTrustCookieCategories", "ketchConsentPurposes"], + "warehouse": ["connectionMode", "oneTrustCookieCategories", "ketchConsentPurposes"], + "web": ["connectionMode", "oneTrustCookieCategories", "ketchConsentPurposes"] } }, "options": { diff --git a/src/configurations/destinations/lytics_bulk_upload/schema.json b/src/configurations/destinations/lytics_bulk_upload/schema.json index 7b7ce1cd2..3c706d5d4 100644 --- a/src/configurations/destinations/lytics_bulk_upload/schema.json +++ b/src/configurations/destinations/lytics_bulk_upload/schema.json @@ -50,15 +50,47 @@ } } } + }, + "web": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "cloud": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } } } }, "connectionMode": { "type": "object", "properties": { + "cloud": { + "type": "string", + "enum": ["cloud"] + }, "warehouse": { "type": "string", "enum": ["cloud"] + }, + "web": { + "type": "string", + "enum": ["cloud"] } } }, @@ -76,6 +108,30 @@ } } } + }, + "web": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "cloud": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } } } } diff --git a/src/configurations/destinations/mp/schema.json b/src/configurations/destinations/mp/schema.json index e492fcadc..72a86ebab 100644 --- a/src/configurations/destinations/mp/schema.json +++ b/src/configurations/destinations/mp/schema.json @@ -8,20 +8,11 @@ "type": "string", "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" }, - "serviceAccountUserName": { - "type": "string" - }, - "serviceAccountSecret": { - "type": "string" - }, "dataResidency": { "type": "string", "enum": ["us", "eu"], "default": "us" }, - "projectId": { - "type": "string" - }, "identityMergeApi": { "type": "string", "enum": ["simplified", "original"], diff --git a/src/configurations/destinations/mp/ui-config.json b/src/configurations/destinations/mp/ui-config.json index 51bd38175..fc0d43b19 100644 --- a/src/configurations/destinations/mp/ui-config.json +++ b/src/configurations/destinations/mp/ui-config.json @@ -13,24 +13,6 @@ "placeholder": "c24f56ed8172XXXX6de5a5f0befe8b92", "secret": true }, - { - "type": "textInput", - "label": "Service Account Username", - "value": "serviceAccountUserName", - "required": false - }, - { - "type": "textInput", - "label": "Service Account Secret", - "value": "serviceAccountSecret", - "required": false - }, - { - "type": "textInput", - "label": "Project ID", - "value": "projectId", - "required": false - }, { "type": "singleSelect", "label": "Data Residency", diff --git a/src/configurations/destinations/mssql/schema.json b/src/configurations/destinations/mssql/schema.json index 1d8244b88..b2774e61c 100644 --- a/src/configurations/destinations/mssql/schema.json +++ b/src/configurations/destinations/mssql/schema.json @@ -43,7 +43,7 @@ }, "syncFrequency": { "type": "string", - "enum": ["30", "60", "180", "360", "720", "1440"], + "enum": ["5", "15", "30", "60", "180", "360", "720", "1440"], "default": "180" }, "syncStartAt": { diff --git a/src/configurations/destinations/mssql/ui-config.json b/src/configurations/destinations/mssql/ui-config.json index 19f2e2d90..21d48b402 100644 --- a/src/configurations/destinations/mssql/ui-config.json +++ b/src/configurations/destinations/mssql/ui-config.json @@ -88,6 +88,16 @@ "label": "Sync Frequency", "value": "syncFrequency", "options": [ + { + "name": "Every 5 minutes", + "value": "5", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, + { + "name": "Every 15 minutes", + "value": "15", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, { "name": "Every 30 minutes", "value": "30" diff --git a/src/configurations/destinations/postgres/schema.json b/src/configurations/destinations/postgres/schema.json index 8682c12c0..7d41fe703 100644 --- a/src/configurations/destinations/postgres/schema.json +++ b/src/configurations/destinations/postgres/schema.json @@ -51,7 +51,7 @@ }, "syncFrequency": { "type": "string", - "pattern": "^(30|60|180|360|720|1440)$", + "pattern": "^(5|15|30|60|180|360|720|1440)$", "default": "180" }, "syncStartAt": { diff --git a/src/configurations/destinations/postgres/ui-config.json b/src/configurations/destinations/postgres/ui-config.json index 55646f5c8..61c26d05b 100644 --- a/src/configurations/destinations/postgres/ui-config.json +++ b/src/configurations/destinations/postgres/ui-config.json @@ -121,6 +121,16 @@ "label": "Sync Frequency", "value": "syncFrequency", "options": [ + { + "name": "Every 5 minutes", + "value": "5", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, + { + "name": "Every 15 minutes", + "value": "15", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, { "name": "Every 30 minutes", "value": "30" diff --git a/src/configurations/destinations/rs/db-config.json b/src/configurations/destinations/rs/db-config.json index 6c6d9de0c..c1fe5e62e 100644 --- a/src/configurations/destinations/rs/db-config.json +++ b/src/configurations/destinations/rs/db-config.json @@ -44,6 +44,8 @@ "iamRoleARNForAuth", "clusterId", "clusterRegion", + "useServerless", + "workgroupName", "bucketName", "iamRoleARN", "roleBasedAuth", diff --git a/src/configurations/destinations/rs/schema.json b/src/configurations/destinations/rs/schema.json index 0e6ac4e0b..d7678a83b 100644 --- a/src/configurations/destinations/rs/schema.json +++ b/src/configurations/destinations/rs/schema.json @@ -26,7 +26,7 @@ }, "syncFrequency": { "type": "string", - "enum": ["30", "60", "180", "360", "720", "1440"], + "enum": ["5", "15", "30", "60", "180", "360", "720", "1440"], "default": "180" }, "syncStartAt": { @@ -941,19 +941,51 @@ "type": "string", "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" }, - "clusterId": { - "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,255})$" - }, "clusterRegion": { "type": "string", "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,255})$" }, "useIAMForAuth": { "const": true + }, + "useServerless": { + "type": "boolean", + "default": false } }, - "required": ["clusterId", "clusterRegion", "iamRoleARNForAuth", "useIAMForAuth"] + "anyOf": [ + { + "properties": { + "clusterId": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,255})$" + }, + "useServerless": { + "const": false + }, + "useIAMForAuth": { + "const": true + } + }, + "required": ["clusterId", "useIAMForAuth"] + }, + { + "properties": { + "workgroupName": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,255})$" + }, + "useIAMForAuth": { + "const": true + }, + "useServerless": { + "const": true + } + }, + "required": ["workgroupName", "useIAMForAuth", "useServerless"] + } + ], + "required": ["clusterRegion", "iamRoleARNForAuth", "useIAMForAuth"] } }, { diff --git a/src/configurations/destinations/rs/ui-config.json b/src/configurations/destinations/rs/ui-config.json index db3e5ac9f..5e8d04057 100644 --- a/src/configurations/destinations/rs/ui-config.json +++ b/src/configurations/destinations/rs/ui-config.json @@ -40,12 +40,29 @@ "placeholder": "5439", "required": true }, + { + "type": "checkbox", + "preRequisiteField": [ + { + "name": "useIAMForAuth", + "selectedValue": true + } + ], + "label": "Use Redshift Serverless", + "value": "useServerless", + "required": false, + "default": false + }, { "type": "textInput", "preRequisiteField": [ { "name": "useIAMForAuth", "selectedValue": true + }, + { + "name": "useServerless", + "selectedValue": false } ], "label": "Cluster Id", @@ -55,6 +72,25 @@ "placeholder": "Cluster Identifier", "required": true }, + { + "type": "textInput", + "preRequisiteField": [ + { + "name": "useIAMForAuth", + "selectedValue": true + }, + { + "name": "useServerless", + "selectedValue": true + } + ], + "label": "Workgroup Name", + "value": "workgroupName", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,255})$", + "regexErrorMessage": "Invalid Workgroup Name", + "placeholder": "Workgroup Name", + "required": true + }, { "type": "textInput", "preRequisiteField": [ @@ -135,6 +171,16 @@ "label": "Sync Frequency", "value": "syncFrequency", "options": [ + { + "name": "Every 5 minutes", + "value": "5", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, + { + "name": "Every 15 minutes", + "value": "15", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, { "name": "Every 30 minutes", "value": "30" diff --git a/src/configurations/destinations/s3_datalake/schema.json b/src/configurations/destinations/s3_datalake/schema.json index dfa025677..dafe0a8d9 100644 --- a/src/configurations/destinations/s3_datalake/schema.json +++ b/src/configurations/destinations/s3_datalake/schema.json @@ -38,7 +38,7 @@ }, "syncFrequency": { "type": "string", - "enum": ["30", "60", "180", "360", "720", "1440"], + "enum": ["5", "15", "30", "60", "180", "360", "720", "1440"], "default": "180" }, "syncStartAt": { diff --git a/src/configurations/destinations/s3_datalake/ui-config.json b/src/configurations/destinations/s3_datalake/ui-config.json index 71bc69bcc..bc48f1a5e 100644 --- a/src/configurations/destinations/s3_datalake/ui-config.json +++ b/src/configurations/destinations/s3_datalake/ui-config.json @@ -116,6 +116,16 @@ "label": "Sync Frequency", "value": "syncFrequency", "options": [ + { + "name": "Every 5 minutes", + "value": "5", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, + { + "name": "Every 15 minutes", + "value": "15", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, { "name": "Every 30 minutes", "value": "30" diff --git a/src/configurations/destinations/singular/ui-config.json b/src/configurations/destinations/singular/ui-config.json index 5f1e76456..83d019335 100644 --- a/src/configurations/destinations/singular/ui-config.json +++ b/src/configurations/destinations/singular/ui-config.json @@ -43,14 +43,6 @@ "label": "Match ID mapping", "value": "match_id", "required": false, - "preRequisites": { - "featureFlags": [ - { - "configKey": "AMP_enable-singular-matchid", - "value": true - } - ] - }, "options": [ { "name": "Map match ID to context.device.advertisingId", diff --git a/src/configurations/destinations/snapchat_conversion/ui-config.json b/src/configurations/destinations/snapchat_conversion/ui-config.json index ff2941edd..3852a8794 100644 --- a/src/configurations/destinations/snapchat_conversion/ui-config.json +++ b/src/configurations/destinations/snapchat_conversion/ui-config.json @@ -61,43 +61,43 @@ "placeholderLeft": "e.g. Product Searched", "options": [ { - "name": "Products Searched", + "name": "Search", "value": "products_searched" }, { - "name": "Product List Viewed", + "name": "View Content ( For List of Products )", "value": "product_list_viewed" }, { - "name": "Promotion Viewed", + "name": "Ad View", "value": "promotion_viewed" }, { - "name": "Promotion Clicked", + "name": "Ad Click", "value": "promotion_clicked" }, { - "name": "Product Viewed", + "name": "View Content ( For Single Product )", "value": "product_viewed" }, { - "name": "Checkout Started", + "name": "Start Checkout", "value": "checkout_started" }, { - "name": "Payment Info Entered", + "name": "Add Billing", "value": "payment_info_entered" }, { - "name": "Order Completed", + "name": "Purchase", "value": "order_completed" }, { - "name": "Product Added", + "name": "Add Cart", "value": "product_added" }, { - "name": "Product Added To Wishlist", + "name": "Add To Wishlist", "value": "product_added_to_wishlist" }, { diff --git a/src/configurations/destinations/snowflake/schema.json b/src/configurations/destinations/snowflake/schema.json index 127660325..ea56e731d 100644 --- a/src/configurations/destinations/snowflake/schema.json +++ b/src/configurations/destinations/snowflake/schema.json @@ -30,7 +30,7 @@ }, "syncFrequency": { "type": "string", - "enum": ["30", "60", "180", "360", "720", "1440"], + "enum": ["5", "15", "30", "60", "180", "360", "720", "1440"], "default": "180" }, "syncStartAt": { diff --git a/src/configurations/destinations/snowflake/ui-config.json b/src/configurations/destinations/snowflake/ui-config.json index ab2eac7ef..a5ab4b86a 100644 --- a/src/configurations/destinations/snowflake/ui-config.json +++ b/src/configurations/destinations/snowflake/ui-config.json @@ -109,6 +109,16 @@ "label": "Sync Frequency", "value": "syncFrequency", "options": [ + { + "name": "Every 5 minutes", + "value": "5", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, + { + "name": "Every 15 minutes", + "value": "15", + "featureFlag": "HIGH_GRANULARITY_WH_SYNC" + }, { "name": "Every 30 minutes", "value": "30" diff --git a/src/configurations/destinations/webhook/schema.json b/src/configurations/destinations/webhook/schema.json index b9f81f31e..eee4570a5 100644 --- a/src/configurations/destinations/webhook/schema.json +++ b/src/configurations/destinations/webhook/schema.json @@ -517,7 +517,7 @@ }, "webhookUrl": { "type": "string", - "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(https?|ftp)://[^\\s/$.?#].[^\\s]*$" + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(https?|ftp):\\/\\/(?!localhost|127\\.0\\.0\\.1|10\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|192\\.168\\.\\d{1,3}\\.\\d{1,3}|172\\.(1[6-9]|2[0-9]|3[0-1])\\.\\d{1,3}\\.\\d{1,3}|0\\.0\\.0\\.0|\\[::1\\]|\\[fc[0-9a-fA-F]{2}:.*\\]|\\[fd[0-9a-fA-F]{2}:.*\\]|\\[::ffff:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\]).*([^\\s/$.?#].[^\\s]*)$" }, "webhookMethod": { "type": "string", diff --git a/src/configurations/destinations/webhook/ui-config.json b/src/configurations/destinations/webhook/ui-config.json index cf61a1a17..c39e5b9c8 100644 --- a/src/configurations/destinations/webhook/ui-config.json +++ b/src/configurations/destinations/webhook/ui-config.json @@ -9,7 +9,7 @@ "value": "webhookUrl", "required": true, "placeholder": "http://www.abcd.com", - "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(https?|ftp)://[^\\s/$.?#].[^\\s]*$" + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(https?|ftp):\\/\\/(?!localhost|127\\.0\\.0\\.1|10\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|192\\.168\\.\\d{1,3}\\.\\d{1,3}|172\\.(1[6-9]|2[0-9]|3[0-1])\\.\\d{1,3}\\.\\d{1,3}|0\\.0\\.0\\.0|\\[::1\\]|\\[fc[0-9a-fA-F]{2}:.*\\]|\\[fd[0-9a-fA-F]{2}:.*\\]|\\[::ffff:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\]).*([^\\s/$.?#].[^\\s]*)$" }, { "type": "singleSelect", diff --git a/src/configurations/sources/singer_google_ads/db-config.json b/src/configurations/sources/singer_google_ads/db-config.json index 50c95b631..e7fd6c94d 100644 --- a/src/configurations/sources/singer_google_ads/db-config.json +++ b/src/configurations/sources/singer_google_ads/db-config.json @@ -6,7 +6,7 @@ "auth": { "provider": "Google" }, - "image": "rudderstack/source-google-ads:v8.2.15" + "image": "rudderstack/source-google-ads:v8.3.6" }, "type": "cloudSource" } diff --git a/src/configurations/sources/singer_shopify/db-config.json b/src/configurations/sources/singer_shopify/db-config.json index 6ef782fc3..a0ac6601d 100644 --- a/src/configurations/sources/singer_shopify/db-config.json +++ b/src/configurations/sources/singer_shopify/db-config.json @@ -3,7 +3,7 @@ "category": "singer-protocol", "displayName": "Shopify", "options": { - "image": "rudderstack/source-shopify:v8.2.9" + "image": "rudderstack/source-shopify:v8.3.7" }, "type": "cloudSource" } diff --git a/test/data/validation/destinations/bq.json b/test/data/validation/destinations/bq.json index 8ecb38f76..feed21204 100644 --- a/test/data/validation/destinations/bq.json +++ b/test/data/validation/destinations/bq.json @@ -13,6 +13,74 @@ }, "result": true }, + { + "config": { + "project": "test-gcs-project", + "location": "", + "bucketName": "test-bucket", + "prefix": "xyzxx", + "namespace": "eu_new3", + "partitionColumn": "_PARTITIONTIME", + "partitionType": "day", + "credentials": "{}", + "syncFrequency": "30", + "testConnection": false, + "testConnectionTS": 1621402528550 + }, + "result": true + }, + { + "config": { + "project": "test-gcs-project", + "location": "", + "bucketName": "test-bucket", + "prefix": "xyzxx", + "namespace": "eu_new3", + "partitionColumn": "received_at", + "partitionType": "hour", + "credentials": "{}", + "syncFrequency": "30", + "testConnection": false, + "testConnectionTS": 1621402528550 + }, + "result": true + }, + { + "config": { + "project": "test-gcs-project", + "location": "", + "bucketName": "test-bucket", + "prefix": "xyzxx", + "namespace": "eu_new3", + "partitionColumn": "invalid", + "partitionType": "day", + "credentials": "{}", + "syncFrequency": "30", + "testConnection": false, + "testConnectionTS": 1621402528550 + }, + "result": false, + "error": [ + "partitionColumn must match pattern \"^(_PARTITIONTIME|loaded_at|received_at|timestamp)$\"" + ] + }, + { + "config": { + "project": "test-gcs-project", + "location": "", + "bucketName": "test-bucket", + "prefix": "xyzxx", + "namespace": "eu_new3", + "partitionColumn": "_PARTITIONTIME", + "partitionType": "invalid", + "credentials": "{}", + "syncFrequency": "30", + "testConnection": false, + "testConnectionTS": 1621402528550 + }, + "result": false, + "error": ["partitionType must match pattern \"^(hour|day|month|year)$\""] + }, { "config": { "project": "test-gcs-project", diff --git a/test/data/validation/destinations/linkedIn_ads.json b/test/data/validation/destinations/linkedIn_ads.json index 4b9499262..ac52f972b 100644 --- a/test/data/validation/destinations/linkedIn_ads.json +++ b/test/data/validation/destinations/linkedIn_ads.json @@ -1,6 +1,79 @@ [ { "config": { + "hashData": true, + "deduplicationKey": "properties.eventId", + "adAccountId": "1234", + "conversionMapping": [ + { + "from": "Order Completed", + "to": "1234567" + } + ] + }, + "result": true + }, + { + "config": { + "hashData": true, + "deduplicationKey": "properties.eventId", + "adAccountId": "1234", + "conversionMapping": [ + { + "from": "Order Completed", + "to": "" + } + ] + }, + "result": false, + "err": [ + "conversionMapping.0.to must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" + ] + }, + { + "config": { + "hashData": true, + "deduplicationKey": "properties.eventId", + "adAccountId": "1234", + "conversionMapping": [ + { + "from": "", + "to": "123454" + } + ] + }, + "result": false, + "err": [ + "conversionMapping.0.from must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" + ] + }, + { + "config": { + "hashData": "", + "adAccountId": "1234" + }, + "result": false, + "err": ["hashData must be boolean"] + }, + { + "config": { + "hashData": true, + "deduplicationKey": "properties.eventId", + "adAccountId": "1234" + }, + "result": true + }, + { + "config": { + "hashData": true, + "deduplicationKey": "properties.eventId", + "adAccountId": "1234", + "conversionMapping": [ + { + "from": "Order Completed", + "to": "1234567" + } + ], "oneTrustCookieCategories": { "android": [ { @@ -128,6 +201,15 @@ }, { "config": { + "hashData": true, + "deduplicationKey": "properties.eventId", + "adAccountId": "1234", + "conversionMapping": [ + { + "from": "Order Completed", + "to": "1234567" + } + ], "oneTrustCookieCategories": [ { "oneTrustCookieCategory": "C0001" @@ -150,6 +232,15 @@ }, { "config": { + "hashData": true, + "deduplicationKey": "properties.eventId", + "adAccountId": "1234", + "conversionMapping": [ + { + "from": "Order Completed", + "to": "1234567" + } + ], "oneTrustCookieCategories": { "android": [ { @@ -209,6 +300,7 @@ ] } }, + "result": false, "err": [ "oneTrustCookieCategories.android.0.oneTrustCookieCategory must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$\"", "oneTrustCookieCategories.ios.0.oneTrustCookieCategory must be string", @@ -218,7 +310,6 @@ "ketchConsentPurposes.ios.0.purpose must be string", "ketchConsentPurposes.web must be array", "ketchConsentPurposes.unity.0 must be object" - ], - "result": false + ] } ] diff --git a/test/data/validation/destinations/linkedin_ads.json b/test/data/validation/destinations/linkedin_ads.json deleted file mode 100644 index 88391ceed..000000000 --- a/test/data/validation/destinations/linkedin_ads.json +++ /dev/null @@ -1,91 +0,0 @@ -[ - { - "config": { - "hashData": true, - "deduplicationKey": "properties.eventId", - "adAccountId": "1234", - "conversionMapping": [ - { - "from": "Order Completed", - "to": "1234567" - } - ], - "oneTrustCookieCategories": [ - { - "oneTrustCookieCategory": "Marketing" - } - ] - }, - "result": true - }, - { - "config": { - "hashData": true, - "deduplicationKey": "properties.eventId", - "adAccountId": "1234", - "conversionMapping": [ - { - "from": "Order Completed", - "to": "" - } - ], - "oneTrustCookieCategories": [ - { - "oneTrustCookieCategory": "Marketing" - } - ] - }, - "result": false, - "err": [ - "conversionMapping.0.to must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" - ] - }, - { - "config": { - "hashData": true, - "deduplicationKey": "properties.eventId", - "adAccountId": "1234", - "conversionMapping": [ - { - "from": "", - "to": "123454" - } - ], - "oneTrustCookieCategories": [ - { - "oneTrustCookieCategory": "Marketing" - } - ] - }, - "result": false, - "err": [ - "conversionMapping.0.from must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\"" - ] - }, - { - "config": { - "hashData": "", - "adAccountId": "1234", - "oneTrustCookieCategories": [ - { - "oneTrustCookieCategory": "Marketing" - } - ] - }, - "result": false, - "err": ["hashData must be boolean"] - }, - { - "config": { - "hashData": true, - "deduplicationKey": "properties.eventId", - "adAccountId": "1234", - "oneTrustCookieCategories": [ - { - "oneTrustCookieCategory": "Marketing" - } - ] - }, - "result": true - } -] diff --git a/test/data/validation/destinations/lytics_bulk_upload.json b/test/data/validation/destinations/lytics_bulk_upload.json index 51795f424..a3b3f68a3 100644 --- a/test/data/validation/destinations/lytics_bulk_upload.json +++ b/test/data/validation/destinations/lytics_bulk_upload.json @@ -60,75 +60,39 @@ { "oneTrustCookieCategory": "C0002" } - ] - }, - "ketchConsentPurposes": { - "warehouse": [ - { - "purpose": "P1" - }, - { - "purpose": "P2" - } - ] - } - }, - "result": true - }, - { - "config": { - "lyticsAccountId": "434342324", - "lyticsApiKey": "447477473", - "lyticsStreamName": "Rudderstack", - "streamTraitsMapping": [ - { - "rudderProperty": "Email", - "lyticsProperty": "Email" - } - ], - "oneTrustCookieCategories": { - "warehouse": [ + ], + "web": [ { "oneTrustCookieCategory": "C0003" }, { "oneTrustCookieCategory": "C0004" } + ], + "cloud": [ + { + "oneTrustCookieCategory": "" + } ] }, "ketchConsentPurposes": { "warehouse": [ { - "purpose": "P3" + "purpose": "P1" }, { - "purpose": "P4" + "purpose": "P2" } - ] - } - }, - "result": true - }, - { - "config": { - "lyticsAccountId": "434342324", - "lyticsApiKey": "447477473", - "lyticsStreamName": "Rudderstack", - "streamTraitsMapping": [ - { - "rudderProperty": "Email", - "lyticsProperty": "Email" - } - ], - "oneTrustCookieCategories": { - "warehouse": [ + ], + "web": [ { - "oneTrustCookieCategory": "" + "purpose": "P3" + }, + { + "purpose": "P4" } - ] - }, - "ketchConsentPurposes": { - "warehouse": [ + ], + "cloud": [ { "purpose": "" } @@ -149,62 +113,26 @@ } ], "oneTrustCookieCategories": { - "warehouse": [] - }, - "ketchConsentPurposes": { - "warehouse": [] - } - }, - "result": true - }, - { - "config": { - "lyticsAccountId": "434342324", - "lyticsApiKey": "447477473", - "lyticsStreamName": "Rudderstack", - "streamTraitsMapping": [ - { - "rudderProperty": "Email", - "lyticsProperty": "Email" - } - ], - "oneTrustCookieCategories": { - "warehouse": [ + "warehouse": [], + "web": [ { "oneTrustCookieCategory": "env.ENVIRONMENT_VARIABLE" } - ] - }, - "ketchConsentPurposes": { - "warehouse": [ - { - "purpose": "env.ENVIRONMENT_VARIABLE" - } - ] - } - }, - "result": true - }, - { - "config": { - "lyticsAccountId": "434342324", - "lyticsApiKey": "447477473", - "lyticsStreamName": "Rudderstack", - "streamTraitsMapping": [ - { - "rudderProperty": "Email", - "lyticsProperty": "Email" - } - ], - "oneTrustCookieCategories": { - "warehouse": [ + ], + "cloud": [ { "oneTrustCookieCategory": "{{ event.properties.prop1 || 'val' }}" } ] }, "ketchConsentPurposes": { - "warehouse": [ + "warehouse": [], + "web": [ + { + "purpose": "env.ENVIRONMENT_VARIABLE" + } + ], + "cloud": [ { "purpose": "{{ event.properties.prop1 || 'val' }}" } @@ -263,38 +191,8 @@ { "oneTrustCookieCategory": "C0004" } - ] - }, - "ketchConsentPurposes": { - "warehouse": [ - { - "purpose": "more than 100 characters string - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" - }, - { - "purpose": "P4" - } - ] - } - }, - "result": false, - "err": [ - "oneTrustCookieCategories.warehouse.0.oneTrustCookieCategory must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$\"", - "ketchConsentPurposes.warehouse.0.purpose must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$\"" - ] - }, - { - "config": { - "lyticsAccountId": "434342324", - "lyticsApiKey": "447477473", - "lyticsStreamName": "Rudderstack", - "streamTraitsMapping": [ - { - "rudderProperty": "Email", - "lyticsProperty": "Email" - } - ], - "oneTrustCookieCategories": { - "warehouse": [ + ], + "web": [ { "oneTrustCookieCategory": { "not": "a string" @@ -303,10 +201,21 @@ { "oneTrustCookieCategory": "C0004" } - ] + ], + "cloud": { + "not": "an array" + } }, "ketchConsentPurposes": { "warehouse": [ + { + "purpose": "more than 100 characters string - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + }, + { + "purpose": "P4" + } + ], + "web": [ { "purpose": { "not": "a string" @@ -315,41 +224,20 @@ { "purpose": "P4" } - ] - } - }, - "result": false, - "err": [ - "oneTrustCookieCategories.warehouse.0.oneTrustCookieCategory must be string", - "ketchConsentPurposes.warehouse.0.purpose must be string" - ] - }, - { - "config": { - "lyticsAccountId": "434342324", - "lyticsApiKey": "447477473", - "lyticsStreamName": "Rudderstack", - "streamTraitsMapping": [ - { - "rudderProperty": "Email", - "lyticsProperty": "Email" - } - ], - "oneTrustCookieCategories": { - "warehouse": { - "not": "an array" - } - }, - "ketchConsentPurposes": { - "warehouse": { + ], + "cloud": { "not": "an array" } } }, "result": false, "err": [ - "oneTrustCookieCategories.warehouse must be array", - "ketchConsentPurposes.warehouse must be array" + "oneTrustCookieCategories.warehouse.0.oneTrustCookieCategory must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$\"", + "oneTrustCookieCategories.web.0.oneTrustCookieCategory must be string", + "oneTrustCookieCategories.cloud must be array", + "ketchConsentPurposes.warehouse.0.purpose must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$\"", + "ketchConsentPurposes.web.0.purpose must be string", + "ketchConsentPurposes.cloud must be array" ] }, { diff --git a/test/data/validation/destinations/rs.json b/test/data/validation/destinations/rs.json index 7419ae921..966873298 100644 --- a/test/data/validation/destinations/rs.json +++ b/test/data/validation/destinations/rs.json @@ -125,11 +125,117 @@ "result": false, "err": [ " must have required property 'clusterId'", + " must have required property 'workgroupName'", + " must have required property 'useServerless'", + " must match a schema in anyOf", " must have required property 'clusterRegion'", " must have required property 'iamRoleARNForAuth'", " must match \"then\" schema" ] }, + { + "config": { + "host": "", + "password": "", + "port": "", + "database": "test-database", + "user": "test-user", + "useIAMForAuth": true, + "useServerless": false, + "iamRoleARNForAuth": "iamRoleARNForAuth", + "clusterId": "clusterId", + "clusterRegion": "clusterRegion", + "useSSH": false, + "syncFrequency": "30", + "enableSSE": false, + "useRudderStorage": false, + "useSTSTokens": false, + "bucketName": "test-bucket", + "roleBasedAuth": false, + "accessKeyID": "", + "accessKey": "" + }, + "result": true + }, + { + "config": { + "host": "", + "password": "", + "port": "", + "database": "test-database", + "user": "test-user", + "useIAMForAuth": true, + "useServerless": true, + "iamRoleARNForAuth": "iamRoleARNForAuth", + "clusterId": "clusterId", + "clusterRegion": "clusterRegion", + "useSSH": false, + "syncFrequency": "30", + "enableSSE": false, + "useRudderStorage": false, + "useSTSTokens": false, + "bucketName": "test-bucket", + "roleBasedAuth": false, + "accessKeyID": "", + "accessKey": "" + }, + "result": false, + "err": [ + "useServerless must be equal to constant", + " must have required property 'workgroupName'", + " must match a schema in anyOf", + " must match \"then\" schema" + ] + }, + { + "config": { + "host": "", + "password": "", + "port": "", + "database": "test-database", + "user": "test-user", + "useIAMForAuth": true, + "useServerless": true, + "iamRoleARNForAuth": "iamRoleARNForAuth", + "workgroupName": "workgroupName", + "clusterRegion": "clusterRegion", + "useSSH": false, + "syncFrequency": "30", + "enableSSE": false, + "useRudderStorage": false, + "useSTSTokens": false, + "bucketName": "test-bucket", + "roleBasedAuth": false, + "accessKeyID": "", + "accessKey": "" + }, + "result": true + }, + { + "config": { + "host": "", + "password": "", + "port": "", + "database": "test-database", + "user": "test-user", + "useIAMForAuth": true, + "useServerless": true, + "iamRoleARNForAuth": "iamRoleARNForAuth", + "clusterId": "clusterId", + "workgroupName": "workgroupName", + "clusterRegion": "clusterRegion", + "useSSH": false, + "syncFrequency": "30", + "enableSSE": false, + "useRudderStorage": false, + "useSTSTokens": false, + "bucketName": "test-bucket", + "roleBasedAuth": false, + "accessKeyID": "", + "accessKey": "" + }, + "result": true + }, { "config": { "host": "test-host", diff --git a/test/data/validation/destinations/webhook.json b/test/data/validation/destinations/webhook.json index 1c27a8bf5..1b7b8f4e0 100644 --- a/test/data/validation/destinations/webhook.json +++ b/test/data/validation/destinations/webhook.json @@ -20,6 +20,27 @@ }, "result": true }, + { + "config": { + "webhookUrl": "http://[2001:4860:4860::8888]", + "webhookMethod": "GET", + "headers": [ + { + "from": "a", + "to": "a" + }, + { + "from": "b", + "to": "b" + }, + { + "from": "c", + "to": "c" + } + ] + }, + "result": true + }, { "config": { "webhookUrl": "http://23452345.ngrok.io/", @@ -41,7 +62,127 @@ }, "result": false, "err": [ - "webhookUrl must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(https?|ftp)://[^\\s/$.?#].[^\\s]*$\"" + "webhookUrl must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(https?|ftp):\\/\\/(?!localhost|127\\.0\\.0\\.1|10\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|192\\.168\\.\\d{1,3}\\.\\d{1,3}|172\\.(1[6-9]|2[0-9]|3[0-1])\\.\\d{1,3}\\.\\d{1,3}|0\\.0\\.0\\.0|\\[::1\\]|\\[fc[0-9a-fA-F]{2}:.*\\]|\\[fd[0-9a-fA-F]{2}:.*\\]|\\[::ffff:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\]).*([^\\s/$.?#].[^\\s]*)$\"" + ] + }, + { + "config": { + "webhookUrl": "http://localhost:3001", + "webhookMethod": "GET", + "headers": [ + { + "from": "a", + "to": "a" + }, + { + "from": "b", + "to": "b" + }, + { + "from": "c", + "to": "c" + } + ] + }, + "result": false, + "err": [ + "webhookUrl must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(https?|ftp):\\/\\/(?!localhost|127\\.0\\.0\\.1|10\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|192\\.168\\.\\d{1,3}\\.\\d{1,3}|172\\.(1[6-9]|2[0-9]|3[0-1])\\.\\d{1,3}\\.\\d{1,3}|0\\.0\\.0\\.0|\\[::1\\]|\\[fc[0-9a-fA-F]{2}:.*\\]|\\[fd[0-9a-fA-F]{2}:.*\\]|\\[::ffff:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\]).*([^\\s/$.?#].[^\\s]*)$\"" + ] + }, + { + "config": { + "webhookUrl": "https://192.168.1.100", + "webhookMethod": "GET", + "headers": [ + { + "from": "a", + "to": "a" + }, + { + "from": "b", + "to": "b" + }, + { + "from": "c", + "to": "c" + } + ] + }, + "result": false, + "err": [ + "webhookUrl must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(https?|ftp):\\/\\/(?!localhost|127\\.0\\.0\\.1|10\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|192\\.168\\.\\d{1,3}\\.\\d{1,3}|172\\.(1[6-9]|2[0-9]|3[0-1])\\.\\d{1,3}\\.\\d{1,3}|0\\.0\\.0\\.0|\\[::1\\]|\\[fc[0-9a-fA-F]{2}:.*\\]|\\[fd[0-9a-fA-F]{2}:.*\\]|\\[::ffff:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\]).*([^\\s/$.?#].[^\\s]*)$\"" + ] + }, + { + "config": { + "webhookUrl": "https://0.0.0.0", + "webhookMethod": "GET", + "headers": [ + { + "from": "a", + "to": "a" + }, + { + "from": "b", + "to": "b" + }, + { + "from": "c", + "to": "c" + } + ] + }, + "result": false, + "err": [ + "webhookUrl must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(https?|ftp):\\/\\/(?!localhost|127\\.0\\.0\\.1|10\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|192\\.168\\.\\d{1,3}\\.\\d{1,3}|172\\.(1[6-9]|2[0-9]|3[0-1])\\.\\d{1,3}\\.\\d{1,3}|0\\.0\\.0\\.0|\\[::1\\]|\\[fc[0-9a-fA-F]{2}:.*\\]|\\[fd[0-9a-fA-F]{2}:.*\\]|\\[::ffff:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\]).*([^\\s/$.?#].[^\\s]*)$\"" + ] + }, + { + "config": { + "webhookUrl": "https://[::1]", + "webhookMethod": "GET", + "headers": [ + { + "from": "a", + "to": "a" + }, + { + "from": "b", + "to": "b" + }, + { + "from": "c", + "to": "c" + } + ] + }, + "result": false, + "err": [ + "webhookUrl must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(https?|ftp):\\/\\/(?!localhost|127\\.0\\.0\\.1|10\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|192\\.168\\.\\d{1,3}\\.\\d{1,3}|172\\.(1[6-9]|2[0-9]|3[0-1])\\.\\d{1,3}\\.\\d{1,3}|0\\.0\\.0\\.0|\\[::1\\]|\\[fc[0-9a-fA-F]{2}:.*\\]|\\[fd[0-9a-fA-F]{2}:.*\\]|\\[::ffff:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\]).*([^\\s/$.?#].[^\\s]*)$\"" + ] + }, + { + "config": { + "webhookUrl": "https://[::ffff:192.168.0.1]", + "webhookMethod": "GET", + "headers": [ + { + "from": "a", + "to": "a" + }, + { + "from": "b", + "to": "b" + }, + { + "from": "c", + "to": "c" + } + ] + }, + "result": false, + "err": [ + "webhookUrl must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|(?!.*\\.ngrok\\.io)^(https?|ftp):\\/\\/(?!localhost|127\\.0\\.0\\.1|10\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}|192\\.168\\.\\d{1,3}\\.\\d{1,3}|172\\.(1[6-9]|2[0-9]|3[0-1])\\.\\d{1,3}\\.\\d{1,3}|0\\.0\\.0\\.0|\\[::1\\]|\\[fc[0-9a-fA-F]{2}:.*\\]|\\[fd[0-9a-fA-F]{2}:.*\\]|\\[::ffff:(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\]).*([^\\s/$.?#].[^\\s]*)$\"" ] }, {