diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f5c84544e..4f15c149d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -12,6 +12,7 @@ - [ ] I have made corresponding changes to the documentation - [ ] The code changed/added as part of this pull request has been covered with tests - [ ] All tests related to the changed code pass in development +- [ ] I have executed schemaGenerator tests and updated schema if needed ### Code review diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c3f2c9c4..f0b1e422a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.59.0](https://github.com/rudderlabs/rudder-config-schema/compare/v1.58.1...v1.59.0) (2023-11-13) + + +### Features + +* add Ketch for Mixpanel, CustomerIO & Snowflake ([#1054](https://github.com/rudderlabs/rudder-config-schema/issues/1054)) ([e1964fb](https://github.com/rudderlabs/rudder-config-schema/commit/e1964fbf74372761c076be00df9eedb285f036ed)) +* added allowUserSuppliedJavascript for braze ([#1070](https://github.com/rudderlabs/rudder-config-schema/issues/1070)) ([6b10504](https://github.com/rudderlabs/rudder-config-schema/commit/6b105041274efbf5e02a8af64c6b212a4910afec)) +* **analytics-js-integrations:** add in-app message in customerIo ([#1017](https://github.com/rudderlabs/rudder-config-schema/issues/1017)) ([7cf7385](https://github.com/rudderlabs/rudder-config-schema/commit/7cf7385d0e019a8844dd10d006f841adb4aae062)) +* custom screen call amplitude ([#1069](https://github.com/rudderlabs/rudder-config-schema/issues/1069)) ([c78780e](https://github.com/rudderlabs/rudder-config-schema/commit/c78780e2be5b3603457ff6c4cd80e027f8a07e04)) +* **INT-503:** hybrid mode braze ([#1030](https://github.com/rudderlabs/rudder-config-schema/issues/1030)) ([4bf4f48](https://github.com/rudderlabs/rudder-config-schema/commit/4bf4f48ae1a312b20aa9493d6c455d157c79b823)) +* **INT-901:** onboard sprig destination ([#1052](https://github.com/rudderlabs/rudder-config-schema/issues/1052)) ([62f089d](https://github.com/rudderlabs/rudder-config-schema/commit/62f089d4741c4b7d7d398412affcd17b886fd35f)) +* marketo: migrate to new UI layout ([#1044](https://github.com/rudderlabs/rudder-config-schema/issues/1044)) ([968310a](https://github.com/rudderlabs/rudder-config-schema/commit/968310a5f72e8480e2e38106b0be8e7b98e531d6)) +* onboarding salesforce with oauth ([#998](https://github.com/rudderlabs/rudder-config-schema/issues/998)) ([f774dfd](https://github.com/rudderlabs/rudder-config-schema/commit/f774dfdc371e12bf8545fc17672fa8420667d4f5)) + + +### Bug Fixes + +* eventFiltering options key and schema ([#1064](https://github.com/rudderlabs/rudder-config-schema/issues/1064)) ([8e0cf6a](https://github.com/rudderlabs/rudder-config-schema/commit/8e0cf6a362b4dae4e3361a8a0789a34c35f0fa28)) +* mp schema inconsistency ([#1068](https://github.com/rudderlabs/rudder-config-schema/issues/1068)) ([ee7cbed](https://github.com/rudderlabs/rudder-config-schema/commit/ee7cbed52b26927c95f9b5204c93b0b2e0a9106a)) +* remove web from Singular supported source ([#1051](https://github.com/rudderlabs/rudder-config-schema/issues/1051)) ([dee78f5](https://github.com/rudderlabs/rudder-config-schema/commit/dee78f54873b00a5802fbc3e34a33e65558b8933)) +* schema generator script for updating tagInput fiel schema ([#1066](https://github.com/rudderlabs/rudder-config-schema/issues/1066)) ([688aa0b](https://github.com/rudderlabs/rudder-config-schema/commit/688aa0b67f2c70da37915407bdc7fc6f18281ebb)) + + ### [1.58.1](https://github.com/rudderlabs/rudder-config-schema/compare/v1.58.0...v1.58.1) (2023-11-09) diff --git a/package-lock.json b/package-lock.json index fd1e992b0..2145b812a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-config-schema", - "version": "1.58.1", + "version": "1.59.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-config-schema", - "version": "1.58.1", + "version": "1.59.0", "license": "MIT", "dependencies": { "ajv": "^8.12.0", diff --git a/package.json b/package.json index 170b5ee93..5a235d34b 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-config-schema", - "version": "1.58.1", + "version": "1.59.0", "description": "", "main": "src/index.ts", "private": true, @@ -26,7 +26,13 @@ "prepare": "husky install", "pre-commit": "npm run test && npx lint-staged", "commit-msg": "commitlint --edit", - "release:github": "DEBUG=conventional-github-releaser npx conventional-github-releaser -p angular --config github-release.config.js" + "release:github": "DEBUG=conventional-github-releaser npx conventional-github-releaser -p angular --config github-release.config.js", + "check:schema:source:all": "python3 scripts/schemaGenerator.py source -all", + "check:schema:source": "python3 scripts/schemaGenerator.py source -name ", + "update:schema:source": "python3 scripts/schemaGenerator.py source -update -name ", + "check:schema:destination:all": "python3 scripts/schemaGenerator.py destination -all", + "check:schema:destination": "python3 scripts/schemaGenerator.py destination -name ", + "update:schema:destination": "python3 scripts/schemaGenerator.py destination -update -name " }, "devDependencies": { "@babel/core": "^7.21.3", diff --git a/scripts/schemaGenerator.py b/scripts/schemaGenerator.py index 6791d65e9..0edc0637d 100644 --- a/scripts/schemaGenerator.py +++ b/scripts/schemaGenerator.py @@ -1,8 +1,9 @@ ''' - Usage: schemaGenerator.py [-h] [-name name | -all] selector + Usage: schemaGenerator.py [-h] [-name name | -all] [-update] selector 1. selector - “source” or “destination” 2. all - runs the validator for all the selector. 3. name - any particular source or destination name such as `google_analytics` + 3. update - updates existing schema with detected changes Example: 1. python3 scripts/schemaGenerator.py -name="adobe_analytics" destination 2. python3 scripts/schemaGenerator.py -all source @@ -414,6 +415,15 @@ def generate_schema_for_tag_input(field, dbConfig, schema_field_name): } tagItem['properties'] = tagItemProps tagObject["items"] = tagItem + isSourceDependent = is_dest_field_dependent_on_source(field, dbConfig, schema_field_name) + if isSourceDependent: + tagObjectCopy = tagObject + tagObject = {} + tagObject = {"type": FieldTypeEnum.OBJECT.value} + tagObject["properties"] = {} + for sourceType in dbConfig["supportedSourceTypes"]: + if sourceType in dbConfig["destConfig"] and field[schema_field_name] in dbConfig["destConfig"][sourceType]: + tagObject["properties"][sourceType] = tagObjectCopy return tagObject @@ -802,7 +812,7 @@ def generate_config_props(config): generate_config_props(config) -def generate_schema(uiConfig, dbConfig, name, selector): +def generate_schema(uiConfig, dbConfig, name, selector, shouldUpdateSchema): """Returns the schema generated from given uiConfig and dbConfig. Args: @@ -810,6 +820,7 @@ def generate_schema(uiConfig, dbConfig, name, selector): dbConfig (object): Configurations of db-config.json. name (string): name of the source or destination. selector (string): either 'source' or 'destination' + shouldUpdateSchema (boolean): if it should update the existing schema with generated one Returns: object: schema @@ -824,7 +835,7 @@ def generate_schema(uiConfig, dbConfig, name, selector): if is_old_format(uiConfig): allOfSchemaObj = generate_schema_for_allOf(uiConfig, dbConfig, "value") if allOfSchemaObj: - # AnyOf occuring separately, not inside of allOf. + # AnyOf occurring separately, not inside allOf. if len(allOfSchemaObj) == 1: if isinstance(allOfSchemaObj[0], list): schemaObject['anyOf'] = allOfSchemaObj[0] @@ -835,6 +846,19 @@ def generate_schema(uiConfig, dbConfig, name, selector): generate_schema_properties(uiConfig, dbConfig, schemaObject, schemaObject['properties'], name, selector) newSchema['configSchema'] = schemaObject + + if shouldUpdateSchema: + # Get the parent directory (one level up) + script_directory = os.path.dirname(os.path.abspath(__file__)) + directory = os.path.dirname(script_directory) + # Define the relative path + relative_path = f'src/configurations/{selector}s/{name.lower()}/schema.json' + file_path = os.path.join(directory, relative_path) + new_content = json.dumps(newSchema) + # Write the new content + with open(file_path, 'w') as file: + file.write(new_content) + return newSchema def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): @@ -924,7 +948,7 @@ def generate_warnings_for_each_type(uiConfig, dbConfig, schema, curUiType): } -def validate_config_consistency(name, selector, uiConfig, dbConfig, schema): +def validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema): """Generates a schema and compares it with an existing one. If schemaDiff is present, it calls for individual warnings by iterating over each ui-type. @@ -934,6 +958,7 @@ def validate_config_consistency(name, selector, uiConfig, dbConfig, schema): uiConfig (object): file content of ui-config.json. dbConfig (object): Configurations of db-config.json. schema (object): Existing schema in schema.json. + shouldUpdateSchema (boolean): if it should update the existing schema with generated one """ if schema == None and uiConfig == None: return @@ -942,7 +967,7 @@ def validate_config_consistency(name, selector, uiConfig, dbConfig, schema): warnings.warn(f"Ui-Config is null for {name} in {selector} \n",UserWarning) print('-'*50) return - generatedSchema = generate_schema(uiConfig, dbConfig, name, selector) + generatedSchema = generate_schema(uiConfig, dbConfig, name, selector, shouldUpdateSchema) if schema: schemaDiff = diff(schema, generatedSchema["configSchema"]) if schemaDiff: @@ -986,12 +1011,13 @@ def validate_config_consistency(name, selector, uiConfig, dbConfig, schema): print(json.dumps(generatedSchema,indent=2)) print('-'*50) -def get_schema_diff(name, selector): +def get_schema_diff(name, selector, shouldUpdateSchema=False): """ Validates the schema for the given name and selector. Args: name (string): name of the source or destination. selector (string): either 'source' or 'destination'. + shouldUpdateSchema (boolean): if it should update the existing schema with generated one """ file_selectors = ['db-config.json', 'ui-config.json', 'schema.json'] directory = f'./{CONFIG_DIR}/{selector}s/{name}' @@ -1005,25 +1031,27 @@ def get_schema_diff(name, selector): schema = file_content.get("configSchema") dbConfig = file_content.get("config") if name not in EXCLUDED_DEST: - validate_config_consistency(name, selector, uiConfig, dbConfig, schema) + validate_config_consistency(name, selector, uiConfig, dbConfig, schema, shouldUpdateSchema) if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Generates schema.json from ui-cofing.json and db-config.json and validates against actual scheme.json') + parser = argparse.ArgumentParser(description='Generates schema.json from ui-config.json and db-config.json and validates against actual scheme.json') group = parser.add_mutually_exclusive_group() - parser.add_argument('selector',metavar='selector',type=str,help='Enter wheather -name is a source or destination') - group.add_argument('-name',metavar='name',type=str,help='Enter the folder name under selector') - group.add_argument('-all',action='store_true', help='will run validation for all entites under selector') - + parser.add_argument('selector', metavar='selector', type=str, help='Enter whether -name is a source or destination') + parser.add_argument('-update', action='store_true', help='Will update existing schema with any changes') + group.add_argument('-name', metavar='name', type=str, help='Enter the folder name under selector') + group.add_argument('-all', action='store_true', help='Will run validation for all entities under selector') args = parser.parse_args() selector = args.selector + shouldUpdateSchema = args.update + if args.all: CONFIG_DIR = 'src/configurations' current_items = os.listdir(f'./{CONFIG_DIR}/{selector}s') for name in current_items: - get_schema_diff(name,selector) + get_schema_diff(name, selector) else: name = args.name - get_schema_diff(name, selector) \ No newline at end of file + get_schema_diff(name, selector, shouldUpdateSchema) diff --git a/src/configurations/destinations/am/db-config.json b/src/configurations/destinations/am/db-config.json index 5b1950dfd..bcbc6bdb2 100644 --- a/src/configurations/destinations/am/db-config.json +++ b/src/configurations/destinations/am/db-config.json @@ -37,7 +37,9 @@ "attribution", "eventUploadThreshold", "eventUploadPeriodMillis", - "trackNewCampaigns" + "trackNewCampaigns", + "userProvidedScreenEventString", + "useUserDefinedScreenEventName" ], "excludeKeys": [], "supportedSourceTypes": [ @@ -84,7 +86,9 @@ "mapDeviceBrand", "oneTrustCookieCategories", "userProvidedPageEventString", - "useUserDefinedPageEventName" + "useUserDefinedPageEventName", + "userProvidedScreenEventString", + "useUserDefinedScreenEventName" ], "web": [ "useNativeSDK", diff --git a/src/configurations/destinations/am/schema.json b/src/configurations/destinations/am/schema.json index acc10ff25..a02e004a6 100644 --- a/src/configurations/destinations/am/schema.json +++ b/src/configurations/destinations/am/schema.json @@ -84,6 +84,11 @@ "mapDeviceBrand": { "type": "boolean", "default": false }, "trackProductsOnce": { "type": "boolean", "default": false }, "trackRevenuePerProduct": { "type": "boolean", "default": false }, + "useUserDefinedScreenEventName": { "type": "boolean", "default": false }, + "userProvidedScreenEventString": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,200})$" + }, "eventFilteringOption": { "type": "string", "enum": ["disable", "whitelistedEvents", "blacklistedEvents"], diff --git a/src/configurations/destinations/am/ui-config.json b/src/configurations/destinations/am/ui-config.json index d7e0dee59..bcb6261af 100644 --- a/src/configurations/destinations/am/ui-config.json +++ b/src/configurations/destinations/am/ui-config.json @@ -176,6 +176,45 @@ } } ] + }, + { + "title": "Screen settings", + "note": "Set how you want to send your screen calls to Amplitude", + "icon": "file", + "fields": [ + { + "type": "checkbox", + "label": "Use Custom Screen Event Name", + "configKey": "useUserDefinedScreenEventName", + "default": false, + "preRequisites": { + "fields": [ + { + "configKey": "connectionModes.cloud", + "value": true + } + ] + } + }, + { + "type": "textInput", + "label": "Screen Event Name Format", + "note": "Assign a event name for your screen calls. Text {{ }} will be replaced with event payload value", + "configKey": "userProvidedScreenEventString", + "regex": "^(.{0,200})$", + "regexErrorMessage": "Invalid Screen Event Name", + "placeholder": "e.g: Viewed a {{ name }}", + "secret": false, + "preRequisites": { + "fields": [ + { + "configKey": "useUserDefinedScreenEventName", + "value": true + } + ] + } + } + ] } ] }, diff --git a/src/configurations/destinations/braze/db-config.json b/src/configurations/destinations/braze/db-config.json index 4037af6dd..693afbb10 100644 --- a/src/configurations/destinations/braze/db-config.json +++ b/src/configurations/destinations/braze/db-config.json @@ -19,7 +19,9 @@ "whitelistedEvents", "oneTrustCookieCategories", "eventFilteringOption", - "connectionMode" + "connectionMode", + "enablePushNotification", + "allowUserSuppliedJavascript" ], "excludeKeys": [], "supportedSourceTypes": [ @@ -37,11 +39,20 @@ ], "supportedConnectionModes": { "android": ["cloud", "device", "hybrid"], - "web": ["cloud", "device"], + "web": ["cloud", "device", "hybrid"], "ios": ["cloud", "device", "hybrid"], "flutter": ["cloud", "device"], "reactnative": ["cloud", "device"] }, + "hybridModeCloudEventsFilter": { + "web": { + "messageType": [ + "identify", + "track", + "page" + ] + } + }, "supportedMessageTypes": ["group", "identify", "page", "screen", "track", "alias"], "destConfig": { "defaultConfig": [ @@ -61,7 +72,7 @@ "ios": ["useNativeSDK", "connectionMode"], "reactnative": ["useNativeSDK", "connectionMode"], "flutter": ["useNativeSDK", "connectionMode"], - "web": ["useNativeSDK", "enableBrazeLogging", "connectionMode"] + "web": ["useNativeSDK", "enableBrazeLogging", "connectionMode", "enablePushNotification", "allowUserSuppliedJavascript"] }, "secretKeys": ["restApiKey"] } diff --git a/src/configurations/destinations/braze/schema.json b/src/configurations/destinations/braze/schema.json index 5b691ee1e..7d34ef7d7 100644 --- a/src/configurations/destinations/braze/schema.json +++ b/src/configurations/destinations/braze/schema.json @@ -90,11 +90,13 @@ "properties": { "android": { "type": "string", "enum": ["cloud", "device", "hybrid"] }, "ios": { "type": "string", "enum": ["cloud", "device", "hybrid"] }, - "web": { "type": "string", "enum": ["cloud", "device"] }, + "web": { "type": "string", "enum": ["cloud", "device", "hybrid"] }, "reactnative": { "type": "string", "enum": ["cloud", "device"] }, "flutter": { "type": "string", "enum": ["cloud", "device"] } } - } + }, + "enablePushNotification": { "type": "object", "properties": { "web": { "type": "boolean" } } }, + "allowUserSuppliedJavascript": { "type": "object", "properties": { "web": { "type": "boolean" } } } } } } diff --git a/src/configurations/destinations/braze/ui-config.json b/src/configurations/destinations/braze/ui-config.json index e08209153..2a7aeaeb9 100644 --- a/src/configurations/destinations/braze/ui-config.json +++ b/src/configurations/destinations/braze/ui-config.json @@ -380,6 +380,27 @@ "configKey": "enableBrazeLogging", "default": false, "note": "Turn on if you want to show braze logs to customer" + }, + { + "type": "checkbox", + "label": "Use web push notifications", + "configKey": "enablePushNotification", + "default": false, + "note": [ + "Turn on if you want to use ", + { + "text": "push notification", + "link": "https://www.braze.com/docs/developer_guide/platform_integration_guides/web/push_notifications/integration/#step-1-configure-your-sites-service-worker" + }, + ". It requires service worker setup by client." + ] + }, + { + "type": "checkbox", + "label": "enable HTML in-app messages", + "configKey": "allowUserSuppliedJavascript", + "default": false, + "note": "Turn on if you want to enable HTML in-app messages" } ] } diff --git a/src/configurations/destinations/customerio/db-config.json b/src/configurations/destinations/customerio/db-config.json index 47207c3c5..15abb5c18 100644 --- a/src/configurations/destinations/customerio/db-config.json +++ b/src/configurations/destinations/customerio/db-config.json @@ -14,8 +14,10 @@ "blacklistedEvents", "whitelistedEvents", "oneTrustCookieCategories", + "ketchConsentPurposes", "eventFilteringOption", - "sendPageNameInSDK" + "sendPageNameInSDK", + "dataUseInApp" ], "excludeKeys": [], "supportedSourceTypes": [ @@ -41,9 +43,10 @@ "blacklistedEvents", "whitelistedEvents", "eventFilteringOption", - "oneTrustCookieCategories" + "oneTrustCookieCategories", + "ketchConsentPurposes" ], - "web": ["useNativeSDK", "sendPageNameInSDK"] + "web": ["dataUseInApp", "useNativeSDK", "sendPageNameInSDK"] }, "secretKeys": [] } diff --git a/src/configurations/destinations/customerio/schema.json b/src/configurations/destinations/customerio/schema.json index b158475ff..29aced2da 100644 --- a/src/configurations/destinations/customerio/schema.json +++ b/src/configurations/destinations/customerio/schema.json @@ -37,6 +37,14 @@ } } }, + "dataUseInApp": { + "type": "object", + "properties": { + "web": { + "type": "boolean" + } + } + }, "eventFilteringOption": { "type": "string", "enum": ["disable", "whitelistedEvents", "blacklistedEvents"], @@ -77,6 +85,13 @@ } } } + }, + "ketchConsentPurposes": { + "type": "array", + "items": { + "type": "object", + "properties": { "purpose": { "type": "string", "pattern": "^(.{0,100})$" } } + } } } } diff --git a/src/configurations/destinations/customerio/ui-config.json b/src/configurations/destinations/customerio/ui-config.json index 82c99cb8b..47fd7f997 100644 --- a/src/configurations/destinations/customerio/ui-config.json +++ b/src/configurations/destinations/customerio/ui-config.json @@ -68,6 +68,13 @@ "label": "Use device-mode to send events", "value": "useNativeSDK", "default": false + }, + { + "type": "checkbox", + "label": "Enable in-app message support", + "value": "dataUseInApp", + "default": false, + "footerNote": "Turn it ON if you want to send in-app messages to your website." } ] }, @@ -142,6 +149,21 @@ "required": false } ] + }, + { + "type": "dynamicCustomForm", + "value": "ketchConsentPurposes", + "label": "Ketch Consent Purposes", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "purpose", + "label": "Purpose ID", + "regex": "^(.{0,100})$", + "required": false + } + ] } ] } diff --git a/src/configurations/destinations/google_adwords_remarketing_lists/db-config.json b/src/configurations/destinations/google_adwords_remarketing_lists/db-config.json index 5d31a81bb..7dbbee326 100644 --- a/src/configurations/destinations/google_adwords_remarketing_lists/db-config.json +++ b/src/configurations/destinations/google_adwords_remarketing_lists/db-config.json @@ -32,20 +32,18 @@ "typeOfList", "oneTrustCookieCategories" ], - "cloud": [ - "audienceId" - ] + "cloud": ["audienceId"] }, "secretKeys": [] }, - "options": { + "options": { "isBeta": false, "destinationFeatures": { - "vdmLabels": { - "newAudienceAlternateLabel": "Create New List", - "existingAudienceAlternateLabel": "Use Existing List", - "audienceAlternateLabel": "List Id" - } - } + "vdmLabels": { + "newAudienceAlternateLabel": "Create New List", + "existingAudienceAlternateLabel": "Use Existing List", + "audienceAlternateLabel": "List Id" + } + } } } diff --git a/src/configurations/destinations/marketo/db-config.json b/src/configurations/destinations/marketo/db-config.json index 2f41a3333..9758f8c5f 100644 --- a/src/configurations/destinations/marketo/db-config.json +++ b/src/configurations/destinations/marketo/db-config.json @@ -7,6 +7,12 @@ "transformAt": "router", "transformAtV1": "router", "saveDestinationResponse": true, + "throttlingCost": { + "eventType": { + "identify": 3, + "track": 3 + } + }, "includeKeys": ["oneTrustCookieCategories"], "excludeKeys": [], "supportedSourceTypes": [ @@ -43,24 +49,66 @@ "responseType": "JSON", "rules": { "retryable": [ - { "success": "false", "errors.0.code": 600 }, - { "success": "false", "errors.0.code": 601 }, - { "success": "false", "errors.0.code": 602 }, - { "success": "false", "errors.0.code": 604 }, - { "success": "false", "errors.0.code": 611 } + { + "success": "false", + "errors.0.code": 600 + }, + { + "success": "false", + "errors.0.code": 601 + }, + { + "success": "false", + "errors.0.code": 602 + }, + { + "success": "false", + "errors.0.code": 604 + }, + { + "success": "false", + "errors.0.code": 611 + } ], "abortable": [ - { "success": "false", "errors.0.code": 603 }, - { "success": "false", "errors.0.code": 605 }, - { "success": "false", "errors.0.code": 609 }, - { "success": "false", "errors.0.code": 610 } + { + "success": "false", + "errors.0.code": 603 + }, + { + "success": "false", + "errors.0.code": 605 + }, + { + "success": "false", + "errors.0.code": 609 + }, + { + "success": "false", + "errors.0.code": 610 + } ], "throttled": [ - { "success": "false", "errors.0.code": 502 }, - { "success": "false", "errors.0.code": 606 }, - { "success": "false", "errors.0.code": 607 }, - { "success": "false", "errors.0.code": 608 }, - { "success": "false", "errors.0.code": 615 } + { + "success": "false", + "errors.0.code": 502 + }, + { + "success": "false", + "errors.0.code": 606 + }, + { + "success": "false", + "errors.0.code": 607 + }, + { + "success": "false", + "errors.0.code": 608 + }, + { + "success": "false", + "errors.0.code": 615 + } ] } } diff --git a/src/configurations/destinations/marketo_bulk_upload/db-config.json b/src/configurations/destinations/marketo_bulk_upload/db-config.json index b477d298d..14ad192ce 100644 --- a/src/configurations/destinations/marketo_bulk_upload/db-config.json +++ b/src/configurations/destinations/marketo_bulk_upload/db-config.json @@ -34,7 +34,7 @@ }, "secretKeys": ["clientId", "clientSecret"] }, - "options": { - "isBeta": true + "options": { + "isBeta": true } } diff --git a/src/configurations/destinations/mp/db-config.json b/src/configurations/destinations/mp/db-config.json index d44f590dc..dfa78ec4e 100644 --- a/src/configurations/destinations/mp/db-config.json +++ b/src/configurations/destinations/mp/db-config.json @@ -28,8 +28,10 @@ "blacklistedEvents", "whitelistedEvents", "oneTrustCookieCategories", + "ketchConsentPurposes", "eventFilteringOption", - "identityMergeApi" + "identityMergeApi", + "ignoreDnt" ], "excludeKeys": [], "supportedSourceTypes": [ @@ -75,6 +77,7 @@ "serviceAccountSecret", "projectId", "oneTrustCookieCategories", + "ketchConsentPurposes", "identityMergeApi", "userDeletionApi", "gdprApiToken", diff --git a/src/configurations/destinations/mp/schema.json b/src/configurations/destinations/mp/schema.json index 80037d134..afdf1512f 100644 --- a/src/configurations/destinations/mp/schema.json +++ b/src/configurations/destinations/mp/schema.json @@ -18,19 +18,24 @@ "serviceAccountSecret": { "type": "string" }, - "projectId": { - "type": "string" - }, "dataResidency": { "type": "string", "enum": ["us", "eu"], "default": "us" }, + "projectId": { + "type": "string" + }, "identityMergeApi": { "type": "string", "enum": ["simplified", "original"], "default": "original" }, + "userDeletionApi": { + "type": "string", + "enum": ["engage", "task"], + "default": "engage" + }, "strictMode": { "type": "boolean", "default": false @@ -39,11 +44,6 @@ "type": "boolean", "default": false }, - "userDeletionApi": { - "type": "string", - "enum": ["engage", "task"], - "default": "engage" - }, "people": { "type": "boolean", "default": false @@ -120,11 +120,6 @@ "type": "boolean", "default": false }, - "persistence": { - "type": "string", - "enum": ["none", "cookie", "localStorage"], - "default": "none" - }, "persistenceType": { "type": "string", "enum": ["none", "cookie", "localStorage"], @@ -202,6 +197,18 @@ } } } + }, + "ketchConsentPurposes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "purpose": { + "type": "string", + "pattern": "^(.{0,100})$" + } + } + } } }, "anyOf": [ diff --git a/src/configurations/destinations/mp/ui-config.json b/src/configurations/destinations/mp/ui-config.json index dfb52da41..276098b5f 100644 --- a/src/configurations/destinations/mp/ui-config.json +++ b/src/configurations/destinations/mp/ui-config.json @@ -406,6 +406,21 @@ "required": false } ] + }, + { + "type": "dynamicCustomForm", + "value": "ketchConsentPurposes", + "label": "Ketch Consent Purposes", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "purpose", + "label": "Purpose ID", + "regex": "^(.{0,100})$", + "required": false + } + ] } ] } diff --git a/src/configurations/destinations/salesforce/db-config.json b/src/configurations/destinations/salesforce/db-config.json index 48a024d3e..da7313d01 100644 --- a/src/configurations/destinations/salesforce/db-config.json +++ b/src/configurations/destinations/salesforce/db-config.json @@ -7,6 +7,11 @@ "transformAt": "router", "transformAtV1": "router", "saveDestinationResponse": true, + "throttlingCost": { + "eventType": { + "identify": 3 + } + }, "includeKeys": ["oneTrustCookieCategories"], "excludeKeys": [], "supportedSourceTypes": [ diff --git a/src/configurations/destinations/salesforce_oauth/db-config.json b/src/configurations/destinations/salesforce_oauth/db-config.json new file mode 100644 index 000000000..f8ce051fc --- /dev/null +++ b/src/configurations/destinations/salesforce_oauth/db-config.json @@ -0,0 +1,44 @@ +{ + "name": "SALESFORCE_OAUTH", + "displayName": "Salesforce V2", + "config": { + "isAudienceSupported": true, + "supportsVisualMapper": true, + "auth": { + "type": "OAuth", + "role": "salesforce", + "rudderScopes": [ + "delivery" + ] + }, + "transformAt": "router", + "transformAtV1": "router", + "saveDestinationResponse": true, + "includeKeys": ["oneTrustCookieCategories"], + "excludeKeys": [], + "supportedSourceTypes": [ + "android", + "ios", + "web", + "unity", + "amp", + "cloud", + "warehouse", + "reactnative", + "flutter", + "cordova" + ], + "supportedMessageTypes": ["identify"], + "destConfig": { + "defaultConfig": [ + "rudderAccountId", + "mapProperties", + "sandbox", + "useContactId", + "oneTrustCookieCategories" + ] + }, + "secretKeys": [] + }, + "options": { "hidden": true } +} diff --git a/src/configurations/destinations/salesforce_oauth/metadata.json b/src/configurations/destinations/salesforce_oauth/metadata.json new file mode 100644 index 000000000..59c4d726b --- /dev/null +++ b/src/configurations/destinations/salesforce_oauth/metadata.json @@ -0,0 +1,25 @@ +{ + "metadata": { + "primaryCategory": "", + "secondaryCategory": [], + "docLink": "", + "logoPath": "", + "connectionMode": { + "cloud-mode": true, + "device-mode": ["web", "android"] + }, + "supportedMethods": { + "cloud-mode": ["track", "identify"], + "device-mode": { + "web": ["track", "identify"] + } + }, + "releaseStatus": "beta", + "sourceCode": { + "android": { + "github": "", + "version": "v1.0.0" + } + } + } +} diff --git a/src/configurations/destinations/salesforce_oauth/schema.json b/src/configurations/destinations/salesforce_oauth/schema.json new file mode 100644 index 000000000..2383092fc --- /dev/null +++ b/src/configurations/destinations/salesforce_oauth/schema.json @@ -0,0 +1,24 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": [], + "type": "object", + "properties": { + "mapProperties": { "type": "boolean", "default": true }, + "sandbox": { "type": "boolean", "default": false }, + "useContactId": { "type": "boolean", "default": false }, + "oneTrustCookieCategories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + } + } +} diff --git a/src/configurations/destinations/salesforce_oauth/ui-config.json b/src/configurations/destinations/salesforce_oauth/ui-config.json new file mode 100644 index 000000000..8c7f52dcc --- /dev/null +++ b/src/configurations/destinations/salesforce_oauth/ui-config.json @@ -0,0 +1,43 @@ +{ + "uiConfig": [ + { + "title": "Other Settings", + "fields": [ + { + "type": "checkbox", + "label": "Map Rudder Properties to Salesforce Properties", + "value": "mapProperties", + "default": true + }, + { "type": "checkbox", "label": "Sandbox mode", "value": "sandbox", "default": false }, + { + "type": "checkbox", + "label": "Use contactId for converted leads", + "value": "useContactId", + "default": false, + "footerNote": "It is suggested to enable this option when both lead and contact field mappings are same." + } + ] + }, + { + "title": "Consent Settings", + "fields": [ + { + "type": "dynamicCustomForm", + "value": "oneTrustCookieCategories", + "label": "OneTrust Cookie Categories", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "oneTrustCookieCategory", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$", + "label": "Category Name/ID", + "required": false + } + ] + } + ] + } + ] +} diff --git a/src/configurations/destinations/sfmc/db-config.json b/src/configurations/destinations/sfmc/db-config.json index 32d81a54a..f586e288b 100644 --- a/src/configurations/destinations/sfmc/db-config.json +++ b/src/configurations/destinations/sfmc/db-config.json @@ -5,6 +5,12 @@ "transformAt": "router", "transformAtV1": "router", "saveDestinationResponse": true, + "throttlingCost": { + "eventType": { + "identify": 1, + "track": 1 + } + }, "includeKeys": ["oneTrustCookieCategories"], "excludeKeys": [], "supportedSourceTypes": [ diff --git a/src/configurations/destinations/singular/db-config.json b/src/configurations/destinations/singular/db-config.json index 9d6cb2c4e..3ba6b1e86 100644 --- a/src/configurations/destinations/singular/db-config.json +++ b/src/configurations/destinations/singular/db-config.json @@ -21,7 +21,6 @@ "flutter", "reactnative", "cordova", - "web", "amp", "cloud", "warehouse", diff --git a/src/configurations/destinations/snowflake/db-config.json b/src/configurations/destinations/snowflake/db-config.json index 58bcb7dbe..db7388ae8 100644 --- a/src/configurations/destinations/snowflake/db-config.json +++ b/src/configurations/destinations/snowflake/db-config.json @@ -6,7 +6,7 @@ "transformAt": "processor", "transformAtV1": "processor", "saveDestinationResponse": true, - "includeKeys": [], + "includeKeys": ["oneTrustCookieCategories", "ketchConsentPurposes"], "excludeKeys": [], "supportedSourceTypes": [ "android", @@ -50,7 +50,8 @@ "excludeWindow", "jsonPaths", "useRudderStorage", - "oneTrustCookieCategories" + "oneTrustCookieCategories", + "ketchConsentPurposes" ] }, "secretKeys": ["password", "accessKeyID", "accessKey", "accountKey", "sasToken"] diff --git a/src/configurations/destinations/snowflake/schema.json b/src/configurations/destinations/snowflake/schema.json index 60fdf5a1c..36b185cba 100644 --- a/src/configurations/destinations/snowflake/schema.json +++ b/src/configurations/destinations/snowflake/schema.json @@ -67,6 +67,13 @@ } } } + }, + "ketchConsentPurposes": { + "type": "array", + "items": { + "type": "object", + "properties": { "purpose": { "type": "string", "pattern": "^(.{0,100})$" } } + } } }, "allOf": [ diff --git a/src/configurations/destinations/snowflake/ui-config.json b/src/configurations/destinations/snowflake/ui-config.json index 932f52902..cbeec9121 100644 --- a/src/configurations/destinations/snowflake/ui-config.json +++ b/src/configurations/destinations/snowflake/ui-config.json @@ -556,11 +556,25 @@ "type": "textInput", "placeholder": "Marketing", "value": "oneTrustCookieCategory", - "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$", "label": "Category Name/ID", "required": false } ] + }, + { + "type": "dynamicCustomForm", + "value": "ketchConsentPurposes", + "label": "Ketch Consent Purposes", + "customFields": [ + { + "type": "textInput", + "placeholder": "Marketing", + "value": "purpose", + "label": "Purpose ID", + "regex": "^(.{0,100})$", + "required": false + } + ] } ] } diff --git a/src/configurations/destinations/sprig/db-config.json b/src/configurations/destinations/sprig/db-config.json new file mode 100644 index 000000000..02bbc65cf --- /dev/null +++ b/src/configurations/destinations/sprig/db-config.json @@ -0,0 +1,28 @@ +{ + "name": "SPRIG", + "displayName": "Sprig", + "config": { + "transformAt": "processor", + "transformAtV1": "processor", + "saveDestinationResponse": false, + "includeKeys": ["apiKey", "whitelistedEvents", "blacklistedEvents", "oneTrustCookieCategories"], + "excludeKeys": [], + "supportedConnectionModes": { + "web": ["device"] + }, + "supportedSourceTypes": ["web"], + "destConfig": { + "defaultConfig": [ + "apiKey", + "whitelistedEvents", + "blacklistedEvents", + "oneTrustCookieCategories" + ], + "web": [ + "useNativeSDK", + "connectionMode" + ] + }, + "secretKeys": ["apiKey"] + } +} diff --git a/src/configurations/destinations/sprig/metadata.json b/src/configurations/destinations/sprig/metadata.json new file mode 100644 index 000000000..e475ab5b6 --- /dev/null +++ b/src/configurations/destinations/sprig/metadata.json @@ -0,0 +1,22 @@ +{ + "metadata": { + "primaryCategory": "", + "secondaryCategory": [], + "docLink": "", + "logoPath": "", + "connectionMode": { + "device-mode": ["web"] + }, + "supportedMethods": { + "device-mode": { + "web": ["track", "identify"] + } + }, + "sourceCode": { + "android": { + "github": "", + "version": "v1.0.0" + } + } + } +} diff --git a/src/configurations/destinations/sprig/schema.json b/src/configurations/destinations/sprig/schema.json new file mode 100644 index 000000000..be16fd9f8 --- /dev/null +++ b/src/configurations/destinations/sprig/schema.json @@ -0,0 +1,66 @@ +{ + "configSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["apiKey"], + "type": "object", + "properties": { + "apiKey": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$" + }, + "useNativeSDK": { + "type": "object", + "properties": { + "web": { + "type": "boolean" + } + } + }, + "connectionMode": { + "type": "object", + "properties": { + "web": { + "type": "string", + "enum": ["device"] + } + } + }, + "whitelistedEvents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "eventName": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "blacklistedEvents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "eventName": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + }, + "oneTrustCookieCategories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "oneTrustCookieCategory": { + "type": "string", + "pattern": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{0,100})$" + } + } + } + } + } + } +} diff --git a/src/configurations/destinations/sprig/ui-config.json b/src/configurations/destinations/sprig/ui-config.json new file mode 100644 index 000000000..9d3277535 --- /dev/null +++ b/src/configurations/destinations/sprig/ui-config.json @@ -0,0 +1,175 @@ +{ + "uiConfig": { + "baseTemplate": [ + { + "title": "Initial setup", + "note": "Review how this destination is set up", + "sections": [ + { + "groups": [ + { + "title": "One click checkout", + "note": "Update your connection settings here", + "icon": "settings", + "fields": [ + { + "type": "textInput", + "label": "API Key", + "note": "Enter your sprig api key. Find this under your sprig dashboard > integrations > installation > javascript", + "configKey": "apiKey", + "regex": "(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$", + "regexErrorMessage": "Invalid API Key", + "placeholder": "e.g: EKfGdrD2IbPr", + "secret": true + } + ] + } + ] + }, + { + "groups": [ + { + "title": "Connection mode", + "note": [ + "Update how you want to route events from your source to destination. ", + { + "text": "Get help deciding", + "link": "https://www.rudderstack.com/docs/destinations/rudderstack-connection-modes/" + } + ], + "icon": "sliders", + "fields": [] + } + ] + } + ] + }, + { + "title": "Configuration settings", + "note": "Manage the settings for your destination", + "sections": [ + { + "title": "Other settings", + "note": "Configure advanced RudderStack features here", + "icon": "otherSettings", + "groups": [ + { + "title": "Client-side event filtering", + "note": "Decide what events are allowed (allowlisting) and blocked (denylisting)", + "preRequisites": { + "fields": [ + { + "configKey": "connectionModes.webDevice", + "value": true + }, + { + "configKey": "connectionModes.mobileDevice", + "value": true + } + ], + "condition": "or" + }, + "fields": [ + { + "type": "singleSelect", + "label": "Choose if you want to turn on events filtering:", + "configKey": "eventFilteringOption", + "note": "You must select either allowlist or denylist to enable events filtering", + "options": [ + { + "label": "No events filtering", + "value": "disable" + }, + { + "label": "Filter via allowlist", + "value": "whitelistedEvents" + }, + { + "label": "Filter via denylist", + "value": "blacklistedEvents" + } + ], + "default": "disable" + }, + { + "type": "tagInput", + "label": "Allowlisted events", + "note": "Input separate events by pressing ‘Enter’.\nInput the events you want to allowlist.", + "configKey": "whitelistedEvents", + "tagKey": "eventName", + "placeholder": "e.g: Anonymous page visit", + "default": [ + { + "eventName": "" + } + ], + "preRequisites": { + "fields": [ + { + "configKey": "eventFilteringOption", + "value": "whitelistedEvents" + } + ] + } + }, + { + "type": "tagInput", + "label": "Denylisted events", + "note": "Input separate events by pressing ‘Enter’.\nInput the events you want to denylist. ", + "configKey": "blacklistedEvents", + "tagKey": "eventName", + "placeholder": "e.g: Anonymous page visit", + "default": [ + { + "eventName": "" + } + ], + "preRequisites": { + "fields": [ + { + "configKey": "eventFilteringOption", + "value": "blacklistedEvents" + } + ] + } + } + ] + }, + { + "title": "OneTrust cookie consent settings", + "note": [ + "Enter your OneTrust category names if you have them configured. ", + { + "text": "Learn more ", + "link": "https://www.rudderstack.com/docs/sources/event-streams/sdks/rudderstack-javascript-sdk/onetrust-consent-manager/" + }, + "about RudderStack’s OneTrust Consent Manager feature." + ], + "fields": [ + { + "type": "tagInput", + "label": "Cookie category name", + "note": "Input your OneTrust category names by pressing ‘Enter’ after each entry", + "configKey": "oneTrustCookieCategories", + "tagKey": "oneTrustCookieCategory", + "placeholder": "e.g: Marketing", + "default": [ + { + "oneTrustCookieCategory": "" + } + ] + } + ] + } + ] + } + ] + } + ], + "sdkTemplate": { + "title": "Web SDK settings", + "note": "not visible in the ui", + "fields": [] + } + } +} diff --git a/src/configurations/sources/singer_google_ads/db-config.json b/src/configurations/sources/singer_google_ads/db-config.json index 4ac8602ae..d59cd195e 100644 --- a/src/configurations/sources/singer_google_ads/db-config.json +++ b/src/configurations/sources/singer_google_ads/db-config.json @@ -6,7 +6,6 @@ "auth": { "provider": "Google" }, - "digest": "sha256:9b52972787a0747792ed43d0d909eee3f0d1c3fe755b2492e109d63cd9731fd6", "image": "rudderstack/source-google-ads:v6.2.0-fix-google-ads-error-handling", "isBeta": false }, diff --git a/test/data/validation/destinations/braze.json b/test/data/validation/destinations/braze.json index 2339b91ca..46ff68054 100644 --- a/test/data/validation/destinations/braze.json +++ b/test/data/validation/destinations/braze.json @@ -166,7 +166,6 @@ "web": "hybrid" } }, - "result": false, - "err": ["connectionMode.web must be equal to one of the allowed values"] + "result": true } ] diff --git a/test/data/validation/destinations/salesforce_oauth.json b/test/data/validation/destinations/salesforce_oauth.json new file mode 100644 index 000000000..df31fa005 --- /dev/null +++ b/test/data/validation/destinations/salesforce_oauth.json @@ -0,0 +1,35 @@ +[ + { + "config": { + "mapProperties": true, + "sandbox": false, + "useContactId": true + }, + "result": true + }, + { + "config": { + "mapProperties": true, + "sandbox": false, + "useContactId": true + }, + "result": true + }, + { + "config": { + "mapProperties": true, + "sandbox": false, + "useContactId": true + }, + "result": true + }, + { + "config": { + "mapProperties": true, + "sandbox": "random", + "useContactId": true + }, + "result": false, + "err": ["sandbox must be boolean"] + } +] diff --git a/test/data/validation/destinations/sprig.json b/test/data/validation/destinations/sprig.json new file mode 100644 index 000000000..350601279 --- /dev/null +++ b/test/data/validation/destinations/sprig.json @@ -0,0 +1,75 @@ +[ + { + "config": { + "whitelistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": true + }, + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": false, + "err": [" must have required property 'apiKey'"] + }, + { + "config": { + "apiKey": "", + "whitelistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": true + }, + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": false, + "err": ["apiKey must match pattern \"(^\\{\\{.*\\|\\|(.*)\\}\\}$)|(^env[.].+)|^(.{1,100})$\""] + }, + { + "config": { + "apiKey": "12345", + "whitelistedEvents": [ + { + "eventName": "" + } + ], + "useNativeSDK": { + "web": true + }, + "blacklistedEvents": [ + { + "eventName": "" + } + ], + "oneTrustCookieCategories": [ + { + "oneTrustCookieCategory": "Marketing" + } + ] + }, + "result": true + } +]