diff --git a/ee/api/hooks.py b/ee/api/hooks.py index 6dd6dfd85e5c3..22d54c4b7bf8e 100644 --- a/ee/api/hooks.py +++ b/ee/api/hooks.py @@ -23,6 +23,7 @@ def create_zapier_hog_function(hook: Hook, serializer_context: dict) -> HogFunct serializer = HogFunctionSerializer( data={ "template_id": template_zapier.id, + "type": "destination", "name": f"Zapier webhook for action {hook.resource_id}", "filters": {"actions": [{"id": str(hook.resource_id), "name": "", "type": "actions", "order": 0}]}, "inputs": { diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-hog-function--dark.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-hog-function--dark.png index 47129e328b11d..d39232bc20b73 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-hog-function--dark.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-hog-function--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-hog-function--light.png b/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-hog-function--light.png index 30c69f8c33417..a128cf65fde2e 100644 Binary files a/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-hog-function--light.png and b/frontend/__snapshots__/scenes-app-pipeline--pipeline-node-new-hog-function--light.png differ diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index 2986d6daa9952..4f5dab68b9942 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -235,6 +235,7 @@ export const FEATURE_FLAGS = { REMOTE_CONFIG: 'remote-config', // owner: @benjackwhite SITE_DESTINATIONS: 'site-destinations', // owner: @mariusandra #team-cdp SITE_APP_FUNCTIONS: 'site-app-functions', // owner: @mariusandra #team-cdp + HOG_TRANSFORMATIONS: 'hog-transformations', // owner: #team-cdp REPLAY_HOGQL_FILTERS: 'replay-hogql-filters', // owner: @pauldambra #team-replay REPLAY_LIST_RECORDINGS_AS_QUERY: 'replay-list-recordings-as-query', // owner: @pauldambra #team-replay BILLING_SKIP_FORECASTING: 'billing-skip-forecasting', // owner: @zach diff --git a/frontend/src/scenes/pipeline/Transformations.tsx b/frontend/src/scenes/pipeline/Transformations.tsx index bd62e177b69f8..586e562f3a42c 100644 --- a/frontend/src/scenes/pipeline/Transformations.tsx +++ b/frontend/src/scenes/pipeline/Transformations.tsx @@ -4,6 +4,7 @@ import { arrayMove, SortableContext, useSortable, verticalListSortingStrategy } import { CSS } from '@dnd-kit/utilities' import { LemonBadge, LemonButton, LemonModal, LemonTable, LemonTableColumn, Link } from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' +import { FlaggedFeature } from 'lib/components/FlaggedFeature' import { PageHeader } from 'lib/components/PageHeader' import { ProductIntroduction } from 'lib/components/ProductIntroduction/ProductIntroduction' import { More } from 'lib/lemon-ui/LemonButton/More' @@ -14,6 +15,8 @@ import { urls } from 'scenes/urls' import { PipelineNodeTab, PipelineStage, ProductKey } from '~/types' import { AppMetricSparkLine } from './AppMetricSparkLine' +import { TRANSFORMATION_TYPES } from './destinations/constants' +import { Destinations } from './destinations/Destinations' import { NewButton } from './NewButton' import { pipelineAccessLogic } from './pipelineAccessLogic' import { PluginImage } from './PipelinePluginImage' @@ -61,7 +64,13 @@ export function Transformations(): JSX.Element { )} + + + +

Experimental transformations

+ +
) } @@ -238,7 +247,7 @@ const MinimalAppView = ({ transformation, order }: { transformation: Transformat return (
} /> - ) : ( - } - /> - )} + ) : null}
@@ -72,7 +67,9 @@ export function Destinations({ types }: DestinationsProps): JSX.Element { ? 'New destinations' : types.includes('site_app') ? 'New site app' - : 'New Hog function'} + : types.includes('transformation') + ? 'New transformation' + : 'New'} {/* Old site-apps until we migrate everyone onto the new ones */} @@ -169,7 +166,7 @@ export function DestinationsTable({ render: function RenderFrequency(_, destination) { return 'interval' in destination ? destination.interval : null }, - } as LemonTableColumn, + } as LemonTableColumn, ] : []), ...(showFrequencyHistory @@ -193,10 +190,10 @@ export function DestinationsTable({ ) }, - } as LemonTableColumn, + } as LemonTableColumn, ] : []), - updatedAtColumn() as LemonTableColumn, + updatedAtColumn() as LemonTableColumn, { title: 'Status', key: 'enabled', diff --git a/frontend/src/scenes/pipeline/destinations/constants.ts b/frontend/src/scenes/pipeline/destinations/constants.ts index dda2e7d0fe3d0..2d353802e5328 100644 --- a/frontend/src/scenes/pipeline/destinations/constants.ts +++ b/frontend/src/scenes/pipeline/destinations/constants.ts @@ -2,3 +2,4 @@ import { HogFunctionTypeType } from '~/types' export const DESTINATION_TYPES = ['destination', 'site_destination'] satisfies HogFunctionTypeType[] export const SITE_APP_TYPES = ['site_app'] satisfies HogFunctionTypeType[] +export const TRANSFORMATION_TYPES = ['transformation'] satisfies HogFunctionTypeType[] diff --git a/frontend/src/scenes/pipeline/destinations/destinationsLogic.tsx b/frontend/src/scenes/pipeline/destinations/destinationsLogic.tsx index 4630e26009cf0..42f8112f99b6c 100644 --- a/frontend/src/scenes/pipeline/destinations/destinationsLogic.tsx +++ b/frontend/src/scenes/pipeline/destinations/destinationsLogic.tsx @@ -20,6 +20,7 @@ import { PluginType, } from '~/types' +import { hogFunctionTypeToPipelineStage } from '../hogfunctions/urls' import { pipelineAccessLogic } from '../pipelineAccessLogic' import { BatchExportDestination, @@ -28,6 +29,7 @@ import { FunctionDestination, PipelineBackend, SiteApp, + Transformation, WebhookDestination, } from '../types' import { captureBatchExportEvent, capturePluginEvent, loadPluginsFromUrl } from '../utils' @@ -35,7 +37,7 @@ import { destinationsFiltersLogic } from './destinationsFiltersLogic' import type { pipelineDestinationsLogicType } from './destinationsLogicType' // Helping kea-typegen navigate the exported default class for Fuse -export interface Fuse extends FuseClass {} +export interface Fuse extends FuseClass {} export interface PipelineDestinationsLogicProps { types: HogFunctionTypeType[] @@ -60,9 +62,12 @@ export const pipelineDestinationsLogic = kea([ ], })), actions({ - toggleNode: (destination: Destination | SiteApp, enabled: boolean) => ({ destination, enabled }), + toggleNode: (destination: Destination | SiteApp | Transformation, enabled: boolean) => ({ + destination, + enabled, + }), toggleNodeHogFunction: (destination: FunctionDestination, enabled: boolean) => ({ destination, enabled }), - deleteNode: (destination: Destination | SiteApp) => ({ destination }), + deleteNode: (destination: Destination | SiteApp | Transformation) => ({ destination }), deleteNodeBatchExport: (destination: BatchExportDestination) => ({ destination }), deleteNodeHogFunction: (destination: FunctionDestination) => ({ destination }), deleteNodeWebhook: (destination: WebhookDestination) => ({ destination }), @@ -240,7 +245,7 @@ export const pipelineDestinationsLogic = kea([ hogFunctions, user, featureFlags - ): (Destination | SiteApp)[] => { + ): (Destination | Transformation | SiteApp)[] => { // Migrations are shown only in impersonation mode, for us to be able to trigger them. const httpEnabled = featureFlags[FEATURE_FLAGS.BATCH_EXPORTS_POSTHOG_HTTP] || user?.is_impersonated || user?.is_staff @@ -262,7 +267,7 @@ export const pipelineDestinationsLogic = kea([ const convertedDestinations = rawDestinations.map((d) => convertToPipelineNode( d, - 'type' in d && d.type === 'site_app' ? PipelineStage.SiteApp : PipelineStage.Destination + 'type' in d ? hogFunctionTypeToPipelineStage(d.type) : PipelineStage.Destination ) ) const enabledFirst = convertedDestinations.sort((a, b) => Number(b.enabled) - Number(a.enabled)) @@ -281,7 +286,7 @@ export const pipelineDestinationsLogic = kea([ filteredDestinations: [ (s) => [s.filters, s.destinations, s.destinationsFuse], - (filters, destinations, destinationsFuse): (Destination | SiteApp)[] => { + (filters, destinations, destinationsFuse): (Destination | Transformation | SiteApp)[] => { const { search, showPaused, kind } = filters return (search ? destinationsFuse.search(search).map((x) => x.item) : destinations).filter((dest) => { @@ -298,7 +303,7 @@ export const pipelineDestinationsLogic = kea([ hiddenDestinations: [ (s) => [s.destinations, s.filteredDestinations], - (destinations, filteredDestinations): (Destination | SiteApp)[] => { + (destinations, filteredDestinations): (Destination | Transformation | SiteApp)[] => { return destinations.filter((dest) => !filteredDestinations.includes(dest)) }, ], diff --git a/frontend/src/scenes/pipeline/hogfunctions/HogFunctionConfiguration.tsx b/frontend/src/scenes/pipeline/hogfunctions/HogFunctionConfiguration.tsx index b84ca60f244ab..cfa5dc06d463c 100644 --- a/frontend/src/scenes/pipeline/hogfunctions/HogFunctionConfiguration.tsx +++ b/frontend/src/scenes/pipeline/hogfunctions/HogFunctionConfiguration.tsx @@ -152,14 +152,13 @@ export function HogFunctionConfiguration({ templateId, id }: HogFunctionConfigur return } - const showFilters = type === 'destination' || type === 'site_destination' || type === 'broadcast' - const showExpectedVolume = type === 'destination' || type === 'site_destination' - const showStatus = type === 'destination' || type === 'email' - const showEnabled = type === 'destination' || type === 'email' || type === 'site_destination' || type === 'site_app' - const canEditSource = - type === 'destination' || type === 'email' || type === 'site_destination' || type === 'site_app' - const showPersonsCount = type === 'broadcast' - const showTesting = type === 'destination' || type === 'broadcast' || type === 'email' + const showFilters = ['destination', 'site_destination', 'broadcast', 'transformation'].includes(type) + const showExpectedVolume = ['destination', 'site_destination'].includes(type) + const showStatus = ['destination', 'email', 'transformation'].includes(type) + const showEnabled = ['destination', 'email', 'site_destination', 'site_app', 'transformation'].includes(type) + const canEditSource = ['destination', 'email', 'site_destination', 'site_app', 'transformation'].includes(type) + const showPersonsCount = ['broadcast'].includes(type) + const showTesting = ['destination', 'transformation', 'broadcast', 'email'].includes(type) return (
@@ -195,10 +194,10 @@ export function HogFunctionConfiguration({ templateId, id }: HogFunctionConfigur formKey="configuration" className="space-y-3" > -
-
-
-
+
+
+
+
{({ value, onChange }) => ( -
+
{configuration.name} {template && }
@@ -244,14 +243,14 @@ export function HogFunctionConfiguration({ templateId, id }: HogFunctionConfigur +

This function was built from the template{' '} {hogFunction.template.name}. If the template is updated, this function is not affected unless you choose to update it.

-
+
Close
@@ -272,8 +271,8 @@ export function HogFunctionConfiguration({ templateId, id }: HogFunctionConfigur
} > -
- +
+ Built from template: {hogFunction?.template.name} @@ -289,7 +288,7 @@ export function HogFunctionConfiguration({ templateId, id }: HogFunctionConfigur {showFilters && } {showPersonsCount && ( -
+
Matching persons
@@ -319,7 +318,7 @@ export function HogFunctionConfiguration({ templateId, id }: HogFunctionConfigur )} {showExpectedVolume && ( -
+
Expected volume {sparkline && !sparklineLoading ? ( <> @@ -359,10 +358,10 @@ export function HogFunctionConfiguration({ templateId, id }: HogFunctionConfigur )}
-
+
{!forcedSubTemplateId && template?.sub_templates && ( <> -
+
Choose template +
{subTemplate.name}
-
+
{subTemplate.description}
@@ -395,7 +394,7 @@ export function HogFunctionConfiguration({ templateId, id }: HogFunctionConfigur )} -
+
-
+

Edit source

{!showSource ?

Click here to edit the function's source code

: null} @@ -507,7 +506,7 @@ export function HogFunctionConfiguration({ templateId, id }: HogFunctionConfigur ) ) : null} -
{saveButtons}
+
{saveButtons}
diff --git a/frontend/src/scenes/pipeline/hogfunctions/HogFunctionTest.tsx b/frontend/src/scenes/pipeline/hogfunctions/HogFunctionTest.tsx index db83344bcbcf1..1861b06f369ed 100644 --- a/frontend/src/scenes/pipeline/hogfunctions/HogFunctionTest.tsx +++ b/frontend/src/scenes/pipeline/hogfunctions/HogFunctionTest.tsx @@ -102,16 +102,14 @@ export function HogFunctionTest(props: HogFunctionTestLogicProps): JSX.Element { ) : ( <> - {type === 'destination' ? ( - - Refresh globals - - ) : null} + + Refresh globals + {({ value, onChange }) => ( +
{({ value, onChange }) => ( +
-
+
Match events and actions
-

+

If set, the destination will only run if the event matches any of the below.

@@ -153,6 +154,33 @@ export function HogFunctionFilters(): JSX.Element { }} buttonCopy="Add event matcher" /> + + {showDropEvents && ( + <> + + + Drop events that don't match + onChange({ ...value, drop_events })} + /> + + + + {!value?.drop_events ? ( +

+ Currently, this will run for all events that match the above + conditions. Any that do not match will be unmodified and ingested as + they are. +

+ ) : ( + + This will drop all events that don't match the above conditions. + Please ensure this is definitely intended. + + )} + + )} ) : null} @@ -162,7 +190,7 @@ export function HogFunctionFilters(): JSX.Element { {showMasking ? ( {({ value, onChange }) => ( -
+
{configuration.masking?.hash ? ( <> -
+
of
-
+
or until { - if (!values.lastEventQuery || values.type !== 'destination') { + if (!values.lastEventQuery) { return values.sampleGlobals } const errorMessage = @@ -799,7 +802,7 @@ export const hogFunctionConfigurationLogic = kea [s.configuration, s.matchingFilters, s.groupTypes, s.type], (configuration, matchingFilters, groupTypes, type): EventsQuery | null => { - if (type !== 'destination') { + if (!TYPES_WITH_GLOBALS.includes(type)) { return null } const query: EventsQuery = { diff --git a/frontend/src/scenes/pipeline/hogfunctions/hogFunctionTestLogic.tsx b/frontend/src/scenes/pipeline/hogfunctions/hogFunctionTestLogic.tsx index 9caa8bc369165..7a06989b5b685 100644 --- a/frontend/src/scenes/pipeline/hogfunctions/hogFunctionTestLogic.tsx +++ b/frontend/src/scenes/pipeline/hogfunctions/hogFunctionTestLogic.tsx @@ -69,9 +69,7 @@ export const hogFunctionTestLogic = kea([ }), listeners(({ values, actions }) => ({ loadSampleGlobalsSuccess: () => { - if (values.type === 'destination') { - actions.setTestInvocationValue('globals', JSON.stringify(values.sampleGlobals, null, 2)) - } + actions.setTestInvocationValue('globals', JSON.stringify(values.sampleGlobals, null, 2)) }, })), forms(({ props, actions, values }) => ({ diff --git a/frontend/src/scenes/pipeline/hogfunctions/urls.ts b/frontend/src/scenes/pipeline/hogfunctions/urls.ts index a26ce4a331d55..fce68f1b7a82b 100644 --- a/frontend/src/scenes/pipeline/hogfunctions/urls.ts +++ b/frontend/src/scenes/pipeline/hogfunctions/urls.ts @@ -26,7 +26,9 @@ export function hogFunctionUrl(type: HogFunctionTypeType | PipelineStage, id?: s } // Supports both hog function types and pipeline stages themselves as input -export function hogFunctionTypeToPipelineStage(type: string): PipelineStage { +export function hogFunctionTypeToPipelineStage( + type: string +): PipelineStage.Destination | PipelineStage.Transformation | PipelineStage.SiteApp { switch (type) { case 'site_destination': return PipelineStage.Destination @@ -38,6 +40,8 @@ export function hogFunctionTypeToPipelineStage(type: string): PipelineStage { return PipelineStage.SiteApp case 'site-app': return PipelineStage.SiteApp + case 'transformation': + return PipelineStage.Transformation default: return PipelineStage.Destination } diff --git a/frontend/src/scenes/pipeline/pipelineAccessLogic.tsx b/frontend/src/scenes/pipeline/pipelineAccessLogic.tsx index 1d8875dbbedfa..41a3af9ee5aae 100644 --- a/frontend/src/scenes/pipeline/pipelineAccessLogic.tsx +++ b/frontend/src/scenes/pipeline/pipelineAccessLogic.tsx @@ -5,7 +5,7 @@ import { AvailableFeature } from '~/types' import { canConfigurePlugins, canGloballyManagePlugins } from './access' import type { pipelineAccessLogicType } from './pipelineAccessLogicType' -import { Destination, NewDestinationItemType, PipelineBackend, SiteApp } from './types' +import { Destination, NewDestinationItemType, PipelineBackend, SiteApp, Transformation } from './types' export const pipelineAccessLogic = kea([ path(['scenes', 'pipeline', 'pipelineAccessLogic']), @@ -25,8 +25,10 @@ export const pipelineAccessLogic = kea([ canEnableDestination: [ (s) => [s.canEnableNewDestinations], - (canEnableNewDestinations): ((destination: Destination | NewDestinationItemType | SiteApp) => boolean) => { - return (destination: Destination | NewDestinationItemType | SiteApp) => { + ( + canEnableNewDestinations + ): ((destination: Destination | NewDestinationItemType | SiteApp | Transformation) => boolean) => { + return (destination: Destination | NewDestinationItemType | SiteApp | Transformation) => { return destination.backend === PipelineBackend.HogFunction ? ('hog_function' in destination ? destination.hog_function.type === 'site_destination' || diff --git a/frontend/src/scenes/pipeline/types.ts b/frontend/src/scenes/pipeline/types.ts index b9621ad253cca..3c1c69cf0f318 100644 --- a/frontend/src/scenes/pipeline/types.ts +++ b/frontend/src/scenes/pipeline/types.ts @@ -124,18 +124,14 @@ export function convertToPipelineNode( ? Source : never { let node: PipelineNode + // check if type is a hog function if ('hog' in candidate) { node = { stage: stage as PipelineStage.Destination, backend: PipelineBackend.HogFunction, interval: 'realtime', - id: - candidate.type === 'destination' || - candidate.type === 'site_destination' || - candidate.type === 'site_app' - ? `hog-${candidate.id}` - : candidate.id, + id: `hog-${candidate.id}`, name: candidate.name, description: candidate.description, enabled: candidate.enabled, diff --git a/frontend/src/types.ts b/frontend/src/types.ts index e2ac6193659cf..514caa3635726 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -4669,6 +4669,7 @@ export type HogFunctionTypeType = | 'destination' | 'site_destination' | 'site_app' + | 'transformation' | 'email' | 'sms' | 'push' diff --git a/posthog/api/hog_function.py b/posthog/api/hog_function.py index 9e47a1da1cdea..4549f4f3a8bb5 100644 --- a/posthog/api/hog_function.py +++ b/posthog/api/hog_function.py @@ -128,8 +128,19 @@ class Meta: "inputs_schema": {"required": False}, "template_id": {"write_only": True}, "deleted": {"write_only": True}, + "type": {"required": True}, } + def validate_type(self, value): + # Ensure it is only set when creating a new function + if self.context.get("view") and self.context["view"].action == "create": + return value + + instance = cast(Optional[HogFunction], self.context.get("instance", self.instance)) + if instance and instance.type != value: + raise serializers.ValidationError("Cannot modify the type of an existing function") + return value + def validate(self, attrs): team = self.context["get_team"]() attrs["team"] = team @@ -137,6 +148,8 @@ def validate(self, attrs): has_addon = team.organization.is_feature_available(AvailableFeature.DATA_PIPELINES) instance = cast(Optional[HogFunction], self.context.get("instance", self.instance)) + hog_type = attrs.get("type", instance.type if instance else "destination") + if not has_addon: template_id = attrs.get("template_id", instance.template_id if instance else None) template = HOG_FUNCTION_TEMPLATES_BY_ID.get(template_id, None) @@ -157,9 +170,6 @@ def validate(self, attrs): attrs["mappings"] = template.mappings attrs["hog"] = template.hog - if "type" not in attrs: - attrs["type"] = "destination" - if self.context.get("view") and self.context["view"].action == "create": # Ensure we have sensible defaults when created attrs["filters"] = attrs.get("filters") or {} @@ -168,7 +178,7 @@ def validate(self, attrs): attrs["mappings"] = attrs.get("mappings") or None # Used for both top level input validation, and mappings input validation - def validate_input_and_filters(attrs: dict, type: str): + def validate_input_and_filters(attrs: dict): if "inputs_schema" in attrs: attrs["inputs_schema"] = validate_inputs_schema(attrs["inputs_schema"]) @@ -180,28 +190,28 @@ def validate_input_and_filters(attrs: dict, type: str): existing_encrypted_inputs = instance.encrypted_inputs attrs["inputs_schema"] = attrs.get("inputs_schema", instance.inputs_schema if instance else []) - attrs["inputs"] = validate_inputs(attrs["inputs_schema"], inputs, existing_encrypted_inputs, type) + attrs["inputs"] = validate_inputs(attrs["inputs_schema"], inputs, existing_encrypted_inputs, hog_type) if "filters" in attrs: - if type in TYPES_WITH_COMPILED_FILTERS: + if hog_type in TYPES_WITH_COMPILED_FILTERS: attrs["filters"] = compile_filters_bytecode(attrs["filters"], team) - elif type in TYPES_WITH_TRANSPILED_FILTERS: + elif hog_type in TYPES_WITH_TRANSPILED_FILTERS: compiler = JavaScriptCompiler() code = compiler.visit(compile_filters_expr(attrs["filters"], team)) attrs["filters"]["transpiled"] = {"lang": "ts", "code": code, "stl": list(compiler.stl_functions)} if "bytecode" in attrs["filters"]: del attrs["filters"]["bytecode"] - validate_input_and_filters(attrs, attrs["type"]) + validate_input_and_filters(attrs) if attrs.get("mappings", None) is not None: - if attrs["type"] != "site_destination": + if hog_type != "site_destination": raise serializers.ValidationError({"mappings": "Mappings are only allowed for site destinations."}) for mapping in attrs["mappings"]: - validate_input_and_filters(mapping, attrs["type"]) + validate_input_and_filters(mapping) if "hog" in attrs: - if attrs["type"] in TYPES_WITH_JAVASCRIPT_SOURCE: + if hog_type in TYPES_WITH_JAVASCRIPT_SOURCE: try: # Validate transpilation using the model instance attrs["transpiled"] = get_transpiled_function( @@ -216,7 +226,7 @@ def validate_input_and_filters(attrs: dict, type: str): raise serializers.ValidationError({"hog": "Error in TypeScript code"}) attrs["bytecode"] = None else: - attrs["bytecode"] = compile_hog(attrs["hog"]) + attrs["bytecode"] = compile_hog(attrs["hog"], hog_type) attrs["transpiled"] = None else: attrs["bytecode"] = None diff --git a/posthog/api/test/test_hog_function.py b/posthog/api/test/test_hog_function.py index b4b72cfc5882f..b988b53fdbbfb 100644 --- a/posthog/api/test/test_hog_function.py +++ b/posthog/api/test/test_hog_function.py @@ -20,6 +20,7 @@ "name": "HogHook", "hog": "fetch(inputs.url, {\n 'headers': inputs.headers,\n 'body': inputs.payload,\n 'method': inputs.method\n});", "type": "destination", + "enabled": True, "inputs_schema": [ {"key": "url", "type": "string", "label": "Webhook URL", "required": True}, {"key": "payload", "type": "json", "label": "JSON Payload", "required": True}, @@ -74,6 +75,7 @@ def _create_slack_function(self, data: Optional[dict] = None): payload = { "name": "Slack", "template_id": template_slack.id, + "type": "destination", "inputs": { "slack_workspace": {"value": 1}, "channel": {"value": "#general"}, @@ -196,7 +198,13 @@ def _filter_expected_keys(self, actual_data, expected_structure): def test_create_hog_function(self, *args): response = self.client.post( f"/api/projects/{self.team.id}/hog_functions/", - data={"name": "Fetch URL", "description": "Test description", "hog": "fetch(inputs.url);", "inputs": {}}, + data={ + "type": "destination", + "name": "Fetch URL", + "description": "Test description", + "hog": "fetch(inputs.url);", + "inputs": {}, + }, ) assert response.status_code == status.HTTP_201_CREATED, response.json() assert response.json()["created_by"]["id"] == self.user.id @@ -257,6 +265,7 @@ def test_creates_with_template_id(self, *args): "description": "Test description", "hog": "fetch(inputs.url);", "template_id": template_webhook.id, + "type": "destination", }, ) assert response.status_code == status.HTTP_201_CREATED, response.json() @@ -365,6 +374,7 @@ def test_inputs_required(self, *args): "inputs_schema": [ {"key": "url", "type": "string", "label": "Webhook URL", "required": True}, ], + "type": "destination", } # Check required res = self.client.post(f"/api/projects/{self.team.id}/hog_functions/", data={**payload}) @@ -385,6 +395,7 @@ def test_inputs_mismatch_type(self, *args): {"key": "dictionary", "type": "dictionary"}, {"key": "boolean", "type": "boolean"}, ], + "type": "destination", } bad_inputs = { @@ -417,6 +428,7 @@ def test_secret_inputs_not_returned(self, *args): "value": "I AM SECRET", }, }, + "type": "destination", } expectation = { "url": { @@ -456,6 +468,7 @@ def test_secret_inputs_not_returned(self, *args): def test_secret_inputs_not_updated_if_not_changed(self, *args): payload = { + "type": "destination", "name": "Fetch URL", "hog": "fetch(inputs.url);", "inputs_schema": [ @@ -492,6 +505,7 @@ def test_secret_inputs_not_updated_if_not_changed(self, *args): def test_secret_inputs_updated_if_changed(self, *args): payload = { + "type": "destination", "name": "Fetch URL", "hog": "fetch(inputs.url);", "inputs_schema": [ @@ -583,6 +597,7 @@ def test_generates_hog_bytecode(self, *args): response = self.client.post( f"/api/projects/{self.team.id}/hog_functions/", data={ + "type": "destination", "name": "Fetch URL", "hog": "let i := 0;\nwhile(i < 3) {\n i := i + 1;\n fetch(inputs.url, {\n 'headers': {\n 'x-count': f'{i}'\n },\n 'body': inputs.payload,\n 'method': inputs.method\n });\n}", }, @@ -793,13 +808,7 @@ def test_loads_status_when_enabled_and_available(self, *args): response = self.client.post( f"/api/projects/{self.team.id}/hog_functions/", - data={ - "name": "Fetch URL", - "description": "Test description", - "hog": "fetch(inputs.url);", - "template_id": template_webhook.id, - "enabled": True, - }, + data=EXAMPLE_FULL, ) assert response.status_code == status.HTTP_201_CREATED, response.json() @@ -813,13 +822,7 @@ def test_does_not_crash_when_status_not_available(self, *args): response = self.client.post( f"/api/projects/{self.team.id}/hog_functions/", - data={ - "name": "Fetch URL", - "description": "Test description", - "hog": "fetch(inputs.url);", - "template_id": template_webhook.id, - "enabled": True, - }, + data=EXAMPLE_FULL, ) assert response.status_code == status.HTTP_201_CREATED, response.json() response = self.client.get(f"/api/projects/{self.team.id}/hog_functions/{response.json()['id']}") @@ -833,7 +836,7 @@ def test_patches_status_on_enabled_update(self, *args): response = self.client.post( f"/api/projects/{self.team.id}/hog_functions/", - data={"name": "Fetch URL", "hog": "fetch(inputs.url);", "enabled": True}, + data={"type": "destination", "name": "Fetch URL", "hog": "fetch(inputs.url);", "enabled": True}, ) id = response.json()["id"] @@ -1058,6 +1061,30 @@ def test_create_hog_function_with_site_destination_type(self): assert response.json()["bytecode"] is None assert "Hello, site_destination" in response.json()["transpiled"] + def test_cannot_modify_type_of_existing_hog_function(self): + response = self.client.post( + f"/api/projects/{self.team.id}/hog_functions/", + data={ + "name": "Site Destination Function", + "hog": "export function onLoad() { console.log('Hello, site_destination'); }", + "type": "site_destination", + }, + ) + + assert response.status_code == status.HTTP_201_CREATED, response.json() + + response = self.client.patch( + f"/api/projects/{self.team.id}/hog_functions/{response.json()['id']}/", + data={"type": "site_app"}, + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json() + assert response.json() == { + "attr": "type", + "detail": "Cannot modify the type of an existing function", + "code": "invalid_input", + "type": "validation_error", + } + def test_transpiled_field_not_populated_for_other_types(self): response = self.client.post( f"/api/projects/{self.team.id}/hog_functions/", diff --git a/posthog/cdp/templates/__init__.py b/posthog/cdp/templates/__init__.py index 57a53d0ab9276..57855bb7ca96f 100644 --- a/posthog/cdp/templates/__init__.py +++ b/posthog/cdp/templates/__init__.py @@ -47,6 +47,7 @@ from ._siteapps.template_debug_posthog import template as debug_posthog from ._internal.template_broadcast import template_new_broadcast as _broadcast from ._internal.template_blank import blank_site_destination, blank_site_app +from ._transformations.template_pass_through import template as pass_through_transformation HOG_FUNCTION_TEMPLATES = [ _broadcast, @@ -97,6 +98,7 @@ hogdesk, notification_bar, pineapple_mode, + pass_through_transformation, debug_posthog, ] diff --git a/posthog/cdp/templates/_transformations/template_pass_through.py b/posthog/cdp/templates/_transformations/template_pass_through.py new file mode 100644 index 0000000000000..5a4e88e003d31 --- /dev/null +++ b/posthog/cdp/templates/_transformations/template_pass_through.py @@ -0,0 +1,18 @@ +from posthog.cdp.templates.hog_function_template import HogFunctionTemplate + +template: HogFunctionTemplate = HogFunctionTemplate( + status="alpha", + type="transformation", + id="template-blank-transformation", + name="Custom transformation", + description="This is a starter template for custom transformations", + icon_url="/static/hedgehog/builder-hog-01.png", + category=["Custom"], + hog=""" +// This is a blank template for custom transformations +// The function receives `event` as a global object and expects it to be returned +// If you return null then the event will be discarded +return event +""".strip(), + inputs_schema=[], +) diff --git a/posthog/cdp/templates/helpers.py b/posthog/cdp/templates/helpers.py index e26f55f842a73..6f1eba04ae75d 100644 --- a/posthog/cdp/templates/helpers.py +++ b/posthog/cdp/templates/helpers.py @@ -16,7 +16,7 @@ class BaseHogFunctionTemplateTest(BaseTest): def setUp(self): super().setUp() - self.compiled_hog = compile_hog(self.template.hog, supported_functions={"fetch", "postHogCapture"}) + self.compiled_hog = compile_hog(self.template.hog, self.template.type) self.mock_print = MagicMock(side_effect=lambda *args: print("[DEBUG HogFunctionPrint]", *args)) # noqa: T201 # Side effect - log the fetch call and return with sensible output diff --git a/posthog/cdp/templates/hog_function_template.py b/posthog/cdp/templates/hog_function_template.py index fea7ca56b6307..0ebfc1f1c37dc 100644 --- a/posthog/cdp/templates/hog_function_template.py +++ b/posthog/cdp/templates/hog_function_template.py @@ -46,6 +46,7 @@ class HogFunctionTemplate: "destination", "site_destination", "site_app", + "transformation", "shared", "email", "sms", diff --git a/posthog/cdp/templates/test_cdp_templates.py b/posthog/cdp/templates/test_cdp_templates.py index 4c873a9a820ec..d4a5520a2fd18 100644 --- a/posthog/cdp/templates/test_cdp_templates.py +++ b/posthog/cdp/templates/test_cdp_templates.py @@ -10,8 +10,9 @@ def setUp(self): def test_templates_are_valid(self): for template in HOG_FUNCTION_TEMPLATES: - assert validate_inputs_schema(template.inputs_schema) + if template.inputs_schema: + assert validate_inputs_schema(template.inputs_schema) if template.type not in TYPES_WITH_TRANSPILED_FILTERS: - bytecode = compile_hog(template.hog) + bytecode = compile_hog(template.hog, template.type) assert bytecode[0] == "_H" diff --git a/posthog/cdp/validation.py b/posthog/cdp/validation.py index a0466d8128dab..ac7f19405cfd5 100644 --- a/posthog/cdp/validation.py +++ b/posthog/cdp/validation.py @@ -185,13 +185,16 @@ def validate_inputs( return validated_inputs -def compile_hog(hog: str, supported_functions: Optional[set[str]] = None, in_repl: Optional[bool] = False) -> list[Any]: +def compile_hog(hog: str, hog_type: str, in_repl: Optional[bool] = False) -> list[Any]: # Attempt to compile the hog try: program = parse_program(hog) - return create_bytecode( - program, supported_functions=supported_functions or {"fetch", "postHogCapture"}, in_repl=in_repl - ).bytecode + supported_functions = set() + + if hog_type == "destination": + supported_functions = {"fetch", "postHogCapture"} + + return create_bytecode(program, supported_functions=supported_functions, in_repl=in_repl).bytecode except Exception as e: logger.error(f"Failed to compile hog {e}", exc_info=True) raise serializers.ValidationError({"hog": "Hog code has errors."}) diff --git a/posthog/management/commands/migrate_action_webhooks.py b/posthog/management/commands/migrate_action_webhooks.py index c35a90f61232b..6dd92d53ad682 100644 --- a/posthog/management/commands/migrate_action_webhooks.py +++ b/posthog/management/commands/migrate_action_webhooks.py @@ -140,7 +140,7 @@ def convert_to_hog_function(action: Action, inert=False) -> Optional[HogFunction inputs_schema=webhook_template.inputs_schema, template_id=webhook_template.id, hog=hog_code, - bytecode=compile_hog(hog_code), + bytecode=compile_hog(hog_code, "destination"), filters=compile_filters_bytecode( {"actions": [{"id": f"{action.id}", "type": "actions", "name": action.name, "order": 0}]}, action.team ), diff --git a/posthog/migrations/0531_alter_hogfunction_type.py b/posthog/migrations/0531_alter_hogfunction_type.py new file mode 100644 index 0000000000000..b3160bc5fbc2d --- /dev/null +++ b/posthog/migrations/0531_alter_hogfunction_type.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.15 on 2024-12-13 13:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("posthog", "0530_convert_dashboard_templates_to_queries"), + ] + + operations = [ + migrations.AlterField( + model_name="hogfunction", + name="type", + field=models.CharField( + blank=True, + choices=[ + ("destination", "Destination"), + ("site_destination", "Site Destination"), + ("site_app", "Site App"), + ("transformation", "Transformation"), + ("email", "Email"), + ("sms", "Sms"), + ("push", "Push"), + ("activity", "Activity"), + ("alert", "Alert"), + ("broadcast", "Broadcast"), + ], + max_length=24, + null=True, + ), + ), + ] diff --git a/posthog/migrations/max_migration.txt b/posthog/migrations/max_migration.txt index 3eb81911c811d..28e2442e211b6 100644 --- a/posthog/migrations/max_migration.txt +++ b/posthog/migrations/max_migration.txt @@ -1 +1 @@ -0530_convert_dashboard_templates_to_queries +0531_alter_hogfunction_type diff --git a/posthog/models/hog_functions/hog_function.py b/posthog/models/hog_functions/hog_function.py index 2f7a33d3ec56e..3ddd0212f21d5 100644 --- a/posthog/models/hog_functions/hog_function.py +++ b/posthog/models/hog_functions/hog_function.py @@ -37,6 +37,7 @@ class HogFunctionType(models.TextChoices): DESTINATION = "destination" SITE_DESTINATION = "site_destination" SITE_APP = "site_app" + TRANSFORMATION = "transformation" EMAIL = "email" SMS = "sms" PUSH = "push" @@ -45,7 +46,7 @@ class HogFunctionType(models.TextChoices): BROADCAST = "broadcast" -TYPES_THAT_RELOAD_PLUGIN_SERVER = (HogFunctionType.DESTINATION, HogFunctionType.EMAIL) +TYPES_THAT_RELOAD_PLUGIN_SERVER = (HogFunctionType.DESTINATION, HogFunctionType.EMAIL, HogFunctionType.TRANSFORMATION) TYPES_WITH_COMPILED_FILTERS = (HogFunctionType.DESTINATION,) TYPES_WITH_TRANSPILED_FILTERS = (HogFunctionType.SITE_DESTINATION, HogFunctionType.SITE_APP) TYPES_WITH_JAVASCRIPT_SOURCE = (HogFunctionType.SITE_DESTINATION, HogFunctionType.SITE_APP)