From 571e20bc5b819a674772a9e89d94299546f743b3 Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Fri, 20 Dec 2024 08:20:30 +0100 Subject: [PATCH] fix(data-warehouse): Pop the incremental value when changing to/from incremental (#27086) --- .../data_imports/workflow_activities/import_data_sync.py | 3 +++ posthog/warehouse/api/external_data_schema.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/posthog/temporal/data_imports/workflow_activities/import_data_sync.py b/posthog/temporal/data_imports/workflow_activities/import_data_sync.py index 135d6b8d5fb89..0c51c8db81fcf 100644 --- a/posthog/temporal/data_imports/workflow_activities/import_data_sync.py +++ b/posthog/temporal/data_imports/workflow_activities/import_data_sync.py @@ -101,6 +101,9 @@ def import_data_activity_sync(inputs: ImportDataActivityInputs): schema.sync_type_config.get("incremental_field_type"), ) + if schema.is_incremental: + logger.debug(f"Incremental last value being used is: {processed_incremental_last_value}") + source = None if model.pipeline.source_type == ExternalDataSource.Type.STRIPE: from posthog.temporal.data_imports.pipelines.stripe import stripe_source diff --git a/posthog/warehouse/api/external_data_schema.py b/posthog/warehouse/api/external_data_schema.py index 9391268bb69d0..56208208d5eaf 100644 --- a/posthog/warehouse/api/external_data_schema.py +++ b/posthog/warehouse/api/external_data_schema.py @@ -141,12 +141,16 @@ def update(self, instance: ExternalDataSchema, validated_data: dict[str, Any]) - payload = instance.sync_type_config payload["incremental_field"] = data.get("incremental_field") payload["incremental_field_type"] = data.get("incremental_field_type") + payload["incremental_field_last_value"] = None + payload["incremental_field_last_value_v2"] = None validated_data["sync_type_config"] = payload else: payload = instance.sync_type_config payload.pop("incremental_field", None) payload.pop("incremental_field_type", None) + payload.pop("incremental_field_last_value", None) + payload.pop("incremental_field_last_value_v2", None) validated_data["sync_type_config"] = payload