From a907e472ee7886875d05f045425e873c37a0350b Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 5 Dec 2023 11:26:27 -0800 Subject: [PATCH 01/17] test out --- packages.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages.yml b/packages.yml index 17d9e104..e36807e9 100644 --- a/packages.yml +++ b/packages.yml @@ -1,6 +1,9 @@ packages: - - package: fivetran/zendesk_source - version: [">=0.10.0", "<0.11.0"] - + # - package: fivetran/zendesk_source + # version: [">=0.10.0", "<0.11.0"] + # - local: ../dbt_zendesk_source + - git: https://github.com/fivetran/dbt_zendesk_source.git + revision: feature/tag-former-agents + warn-unpinned: false - package: calogica/dbt_date version: [">=0.9.0", "<1.0.0"] From e3b4daf6f9a54b268fdcb5cc545d7824fc2bb5da Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 18 Dec 2023 15:26:13 -0800 Subject: [PATCH 02/17] first pass --- ...int_zendesk__ticket_work_time_business.sql | 11 +++- ...int_zendesk__ticket_work_time_calendar.sql | 4 +- .../int_zendesk__assignee_updates.sql | 5 +- .../int_zendesk__comment_metrics.sql | 3 +- .../int_zendesk__latest_ticket_form.sql | 3 +- .../int_zendesk__organization_aggregates.sql | 18 ++++--- .../int_zendesk__requester_updates.sql | 5 +- .../int_zendesk__schedule_spine.sql | 43 ++++++++++----- .../int_zendesk__ticket_aggregates.sql | 7 ++- ...nt_zendesk__ticket_historical_assignee.sql | 24 +++++---- .../int_zendesk__ticket_historical_group.sql | 4 +- ...endesk__ticket_historical_satisfaction.sql | 29 +++++++---- .../int_zendesk__ticket_historical_status.sql | 5 +- .../int_zendesk__ticket_schedules.sql | 34 +++++++++--- models/intermediate/int_zendesk__updates.sql | 5 +- .../int_zendesk__user_aggregates.sql | 6 ++- .../int_zendesk__comments_enriched.sql | 7 +-- ...desk__ticket_first_reply_time_business.sql | 13 +++-- .../int_zendesk__ticket_reply_times.sql | 7 ++- ...t_zendesk__ticket_reply_times_calendar.sql | 6 ++- ..._ticket_first_resolution_time_business.sql | 12 +++-- ...__ticket_full_resolution_time_business.sql | 12 +++-- ...desk__ticket_resolution_times_calendar.sql | 14 +++-- ...endesk__agent_work_time_business_hours.sql | 15 ++++-- ...endesk__agent_work_time_calendar_hours.sql | 6 +-- ...esk__agent_work_time_filtered_statuses.sql | 2 + .../int_zendesk__sla_policy_applied.sql | 5 +- ...int_zendesk__reply_time_business_hours.sql | 16 ++++-- .../int_zendesk__reply_time_combined.sql | 20 ++++--- ...sk__requester_wait_time_business_hours.sql | 16 ++++-- ...sk__requester_wait_time_calendar_hours.sql | 6 +-- ..._requester_wait_time_filtered_statuses.sql | 2 + .../int_zendesk__field_calendar_spine.sql | 5 +- .../int_zendesk__field_history_enriched.sql | 1 + .../int_zendesk__field_history_pivot.sql | 8 +-- .../int_zendesk__field_history_scd.sql | 14 ++--- .../int_zendesk__updater_information.sql | 4 +- models/utils/int_zendesk__calendar_spine.sql | 4 +- models/zendesk.yml | 52 ++++++++++++++++--- models/zendesk__sla_policies.sql | 14 +++-- models/zendesk__ticket_backlog.sql | 8 +++ models/zendesk__ticket_enriched.sql | 10 ++++ models/zendesk__ticket_field_history.sql | 30 ++++++----- models/zendesk__ticket_metrics.sql | 28 ++++++---- models/zendesk__ticket_summary.sql | 10 ++-- packages.yml | 2 +- 46 files changed, 396 insertions(+), 159 deletions(-) diff --git a/models/agent_work_time/int_zendesk__ticket_work_time_business.sql b/models/agent_work_time/int_zendesk__ticket_work_time_business.sql index 01c0ae18..f5945e7d 100644 --- a/models/agent_work_time/int_zendesk__ticket_work_time_business.sql +++ b/models/agent_work_time/int_zendesk__ticket_work_time_business.sql @@ -19,6 +19,7 @@ with ticket_historical_status as ( select ticket_historical_status.ticket_id, + ticket_historical_status.source_relation, ticket_historical_status.status as ticket_status, ticket_schedules.schedule_id, @@ -33,6 +34,7 @@ with ticket_historical_status as ( from ticket_historical_status left join ticket_schedules on ticket_historical_status.ticket_id = ticket_schedules.ticket_id + and ticket_historical_status.source_relation = ticket_schedules.source_relation -- making sure there is indeed real overlap where {{ dbt.datediff('greatest(valid_starting_at, schedule_created_at)', 'least(valid_ending_at, schedule_invalidated_at)', 'second') }} > 0 @@ -40,6 +42,7 @@ with ticket_historical_status as ( select ticket_id, + source_relation, ticket_status, schedule_id, status_schedule_start, @@ -59,7 +62,7 @@ with ticket_historical_status as ( {{ dbt_date.week_start('ticket_status_crossed_with_schedule.status_schedule_start','UTC') }} as start_week_date from ticket_status_crossed_with_schedule - {{ dbt_utils.group_by(n=7) }} + {{ dbt_utils.group_by(n=8) }} ), weeks as ( @@ -90,6 +93,7 @@ with ticket_historical_status as ( select weekly_periods.ticket_id, + weekly_periods.source_relation, weekly_periods.week_number, weekly_periods.schedule_id, weekly_periods.ticket_status, @@ -103,6 +107,7 @@ with ticket_historical_status as ( ticket_week_start_time <= schedule.end_time_utc and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id + and weekly_periods.source_relation = schedule.source_relation -- this chooses the Daylight Savings Time or Standard Time version of the schedule -- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }}) @@ -112,6 +117,7 @@ with ticket_historical_status as ( select ticket_id, + source_relation, ticket_status, case when ticket_status in ('pending') then scheduled_minutes else 0 end as agent_wait_time_in_minutes, @@ -133,6 +139,7 @@ with ticket_historical_status as ( select ticket_id, + source_relation, sum(agent_wait_time_in_minutes) as agent_wait_time_in_business_minutes, sum(requester_wait_time_in_minutes) as requester_wait_time_in_business_minutes, sum(solve_time_in_minutes) as solve_time_in_business_minutes, @@ -141,4 +148,4 @@ with ticket_historical_status as ( sum(new_status_duration_minutes) as new_status_duration_in_business_minutes, sum(open_status_duration_minutes) as open_status_duration_in_business_minutes from business_minutes - group by 1 + group by 1,2 diff --git a/models/agent_work_time/int_zendesk__ticket_work_time_calendar.sql b/models/agent_work_time/int_zendesk__ticket_work_time_calendar.sql index b49c074f..1b8ee070 100644 --- a/models/agent_work_time/int_zendesk__ticket_work_time_calendar.sql +++ b/models/agent_work_time/int_zendesk__ticket_work_time_calendar.sql @@ -7,6 +7,7 @@ with ticket_historical_status as ( select ticket_id, + source_relation, status, case when status in ('pending') then status_duration_calendar_minutes else 0 end as agent_wait_time_in_minutes, @@ -36,6 +37,7 @@ with ticket_historical_status as ( select ticket_id, + source_relation, last_status_assignment_date, sum(ticket_deleted) as ticket_deleted_count, sum(agent_wait_time_in_minutes) as agent_wait_time_in_calendar_minutes, @@ -47,4 +49,4 @@ select sum(open_status_duration_minutes) as open_status_duration_in_calendar_minutes, sum(ticket_recoveries) as total_ticket_recoveries from calendar_minutes -group by 1, 2 \ No newline at end of file +group by 1, 2, 3 \ No newline at end of file diff --git a/models/intermediate/int_zendesk__assignee_updates.sql b/models/intermediate/int_zendesk__assignee_updates.sql index 48e4afbd..8caa03a9 100644 --- a/models/intermediate/int_zendesk__assignee_updates.sql +++ b/models/intermediate/int_zendesk__assignee_updates.sql @@ -9,6 +9,7 @@ with ticket_updates as ( ), ticket_requester as ( select ticket.ticket_id, + ticket.source_relation, ticket.assignee_id, ticket_updates.valid_starting_at @@ -17,16 +18,18 @@ with ticket_updates as ( left join ticket_updates on ticket_updates.ticket_id = ticket.ticket_id and ticket_updates.user_id = ticket.assignee_id + and ticket_updates.source_relation = ticket.source_relation ), final as ( select ticket_id, + source_relation, assignee_id, max(valid_starting_at) as last_updated, count(*) as total_updates from ticket_requester - group by 1, 2 + group by 1, 2, 3 ) select * diff --git a/models/intermediate/int_zendesk__comment_metrics.sql b/models/intermediate/int_zendesk__comment_metrics.sql index 868595c1..8dac0772 100644 --- a/models/intermediate/int_zendesk__comment_metrics.sql +++ b/models/intermediate/int_zendesk__comment_metrics.sql @@ -7,6 +7,7 @@ with ticket_comments as ( comment_counts as ( select ticket_id, + source_relation, last_comment_added_at, sum(case when commenter_role = 'internal_comment' and is_public = true then 1 @@ -38,7 +39,7 @@ comment_counts as ( end) as count_agent_replies from ticket_comments - group by 1, 2 + group by 1, 2, 3 ), final as ( diff --git a/models/intermediate/int_zendesk__latest_ticket_form.sql b/models/intermediate/int_zendesk__latest_ticket_form.sql index 060f0440..0af62895 100644 --- a/models/intermediate/int_zendesk__latest_ticket_form.sql +++ b/models/intermediate/int_zendesk__latest_ticket_form.sql @@ -9,13 +9,14 @@ with ticket_form_history as ( latest_ticket_form as ( select *, - row_number() over(partition by ticket_form_id order by updated_at desc) as latest_form_index + row_number() over(partition by ticket_form_id, source_relation order by updated_at desc) as latest_form_index from ticket_form_history ), final as ( select ticket_form_id, + source_relation, created_at, updated_at, display_name, diff --git a/models/intermediate/int_zendesk__organization_aggregates.sql b/models/intermediate/int_zendesk__organization_aggregates.sql index 50f25715..1b5d5f37 100644 --- a/models/intermediate/int_zendesk__organization_aggregates.sql +++ b/models/intermediate/int_zendesk__organization_aggregates.sql @@ -11,13 +11,15 @@ with organizations as ( ), tag_aggregates as ( select organizations.organization_id, + organizations.source_relation, {{ fivetran_utils.string_agg('organization_tags.tags', "', '" ) }} as organization_tags from organizations left join organization_tags - using (organization_id) + on organizations.organization_id = organization_tags.organization_id + and organizations.source_relation = organization_tags.source_relation - group by 1 + group by 1, 2 {% endif %} --If you use using_domain_names tags this will be included, if not it will be ignored. @@ -30,13 +32,15 @@ with organizations as ( ), domain_aggregates as ( select organizations.organization_id, + organizations.source_relation, {{ fivetran_utils.string_agg('domain_names.domain_name', "', '" ) }} as domain_names from organizations left join domain_names - using(organization_id) + on organizations.organization_id = domain_names.organization_id + and organizations.source_relation = domain_names.source_relation - group by 1 + group by 1, 2 {% endif %} @@ -59,13 +63,15 @@ with organizations as ( --If you use using_domain_names tags this will be included, if not it will be ignored. {% if var('using_domain_names', True) %} left join domain_aggregates - using(organization_id) + on organizations.organization_id = domain_aggregates.organization_id + and organizations.source_relation = domain_aggregates.source_relation {% endif %} --If you use organization tags this will be included, if not it will be ignored. {% if var('using_organization_tags', True) %} left join tag_aggregates - using(organization_id) + on organizations.organization_id = tag_aggregates.organization_id + and organizations.source_relation = tag_aggregates.source_relation {% endif %} ) diff --git a/models/intermediate/int_zendesk__requester_updates.sql b/models/intermediate/int_zendesk__requester_updates.sql index 23c69f91..b3702ece 100644 --- a/models/intermediate/int_zendesk__requester_updates.sql +++ b/models/intermediate/int_zendesk__requester_updates.sql @@ -9,6 +9,7 @@ with ticket_updates as ( ), ticket_requester as ( select ticket.ticket_id, + ticket.source_relation, ticket.requester_id, ticket_updates.valid_starting_at @@ -17,16 +18,18 @@ with ticket_updates as ( left join ticket_updates on ticket_updates.ticket_id = ticket.ticket_id and ticket_updates.user_id = ticket.requester_id + and ticket_updates.source_relation = ticket.source_relation ), final as ( select ticket_id, + source_relation, requester_id, max(valid_starting_at) as last_updated, count(*) as total_updates from ticket_requester - group by 1, 2 + group by 1, 2, 3 ) select * diff --git a/models/intermediate/int_zendesk__schedule_spine.sql b/models/intermediate/int_zendesk__schedule_spine.sql index b242ceae..b54359c2 100644 --- a/models/intermediate/int_zendesk__schedule_spine.sql +++ b/models/intermediate/int_zendesk__schedule_spine.sql @@ -36,6 +36,7 @@ with timezone as ( from timezone left join daylight_time on timezone.time_zone = daylight_time.time_zone + and timezone.source_relation = daylight_time.source_relation ), order_timezone_dt as ( @@ -43,10 +44,10 @@ with timezone as ( *, -- will be null for timezones without any daylight savings records (and the first entry) -- we will coalesce the first entry date with .... the X years ago - lag(daylight_end_utc, 1) over (partition by time_zone order by daylight_end_utc asc) as last_daylight_end_utc, + lag(daylight_end_utc, 1) over (partition by time_zone, source_relation order by daylight_end_utc asc) as last_daylight_end_utc, -- will be null for timezones without any daylight savings records (and the last entry) -- we will coalesce the last entry date with the current date - lead(daylight_start_utc, 1) over (partition by time_zone order by daylight_start_utc asc) as next_daylight_start_utc + lead(daylight_start_utc, 1) over (partition by time_zone, source_relation order by daylight_start_utc asc) as next_daylight_start_utc from timezone_with_dt @@ -57,6 +58,7 @@ with timezone as ( -- ends: when the next Daylight Savings starts select time_zone, + source_relation, standard_offset_minutes as offset_minutes, -- last_daylight_end_utc is null for the first record of the time_zone's daylight time, or if the TZ doesn't use DT @@ -74,6 +76,7 @@ with timezone as ( -- ends: when this Daylight Savings ends select time_zone, + source_relation, -- Pacific Time is -8h during standard time and -7h during DT standard_offset_minutes + daylight_offset_minutes as offset_minutes, daylight_start_utc as valid_from, @@ -86,6 +89,7 @@ with timezone as ( select schedule.schedule_id, + schedule.source_relation, schedule.time_zone, schedule.start_time, schedule.end_time, @@ -101,6 +105,7 @@ with timezone as ( from schedule left join split_timezones on split_timezones.time_zone = schedule.time_zone + and split_timezones.source_relation = schedule.source_relation -- Now we need take holiday's into consideration and perform the following transformations to account for Holidays in existing schedules ), holiday_start_end_times as ( @@ -117,6 +122,7 @@ with timezone as ( on calculate_schedules.schedule_id = schedule_holiday.schedule_id and schedule_holiday.holiday_start_date_at >= calculate_schedules.valid_from and schedule_holiday.holiday_start_date_at < calculate_schedules.valid_until + and schedule_holiday.source_relation = calculate_schedules.source_relation -- Let's calculate the start and end date of the Holiday in terms of minutes from Sunday (like other Zendesk schedules) ), holiday_minutes as( @@ -128,6 +134,7 @@ with timezone as ( from holiday_start_end_times left join timezone on timezone.time_zone = holiday_start_end_times.time_zone + and timezone.source_relation = holiday_start_end_times.source_relation -- Determine which schedule days include a holiday ), holiday_check as ( @@ -144,6 +151,7 @@ with timezone as ( select schedule_id, + source_relation, time_zone, schedule_name, valid_from, @@ -154,13 +162,14 @@ with timezone as ( cast({{ dbt.dateadd("second", "86400", "holiday_week_end") }} as {{ dbt.type_timestamp() }}) as holiday_week_end, max(holiday_name_check) as holiday_name_check from holiday_check - {{ dbt_utils.group_by(n=9) }} + {{ dbt_utils.group_by(n=10) }} -- Since we have holiday schedules and normal schedules, we need to union them into a holistic schedule spine ), spine_union as ( select schedule_id, + source_relation, time_zone, schedule_name, valid_from, @@ -176,6 +185,7 @@ with timezone as ( select schedule_id, + source_relation, time_zone, schedule_name, valid_from, @@ -192,6 +202,7 @@ with timezone as ( select distinct schedule_id, + source_relation, holiday_week_start as period_start, holiday_week_end as period_end, start_time_utc, @@ -206,6 +217,7 @@ with timezone as ( select distinct schedule_id, + source_relation, valid_from as period_start, valid_until as period_end, start_time_utc, @@ -219,8 +231,8 @@ with timezone as ( select distinct *, - lag(period_end) over (partition by schedule_id order by period_start, start_time_utc) as prev_end, - lead(period_start) over (partition by schedule_id order by period_start, start_time_utc) as next_start + lag(period_end) over (partition by schedule_id, source_relation order by period_start, start_time_utc) as prev_end, + lead(period_start) over (partition by schedule_id, source_relation order by period_start, start_time_utc) as next_start from all_periods -- We need to adjust some non holiday schedules in order to properly fill holiday gaps in the schedules later down the transformation @@ -228,17 +240,18 @@ with timezone as ( select schedule_id, + source_relation, period_start, period_end, prev_end, next_start, -- taking first_value/last_value because prev_end and next_start are inconsistent within the schedule partitions -- they all include a record that is outside the partition. so we need to ignore those erroneous records that slip in coalesce(greatest(case - when not is_holiday_week and prev_end is not null then first_value(prev_end) over (partition by schedule_id, period_start order by start_time_utc rows between unbounded preceding and unbounded following) + when not is_holiday_week and prev_end is not null then first_value(prev_end) over (partition by schedule_id, period_start, source_relation order by start_time_utc rows between unbounded preceding and unbounded following) else period_start end, period_start), period_start) as valid_from, coalesce(case - when not is_holiday_week and next_start is not null then last_value(next_start) over (partition by schedule_id, period_start order by start_time_utc rows between unbounded preceding and unbounded following) + when not is_holiday_week and next_start is not null then last_value(next_start) over (partition by schedule_id, period_start, source_relation order by start_time_utc rows between unbounded preceding and unbounded following) else period_end end, period_end) as valid_until, start_time_utc, @@ -251,9 +264,9 @@ with timezone as ( ), gap_starter as ( select *, - max(period_end) over (partition by schedule_id) as max_valid_until, - last_value(next_start) over (partition by schedule_id, period_start order by valid_until rows between unbounded preceding and unbounded following) as lead_next_start, - first_value(prev_end) over (partition by schedule_id, valid_from order by start_time_utc rows between unbounded preceding and unbounded following) as first_prev_end + max(period_end) over (partition by schedule_id, source_relation) as max_valid_until, + last_value(next_start) over (partition by schedule_id, period_start, source_relation order by valid_until rows between unbounded preceding and unbounded following) as lead_next_start, + first_value(prev_end) over (partition by schedule_id, valid_from, source_relation order by start_time_utc rows between unbounded preceding and unbounded following) as first_prev_end from non_holiday_period_adjustments -- There may be gaps in holiday and non holiday schedules, so we need to identify where these gaps are @@ -278,6 +291,7 @@ with timezone as ( select schedule_id, + source_relation, valid_from, valid_until, start_time_utc, @@ -286,8 +300,8 @@ with timezone as ( max_valid_until, holiday_name_check, is_holiday_week, - max(is_schedule_gap) over (partition by schedule_id, valid_until) as is_gap_period, - lead(valid_from) over (partition by schedule_id order by valid_from, start_time_utc) as fill_primer + max(is_schedule_gap) over (partition by schedule_id, valid_until, source_relation) as is_gap_period, + lead(valid_from) over (partition by schedule_id, source_relation order by valid_from, start_time_utc) as fill_primer from gap_adjustments -- We know the gaps and where they are, so let's fill them with the following union @@ -296,8 +310,9 @@ with timezone as ( -- For all gap periods, let's properly create a schedule filled before the holiday. select schedule_id, + source_relation, valid_until as valid_from, - coalesce(last_value(fill_primer) over (partition by schedule_id, valid_until order by start_time_utc rows between unbounded preceding and unbounded following), max_valid_until) as valid_until, + coalesce(last_value(fill_primer) over (partition by schedule_id, valid_until, source_relation order by start_time_utc rows between unbounded preceding and unbounded following), max_valid_until) as valid_until, start_time_utc, end_time_utc, cast(null as {{ dbt.type_string() }}) as holiday_name_check, @@ -310,6 +325,7 @@ with timezone as ( -- Fill all other normal schedules. select schedule_id, + source_relation, valid_from, valid_until, start_time_utc, @@ -323,6 +339,7 @@ with timezone as ( select schedule_id, + source_relation, valid_from, valid_until, start_time_utc, diff --git a/models/intermediate/int_zendesk__ticket_aggregates.sql b/models/intermediate/int_zendesk__ticket_aggregates.sql index b1e5e3e1..459dede7 100644 --- a/models/intermediate/int_zendesk__ticket_aggregates.sql +++ b/models/intermediate/int_zendesk__ticket_aggregates.sql @@ -15,9 +15,10 @@ with tickets as ( ), ticket_tag_aggregate as ( select ticket_tags.ticket_id, + ticket_tags.source_relation, {{ fivetran_utils.string_agg( 'ticket_tags.tags', "', '" )}} as ticket_tags from ticket_tags - group by 1 + group by 1, 2 ), final as ( select @@ -31,10 +32,12 @@ with tickets as ( from tickets left join ticket_tag_aggregate - using(ticket_id) + on tickets.ticket_id = ticket_tag_aggregate.ticket_id + and tickets.source_relation = ticket_tag_aggregate.source_relation left join brands on brands.brand_id = tickets.brand_id + and brands.source_relation = tickets.source_relation ) select * diff --git a/models/intermediate/int_zendesk__ticket_historical_assignee.sql b/models/intermediate/int_zendesk__ticket_historical_assignee.sql index 9227ead4..4ea7ce58 100644 --- a/models/intermediate/int_zendesk__ticket_historical_assignee.sql +++ b/models/intermediate/int_zendesk__ticket_historical_assignee.sql @@ -7,22 +7,24 @@ with assignee_updates as ( ), calculate_metrics as ( select ticket_id, + source_relation, field_name as assignee_id, value, ticket_created_date, valid_starting_at, - lag(valid_starting_at) over (partition by ticket_id order by valid_starting_at) as previous_update, - lag(value) over (partition by ticket_id order by valid_starting_at) as previous_assignee, - first_value(valid_starting_at) over (partition by ticket_id order by valid_starting_at, ticket_id rows unbounded preceding) as first_agent_assignment_date, - first_value(value) over (partition by ticket_id order by valid_starting_at, ticket_id rows unbounded preceding) as first_assignee_id, - first_value(valid_starting_at) over (partition by ticket_id order by valid_starting_at desc, ticket_id rows unbounded preceding) as last_agent_assignment_date, - first_value(value) over (partition by ticket_id order by valid_starting_at desc, ticket_id rows unbounded preceding) as last_assignee_id, - count(value) over (partition by ticket_id) as assignee_stations_count + lag(valid_starting_at) over (partition by ticket_id, source_relation order by valid_starting_at) as previous_update, + lag(value) over (partition by ticket_id, source_relation order by valid_starting_at) as previous_assignee, + first_value(valid_starting_at) over (partition by ticket_id, source_relation order by valid_starting_at, ticket_id rows unbounded preceding) as first_agent_assignment_date, + first_value(value) over (partition by ticket_id, source_relation order by valid_starting_at, ticket_id rows unbounded preceding) as first_assignee_id, + first_value(valid_starting_at) over (partition by ticket_id, source_relation order by valid_starting_at desc, ticket_id rows unbounded preceding) as last_agent_assignment_date, + first_value(value) over (partition by ticket_id, source_relation order by valid_starting_at desc, ticket_id rows unbounded preceding) as last_assignee_id, + count(value) over (partition by ticket_id, source_relation) as assignee_stations_count from assignee_updates ), unassigned_time as ( select ticket_id, + source_relation, sum(case when assignee_id is not null and previous_assignee is null then {{ dbt.datediff("coalesce(previous_update, ticket_created_date)", "valid_starting_at", 'second') }} / 60 else 0 @@ -30,11 +32,12 @@ with assignee_updates as ( count(distinct value) as unique_assignee_count from calculate_metrics - group by 1 + group by 1, 2 ), window_group as ( select calculate_metrics.ticket_id, + calculate_metrics.source_relation, calculate_metrics.first_agent_assignment_date, calculate_metrics.first_assignee_id, calculate_metrics.last_agent_assignment_date, @@ -42,7 +45,7 @@ with assignee_updates as ( calculate_metrics.assignee_stations_count from calculate_metrics - {{ dbt_utils.group_by(n=6) }} + {{ dbt_utils.group_by(n=7) }} ), final as ( select @@ -52,7 +55,8 @@ with assignee_updates as ( from window_group left join unassigned_time - using(ticket_id) + on window_group.ticket_id = unassigned_time.ticket_id + and window_group.source_relation = unassigned_time.source_relation ) select * diff --git a/models/intermediate/int_zendesk__ticket_historical_group.sql b/models/intermediate/int_zendesk__ticket_historical_group.sql index 67000590..b9c85221 100644 --- a/models/intermediate/int_zendesk__ticket_historical_group.sql +++ b/models/intermediate/int_zendesk__ticket_historical_group.sql @@ -8,6 +8,7 @@ with ticket_group_history as ( select ticket_id, + source_relation, valid_starting_at, valid_ending_at, value as group_id @@ -16,10 +17,11 @@ with ticket_group_history as ( ), final as ( select ticket_id, + source_relation, count(group_id) as group_stations_count from group_breakdown - group by 1 + group by 1, 2 ) select * diff --git a/models/intermediate/int_zendesk__ticket_historical_satisfaction.sql b/models/intermediate/int_zendesk__ticket_historical_satisfaction.sql index a75bd802..b4c5d7b6 100644 --- a/models/intermediate/int_zendesk__ticket_historical_satisfaction.sql +++ b/models/intermediate/int_zendesk__ticket_historical_satisfaction.sql @@ -7,7 +7,8 @@ with satisfaction_updates as ( ), latest_reason as ( select ticket_id, - first_value(value) over (partition by ticket_id order by valid_starting_at desc, ticket_id rows unbounded preceding) as latest_satisfaction_reason + source_relation, + first_value(value) over (partition by ticket_id, source_relation order by valid_starting_at desc, ticket_id rows unbounded preceding) as latest_satisfaction_reason from satisfaction_updates where field_name = 'satisfaction_reason_code' @@ -15,7 +16,8 @@ with satisfaction_updates as ( ), latest_comment as ( select ticket_id, - first_value(value) over (partition by ticket_id order by valid_starting_at desc, ticket_id rows unbounded preceding) as latest_satisfaction_comment + source_relation, + first_value(value) over (partition by ticket_id, source_relation order by valid_starting_at desc, ticket_id rows unbounded preceding) as latest_satisfaction_comment from satisfaction_updates where field_name = 'satisfaction_comment' @@ -23,8 +25,9 @@ with satisfaction_updates as ( ), first_and_latest_score as ( select ticket_id, - first_value(value) over (partition by ticket_id order by valid_starting_at, ticket_id rows unbounded preceding) as first_satisfaction_score, - first_value(value) over (partition by ticket_id order by valid_starting_at desc, ticket_id rows unbounded preceding) as latest_satisfaction_score + source_relation, + first_value(value) over (partition by ticket_id, source_relation order by valid_starting_at, ticket_id rows unbounded preceding) as first_satisfaction_score, + first_value(value) over (partition by ticket_id, source_relation order by valid_starting_at desc, ticket_id rows unbounded preceding) as latest_satisfaction_score from satisfaction_updates where field_name = 'satisfaction_score' and value != 'offered' @@ -32,12 +35,13 @@ with satisfaction_updates as ( ), satisfaction_scores as ( select ticket_id, - count(value) over (partition by ticket_id) as count_satisfaction_scores, - case when lag(value) over (partition by ticket_id order by valid_starting_at desc) = 'good' and value = 'bad' + source_relation, + count(value) over (partition by ticket_id, source_relation) as count_satisfaction_scores, + case when lag(value) over (partition by ticket_id, source_relation order by valid_starting_at desc) = 'good' and value = 'bad' then 1 else 0 end as good_to_bad_score, - case when lag(value) over (partition by ticket_id order by valid_starting_at desc) = 'bad' and value = 'good' + case when lag(value) over (partition by ticket_id, source_relation order by valid_starting_at desc) = 'bad' and value = 'good' then 1 else 0 end as bad_to_good_score @@ -47,16 +51,18 @@ with satisfaction_updates as ( ), score_group as ( select ticket_id, + source_relation, count_satisfaction_scores, sum(good_to_bad_score) as total_good_to_bad_score, sum(bad_to_good_score) as total_bad_to_good_score from satisfaction_scores - group by 1, 2 + group by 1, 2, 3 ), window_group as ( select satisfaction_updates.ticket_id, + satisfaction_updates.source_relation, latest_reason.latest_satisfaction_reason, latest_comment.latest_satisfaction_comment, first_and_latest_score.first_satisfaction_score, @@ -69,21 +75,26 @@ with satisfaction_updates as ( left join latest_reason on satisfaction_updates.ticket_id = latest_reason.ticket_id + and satisfaction_updates.source_relation = latest_reason.source_relation left join latest_comment on satisfaction_updates.ticket_id = latest_comment.ticket_id + and satisfaction_updates.source_relation = latest_comment.source_relation left join first_and_latest_score on satisfaction_updates.ticket_id = first_and_latest_score.ticket_id + and satisfaction_updates.source_relation = first_and_latest_score.source_relation left join score_group on satisfaction_updates.ticket_id = score_group.ticket_id + and satisfaction_updates.source_relation = score_group.source_relation - group by 1, 2, 3, 4, 5, 6, 7, 8 + {{ dbt_utils.group_by(n=9) }} ), final as ( select ticket_id, + source_relation, latest_satisfaction_reason, latest_satisfaction_comment, first_satisfaction_score, diff --git a/models/intermediate/int_zendesk__ticket_historical_status.sql b/models/intermediate/int_zendesk__ticket_historical_status.sql index 5d5911a9..5bbfbf3b 100644 --- a/models/intermediate/int_zendesk__ticket_historical_status.sql +++ b/models/intermediate/int_zendesk__ticket_historical_status.sql @@ -11,6 +11,7 @@ with ticket_status_history as ( select ticket_id, + source_relation, valid_starting_at, valid_ending_at, {{ dbt.datediff( @@ -19,7 +20,7 @@ with ticket_status_history as ( 'minute') }} as status_duration_calendar_minutes, value as status, -- MIGHT BE ABLE TO DELETE ROWS BELOW - row_number() over (partition by ticket_id order by valid_starting_at) as ticket_status_counter, - row_number() over (partition by ticket_id, value order by valid_starting_at) as unique_status_counter + row_number() over (partition by ticket_id, source_relation order by valid_starting_at) as ticket_status_counter, + row_number() over (partition by ticket_id, value, source_relation order by valid_starting_at) as unique_status_counter from ticket_status_history \ No newline at end of file diff --git a/models/intermediate/int_zendesk__ticket_schedules.sql b/models/intermediate/int_zendesk__ticket_schedules.sql index 9f160524..20174b76 100644 --- a/models/intermediate/int_zendesk__ticket_schedules.sql +++ b/models/intermediate/int_zendesk__ticket_schedules.sql @@ -15,15 +15,29 @@ with ticket as ( select * from {{ ref('stg_zendesk__schedule') }} - -), default_schedule_events as ( +), default_schedules as ( -- Goal: understand the working schedules applied to tickets, so that we can then determine the applicable business hours/schedule. -- Your default schedule is used for all tickets, unless you set up a trigger to apply a specific schedule to specific tickets. -- This portion of the query creates ticket_schedules for these "default" schedules, as the ticket_schedule table only includes -- trigger schedules + select + schedule_id, + source_relation + from ( + + select + schedule_id, + source_relation, + row_number() over (partition by source_relation order by created_at) = 1 as is_default_schedule + from schedule + + ) as order_schedules + where is_default_schedule + +), default_schedule_events as ( -{% if execute %} +{# {% if execute %} {% set default_schedule_id_query %} with set_default_schedule_flag as ( @@ -41,17 +55,21 @@ with ticket as ( {% set default_schedule_id = run_query(default_schedule_id_query).columns[0][0]|string %} - {% endif %} + {% endif %} #} select ticket.ticket_id, + ticket.source_relation, ticket.created_at as schedule_created_at, - '{{default_schedule_id}}' as schedule_id + default_schedules.schedule_id from ticket + join default_schedules + on ticket.source_relation = default_schedules.source_relation left join ticket_schedule as first_schedule on first_schedule.ticket_id = ticket.ticket_id and {{ fivetran_utils.timestamp_add('second', -5, 'first_schedule.created_at') }} <= ticket.created_at - and first_schedule.created_at >= ticket.created_at + and first_schedule.created_at >= ticket.created_at + and first_schedule.source_relation = ticket.source_relation where first_schedule.ticket_id is null ), schedule_events as ( @@ -64,6 +82,7 @@ with ticket as ( select ticket_id, + source_relation, created_at as schedule_created_at, schedule_id from ticket_schedule @@ -72,9 +91,10 @@ with ticket as ( select ticket_id, + source_relation, schedule_id, schedule_created_at, - coalesce(lead(schedule_created_at) over (partition by ticket_id order by schedule_created_at) + coalesce(lead(schedule_created_at) over (partition by ticket_id, source_relation order by schedule_created_at) , {{ fivetran_utils.timestamp_add("hour", 1000, "" ~ dbt.current_timestamp_backcompat() ~ "") }} ) as schedule_invalidated_at from schedule_events diff --git a/models/intermediate/int_zendesk__updates.sql b/models/intermediate/int_zendesk__updates.sql index e608d747..74ad8f66 100644 --- a/models/intermediate/int_zendesk__updates.sql +++ b/models/intermediate/int_zendesk__updates.sql @@ -13,6 +13,7 @@ with ticket_history as ( ), updates_union as ( select ticket_id, + source_relation, field_name, value, null as is_public, @@ -25,12 +26,13 @@ with ticket_history as ( select ticket_id, + source_relation, cast('comment' as {{ dbt.type_string() }}) as field_name, body as value, is_public, user_id, created_at as valid_starting_at, - lead(created_at) over (partition by ticket_id order by created_at) as valid_ending_at + lead(created_at) over (partition by ticket_id, source_relation order by created_at) as valid_ending_at from ticket_comment ), final as ( @@ -41,6 +43,7 @@ with ticket_history as ( left join tickets on tickets.ticket_id = updates_union.ticket_id + and tickets.source_relation = updates_union.source_relation ) select * diff --git a/models/intermediate/int_zendesk__user_aggregates.sql b/models/intermediate/int_zendesk__user_aggregates.sql index 7ea7953d..33b15461 100644 --- a/models/intermediate/int_zendesk__user_aggregates.sql +++ b/models/intermediate/int_zendesk__user_aggregates.sql @@ -12,9 +12,10 @@ with users as ( ), user_tag_aggregate as ( select user_tags.user_id, + user_tags.source_relation, {{ fivetran_utils.string_agg( 'user_tags.tags', "', '" )}} as user_tags from user_tags - group by 1 + group by 1, 2 {% endif %} @@ -31,7 +32,8 @@ with users as ( --If you use user tags this will be included, if not it will be ignored. {% if var('using_user_tags', True) %} left join user_tag_aggregate - using(user_id) + on users.user_id = user_tag_aggregate.user_id + and users.source_relation = user_tag_aggregate.source_relation {% endif %} ) diff --git a/models/reply_times/int_zendesk__comments_enriched.sql b/models/reply_times/int_zendesk__comments_enriched.sql index dfc50c7f..719c01a4 100644 --- a/models/reply_times/int_zendesk__comments_enriched.sql +++ b/models/reply_times/int_zendesk__comments_enriched.sql @@ -22,6 +22,7 @@ with ticket_comment as ( join users as commenter on commenter.user_id = ticket_comment.user_id + and commenter.source_relation = ticket_comment.source_relation ), add_previous_commenter_role as ( /* @@ -31,7 +32,7 @@ with ticket_comment as ( select *, coalesce( - lag(commenter_role) over (partition by ticket_id order by valid_starting_at, commenter_role) + lag(commenter_role) over (partition by ticket_id, source_relation order by valid_starting_at, commenter_role) , 'first_comment') as previous_commenter_role from joined @@ -48,6 +49,6 @@ with ticket_comment as ( select *, - first_value(valid_starting_at) over (partition by ticket_id order by valid_starting_at desc, ticket_id rows unbounded preceding) as last_comment_added_at, - sum(case when not is_public then 1 else 0 end) over (partition by ticket_id order by valid_starting_at rows between unbounded preceding and current row) as previous_internal_comment_count + first_value(valid_starting_at) over (partition by ticket_id, source_relation order by valid_starting_at desc, ticket_id rows unbounded preceding) as last_comment_added_at, + sum(case when not is_public then 1 else 0 end) over (partition by ticket_id, source_relation order by valid_starting_at rows between unbounded preceding and current row) as previous_internal_comment_count from add_previous_commenter_role \ No newline at end of file diff --git a/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql b/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql index ba305ce5..6646aec9 100644 --- a/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql +++ b/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql @@ -20,6 +20,7 @@ with ticket_reply_times as ( select ticket_id, + source_relation, end_user_comment_created_at, agent_responded_at @@ -30,6 +31,7 @@ with ticket_reply_times as ( select first_reply_time.ticket_id, + first_reply_time.source_relation, ticket_schedules.schedule_created_at, ticket_schedules.schedule_invalidated_at, ticket_schedules.schedule_id, @@ -52,8 +54,10 @@ with ticket_reply_times as ( {{ dbt_date.week_start('ticket_schedules.schedule_created_at','UTC') }} as start_week_date from first_reply_time - join ticket_schedules on first_reply_time.ticket_id = ticket_schedules.ticket_id - group by 1, 2, 3, 4 + join ticket_schedules + on first_reply_time.ticket_id = ticket_schedules.ticket_id + and first_reply_time.source_relation = ticket_schedules.source_relation + {{ dbt_utils.group_by(n=5) }} ), weeks as ( @@ -83,6 +87,7 @@ with ticket_reply_times as ( ), intercepted_periods as ( select ticket_id, + source_relation, week_number, weekly_periods.schedule_id, ticket_week_start_time, @@ -94,6 +99,7 @@ with ticket_reply_times as ( join schedule on ticket_week_start_time <= schedule.end_time_utc and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id + and weekly_periods.source_relation = schedule.source_relation -- this chooses the Daylight Savings Time or Standard Time version of the schedule -- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }}) @@ -102,6 +108,7 @@ with ticket_reply_times as ( ) select ticket_id, + source_relation, sum(scheduled_minutes) as first_reply_time_business_minutes from intercepted_periods - group by 1 \ No newline at end of file + group by 1, 2 \ No newline at end of file diff --git a/models/reply_times/int_zendesk__ticket_reply_times.sql b/models/reply_times/int_zendesk__ticket_reply_times.sql index cb6cd720..19dbf186 100644 --- a/models/reply_times/int_zendesk__ticket_reply_times.sql +++ b/models/reply_times/int_zendesk__ticket_reply_times.sql @@ -8,6 +8,7 @@ with ticket_public_comments as ( select ticket_id, + source_relation, valid_starting_at as end_user_comment_created_at, ticket_created_date, commenter_role, @@ -22,6 +23,7 @@ with ticket_public_comments as ( select end_user_comments.ticket_id, + end_user_comments.source_relation, -- If the commentor was internal, a first comment, and had previous non public internal comments then we want the ticket created date to be the end user comment created date -- Otherwise we will want to end user comment created date case when is_first_comment then end_user_comments.ticket_created_date else end_user_comments.end_user_comment_created_at end as end_user_comment_created_at, @@ -36,7 +38,8 @@ with ticket_public_comments as ( on agent_comments.ticket_id = end_user_comments.ticket_id and agent_comments.commenter_role = 'internal_comment' and agent_comments.valid_starting_at > end_user_comments.end_user_comment_created_at - group by 1,2,3 + and agent_comments.source_relation = end_user_comments.source_relation + {{ dbt_utils.group_by(n=4) }} ) @@ -47,4 +50,4 @@ with ticket_public_comments as ( 'agent_responded_at', 'second') }} / 60) as reply_time_calendar_minutes from reply_timestamps - order by 1,2 \ No newline at end of file + order by 1,2,3 \ No newline at end of file diff --git a/models/reply_times/int_zendesk__ticket_reply_times_calendar.sql b/models/reply_times/int_zendesk__ticket_reply_times_calendar.sql index f7b65290..786b09d3 100644 --- a/models/reply_times/int_zendesk__ticket_reply_times_calendar.sql +++ b/models/reply_times/int_zendesk__ticket_reply_times_calendar.sql @@ -13,12 +13,14 @@ with ticket as ( select ticket.ticket_id, + ticket.source_relation, sum(case when is_first_comment then reply_time_calendar_minutes else null end) as first_reply_time_calendar_minutes, sum(reply_time_calendar_minutes) as total_reply_time_calendar_minutes --total combined time the customer waits for internal response from ticket left join ticket_reply_times - using (ticket_id) + on ticket.ticket_id = ticket_reply_times.ticket_id + and ticket.source_relation = ticket_reply_times.source_relation -group by 1 \ No newline at end of file +group by 1, 2 \ No newline at end of file diff --git a/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql b/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql index 90a3fb13..62a8a81c 100644 --- a/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql +++ b/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql @@ -19,6 +19,7 @@ with ticket_resolution_times_calendar as ( select ticket_resolution_times_calendar.ticket_id, + ticket_resolution_times_calendar.source_relation, ticket_schedules.schedule_created_at, ticket_schedules.schedule_invalidated_at, ticket_schedules.schedule_id, @@ -41,8 +42,10 @@ with ticket_resolution_times_calendar as ( {{ dbt_date.week_start('ticket_schedules.schedule_created_at','UTC') }} as start_week_date from ticket_resolution_times_calendar - join ticket_schedules on ticket_resolution_times_calendar.ticket_id = ticket_schedules.ticket_id - group by 1, 2, 3, 4 + join ticket_schedules + on ticket_resolution_times_calendar.ticket_id = ticket_schedules.ticket_id + and ticket_resolution_times_calendar.source_relation = ticket_schedules.source_relation + {{ dbt_utils.group_by(n=5) }} ), weeks as ( @@ -73,6 +76,7 @@ with ticket_resolution_times_calendar as ( ), intercepted_periods as ( select ticket_id, + source_relation, week_number, weekly_periods.schedule_id, ticket_week_start_time, @@ -84,6 +88,7 @@ with ticket_resolution_times_calendar as ( join schedule on ticket_week_start_time <= schedule.end_time_utc and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id + and weekly_periods.source_relation = schedule.source_relation -- this chooses the Daylight Savings Time or Standard Time version of the schedule -- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }}) @@ -93,6 +98,7 @@ with ticket_resolution_times_calendar as ( select ticket_id, + source_relation, sum(scheduled_minutes) as first_resolution_business_minutes from intercepted_periods - group by 1 \ No newline at end of file + group by 1, 2 \ No newline at end of file diff --git a/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql b/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql index 414ab654..8deb3be3 100644 --- a/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql +++ b/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql @@ -19,6 +19,7 @@ with ticket_resolution_times_calendar as ( select ticket_resolution_times_calendar.ticket_id, + ticket_resolution_times_calendar.source_relation, ticket_schedules.schedule_created_at, ticket_schedules.schedule_invalidated_at, ticket_schedules.schedule_id, @@ -40,8 +41,10 @@ with ticket_resolution_times_calendar as ( {{ dbt_date.week_start('ticket_schedules.schedule_created_at','UTC') }} as start_week_date from ticket_resolution_times_calendar - join ticket_schedules on ticket_resolution_times_calendar.ticket_id = ticket_schedules.ticket_id - group by 1, 2, 3, 4 + join ticket_schedules + on ticket_resolution_times_calendar.ticket_id = ticket_schedules.ticket_id + and ticket_resolution_times_calendar.source_relation = ticket_schedules.source_relation + {{ dbt_utils.group_by(n=5) }} ), weeks as ( @@ -72,6 +75,7 @@ with ticket_resolution_times_calendar as ( select ticket_id, + source_relation, week_number, weekly_periods.schedule_id, ticket_week_start_time, @@ -83,6 +87,7 @@ with ticket_resolution_times_calendar as ( join schedule on ticket_week_start_time <= schedule.end_time_utc and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id + and weekly_periods.source_relation = schedule.source_relation -- this chooses the Daylight Savings Time or Standard Time version of the schedule -- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }}) @@ -92,6 +97,7 @@ with ticket_resolution_times_calendar as ( select ticket_id, + source_relation, sum(scheduled_minutes) as full_resolution_business_minutes from intercepted_periods - group by 1 \ No newline at end of file + group by 1, 2 \ No newline at end of file diff --git a/models/resolution_times/int_zendesk__ticket_resolution_times_calendar.sql b/models/resolution_times/int_zendesk__ticket_resolution_times_calendar.sql index 9626f114..959b7d2a 100644 --- a/models/resolution_times/int_zendesk__ticket_resolution_times_calendar.sql +++ b/models/resolution_times/int_zendesk__ticket_resolution_times_calendar.sql @@ -24,18 +24,20 @@ with historical_solved_status as ( select ticket_id, + source_relation, min(valid_starting_at) as first_solved_at, max(valid_starting_at) as last_solved_at, count(status) as solved_count from historical_solved_status - group by 1 + group by 1, 2 ) select ticket.ticket_id, + ticket.source_relation, ticket.created_at, solved_times.first_solved_at, solved_times.last_solved_at, @@ -73,11 +75,13 @@ with historical_solved_status as ( from ticket left join ticket_historical_assignee - using(ticket_id) + on ticket.ticket_id = ticket_historical_assignee.ticket_id + and ticket.source_relation = ticket_historical_assignee.source_relation left join ticket_historical_group - using(ticket_id) + on ticket.ticket_id = ticket_historical_group.ticket_id + and ticket.source_relation = ticket_historical_group.source_relation left join solved_times - using(ticket_id) - + on ticket.ticket_id = solved_times.ticket_id + and ticket.source_relation = solved_times.source_relation \ No newline at end of file diff --git a/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql b/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql index 5abad813..15f34d34 100644 --- a/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql +++ b/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql @@ -25,6 +25,7 @@ with agent_work_time_filtered_statuses as ( select agent_work_time_filtered_statuses.ticket_id, + agent_work_time_filtered_statuses.source_relation, agent_work_time_filtered_statuses.sla_applied_at, agent_work_time_filtered_statuses.target, agent_work_time_filtered_statuses.sla_policy_name, @@ -41,6 +42,7 @@ with agent_work_time_filtered_statuses as ( from agent_work_time_filtered_statuses left join ticket_schedules on agent_work_time_filtered_statuses.ticket_id = ticket_schedules.ticket_id + and agent_work_time_filtered_statuses.source_relation = ticket_schedules.source_relation where {{ dbt.datediff( 'greatest(valid_starting_at, schedule_created_at)', 'least(valid_ending_at, schedule_invalidated_at)', @@ -50,6 +52,7 @@ with agent_work_time_filtered_statuses as ( select ticket_id, + source_relation, sla_applied_at, target, sla_policy_name, @@ -71,7 +74,7 @@ with agent_work_time_filtered_statuses as ( {{ dbt_date.week_start('ticket_status_crossed_with_schedule.valid_starting_at','UTC') }} as start_week_date from ticket_status_crossed_with_schedule - {{ dbt_utils.group_by(n=10) }} + {{ dbt_utils.group_by(n=11) }} ), weeks as ( @@ -91,6 +94,7 @@ with agent_work_time_filtered_statuses as ( select ticket_id, + source_relation, sla_applied_at, valid_starting_at, valid_ending_at, @@ -127,6 +131,7 @@ with agent_work_time_filtered_statuses as ( join schedule on ticket_week_start_time_minute <= schedule.end_time_utc and ticket_week_end_time_minute >= schedule.start_time_utc and weekly_period_agent_work_time.schedule_id = schedule.schedule_id + and weekly_period_agent_work_time.source_relation = schedule.source_relation -- this chooses the Daylight Savings Time or Standard Time version of the schedule -- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time_minute', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }}) @@ -137,7 +142,7 @@ with agent_work_time_filtered_statuses as ( select *, sum(scheduled_minutes) over - (partition by ticket_id, sla_applied_at + (partition by ticket_id, sla_applied_at, source_relation order by valid_starting_at, week_number, schedule_end_time rows between unbounded preceding and current row) as running_total_scheduled_minutes @@ -150,15 +155,15 @@ with agent_work_time_filtered_statuses as ( intercepted_periods_with_running_total.*, target - running_total_scheduled_minutes as remaining_target_minutes, lag(target - running_total_scheduled_minutes) over - (partition by ticket_id, sla_applied_at order by valid_starting_at, week_number, schedule_end_time) as lag_check, + (partition by ticket_id, sla_applied_at, source_relation order by valid_starting_at, week_number, schedule_end_time) as lag_check, case when (target - running_total_scheduled_minutes) = 0 then true when (target - running_total_scheduled_minutes) < 0 and (lag(target - running_total_scheduled_minutes) over - (partition by ticket_id, sla_applied_at order by valid_starting_at, week_number, schedule_end_time) > 0 + (partition by ticket_id, sla_applied_at, source_relation order by valid_starting_at, week_number, schedule_end_time) > 0 or lag(target - running_total_scheduled_minutes) over - (partition by ticket_id, sla_applied_at order by valid_starting_at, week_number, schedule_end_time) is null) + (partition by ticket_id, sla_applied_at, source_relation order by valid_starting_at, week_number, schedule_end_time) is null) then true else false end as is_breached_during_schedule from intercepted_periods_with_running_total diff --git a/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_calendar_hours.sql b/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_calendar_hours.sql index bd402041..e7dcbb28 100644 --- a/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_calendar_hours.sql +++ b/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_calendar_hours.sql @@ -17,7 +17,7 @@ with agent_work_time_filtered_statuses as ( 'valid_starting_at', 'valid_ending_at', 'minute') }} ) - over (partition by ticket_id, sla_applied_at order by valid_starting_at rows between unbounded preceding and current row) as running_total_calendar_minutes + over (partition by ticket_id, sla_applied_at, source_relation order by valid_starting_at rows between unbounded preceding and current row) as running_total_calendar_minutes from agent_work_time_filtered_statuses ), agent_work_time_calendar_minutes_flagged as ( @@ -28,10 +28,10 @@ select case when (target - running_total_calendar_minutes) < 0 and (lag(target - running_total_calendar_minutes) over - (partition by ticket_id, sla_applied_at order by valid_starting_at) >= 0 + (partition by ticket_id, sla_applied_at, source_relation order by valid_starting_at) >= 0 or lag(target - running_total_calendar_minutes) over - (partition by ticket_id, sla_applied_at order by valid_starting_at) is null) + (partition by ticket_id, sla_applied_at, source_relation order by valid_starting_at) is null) then true else false end as is_breached_during_schedule from agent_work_time_calendar_minutes diff --git a/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_filtered_statuses.sql b/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_filtered_statuses.sql index 008f6878..0cb0bc92 100644 --- a/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_filtered_statuses.sql +++ b/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_filtered_statuses.sql @@ -14,6 +14,7 @@ with agent_work_time_sla as ( select ticket_historical_status.ticket_id, + ticket_historical_status.source_relation, greatest(ticket_historical_status.valid_starting_at, agent_work_time_sla.sla_applied_at) as valid_starting_at, coalesce( ticket_historical_status.valid_ending_at, @@ -27,6 +28,7 @@ with agent_work_time_sla as ( from ticket_historical_status join agent_work_time_sla on ticket_historical_status.ticket_id = agent_work_time_sla.ticket_id + and ticket_historical_status.source_relation = agent_work_time_sla.source_relation where ticket_historical_status.status in ('new', 'open') -- these are the only statuses that count as "agent work time" and sla_applied_at < valid_ending_at diff --git a/models/sla_policy/int_zendesk__sla_policy_applied.sql b/models/sla_policy/int_zendesk__sla_policy_applied.sql index 2602e7e7..e5ca06ba 100644 --- a/models/sla_policy/int_zendesk__sla_policy_applied.sql +++ b/models/sla_policy/int_zendesk__sla_policy_applied.sql @@ -25,17 +25,19 @@ with ticket_field_history as ( select ticket_field_history.ticket_id, + ticket_field_history.source_relation, ticket.created_at as ticket_created_at, ticket_field_history.valid_starting_at, ticket.status as ticket_current_status, ticket_field_history.field_name as metric, - case when ticket_field_history.field_name = 'first_reply_time' then row_number() over (partition by ticket_field_history.ticket_id, ticket_field_history.field_name order by ticket_field_history.valid_starting_at desc) else 1 end as latest_sla, + case when ticket_field_history.field_name = 'first_reply_time' then row_number() over (partition by ticket_field_history.ticket_id, ticket_field_history.field_name, ticket_field_history.source_relation order by ticket_field_history.valid_starting_at desc) else 1 end as latest_sla, case when ticket_field_history.field_name = 'first_reply_time' then ticket.created_at else ticket_field_history.valid_starting_at end as sla_applied_at, cast({{ fivetran_utils.json_parse('ticket_field_history.value', ['minutes']) }} as {{ dbt.type_int() }} ) as target, {{ fivetran_utils.json_parse('ticket_field_history.value', ['in_business_hours']) }} = 'true' as in_business_hours from ticket_field_history join ticket on ticket.ticket_id = ticket_field_history.ticket_id + and ticket.source_relation = ticket_field_history.source_relation where ticket_field_history.value is not null and ticket_field_history.field_name in ('next_reply_time', 'first_reply_time', 'agent_work_time', 'requester_wait_time') @@ -48,6 +50,7 @@ with ticket_field_history as ( on sla_policy_name.ticket_id = sla_policy_applied.ticket_id and sla_policy_applied.valid_starting_at >= sla_policy_name.valid_starting_at and sla_policy_applied.valid_starting_at < coalesce(sla_policy_name.valid_ending_at, {{ dbt.current_timestamp_backcompat() }}) + and sla_policy_applied.source_relation = sla_policy_name.source_relation where sla_policy_applied.latest_sla = 1 ) diff --git a/models/sla_policy/reply_time/int_zendesk__reply_time_business_hours.sql b/models/sla_policy/reply_time/int_zendesk__reply_time_business_hours.sql index ac5820e9..8b01c466 100644 --- a/models/sla_policy/reply_time/int_zendesk__reply_time_business_hours.sql +++ b/models/sla_policy/reply_time/int_zendesk__reply_time_business_hours.sql @@ -22,10 +22,11 @@ with ticket_schedules as ( select schedule_id, + source_relation, sum(end_time - start_time) as total_schedule_weekly_business_minutes -- referring to stg_zendesk__schedule instead of int_zendesk__schedule_spine just to calculate total minutes from {{ ref('stg_zendesk__schedule') }} - group by 1 + group by 1, 2 ), ticket_sla_applied_with_schedules as ( @@ -41,11 +42,14 @@ with ticket_schedules as ( {{ dbt_date.week_start('sla_policy_applied.sla_applied_at','UTC') }} as start_week_date from sla_policy_applied - left join ticket_schedules on sla_policy_applied.ticket_id = ticket_schedules.ticket_id + left join ticket_schedules + on sla_policy_applied.ticket_id = ticket_schedules.ticket_id + and sla_policy_applied.source_relation = ticket_schedules.source_relation and {{ fivetran_utils.timestamp_add('second', -1, 'ticket_schedules.schedule_created_at') }} <= sla_policy_applied.sla_applied_at and {{ fivetran_utils.timestamp_add('second', -1, 'ticket_schedules.schedule_invalidated_at') }} > sla_policy_applied.sla_applied_at left join schedule_business_hours on ticket_schedules.schedule_id = schedule_business_hours.schedule_id + and ticket_schedules.source_relation = schedule_business_hours.source_relation where sla_policy_applied.in_business_hours and metric in ('next_reply_time', 'first_reply_time') @@ -80,13 +84,14 @@ with ticket_schedules as ( schedule.end_time_utc as schedule_end_time, (schedule.end_time_utc - greatest(ticket_week_start_time,schedule.start_time_utc)) as lapsed_business_minutes, sum(schedule.end_time_utc - greatest(ticket_week_start_time,schedule.start_time_utc)) over - (partition by ticket_id, metric, sla_applied_at + (partition by ticket_id, metric, sla_applied_at, source_relation order by week_number, schedule.start_time_utc rows between unbounded preceding and current row) as sum_lapsed_business_minutes from weekly_periods join schedule on ticket_week_start_time <= schedule.end_time_utc and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id + and weekly_periods.source_relation = schedule.source_relation -- this chooses the Daylight Savings Time or Standard Time version of the schedule -- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }}) @@ -100,10 +105,10 @@ with ticket_schedules as ( case when (target - sum_lapsed_business_minutes) < 0 and (lag(target - sum_lapsed_business_minutes) over - (partition by ticket_id, metric, sla_applied_at order by week_number, schedule_start_time) >= 0 + (partition by ticket_id, metric, sla_applied_at, source_relation order by week_number, schedule_start_time) >= 0 or lag(target - sum_lapsed_business_minutes) over - (partition by ticket_id, metric, sla_applied_at order by week_number, schedule_start_time) is null) + (partition by ticket_id, metric, sla_applied_at, source_relation order by week_number, schedule_start_time) is null) then true else false end as is_breached_during_schedule -- this flags the scheduled period on which the breach took place from intercepted_periods @@ -132,6 +137,7 @@ with ticket_schedules as ( select ticket_id, + source_relation, sla_policy_name, metric, ticket_created_at, diff --git a/models/sla_policy/reply_time/int_zendesk__reply_time_combined.sql b/models/sla_policy/reply_time/int_zendesk__reply_time_combined.sql index b145570b..f87fa0fe 100644 --- a/models/sla_policy/reply_time/int_zendesk__reply_time_combined.sql +++ b/models/sla_policy/reply_time/int_zendesk__reply_time_combined.sql @@ -25,6 +25,7 @@ with reply_time_calendar_hours_sla as ( select ticket_id, + source_relation, sla_policy_name, metric, ticket_created_at, @@ -43,6 +44,7 @@ with reply_time_calendar_hours_sla as ( select ticket_id, + source_relation, sla_policy_name, metric, ticket_created_at, @@ -60,6 +62,7 @@ with reply_time_calendar_hours_sla as ( ), ticket_solved_times as ( select ticket_id, + source_relation, valid_starting_at as solved_at from ticket_updates where field_name = 'status' @@ -68,11 +71,13 @@ with reply_time_calendar_hours_sla as ( ), reply_time as ( select ticket_comment.ticket_id, + ticket_comment.source_relation, ticket_comment.valid_starting_at as reply_at, commenter.role from ticket_updates as ticket_comment join users as commenter on commenter.user_id = ticket_comment.user_id + and commenter.source_relation = ticket_comment.source_relation where field_name = 'comment' and ticket_comment.is_public and commenter.role in ('agent','admin') @@ -81,6 +86,7 @@ with reply_time_calendar_hours_sla as ( select reply_time_breached_at.ticket_id, + reply_time_breached_at.source_relation, reply_time_breached_at.sla_policy_name, reply_time_breached_at.metric, reply_time_breached_at.ticket_created_at, @@ -97,17 +103,19 @@ with reply_time_calendar_hours_sla as ( left join reply_time on reply_time.ticket_id = reply_time_breached_at.ticket_id and reply_time.reply_at > reply_time_breached_at.sla_applied_at + and reply_time.source_relation = reply_time_breached_at.source_relation left join ticket_solved_times on reply_time_breached_at.ticket_id = ticket_solved_times.ticket_id and ticket_solved_times.solved_at > reply_time_breached_at.sla_applied_at - {{ dbt_utils.group_by(n=8) }} + and ticket_solved_times.source_relation = reply_time_breached_at.source_relation + {{ dbt_utils.group_by(n=9) }} ), lagging_time_block as ( select *, - lead(sla_schedule_start_at) over (partition by ticket_id, sla_policy_name, metric, sla_applied_at order by sla_schedule_start_at) as next_schedule_start, - min(sla_breach_at) over (partition by sla_policy_name, metric, sla_applied_at order by sla_schedule_start_at rows unbounded preceding) as first_sla_breach_at, - coalesce(lag(sum_lapsed_business_minutes) over (partition by sla_policy_name, metric, sla_applied_at order by sla_schedule_start_at), 0) as sum_lapsed_business_minutes_new + lead(sla_schedule_start_at) over (partition by ticket_id, sla_policy_name, metric, sla_applied_at, source_relation order by sla_schedule_start_at) as next_schedule_start, + min(sla_breach_at) over (partition by sla_policy_name, metric, sla_applied_at, source_relation order by sla_schedule_start_at rows unbounded preceding) as first_sla_breach_at, + coalesce(lag(sum_lapsed_business_minutes) over (partition by sla_policy_name, metric, sla_applied_at, source_relation order by sla_schedule_start_at), 0) as sum_lapsed_business_minutes_new from reply_time_breached_at_with_next_reply_timestamp ), filtered_reply_times as ( @@ -127,9 +135,9 @@ with reply_time_calendar_hours_sla as ( select *, {{ dbt.current_timestamp() }} as current_time_check, - lead(sla_applied_at) over (partition by ticket_id, metric, in_business_hours order by sla_applied_at) as updated_sla_policy_starts_at, + lead(sla_applied_at) over (partition by ticket_id, metric, in_business_hours, source_relation order by sla_applied_at) as updated_sla_policy_starts_at, case when - lead(sla_applied_at) over (partition by ticket_id, metric, in_business_hours order by sla_applied_at) --updated sla policy start at time + lead(sla_applied_at) over (partition by ticket_id, metric, in_business_hours, source_relation order by sla_applied_at) --updated sla policy start at time < sla_breach_at then true else false end as is_stale_sla_policy, case when (sla_breach_at < agent_reply_at and sla_breach_at < next_solved_at) or (sla_breach_at < agent_reply_at and next_solved_at is null) diff --git a/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql b/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql index 53b09dca..172ab073 100644 --- a/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql +++ b/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql @@ -25,6 +25,7 @@ with requester_wait_time_filtered_statuses as ( select requester_wait_time_filtered_statuses.ticket_id, + requester_wait_time_filtered_statuses.source_relation, requester_wait_time_filtered_statuses.sla_applied_at, requester_wait_time_filtered_statuses.target, requester_wait_time_filtered_statuses.sla_policy_name, @@ -41,6 +42,7 @@ with requester_wait_time_filtered_statuses as ( from requester_wait_time_filtered_statuses left join ticket_schedules on requester_wait_time_filtered_statuses.ticket_id = ticket_schedules.ticket_id + and requester_wait_time_filtered_statuses.source_relation = ticket_schedules.source_relation where {{ dbt.datediff( 'greatest(valid_starting_at, schedule_created_at)', 'least(valid_ending_at, schedule_invalidated_at)', @@ -50,6 +52,7 @@ with requester_wait_time_filtered_statuses as ( select ticket_id, + source_relation, sla_applied_at, target, sla_policy_name, @@ -71,7 +74,7 @@ with requester_wait_time_filtered_statuses as ( {{ dbt_date.week_start('ticket_status_crossed_with_schedule.valid_starting_at','UTC') }} as start_week_date from ticket_status_crossed_with_schedule - {{ dbt_utils.group_by(n=10) }} + {{ dbt_utils.group_by(n=11) }} ), weeks as ( @@ -91,6 +94,7 @@ with requester_wait_time_filtered_statuses as ( select ticket_id, + source_relation, sla_applied_at, valid_starting_at, valid_ending_at, @@ -112,6 +116,7 @@ with requester_wait_time_filtered_statuses as ( select weekly_period_requester_wait_time.ticket_id, + weekly_period_requester_wait_time.source_relation, weekly_period_requester_wait_time.sla_applied_at, weekly_period_requester_wait_time.target, weekly_period_requester_wait_time.sla_policy_name, @@ -127,6 +132,7 @@ with requester_wait_time_filtered_statuses as ( join schedule on ticket_week_start_time_minute <= schedule.end_time_utc and ticket_week_end_time_minute >= schedule.start_time_utc and weekly_period_requester_wait_time.schedule_id = schedule.schedule_id + and weekly_period_requester_wait_time.source_relation = schedule.source_relation -- this chooses the Daylight Savings Time or Standard Time version of the schedule -- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time_minute', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }}) @@ -137,7 +143,7 @@ with requester_wait_time_filtered_statuses as ( select *, sum(scheduled_minutes) over - (partition by ticket_id, sla_applied_at + (partition by ticket_id, sla_applied_at, source_relation order by valid_starting_at, week_number, schedule_end_time rows between unbounded preceding and current row) as running_total_scheduled_minutes @@ -150,15 +156,15 @@ with requester_wait_time_filtered_statuses as ( intercepted_periods_with_running_total.*, target - running_total_scheduled_minutes as remaining_target_minutes, lag(target - running_total_scheduled_minutes) over - (partition by ticket_id, sla_applied_at order by valid_starting_at, week_number, schedule_end_time) as lag_check, + (partition by ticket_id, sla_applied_at, source_relation order by valid_starting_at, week_number, schedule_end_time) as lag_check, case when (target - running_total_scheduled_minutes) = 0 then true when (target - running_total_scheduled_minutes) < 0 and (lag(target - running_total_scheduled_minutes) over - (partition by ticket_id, sla_applied_at order by valid_starting_at, week_number, schedule_end_time) > 0 + (partition by ticket_id, sla_applied_at, source_relation order by valid_starting_at, week_number, schedule_end_time) > 0 or lag(target - running_total_scheduled_minutes) over - (partition by ticket_id, sla_applied_at order by valid_starting_at, week_number, schedule_end_time) is null) + (partition by ticket_id, sla_applied_at, source_relation order by valid_starting_at, week_number, schedule_end_time) is null) then true else false end as is_breached_during_schedule from intercepted_periods_with_running_total diff --git a/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_calendar_hours.sql b/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_calendar_hours.sql index 80271cca..b10ba539 100644 --- a/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_calendar_hours.sql +++ b/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_calendar_hours.sql @@ -17,7 +17,7 @@ with requester_wait_time_filtered_statuses as ( 'valid_starting_at', 'valid_ending_at', 'minute') }} ) - over (partition by ticket_id, sla_applied_at order by valid_starting_at rows between unbounded preceding and current row) as running_total_calendar_minutes + over (partition by ticket_id, sla_applied_at, source_relation order by valid_starting_at rows between unbounded preceding and current row) as running_total_calendar_minutes from requester_wait_time_filtered_statuses ), requester_wait_time_calendar_minutes_flagged as ( @@ -28,10 +28,10 @@ select case when (target - running_total_calendar_minutes) < 0 and (lag(target - running_total_calendar_minutes) over - (partition by ticket_id, sla_applied_at order by valid_starting_at) >= 0 + (partition by ticket_id, sla_applied_at, source_relation order by valid_starting_at) >= 0 or lag(target - running_total_calendar_minutes) over - (partition by ticket_id, sla_applied_at order by valid_starting_at) is null) + (partition by ticket_id, sla_applied_at, source_relation order by valid_starting_at) is null) then true else false end as is_breached_during_schedule from requester_wait_time_calendar_minutes diff --git a/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_filtered_statuses.sql b/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_filtered_statuses.sql index 8ea160fb..3e4150db 100644 --- a/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_filtered_statuses.sql +++ b/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_filtered_statuses.sql @@ -14,6 +14,7 @@ with requester_wait_time_sla as ( select ticket_historical_status.ticket_id, + ticket_historical_status.source_relation, greatest(ticket_historical_status.valid_starting_at, requester_wait_time_sla.sla_applied_at) as valid_starting_at, coalesce( ticket_historical_status.valid_ending_at, @@ -27,6 +28,7 @@ with requester_wait_time_sla as ( from ticket_historical_status join requester_wait_time_sla on ticket_historical_status.ticket_id = requester_wait_time_sla.ticket_id + and ticket_historical_status.source_relation = requester_wait_time_sla.source_relation where ticket_historical_status.status in ('new', 'open', 'on-hold', 'hold') -- these are the only statuses that count as "requester wait time" and sla_applied_at < valid_ending_at diff --git a/models/ticket_history/int_zendesk__field_calendar_spine.sql b/models/ticket_history/int_zendesk__field_calendar_spine.sql index 1e821310..3810ba03 100644 --- a/models/ticket_history/int_zendesk__field_calendar_spine.sql +++ b/models/ticket_history/int_zendesk__field_calendar_spine.sql @@ -28,7 +28,8 @@ with calendar as ( select calendar.date_day, - ticket.ticket_id + ticket.ticket_id, + ticket.source_relation from calendar inner join ticket on calendar.date_day >= cast(ticket.created_at as date) @@ -39,7 +40,7 @@ with calendar as ( select *, - {{ dbt_utils.generate_surrogate_key(['date_day','ticket_id']) }} as ticket_day_id + {{ dbt_utils.generate_surrogate_key(['date_day','ticket_id','source_relation']) }} as ticket_day_id from joined ) diff --git a/models/ticket_history/int_zendesk__field_history_enriched.sql b/models/ticket_history/int_zendesk__field_history_enriched.sql index 3b8ab949..4d9a70e5 100644 --- a/models/ticket_history/int_zendesk__field_history_enriched.sql +++ b/models/ticket_history/int_zendesk__field_history_enriched.sql @@ -35,6 +35,7 @@ with ticket_field_history as ( left join updater_info on ticket_field_history.user_id = updater_info.updater_user_id + and ticket_field_history.source_relation = updater_info.source_relation ) select * from final diff --git a/models/ticket_history/int_zendesk__field_history_pivot.sql b/models/ticket_history/int_zendesk__field_history_pivot.sql index 56600191..cb3b1218 100644 --- a/models/ticket_history/int_zendesk__field_history_pivot.sql +++ b/models/ticket_history/int_zendesk__field_history_pivot.sql @@ -19,6 +19,7 @@ with field_history as ( select ticket_id, + source_relation, field_name, valid_ending_at, valid_starting_at @@ -43,7 +44,7 @@ with field_history as ( select *, row_number() over ( - partition by cast(valid_starting_at as date), ticket_id, field_name + partition by cast(valid_starting_at as date), ticket_id, field_name, source_relation order by valid_starting_at desc ) as row_num from field_history @@ -63,6 +64,7 @@ with field_history as ( select ticket_id, + source_relation, cast({{ dbt.date_trunc('day', 'valid_starting_at') }} as date) as date_day {% for col in results_list if col in var('ticket_field_history_columns') %} @@ -90,13 +92,13 @@ with field_history as ( {% endfor %} from filtered - group by 1,2 + group by 1,2,3 ), surrogate_key as ( select *, - {{ dbt_utils.generate_surrogate_key(['ticket_id','date_day'])}} as ticket_day_id + {{ dbt_utils.generate_surrogate_key(['ticket_id','date_day','source_relation'])}} as ticket_day_id from pivots ) diff --git a/models/ticket_history/int_zendesk__field_history_scd.sql b/models/ticket_history/int_zendesk__field_history_scd.sql index 94f44e82..0f12dcd4 100644 --- a/models/ticket_history/int_zendesk__field_history_scd.sql +++ b/models/ticket_history/int_zendesk__field_history_scd.sql @@ -15,15 +15,16 @@ with change_data as ( select date_day as valid_from, ticket_id, - ticket_day_id + ticket_day_id, + source_relation - {% for col in ticket_columns if col.name|lower not in ['date_day','ending_day','ticket_id','ticket_day_id'] %} + {% for col in ticket_columns if col.name|lower not in ['date_day','ending_day','ticket_id','ticket_day_id','source_relation'] %} ,{{ col.name }} ,sum(case when {{ col.name }} is null then 0 else 1 - end) over (order by ticket_id, date_day rows unbounded preceding) as {{ col.name }}_field_partition + end) over (order by ticket_id, date_day, source_relation rows unbounded preceding) as {{ col.name }}_field_partition {% endfor %} from change_data @@ -32,11 +33,12 @@ with change_data as ( select valid_from, ticket_id, - ticket_day_id + ticket_day_id, + source_relation - {% for col in ticket_columns if col.name|lower not in ['date_day','ending_day','ticket_id','ticket_day_id'] %} + {% for col in ticket_columns if col.name|lower not in ['date_day','ending_day','ticket_id','ticket_day_id','source_relation'] %} - ,first_value( {{ col.name }} ) over (partition by {{ col.name }}_field_partition, ticket_id order by valid_from asc rows between unbounded preceding and current row) as {{ col.name }} + ,first_value( {{ col.name }} ) over (partition by {{ col.name }}_field_partition, ticket_id, source_relation order by valid_from asc rows between unbounded preceding and current row) as {{ col.name }} {% endfor %} from set_values diff --git a/models/ticket_history/int_zendesk__updater_information.sql b/models/ticket_history/int_zendesk__updater_information.sql index 85229c7b..e2285e86 100644 --- a/models/ticket_history/int_zendesk__updater_information.sql +++ b/models/ticket_history/int_zendesk__updater_information.sql @@ -9,6 +9,7 @@ with users as ( ), final as ( select users.user_id as updater_user_id + ,users.source_relation ,users.name as updater_name ,users.role as updater_role ,users.email as updater_email @@ -37,7 +38,8 @@ with users as ( from users left join organizations - using(organization_id) + on users.organization_id = organizations.organization_id + and users.source_relation = organizations.source_relation ) select * diff --git a/models/utils/int_zendesk__calendar_spine.sql b/models/utils/int_zendesk__calendar_spine.sql index 393b6a48..f40e1f76 100644 --- a/models/utils/int_zendesk__calendar_spine.sql +++ b/models/utils/int_zendesk__calendar_spine.sql @@ -1,11 +1,11 @@ --- depends_on: {{ source('zendesk', 'ticket') }} +-- depends_on: {{ ref('stg_zendesk__ticket') }} with spine as ( {% if execute %} {% set current_ts = dbt.current_timestamp_backcompat() %} {% set first_date_query %} - select min( created_at ) as min_date from {{ source('zendesk', 'ticket') }} + select min( created_at ) as min_date from {{ var('ticket') }} -- by default take all the data where cast(created_at as date) >= {{ dbt.dateadd('year', - var('ticket_field_history_timeframe_years', 50), current_ts ) }} {% endset %} diff --git a/models/zendesk.yml b/models/zendesk.yml index a0c19151..7acae3ab 100644 --- a/models/zendesk.yml +++ b/models/zendesk.yml @@ -3,11 +3,15 @@ version: 2 models: - name: zendesk__ticket_enriched description: Each record represents a Zendesk ticket, enriched with data about it's tags, assignees, requester, submitter, organization and group. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ticket_id + - source_relation columns: - name: ticket_id description: Automatically assigned when the ticket is created tests: - - unique - not_null - name: url description: The API url of this ticket @@ -171,14 +175,21 @@ models: description: Boolean indicating if the ticket had a satisfaction score went from good to bad. - name: is_bad_to_good_satisfaction_score description: Boolean indicating if the ticket had a satisfaction score went from bad to good. + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: zendesk__sla_policies description: Each record represents an SLA policy event and additional sla breach and achievement metrics. Calendar and business hour SLA breaches are supported. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - sla_event_id + - source_relation columns: - name: sla_event_id description: A surrogate key generated from the combination of ticket_id, metric, and sla_applied_at fields - tests: - - unique - name: ticket_id description: A ticket's unique identifier, it is automatically assigned when the ticket is created - name: sla_policy_name @@ -199,7 +210,11 @@ models: description: Boolean field indicating that the SLA event is currently active and not breached (true) or past (false) - name: is_sla_breach description: Boolean field indicating if the SLA has been breached (true) or was achieved (false) - + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. + - name: zendesk__ticket_field_history description: > A daily historical view of the ticket field values defined in the `ticket_field_history_columns` variable @@ -210,16 +225,28 @@ models: - name: ticket_id description: A ticket's unique identifier, it is automatically assigned when the ticket is created - name: ticket_day_id - description: The unique key of the table, a surrogate key of date_day and ticket_id. + description: The unique key of the table, a surrogate key of `date_day`, `ticket_id`, and `source_relation` + tests: + - unique + - not_null - name: assignee_id description: The assignee id assigned to the ticket - name: status description: The status of the ticket - name: priority description: The tickets priority ranking + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: zendesk__ticket_metrics description: Each record represents a Zendesk ticket, enriched with metrics about reply times, resolution times and work times. Calendar and business hours are supported + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ticket_id + - source_relation columns: - name: first_reply_time_calendar_minutes description: The number of calendar minutes between when the ticket was created and when the first public agent response occurred @@ -268,7 +295,6 @@ models: - name: ticket_id description: Automatically assigned when the ticket is created tests: - - unique - not_null - name: url description: The API url of this ticket @@ -492,6 +518,10 @@ models: description: The time in minutes the ticket was in an unassigned state - name: last_status_assignment_date description: The time the status was last changed on the ticket + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: zendesk__ticket_summary description: A single record table containing Zendesk ticket and user summary metrics. These metrics are updated for the current day the model is run. @@ -538,6 +568,10 @@ models: description: Total count of deleted tickets - name: recovered_ticket_count description: Total count of tickets that were deleted then reopened + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. - name: zendesk__ticket_backlog description: > @@ -555,4 +589,8 @@ models: - name: assignee_name description: The assignee name assigned to the ticket - name: priority - description: The tickets priority ranking \ No newline at end of file + description: The tickets priority ranking + - name: source_relation + description: > + The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. + Empty string if you are not. \ No newline at end of file diff --git a/models/zendesk__sla_policies.sql b/models/zendesk__sla_policies.sql index fd0aff89..53ce70cb 100644 --- a/models/zendesk__sla_policies.sql +++ b/models/zendesk__sla_policies.sql @@ -31,6 +31,7 @@ with reply_time_sla as ( ), all_slas_unioned as ( select ticket_id, + source_relation, sla_policy_name, metric, sla_applied_at, @@ -45,6 +46,7 @@ union all select ticket_id, + source_relation, sla_policy_name, 'agent_work_time' as metric, sla_applied_at, @@ -55,12 +57,13 @@ union all {{ fivetran_utils.max_bool("is_breached_during_schedule") }} from agent_work_calendar_sla - group by 1, 2, 3, 4, 5, 6 + {{ dbt_utils.group_by(n=7) }} union all select ticket_id, + source_relation, sla_policy_name, 'requester_wait_time' as metric, sla_applied_at, @@ -71,7 +74,7 @@ union all {{ fivetran_utils.max_bool("is_breached_during_schedule") }} from requester_wait_calendar_sla - group by 1, 2, 3, 4, 5, 6 + {{ dbt_utils.group_by(n=7) }} {% if var('using_schedules', True) %} @@ -80,6 +83,7 @@ union all select ticket_id, + source_relation, sla_policy_name, 'agent_work_time' as metric, sla_applied_at, @@ -90,12 +94,13 @@ union all {{ fivetran_utils.max_bool("is_breached_during_schedule") }} from agent_work_business_sla - group by 1, 2, 3, 4, 5, 6 + {{ dbt_utils.group_by(n=7) }} union all select ticket_id, + source_relation, sla_policy_name, 'requester_wait_time' as metric, sla_applied_at, @@ -107,7 +112,7 @@ union all from requester_wait_business_sla - group by 1, 2, 3, 4, 5, 6 + {{ dbt_utils.group_by(n=7) }} {% endif %} @@ -116,6 +121,7 @@ union all select {{ dbt_utils.generate_surrogate_key(['ticket_id', 'metric', 'sla_applied_at']) }} as sla_event_id, ticket_id, + source_relation, sla_policy_name, metric, sla_applied_at, diff --git a/models/zendesk__ticket_backlog.sql b/models/zendesk__ticket_backlog.sql index fca41698..6d96da0d 100644 --- a/models/zendesk__ticket_backlog.sql +++ b/models/zendesk__ticket_backlog.sql @@ -35,6 +35,7 @@ with ticket_field_history as ( ), backlog as ( select ticket_field_history.date_day + ,ticket_field_history.source_relation ,ticket_field_history.ticket_id ,ticket_field_history.status ,tickets.created_channel @@ -69,35 +70,42 @@ with ticket_field_history as ( left join tickets on tickets.ticket_id = ticket_field_history.ticket_id + and tickets.source_relation = ticket_field_history.source_relation {% if 'ticket_form_id' in var('ticket_field_history_columns') %} --Join not needed if field is not located in variable, otherwise it is included. left join ticket_forms on ticket_forms.ticket_form_id = cast(ticket_field_history.ticket_form_id as {{ dbt.type_bigint() }}) + and ticket_forms.source_relation = ticket_field_history.source_relation {% endif %} {% if 'group_id' in var('ticket_field_history_columns') %}--Join not needed if field is not located in variable, otherwise it is included. left join group_names on group_names.group_id = cast(ticket_field_history.group_id as {{ dbt.type_bigint() }}) + and group_names.source_relation = ticket_field_history.source_relation {% endif %} {% if 'assignee_id' in var('ticket_field_history_columns') or 'requester_id' in var('ticket_field_history_columns') or 'locale_id' in var('ticket_field_history_columns')%} --Join not needed if fields is not located in variable, otherwise it is included. left join users as assignee on assignee.user_id = cast(ticket_field_history.assignee_id as {{ dbt.type_bigint() }}) + and assignee.source_relation = ticket_field_history.source_relation {% endif %} {% if 'requester_id' in var('ticket_field_history_columns') %} --Join not needed if field is not located in variable, otherwise it is included. left join users as requester on requester.user_id = cast(ticket_field_history.requester_id as {{ dbt.type_bigint() }}) + and requester.source_relation = ticket_field_history.source_relation {% endif %} {% if 'brand_id' in var('ticket_field_history_columns') %} --Join not needed if field is not located in variable, otherwise it is included. left join brands on brands.brand_id = cast(ticket_field_history.brand_id as {{ dbt.type_bigint() }}) + and brands.source_relation = ticket_field_history.source_relation {% endif %} {% if 'organization_id' in var('ticket_field_history_columns') %} --Join not needed if field is not located in variable, otherwise it is included. left join organizations on organizations.organization_id = cast(ticket_field_history.organization_id as {{ dbt.type_bigint() }}) + and organizations.source_relation = ticket_field_history.source_relation {% endif %} where ticket_field_history.status not in ('closed', 'solved', 'deleted') diff --git a/models/zendesk__ticket_enriched.sql b/models/zendesk__ticket_enriched.sql index f772b7be..38b09c61 100644 --- a/models/zendesk__ticket_enriched.sql +++ b/models/zendesk__ticket_enriched.sql @@ -128,41 +128,51 @@ with ticket as ( --Requester Joins join users as requester on requester.user_id = ticket.requester_id + and requester.source_relation = ticket.source_relation left join organization as requester_org on requester_org.organization_id = requester.organization_id + and requester_org.source_relation = requester.source_relation left join requester_updates on requester_updates.ticket_id = ticket.ticket_id and requester_updates.requester_id = ticket.requester_id + and requester_updates.source_relation = ticket.source_relation --Submitter Joins join users as submitter on submitter.user_id = ticket.submitter_id + and submitter.source_relation = ticket.source_relation --Assignee Joins left join users as assignee on assignee.user_id = ticket.assignee_id + and assignee.source_relation = ticket.source_relation left join assignee_updates on assignee_updates.ticket_id = ticket.ticket_id and assignee_updates.assignee_id = ticket.assignee_id + and assignee_updates.source_relation = ticket.source_relation --Ticket, Org, and Brand Joins left join ticket_group on ticket_group.group_id = ticket.group_id + and ticket_group.source_relation = ticket.source_relation --If you use using_ticket_form_history this will be included, if not it will be ignored. {% if var('using_ticket_form_history', True) %} left join latest_ticket_form on latest_ticket_form.ticket_form_id = ticket.ticket_form_id + and latest_ticket_form.source_relation = ticket.source_relation {% endif %} left join organization on organization.organization_id = ticket.organization_id + and organization.source_relation = ticket.source_relation left join latest_satisfaction_ratings on latest_satisfaction_ratings.ticket_id = ticket.ticket_id + and latest_satisfaction_ratings.source_relation = ticket.source_relation ) select * diff --git a/models/zendesk__ticket_field_history.sql b/models/zendesk__ticket_field_history.sql index 5879e458..e3f6ca1e 100644 --- a/models/zendesk__ticket_field_history.sql +++ b/models/zendesk__ticket_field_history.sql @@ -44,14 +44,15 @@ with change_data as ( select calendar.date_day, - calendar.ticket_id + calendar.ticket_id, + calendar.source_relation {% if is_incremental() %} - {% for col in change_data_columns if col.name|lower not in ['ticket_id','valid_from','valid_to','ticket_day_id'] %} + {% for col in change_data_columns if col.name|lower not in ['ticket_id','valid_from','valid_to','ticket_day_id','source_relation'] %} , coalesce(change_data.{{ col.name }}, most_recent_data.{{ col.name }}) as {{ col.name }} {% endfor %} {% else %} - {% for col in change_data_columns if col.name|lower not in ['ticket_id','valid_from','valid_to','ticket_day_id'] %} + {% for col in change_data_columns if col.name|lower not in ['ticket_id','valid_from','valid_to','ticket_day_id','source_relation'] %} , {{ col.name }} {% endfor %} {% endif %} @@ -60,23 +61,26 @@ with change_data as ( left join change_data on calendar.ticket_id = change_data.ticket_id and calendar.date_day = change_data.valid_from + and calendar.source_relation = change_data.source_relation {% if is_incremental() %} left join most_recent_data on calendar.ticket_id = most_recent_data.ticket_id and calendar.date_day = most_recent_data.date_day + and calendar.source_relation = most_recent_data.source_relation {% endif %} ), set_values as ( select date_day, - ticket_id + ticket_id, + source_relation - {% for col in change_data_columns if col.name|lower not in ['ticket_id','valid_from','valid_to','ticket_day_id'] %} + {% for col in change_data_columns if col.name|lower not in ['ticket_id','valid_from','valid_to','ticket_day_id','source_relation'] %} , {{ col.name }} -- create a batch/partition once a new value is provided - , sum( case when {{ col.name }} is null then 0 else 1 end) over ( partition by ticket_id + , sum( case when {{ col.name }} is null then 0 else 1 end) over ( partition by ticket_id, source_relation order by date_day rows unbounded preceding) as {{ col.name }}_field_partition {% endfor %} @@ -88,12 +92,13 @@ fill_values as ( select date_day, - ticket_id + ticket_id, + source_relation - {% for col in change_data_columns if col.name|lower not in ['ticket_id','valid_from','valid_to','ticket_day_id'] %} + {% for col in change_data_columns if col.name|lower not in ['ticket_id','valid_from','valid_to','ticket_day_id','source_relation'] %} -- grab the value that started this batch/partition , first_value( {{ col.name }} ) over ( - partition by ticket_id, {{ col.name }}_field_partition + partition by ticket_id, {{ col.name }}_field_partition, source_relation order by date_day asc rows between unbounded preceding and current row) as {{ col.name }} {% endfor %} @@ -103,8 +108,9 @@ fill_values as ( select date_day, - ticket_id - {% for col in change_data_columns if col.name|lower not in ['ticket_id','valid_from','valid_to','ticket_day_id'] %} + ticket_id, + source_relation + {% for col in change_data_columns if col.name|lower not in ['ticket_id','valid_from','valid_to','ticket_day_id','source_relation'] %} -- we de-nulled the true null values earlier in order to differentiate them from nulls that just needed to be backfilled , case when cast( {{ col.name }} as {{ dbt.type_string() }} ) = 'is_null' then null else {{ col.name }} end as {{ col.name }} @@ -115,7 +121,7 @@ fill_values as ( ), surrogate_key as ( select - {{ dbt_utils.generate_surrogate_key(['date_day','ticket_id']) }} as ticket_day_id, + {{ dbt_utils.generate_surrogate_key(['date_day','ticket_id','source_relation']) }} as ticket_day_id, * from fix_null_values diff --git a/models/zendesk__ticket_metrics.sql b/models/zendesk__ticket_metrics.sql index c22fe52f..fdfeabb5 100644 --- a/models/zendesk__ticket_metrics.sql +++ b/models/zendesk__ticket_metrics.sql @@ -133,16 +133,20 @@ select from ticket_enriched left join ticket_reply_times_calendar - using (ticket_id) + on ticket_enriched.ticket_id = ticket_reply_times_calendar.ticket_id + and ticket_enriched.source_relation = ticket_reply_times_calendar.source_relation left join ticket_resolution_times_calendar - using (ticket_id) + on ticket_enriched.ticket_id = ticket_resolution_times_calendar.ticket_id + and ticket_enriched.source_relation = ticket_resolution_times_calendar.source_relation left join ticket_work_time_calendar - using (ticket_id) + on ticket_enriched.ticket_id = ticket_work_time_calendar.ticket_id + and ticket_enriched.source_relation = ticket_work_time_calendar.source_relation left join ticket_comments - using(ticket_id) +on ticket_enriched.ticket_id = ticket_comments.ticket_id +and ticket_enriched.source_relation = ticket_comments.source_relation {% if var('using_schedules', True) %} @@ -150,6 +154,7 @@ left join ticket_comments select ticket_enriched.ticket_id, + ticket_enriched.source_relation, ticket_first_resolution_time_business.first_resolution_business_minutes, ticket_full_resolution_time_business.full_resolution_business_minutes, ticket_first_reply_time_business.first_reply_time_business_minutes, @@ -164,16 +169,20 @@ left join ticket_comments from ticket_enriched left join ticket_first_resolution_time_business - using (ticket_id) + on ticket_enriched.ticket_id = ticket_first_resolution_time_business.ticket_id + and ticket_enriched.source_relation = ticket_first_resolution_time_business.source_relation left join ticket_full_resolution_time_business - using (ticket_id) + on ticket_enriched.ticket_id = ticket_full_resolution_time_business.ticket_id + and ticket_enriched.source_relation = ticket_full_resolution_time_business.source_relation left join ticket_first_reply_time_business - using (ticket_id) + on ticket_enriched.ticket_id = ticket_first_reply_time_business.ticket_id + and ticket_enriched.source_relation = ticket_first_reply_time_business.source_relation left join ticket_work_time_business - using (ticket_id) + on ticket_enriched.ticket_id = ticket_work_time_business.ticket_id + and ticket_enriched.source_relation = ticket_work_time_business.source_relation ) @@ -202,7 +211,8 @@ select from calendar_hour_metrics left join business_hour_metrics - using (ticket_id) + on calendar_hour_metrics.ticket_id = business_hour_metrics.ticket_id + and calendar_hour_metrics.source_relation = business_hour_metrics.source_relation {% else %} diff --git a/models/zendesk__ticket_summary.sql b/models/zendesk__ticket_summary.sql index a32958af..d4c20f37 100644 --- a/models/zendesk__ticket_summary.sql +++ b/models/zendesk__ticket_summary.sql @@ -9,6 +9,7 @@ with ticket_metrics as ( ), user_sum as ( select cast(1 as {{ dbt.type_int() }}) as summary_helper, + source_relation, sum(case when is_active = true then 1 else 0 @@ -31,11 +32,12 @@ with ticket_metrics as ( end) as suspended_user_count from user_table - group by 1 + group by 1,2 ), ticket_metric_sum as ( select cast(1 as {{ dbt.type_int() }}) as summary_helper, + source_relation, sum(case when lower(status) = 'new' then 1 else 0 @@ -107,11 +109,12 @@ with ticket_metrics as ( count(total_comments) from ticket_metrics - group by 1 + group by 1,2 ), final as ( select + user_sum.source_relation, user_sum.user_count, user_sum.active_agent_count, user_sum.deleted_user_count, @@ -136,7 +139,8 @@ with ticket_metrics as ( from user_sum left join ticket_metric_sum - using(summary_helper) + on user_sum.summary_helper = ticket_metric_sum.summary_helper + and user_sum.source_relation = ticket_metric_sum.source_relation ) select * diff --git a/packages.yml b/packages.yml index e36807e9..e89332f6 100644 --- a/packages.yml +++ b/packages.yml @@ -3,7 +3,7 @@ packages: # version: [">=0.10.0", "<0.11.0"] # - local: ../dbt_zendesk_source - git: https://github.com/fivetran/dbt_zendesk_source.git - revision: feature/tag-former-agents + revision: feature/add-union-data warn-unpinned: false - package: calogica/dbt_date version: [">=0.9.0", "<1.0.0"] From 1d742471aa02765731c68862299f3543db337888 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 18 Dec 2023 16:14:01 -0800 Subject: [PATCH 03/17] other warehouse issues --- .../int_zendesk__ticket_first_reply_time_business.sql | 2 +- .../int_zendesk__ticket_full_resolution_time_business.sql | 2 +- .../int_zendesk__agent_work_time_business_hours.sql | 1 + .../reply_time/int_zendesk__reply_time_business_hours.sql | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql b/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql index 6646aec9..eb2c96bf 100644 --- a/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql +++ b/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql @@ -87,7 +87,7 @@ with ticket_reply_times as ( ), intercepted_periods as ( select ticket_id, - source_relation, + weekly_periods.source_relation, week_number, weekly_periods.schedule_id, ticket_week_start_time, diff --git a/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql b/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql index 8deb3be3..efd7aa43 100644 --- a/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql +++ b/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql @@ -75,7 +75,7 @@ with ticket_resolution_times_calendar as ( select ticket_id, - source_relation, + weekly_periods.source_relation, week_number, weekly_periods.schedule_id, ticket_week_start_time, diff --git a/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql b/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql index 15f34d34..cb8b2e0e 100644 --- a/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql +++ b/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql @@ -116,6 +116,7 @@ with agent_work_time_filtered_statuses as ( select weekly_period_agent_work_time.ticket_id, + weekly_period_agent_work_time.source_relation, weekly_period_agent_work_time.sla_applied_at, weekly_period_agent_work_time.target, weekly_period_agent_work_time.sla_policy_name, diff --git a/models/sla_policy/reply_time/int_zendesk__reply_time_business_hours.sql b/models/sla_policy/reply_time/int_zendesk__reply_time_business_hours.sql index 8b01c466..b71f5920 100644 --- a/models/sla_policy/reply_time/int_zendesk__reply_time_business_hours.sql +++ b/models/sla_policy/reply_time/int_zendesk__reply_time_business_hours.sql @@ -84,7 +84,7 @@ with ticket_schedules as ( schedule.end_time_utc as schedule_end_time, (schedule.end_time_utc - greatest(ticket_week_start_time,schedule.start_time_utc)) as lapsed_business_minutes, sum(schedule.end_time_utc - greatest(ticket_week_start_time,schedule.start_time_utc)) over - (partition by ticket_id, metric, sla_applied_at, source_relation + (partition by ticket_id, metric, sla_applied_at, weekly_periods.source_relation order by week_number, schedule.start_time_utc rows between unbounded preceding and current row) as sum_lapsed_business_minutes from weekly_periods From 20ea717b97544582d87e1e555115c9a13dfacd01 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 18 Dec 2023 16:34:43 -0800 Subject: [PATCH 04/17] ambigious cols --- .../int_zendesk__ticket_work_time_calendar.sql | 4 ++-- .../int_zendesk__ticket_first_resolution_time_business.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/agent_work_time/int_zendesk__ticket_work_time_calendar.sql b/models/agent_work_time/int_zendesk__ticket_work_time_calendar.sql index 1b8ee070..4ea04680 100644 --- a/models/agent_work_time/int_zendesk__ticket_work_time_calendar.sql +++ b/models/agent_work_time/int_zendesk__ticket_work_time_calendar.sql @@ -25,8 +25,8 @@ with ticket_historical_status as ( else 0 end as open_status_duration_minutes, case when status = 'deleted' then 1 else 0 end as ticket_deleted, - first_value(valid_starting_at) over (partition by ticket_id order by valid_starting_at desc, ticket_id rows unbounded preceding) as last_status_assignment_date, - case when lag(status) over (partition by ticket_id order by valid_starting_at) = 'deleted' and status != 'deleted' + first_value(valid_starting_at) over (partition by ticket_id, source_relation order by valid_starting_at desc, ticket_id rows unbounded preceding) as last_status_assignment_date, + case when lag(status) over (partition by ticket_id, source_relation order by valid_starting_at) = 'deleted' and status != 'deleted' then 1 else 0 end as ticket_recoveries diff --git a/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql b/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql index 62a8a81c..ddb1f2bb 100644 --- a/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql +++ b/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql @@ -76,7 +76,7 @@ with ticket_resolution_times_calendar as ( ), intercepted_periods as ( select ticket_id, - source_relation, + weekly_periods.source_relation, week_number, weekly_periods.schedule_id, ticket_week_start_time, From f111096e855d5dd3d7fe721e12177ebe53cf4a0a Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 20 Dec 2023 11:24:58 -0800 Subject: [PATCH 05/17] surrogate keys --- .../int_zendesk__field_history_pivot.sql | 4 ++-- models/zendesk.yml | 12 +++++------- models/zendesk__sla_policies.sql | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/models/ticket_history/int_zendesk__field_history_pivot.sql b/models/ticket_history/int_zendesk__field_history_pivot.sql index cb3b1218..23cf2c55 100644 --- a/models/ticket_history/int_zendesk__field_history_pivot.sql +++ b/models/ticket_history/int_zendesk__field_history_pivot.sql @@ -1,4 +1,4 @@ --- depends_on: {{ source('zendesk', 'ticket_field_history') }} +-- depends_on: {{ ref('stg_zendesk__ticket_field_history') }} {{ config( @@ -11,7 +11,7 @@ }} {% if execute -%} - {% set results = run_query('select distinct field_name from ' ~ source('zendesk', 'ticket_field_history') ) %} + {% set results = run_query('select distinct field_name from ' ~ var('field_history') ) %} {% set results_list = results.columns[0].values() %} {% endif -%} diff --git a/models/zendesk.yml b/models/zendesk.yml index 7acae3ab..b69e195b 100644 --- a/models/zendesk.yml +++ b/models/zendesk.yml @@ -182,14 +182,12 @@ models: - name: zendesk__sla_policies description: Each record represents an SLA policy event and additional sla breach and achievement metrics. Calendar and business hour SLA breaches are supported. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - sla_event_id - - source_relation columns: - name: sla_event_id - description: A surrogate key generated from the combination of ticket_id, metric, and sla_applied_at fields + description: A surrogate key generated from the combination of `ticket_id`, `metric`, `sla_applied_at`, and `source_relation` fields. + tests: + - unique + - not_null - name: ticket_id description: A ticket's unique identifier, it is automatically assigned when the ticket is created - name: sla_policy_name @@ -225,7 +223,7 @@ models: - name: ticket_id description: A ticket's unique identifier, it is automatically assigned when the ticket is created - name: ticket_day_id - description: The unique key of the table, a surrogate key of `date_day`, `ticket_id`, and `source_relation` + description: The unique key of the table, a surrogate key of `date_day`, `ticket_id`, and `source_relation`. tests: - unique - not_null diff --git a/models/zendesk__sla_policies.sql b/models/zendesk__sla_policies.sql index 53ce70cb..4470d819 100644 --- a/models/zendesk__sla_policies.sql +++ b/models/zendesk__sla_policies.sql @@ -119,7 +119,7 @@ union all ) select - {{ dbt_utils.generate_surrogate_key(['ticket_id', 'metric', 'sla_applied_at']) }} as sla_event_id, + {{ dbt_utils.generate_surrogate_key(['ticket_id', 'metric', 'sla_applied_at', 'source_relation']) }} as sla_event_id, ticket_id, source_relation, sla_policy_name, From 20df3e2f0df95059525bc16646287f174f4bb961 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 27 Dec 2023 10:03:16 -0800 Subject: [PATCH 06/17] docs --- CHANGELOG.md | 5 + README.md | 367 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 371 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f73c5ad3..77f5e136 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# dbt_zendesk v0.14.0 + +## ๐ŸŽ‰ Feature Update ๐ŸŽ‰ +This release supports running the package on multiple Zendesk sources at once! See the [README](https://github.com/fivetran/dbt_zendesk?tab=readme-ov-file#step-3-define-database-and-schema-variables) for details on how to leverage this feature ([PR #44](https://github.com/fivetran/dbt_zendesk/pull/132)). + # dbt_zendesk v0.13.0 ## ๐Ÿšจ Breaking Change (Snowflake users) ๐Ÿšจ diff --git a/README.md b/README.md index 23dcfe9e..71bfa726 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ packages: > **Note**: Do not include the Zendesk source package. The Zendesk transform package already has a dependency on the source in its own `packages.yml` file. ## Step 3: Define database and schema variables +### Option 1: Single connector By default, this package runs using your destination and the `zendesk` schema. If this is not where your zendesk data is (for example, if your zendesk schema is named `zendesk_fivetran`), update the following variables in your root `dbt_project.yml` file accordingly: ```yml @@ -72,6 +73,370 @@ vars: zendesk_database: your_destination_name zendesk_schema: your_schema_name ``` +> **Note**: If you are running the package on one source connector, each model will have a `source_relation` column that is just an empty string. + +### Option 2: Union multiple connectors +If you have multiple Zendesk connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `zendesk_union_schemas` OR `zendesk_union_databases` variables (cannot do both, though a more flexible approach is in the works...) in your root `dbt_project.yml` file: + +```yml +# dbt_project.yml + +vars: + zendesk_union_schemas: ['zendesk_usa','zendesk_canada'] # use this if the data is in different schemas/datasets of the same database/project + zendesk_union_databases: ['zendesk_usa','zendesk_canada'] # use this if the data is in different databases/projects but uses the same schema name +``` + +#### Recommended: Incorporate unioned sources into DAG +By default, this package defines one single-connector source, called `zendesk`, which will be disabled if you are unioning multiple connectors. This means that your DAG will not include your Zendesk sources, though the package will run successfully. + +To properly incorporate all of your Zendesk connectors into your project's DAG: +1. Define each of your sources in a `.yml` file in your project. Utilize the following template to leverage our table and column documentation. + +
Expand for source configuration template

+ +> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_zendesk?tab=readme-ov-file#step-4-disable-models-for-non-existent-sources)), you may still include them, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definitions. + +```yml +sources: + - name: + schema: + database: + loader: fivetran + loaded_at_field: _fivetran_synced + + freshness: + warn_after: {count: 72, period: hour} + error_after: {count: 168, period: hour} + + tables: &zendesk_table_defs # <- see https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/ + - name: ticket + description: > + Tickets are the means through which your end users (customers) communicate with agents in Zendesk Support. Tickets can + originate from a number of channels, including email, Help Center, chat, phone call, Twitter, Facebook, or the API. + columns: + - name: id + description: Automatically assigned when the ticket is created + - name: url + description: The API url of this ticket + - name: assignee_id + description: The agent currently assigned to the ticket + - name: brand_id + description: Enterprise only. The id of the brand this ticket is associated with + - name: created_at + description: When this record was created + - name: type + description: The type of this ticket, possible values are problem, incident, question or task + - name: subject + description: The value of the subject field for this ticket + - name: description + description: Read-only first comment on the ticket + - name: priority + description: The urgency with which the ticket should be addressed, possible values are urgent, high, normal and low + - name: status + description: The state of the ticket, possible values are new, open, pending, hold, solved and closed + - name: recipient + description: The original recipient e-mail address of the ticket + - name: requester_id + description: The user who requested this ticket + - name: submitter_id + description: The user who submitted the ticket. The submitter always becomes the author of the first comment on the ticket + - name: organization_id + description: The organization of the requester + - name: group_id + description: The group this ticket is assigned to + - name: due_at + description: If this is a ticket of type "task" it has a due date. Due date format uses ISO 8601 format. + - name: ticket_form_id + description: Enterprise only. The id of the ticket form to render for the ticket + - name: is_public + description: Is true if any comments are public, false otherwise + - name: updated_at + description: When this record last got updated + - name: via_channel + description: The channel the ticket was created from + - name: via_source_from_id + description: The channel the ticket was created from + - name: via_source_from_title + description: The channel the ticket was created from + - name: via_source_rel + description: The rel the ticket was created from + - name: via_source_to_address + description: The address of the source the ticket was created from + - name: via_source_to_name + description: The name of the source the ticket was created from + + - name: brand + description: > + Brands are your customer-facing identities. They might represent multiple products or services, or they + might literally be multiple brands owned and represented by your company. + columns: + - name: id + description: The ID automatically assigned when the brand is created + - name: brand_url + description: The url of the brand + - name: name + description: The name of the brand + - name: subdomain + description: The subdomain of the brand + - name: active + description: If the brand is set as active + + - name: domain_name + description: Domain names associated with an organization. An organization may have multiple domain names. + config: + enabled: "{{ var('using_domain_names', true) }}" + columns: + - name: organization_id + description: Reference to the organization + - name: domain_name + description: The name of the domain associated with the organization + - name: index + description: Index number of the domain name associated with the organization + + - name: group + identifier: > + {% if target.type == 'snowflake' %}"GROUP"{% else %}group{% endif %} + description: > + When support requests arrive in Zendesk Support, they can be assigned to a Group. Groups serve as the core + element of ticket workflow; support agents are organized into Groups and tickets can be assigned to a Group + only, or to an assigned agent within a Group. A ticket can never be assigned to an agent without also being + assigned to a Group. + freshness: null + columns: + - name: id + description: Automatically assigned when creating groups + - name: name + description: The name of the group + + - name: organization_tag + description: The tags associated with an organization. An organization may have multiple tags. + config: + enabled: "{{ var('using_organization_tags', true) }}" + columns: + - name: organization_id + description: Reference to the organization + - name: tag + description: Tag associated with the organization + + - name: organization + description: > + Just as agents can be segmented into groups in Zendesk Support, your customers (end-users) can be segmented into + organizations. You can manually assign customers to an organization or automatically assign them to an organization + by their email address domain. Organizations can be used in business rules to route tickets to groups of agents or + to send email notifications. + freshness: null + columns: + - name: id + description: Automatically assigned when the organization is created + - name: name + description: A unique name for the organization + - name: details + description: Any details obout the organization, such as the address + - name: url + description: The API url of this organization + - name: external_id + description: A unique external id to associate organizations to an external record + - name: created_at + description: The time the organization was created + - name: updated_at + description: The time of the last update of the organization + - name: domain_names + description: An array of domain names associated with this organization + - name: details + description: Any details obout the organization, such as the address + - name: notes + description: Any notes you have about the organization + - name: group_id + description: New tickets from users in this organization are automatically put in this group + - name: shared_tickets + description: End users in this organization are able to see each other's tickets + - name: shared_comments + description: End users in this organization are able to see each other's comments on tickets + - name: tags + description: The tags of the organization + - name: organization_fields + description: Custom fields for this organization + + - name: ticket_comment + description: Ticket comments represent the conversation between requesters, collaborators, and agents. Comments can be public or private. + columns: + - name: id + description: Automatically assigned when the comment is created + - name: body + description: The comment string + - name: created + description: The time the comment was created + - name: public + description: Boolean field indicating if the comment is public (true), or if it is an internal note (false) + - name: ticket_id + description: The ticket id associated with this comment + - name: user_id + description: The id of the comment author + - name: facebook_comment + description: Boolean field indicating if the comment is a facebook comment + - name: tweet + description: Boolean field indicating if the comment is a twitter tweet + - name: voice_comment + description: Boolean field indicating if the comment is a voice comment + + - name: user_tag + description: Table containing all tags associated with a user. Only present if your account has user tagging enabled. + config: + enabled: "{{ var('using_user_tags', true) }}" + columns: + - name: user_id + description: Reference to the user + - name: tag + description: Tag associated with the user + + - name: user + description: Zendesk has three types of users, end-users (your customers), agents, and administrators. + freshness: null + columns: + - name: id + description: Automatically assigned when the user is created + - name: email + description: The user's primary email address. *Writeable on create only. On update, a secondary email is added. See Email Address + - name: name + description: The user's name + - name: active + description: false if the user has been deleted + - name: created_at + description: The time the user was created + - name: organization_id + description: The id of the user's organization. If the user has more than one organization memberships, the id of the user's default organization + - name: role + description: The user's role. Possible values are "end-user", "agent", or "admin" + - name: time_zone + description: The user's time zone. See Time Zone + - name: ticket_restriction + description: Specifies which tickets the user has access to. Possible values are organization, groups, assigned, requested and null + + - name: schedule + description: The support schedules created with different business hours and holidays. + freshness: null + config: + enabled: "{{ var('using_schedules', true) }}" + columns: + - name: id + description: ID automatically assigned to the schedule upon creation + - name: name + description: Name of the schedule + - name: created_at + description: Time the schedule was created + - name: start_time + description: Start time of the schedule, in the schedule's time zone. + - name: end_time + description: End time of the schedule, in the schedule's time zone. + - name: time_zone + description: Timezone in which the schedule operates. + + - name: ticket_schedule + description: The schedules applied to tickets through a trigger. + freshness: null + columns: + - name: ticket_id + description: The ID of the ticket assigned to the schedule + - name: created_at + description: The time the schedule was assigned to the ticket + - name: schedule_id + description: The ID of the schedule applied to the ticket + + - name: ticket_form_history + description: Ticket forms allow an admin to define a subset of ticket fields for display to both agents and end users. + config: + enabled: "{{ var('using_ticket_form_history', true) }}" + columns: + - name: id + description: Automatically assigned when creating ticket form + - name: created_at + description: The time the ticket form was created + - name: updated_at + description: The time of the last update of the ticket form + - name: display_name + description: The name of the form that is displayed to an end user + - name: active + description: If the form is set as active + - name: name + description: The name of the form + + - name: ticket_tag + description: > + Tags are words, or combinations of words, you can use to add more context to tickets. The table lists all + tags currently associated with a ticket. + freshness: null + columns: + - name: ticket_id + description: The ID of the ticket associated with the tag + - name: tags + description: The tag, or word(s), associated with the ticket + + - name: ticket_field_history + description: All fields and field values associated with tickets. + freshness: null + columns: + - name: ticket_id + description: The ID of the ticket associated with the field + - name: field_name + description: The name of the ticket field + - name: updated + description: The time the ticket field value was created + - name: value + description: The value of the field + - name: user_id + description: The id of the user who made the update + + - name: daylight_time + description: > + Appropriate offsets (from UTC) for timezones that engage or have engaged with Daylight Savings at some point since 1970. + freshness: null + columns: + - name: daylight_end_utc + description: UTC timestamp of when Daylight Time ended in this year. + - name: daylight_offset + description: Number of **hours** added during Daylight Savings Time. + - name: daylight_start_utc + description: UTC timestamp of when Daylight Time began in this year. + - name: time_zone + description: Name of the timezone. + - name: year + description: Year in which daylight savings occurred. + + - name: time_zone + description: Offsets (from UTC) for each timezone. + freshness: null + columns: + - name: time_zone + description: Name of the time zone. + - name: standard_offset + description: Standard offset of the timezone (non-daylight savings hours). In `+/-hh:mm` format. + + - name: schedule_holiday + description: Information about holidays for each specified schedule. + freshness: null + config: + enabled: "{{ var('using_schedules', true) }}" + columns: + - name: end_date + description: ISO 8601 representation of the holiday end date. + - name: id + description: The ID of the scheduled holiday. + - name: name + description: Name of the holiday. + - name: schedule_id + description: The ID of the schedule. + - name: start_date + description: ISO 8601 representation of the holiday start date. +``` +

+ +2. Set the `has_defined_sources` variable (scoped to the `zendesk_source` package) to true, like such: +```yml +# dbt_project.yml +vars: + zendesk_source: + has_defined_sources: true +``` ## Step 4: Disable models for non-existent sources This package takes into consideration that not every Zendesk account utilizes the `schedule`, `schedule_holiday`, `ticket_schedule` `daylight_time`, `time_zone`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable: @@ -175,7 +540,7 @@ This dbt package is dependent on the following dbt packages. Please be aware tha ```yml packages: - package: fivetran/zendesk_source - version: [">=0.10.0", "<0.11.0"] + version: [">=0.11.0", "<0.12.0"] - package: fivetran/fivetran_utils version: [">=0.4.0", "<0.5.0"] From b7aa8b62f6ac2baf37151bb02f5a55f70ee178da Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 27 Dec 2023 10:58:10 -0800 Subject: [PATCH 07/17] update package versions --- README.md | 2 +- dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 71bfa726..2a4b0f6d 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Include the following zendesk package version in your `packages.yml` file: ```yml packages: - package: fivetran/zendesk - version: [">=0.13.0", "<0.14.0"] + version: [">=0.14.0", "<0.15.0"] ``` > **Note**: Do not include the Zendesk source package. The Zendesk transform package already has a dependency on the source in its own `packages.yml` file. diff --git a/dbt_project.yml b/dbt_project.yml index 409883da..25008664 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'zendesk' -version: '0.13.0' +version: '0.14.0' config-version: 2 diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index c33d0b73..a920d156 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'zendesk_integration_tests' -version: '0.13.0' +version: '0.14.0' profile: 'integration_tests' From e7ee8a37b85d631e51d46f21f1cbc6d2d80ce2f7 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 27 Dec 2023 11:00:44 -0800 Subject: [PATCH 08/17] changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77f5e136..c22b2768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # dbt_zendesk v0.14.0 ## ๐ŸŽ‰ Feature Update ๐ŸŽ‰ -This release supports running the package on multiple Zendesk sources at once! See the [README](https://github.com/fivetran/dbt_zendesk?tab=readme-ov-file#step-3-define-database-and-schema-variables) for details on how to leverage this feature ([PR #44](https://github.com/fivetran/dbt_zendesk/pull/132)). +This release supports running the package on multiple Zendesk sources at once! See the [README](https://github.com/fivetran/dbt_zendesk?tab=readme-ov-file#step-3-define-database-and-schema-variables) for details on how to leverage this feature ([PR #132](https://github.com/fivetran/dbt_zendesk/pull/132)). # dbt_zendesk v0.13.0 From d5609b4becca8860623eed7464005638a1538ba4 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 27 Dec 2023 15:17:01 -0800 Subject: [PATCH 09/17] remove commented out code --- .../int_zendesk__ticket_schedules.sql | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/models/intermediate/int_zendesk__ticket_schedules.sql b/models/intermediate/int_zendesk__ticket_schedules.sql index 20174b76..a5e2be17 100644 --- a/models/intermediate/int_zendesk__ticket_schedules.sql +++ b/models/intermediate/int_zendesk__ticket_schedules.sql @@ -37,26 +37,6 @@ with ticket as ( ), default_schedule_events as ( -{# {% if execute %} - - {% set default_schedule_id_query %} - with set_default_schedule_flag as ( - select - row_number() over (order by created_at) = 1 as is_default_schedule, - id - from {{ source('zendesk','schedule') }} - ) - select - id - from set_default_schedule_flag - where is_default_schedule - - {% endset %} - - {% set default_schedule_id = run_query(default_schedule_id_query).columns[0][0]|string %} - - {% endif %} #} - select ticket.ticket_id, ticket.source_relation, From c7518bcd866259e8c02793a98601ae7b3562f2ad Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 31 Jan 2024 15:17:27 -0400 Subject: [PATCH 10/17] update readme --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 17b74fad..4933e9da 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ packages: > **Note**: Do not include the Zendesk source package. The Zendesk transform package already has a dependency on the source in its own `packages.yml` file. ## Step 3: Define database and schema variables -### Option 1: Single connector +### Option 1: Single connector ๐Ÿ’ƒ By default, this package runs using your destination and the `zendesk` schema. If this is not where your zendesk data is (for example, if your zendesk schema is named `zendesk_fivetran`), update the following variables in your root `dbt_project.yml` file accordingly: ```yml @@ -75,7 +75,7 @@ vars: ``` > **Note**: If you are running the package on one source connector, each model will have a `source_relation` column that is just an empty string. -### Option 2: Union multiple connectors +### Option 2: Union multiple connectors ๐Ÿ‘ฏ If you have multiple Zendesk connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `zendesk_union_schemas` OR `zendesk_union_databases` variables (cannot do both, though a more flexible approach is in the works...) in your root `dbt_project.yml` file: ```yml @@ -439,6 +439,8 @@ vars: ``` ## Step 4: Disable models for non-existent sources +> _This step is unnecessary (but still available for use) if you are unioning multiple connectors together in the previous step. That is, the `union_data` macro we use will create completely empty staging models for sources that are not found in any of your Zendesk schemas/databases. However, you can still leverage the below variables if you would like to avoid this behavior._ + This package takes into consideration that not every Zendesk account utilizes the `schedule`, `schedule_holiday`, `ticket_schedule` `daylight_time`, `time_zone`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable: ```yml vars: @@ -450,6 +452,7 @@ vars: ``` ## (Optional) Step 5: Additional configurations +
Expand/collapse configurations ### Adding passthrough columns This package includes all source columns defined in the staging models. However, the `stg_zendesk__ticket` model allows for additional columns to be added using a pass-through column variable. This is extremely useful if you'd like to include custom fields to the package. @@ -529,7 +532,7 @@ models: +schema: my_new_schema_name # leave blank for just the target_schema ``` -### Change the source table references +### Change the source table references (only if using a single connector) If an individual source table has a different name than the package expects, add the table name as it appears in your destination to the respective variable: > IMPORTANT: See this project's [`dbt_project.yml`](https://github.com/fivetran/dbt_zendesk_source/blob/main/dbt_project.yml) variable declarations to see the expected names. @@ -538,6 +541,7 @@ If an individual source table has a different name than the package expects, add vars: zendesk__identifier: your_table_name ``` +
## (Optional) Step 6: Orchestrate your models with Fivetran Transformations for dbt Coreโ„ข
Expand for details From 46f0470f0844dd6014341ea8ed5ec0c36c800bd6 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 31 Jan 2024 16:33:02 -0400 Subject: [PATCH 11/17] missing source_relation --- models/intermediate/int_zendesk__schedule_spine.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/intermediate/int_zendesk__schedule_spine.sql b/models/intermediate/int_zendesk__schedule_spine.sql index fcaa6faf..f7532624 100644 --- a/models/intermediate/int_zendesk__schedule_spine.sql +++ b/models/intermediate/int_zendesk__schedule_spine.sql @@ -89,6 +89,7 @@ with timezone as ( select time_zone, + source_relation, standard_offset_minutes as offset_minutes, -- Get the latest daylight_end_utc time and set that as the valid_from From 630325716cdbfea88ae4eabb4d5c7ef125171c1a Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 31 Jan 2024 16:38:28 -0400 Subject: [PATCH 12/17] grouping --- models/intermediate/int_zendesk__schedule_spine.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/intermediate/int_zendesk__schedule_spine.sql b/models/intermediate/int_zendesk__schedule_spine.sql index f7532624..437a0606 100644 --- a/models/intermediate/int_zendesk__schedule_spine.sql +++ b/models/intermediate/int_zendesk__schedule_spine.sql @@ -99,7 +99,8 @@ with timezone as ( cast( {{ dbt.dateadd('year', 1, dbt.current_timestamp_backcompat()) }} as date) as valid_until from order_timezone_dt - group by 1, 2 + + group by 1, 2, 3 -- We only want to apply this logic to time_zone's that had daylight saving time and it ended at a point. For example, Hong Kong ended DST in 1979. having cast(max(daylight_end_utc) as date) < cast({{ dbt.current_timestamp_backcompat() }} as date) From b7f1b7e42bb96e93b1db3bc020b139cd1badd017 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 5 Feb 2024 11:03:54 -0500 Subject: [PATCH 13/17] joe feedback --- models/ticket_history/int_zendesk__field_history_pivot.sql | 3 ++- models/utils/int_zendesk__calendar_spine.sql | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/models/ticket_history/int_zendesk__field_history_pivot.sql b/models/ticket_history/int_zendesk__field_history_pivot.sql index 23cf2c55..90dcf235 100644 --- a/models/ticket_history/int_zendesk__field_history_pivot.sql +++ b/models/ticket_history/int_zendesk__field_history_pivot.sql @@ -11,7 +11,8 @@ }} {% if execute -%} - {% set results = run_query('select distinct field_name from ' ~ var('field_history') ) %} + -- if you are unioning multiple connectors, you may not be able to `dbt compile` before `dbt run`ing on a new schema. + {% set results = run_query('select distinct field_name from ' ~ source('field_history') if var('zendesk_union_schemas', []) == [] and var('zendesk_union_databases', []) == [] else var('field_history') ) %} {% set results_list = results.columns[0].values() %} {% endif -%} diff --git a/models/utils/int_zendesk__calendar_spine.sql b/models/utils/int_zendesk__calendar_spine.sql index f40e1f76..fc5ec664 100644 --- a/models/utils/int_zendesk__calendar_spine.sql +++ b/models/utils/int_zendesk__calendar_spine.sql @@ -5,7 +5,8 @@ with spine as ( {% if execute %} {% set current_ts = dbt.current_timestamp_backcompat() %} {% set first_date_query %} - select min( created_at ) as min_date from {{ var('ticket') }} + -- if you are unioning multiple connectors, you may not be able to `dbt compile` before `dbt run`ing on a new schema. + select min( created_at ) as min_date from {{ source('ticket') if var('zendesk_union_schemas', []) == [] and var('zendesk_union_databases', []) == [] else var('ticket') }} -- by default take all the data where cast(created_at as date) >= {{ dbt.dateadd('year', - var('ticket_field_history_timeframe_years', 50), current_ts ) }} {% endset %} From 19b4694ea0c8c4357e14aedcf96ef7ad2e15976b Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 5 Feb 2024 13:21:01 -0500 Subject: [PATCH 14/17] more joe feedback --- CHANGELOG.md | 2 ++ models/utils/int_zendesk__calendar_spine.sql | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 518519ed..4f68012d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## ๐ŸŽ‰ Feature Update ๐ŸŽ‰ This release supports running the package on multiple Zendesk sources at once! See the [README](https://github.com/fivetran/dbt_zendesk?tab=readme-ov-file#step-3-define-database-and-schema-variables) for details on how to leverage this feature ([PR #132](https://github.com/fivetran/dbt_zendesk/pull/132)). +> Please note: This is a **๐ŸšจBreaking Change๐Ÿšจ** in that we have a added a new field, `source_relation`, that points to the source connector from which the record originated. This field addition will require a `dbt run --full-refresh`. + # dbt_zendesk v0.13.1 [PR #128](https://github.com/fivetran/dbt_zendesk/pull/128) includes the following changes: diff --git a/models/utils/int_zendesk__calendar_spine.sql b/models/utils/int_zendesk__calendar_spine.sql index fc5ec664..1fade651 100644 --- a/models/utils/int_zendesk__calendar_spine.sql +++ b/models/utils/int_zendesk__calendar_spine.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('stg_zendesk__ticket') }} +-- depends_on: {{ ref('stg_zendesk__ticket') }} and {{ source('zendesk', 'ticket') }} with spine as ( @@ -6,7 +6,7 @@ with spine as ( {% set current_ts = dbt.current_timestamp_backcompat() %} {% set first_date_query %} -- if you are unioning multiple connectors, you may not be able to `dbt compile` before `dbt run`ing on a new schema. - select min( created_at ) as min_date from {{ source('ticket') if var('zendesk_union_schemas', []) == [] and var('zendesk_union_databases', []) == [] else var('ticket') }} + select min( created_at ) as min_date from {{ source('ticket') if var('zendesk_union_schemas', []) == [] and var('zendesk_union_databases', []) == [] else var('ticket') }} -- by default take all the data where cast(created_at as date) >= {{ dbt.dateadd('year', - var('ticket_field_history_timeframe_years', 50), current_ts ) }} {% endset %} From a6db1820c2f0aa0696a82091a5274e0d7c64eb89 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 5 Feb 2024 13:23:21 -0500 Subject: [PATCH 15/17] fix source syntacx --- models/ticket_history/int_zendesk__field_history_pivot.sql | 2 +- models/utils/int_zendesk__calendar_spine.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ticket_history/int_zendesk__field_history_pivot.sql b/models/ticket_history/int_zendesk__field_history_pivot.sql index 90dcf235..6406ab23 100644 --- a/models/ticket_history/int_zendesk__field_history_pivot.sql +++ b/models/ticket_history/int_zendesk__field_history_pivot.sql @@ -12,7 +12,7 @@ {% if execute -%} -- if you are unioning multiple connectors, you may not be able to `dbt compile` before `dbt run`ing on a new schema. - {% set results = run_query('select distinct field_name from ' ~ source('field_history') if var('zendesk_union_schemas', []) == [] and var('zendesk_union_databases', []) == [] else var('field_history') ) %} + {% set results = run_query('select distinct field_name from ' ~ source('zendesk', 'field_history') if var('zendesk_union_schemas', []) == [] and var('zendesk_union_databases', []) == [] else var('field_history') ) %} {% set results_list = results.columns[0].values() %} {% endif -%} diff --git a/models/utils/int_zendesk__calendar_spine.sql b/models/utils/int_zendesk__calendar_spine.sql index 1fade651..75a4e53c 100644 --- a/models/utils/int_zendesk__calendar_spine.sql +++ b/models/utils/int_zendesk__calendar_spine.sql @@ -6,7 +6,7 @@ with spine as ( {% set current_ts = dbt.current_timestamp_backcompat() %} {% set first_date_query %} -- if you are unioning multiple connectors, you may not be able to `dbt compile` before `dbt run`ing on a new schema. - select min( created_at ) as min_date from {{ source('ticket') if var('zendesk_union_schemas', []) == [] and var('zendesk_union_databases', []) == [] else var('ticket') }} + select min( created_at ) as min_date from {{ source('zendesk', 'ticket') if var('zendesk_union_schemas', []) == [] and var('zendesk_union_databases', []) == [] else var('ticket') }} -- by default take all the data where cast(created_at as date) >= {{ dbt.dateadd('year', - var('ticket_field_history_timeframe_years', 50), current_ts ) }} {% endset %} From 9ccb0ff65a5488d6b301c576d0cad073bcf7d7f6 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 5 Feb 2024 18:54:16 -0500 Subject: [PATCH 16/17] swap source template and add changelog bug fix note --- CHANGELOG.md | 5 +- README.md | 340 ++------------------------------------------------- 2 files changed, 14 insertions(+), 331 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f68012d..ece1d79c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,13 @@ # dbt_zendesk v0.14.0 ## ๐ŸŽ‰ Feature Update ๐ŸŽ‰ -This release supports running the package on multiple Zendesk sources at once! See the [README](https://github.com/fivetran/dbt_zendesk?tab=readme-ov-file#step-3-define-database-and-schema-variables) for details on how to leverage this feature ([PR #132](https://github.com/fivetran/dbt_zendesk/pull/132)). +- This release supports running the package on multiple Zendesk sources at once! See the [README](https://github.com/fivetran/dbt_zendesk?tab=readme-ov-file#step-3-define-database-and-schema-variables) for details on how to leverage this feature ([PR #132](https://github.com/fivetran/dbt_zendesk/pull/132)). > Please note: This is a **๐ŸšจBreaking Change๐Ÿšจ** in that we have a added a new field, `source_relation`, that points to the source connector from which the record originated. This field addition will require a `dbt run --full-refresh`. +## ๐Ÿž Bug Fixes ๐Ÿž +- Previous versions of this package determined the [default ticket schedule](https://github.com/fivetran/dbt_zendesk/blob/v0.13.1/models/intermediate/int_zendesk__ticket_schedules.sql#L19-L55) without considering whether schedules have been deleted or not. We now only consider non-deleted schedules in our default ticket schedule logic ([PR #132](https://github.com/fivetran/dbt_zendesk/pull/132)). + # dbt_zendesk v0.13.1 [PR #128](https://github.com/fivetran/dbt_zendesk/pull/128) includes the following changes: diff --git a/README.md b/README.md index 4933e9da..ab665bcc 100644 --- a/README.md +++ b/README.md @@ -90,347 +90,27 @@ vars: By default, this package defines one single-connector source, called `zendesk`, which will be disabled if you are unioning multiple connectors. This means that your DAG will not include your Zendesk sources, though the package will run successfully. To properly incorporate all of your Zendesk connectors into your project's DAG: -1. Define each of your sources in a `.yml` file in your project. Utilize the following template to leverage our table and column documentation. - -
Expand for source configuration template

- -> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_zendesk?tab=readme-ov-file#step-4-disable-models-for-non-existent-sources)), you may still include them, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definitions. +1. Define each of your sources in a `.yml` file in your project. Utilize the following template for the `source`-level configurations, and, **most importantly**, copy and paste the table and column-level definitions from the package's `src_zendesk.yml` [file](https://github.com/fivetran/dbt_zendesk_source/blob/main/models/src_zendesk.yml#L15-L351). ```yml +# a .yml file in your root project sources: - - name: - schema: - database: + - name: # ex: zendesk_usa + schema: # one of var('zendesk_union_schemas') if unioning schemas, otherwise just 'zendesk' + database: # one of var('zendesk_union_databases') if unioning databases, otherwise whatever DB your zendesk schemas all live in loader: fivetran loaded_at_field: _fivetran_synced - freshness: + freshness: # feel free to adjust to your liking warn_after: {count: 72, period: hour} error_after: {count: 168, period: hour} - tables: &zendesk_table_defs # <- see https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/ - - name: ticket - description: > - Tickets are the means through which your end users (customers) communicate with agents in Zendesk Support. Tickets can - originate from a number of channels, including email, Help Center, chat, phone call, Twitter, Facebook, or the API. - columns: - - name: id - description: Automatically assigned when the ticket is created - - name: url - description: The API url of this ticket - - name: assignee_id - description: The agent currently assigned to the ticket - - name: brand_id - description: Enterprise only. The id of the brand this ticket is associated with - - name: created_at - description: When this record was created - - name: type - description: The type of this ticket, possible values are problem, incident, question or task - - name: subject - description: The value of the subject field for this ticket - - name: description - description: Read-only first comment on the ticket - - name: priority - description: The urgency with which the ticket should be addressed, possible values are urgent, high, normal and low - - name: status - description: The state of the ticket, possible values are new, open, pending, hold, solved and closed - - name: recipient - description: The original recipient e-mail address of the ticket - - name: requester_id - description: The user who requested this ticket - - name: submitter_id - description: The user who submitted the ticket. The submitter always becomes the author of the first comment on the ticket - - name: organization_id - description: The organization of the requester - - name: group_id - description: The group this ticket is assigned to - - name: due_at - description: If this is a ticket of type "task" it has a due date. Due date format uses ISO 8601 format. - - name: ticket_form_id - description: Enterprise only. The id of the ticket form to render for the ticket - - name: is_public - description: Is true if any comments are public, false otherwise - - name: updated_at - description: When this record last got updated - - name: via_channel - description: The channel the ticket was created from - - name: via_source_from_id - description: The channel the ticket was created from - - name: via_source_from_title - description: The channel the ticket was created from - - name: via_source_rel - description: The rel the ticket was created from - - name: via_source_to_address - description: The address of the source the ticket was created from - - name: via_source_to_name - description: The name of the source the ticket was created from - - - name: brand - description: > - Brands are your customer-facing identities. They might represent multiple products or services, or they - might literally be multiple brands owned and represented by your company. - columns: - - name: id - description: The ID automatically assigned when the brand is created - - name: brand_url - description: The url of the brand - - name: name - description: The name of the brand - - name: subdomain - description: The subdomain of the brand - - name: active - description: If the brand is set as active - - - name: domain_name - description: Domain names associated with an organization. An organization may have multiple domain names. - config: - enabled: "{{ var('using_domain_names', true) }}" - columns: - - name: organization_id - description: Reference to the organization - - name: domain_name - description: The name of the domain associated with the organization - - name: index - description: Index number of the domain name associated with the organization - - - name: group - identifier: > - {% if target.type == 'snowflake' %}"GROUP"{% else %}group{% endif %} - description: > - When support requests arrive in Zendesk Support, they can be assigned to a Group. Groups serve as the core - element of ticket workflow; support agents are organized into Groups and tickets can be assigned to a Group - only, or to an assigned agent within a Group. A ticket can never be assigned to an agent without also being - assigned to a Group. - freshness: null - columns: - - name: id - description: Automatically assigned when creating groups - - name: name - description: The name of the group - - - name: organization_tag - description: The tags associated with an organization. An organization may have multiple tags. - config: - enabled: "{{ var('using_organization_tags', true) }}" - columns: - - name: organization_id - description: Reference to the organization - - name: tag - description: Tag associated with the organization - - - name: organization - description: > - Just as agents can be segmented into groups in Zendesk Support, your customers (end-users) can be segmented into - organizations. You can manually assign customers to an organization or automatically assign them to an organization - by their email address domain. Organizations can be used in business rules to route tickets to groups of agents or - to send email notifications. - freshness: null - columns: - - name: id - description: Automatically assigned when the organization is created - - name: name - description: A unique name for the organization - - name: details - description: Any details obout the organization, such as the address - - name: url - description: The API url of this organization - - name: external_id - description: A unique external id to associate organizations to an external record - - name: created_at - description: The time the organization was created - - name: updated_at - description: The time of the last update of the organization - - name: domain_names - description: An array of domain names associated with this organization - - name: details - description: Any details obout the organization, such as the address - - name: notes - description: Any notes you have about the organization - - name: group_id - description: New tickets from users in this organization are automatically put in this group - - name: shared_tickets - description: End users in this organization are able to see each other's tickets - - name: shared_comments - description: End users in this organization are able to see each other's comments on tickets - - name: tags - description: The tags of the organization - - name: organization_fields - description: Custom fields for this organization - - - name: ticket_comment - description: Ticket comments represent the conversation between requesters, collaborators, and agents. Comments can be public or private. - columns: - - name: id - description: Automatically assigned when the comment is created - - name: body - description: The comment string - - name: created - description: The time the comment was created - - name: public - description: Boolean field indicating if the comment is public (true), or if it is an internal note (false) - - name: ticket_id - description: The ticket id associated with this comment - - name: user_id - description: The id of the comment author - - name: facebook_comment - description: Boolean field indicating if the comment is a facebook comment - - name: tweet - description: Boolean field indicating if the comment is a twitter tweet - - name: voice_comment - description: Boolean field indicating if the comment is a voice comment - - - name: user_tag - description: Table containing all tags associated with a user. Only present if your account has user tagging enabled. - config: - enabled: "{{ var('using_user_tags', true) }}" - columns: - - name: user_id - description: Reference to the user - - name: tag - description: Tag associated with the user - - - name: user - description: Zendesk has three types of users, end-users (your customers), agents, and administrators. - freshness: null - columns: - - name: id - description: Automatically assigned when the user is created - - name: email - description: The user's primary email address. *Writeable on create only. On update, a secondary email is added. See Email Address - - name: name - description: The user's name - - name: active - description: false if the user has been deleted - - name: created_at - description: The time the user was created - - name: organization_id - description: The id of the user's organization. If the user has more than one organization memberships, the id of the user's default organization - - name: role - description: The user's role. Possible values are "end-user", "agent", or "admin" - - name: time_zone - description: The user's time zone. See Time Zone - - name: ticket_restriction - description: Specifies which tickets the user has access to. Possible values are organization, groups, assigned, requested and null - - - name: schedule - description: The support schedules created with different business hours and holidays. - freshness: null - config: - enabled: "{{ var('using_schedules', true) }}" - columns: - - name: id - description: ID automatically assigned to the schedule upon creation - - name: name - description: Name of the schedule - - name: created_at - description: Time the schedule was created - - name: start_time - description: Start time of the schedule, in the schedule's time zone. - - name: end_time - description: End time of the schedule, in the schedule's time zone. - - name: time_zone - description: Timezone in which the schedule operates. - - - name: ticket_schedule - description: The schedules applied to tickets through a trigger. - freshness: null - columns: - - name: ticket_id - description: The ID of the ticket assigned to the schedule - - name: created_at - description: The time the schedule was assigned to the ticket - - name: schedule_id - description: The ID of the schedule applied to the ticket - - - name: ticket_form_history - description: Ticket forms allow an admin to define a subset of ticket fields for display to both agents and end users. - config: - enabled: "{{ var('using_ticket_form_history', true) }}" - columns: - - name: id - description: Automatically assigned when creating ticket form - - name: created_at - description: The time the ticket form was created - - name: updated_at - description: The time of the last update of the ticket form - - name: display_name - description: The name of the form that is displayed to an end user - - name: active - description: If the form is set as active - - name: name - description: The name of the form - - - name: ticket_tag - description: > - Tags are words, or combinations of words, you can use to add more context to tickets. The table lists all - tags currently associated with a ticket. - freshness: null - columns: - - name: ticket_id - description: The ID of the ticket associated with the tag - - name: tags - description: The tag, or word(s), associated with the ticket - - - name: ticket_field_history - description: All fields and field values associated with tickets. - freshness: null - columns: - - name: ticket_id - description: The ID of the ticket associated with the field - - name: field_name - description: The name of the ticket field - - name: updated - description: The time the ticket field value was created - - name: value - description: The value of the field - - name: user_id - description: The id of the user who made the update - - - name: daylight_time - description: > - Appropriate offsets (from UTC) for timezones that engage or have engaged with Daylight Savings at some point since 1970. - freshness: null - columns: - - name: daylight_end_utc - description: UTC timestamp of when Daylight Time ended in this year. - - name: daylight_offset - description: Number of **hours** added during Daylight Savings Time. - - name: daylight_start_utc - description: UTC timestamp of when Daylight Time began in this year. - - name: time_zone - description: Name of the timezone. - - name: year - description: Year in which daylight savings occurred. - - - name: time_zone - description: Offsets (from UTC) for each timezone. - freshness: null - columns: - - name: time_zone - description: Name of the time zone. - - name: standard_offset - description: Standard offset of the timezone (non-daylight savings hours). In `+/-hh:mm` format. - - - name: schedule_holiday - description: Information about holidays for each specified schedule. - freshness: null - config: - enabled: "{{ var('using_schedules', true) }}" - columns: - - name: end_date - description: ISO 8601 representation of the holiday end date. - - name: id - description: The ID of the scheduled holiday. - - name: name - description: Name of the holiday. - - name: schedule_id - description: The ID of the schedule. - - name: start_date - description: ISO 8601 representation of the holiday start date. + tables: # copy and paste from zendesk_source/models/src_zendesk.yml ``` -

-2. Set the `has_defined_sources` variable (scoped to the `zendesk_source` package) to true, like such: +> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_zendesk?tab=readme-ov-file#step-4-disable-models-for-non-existent-sources)), you may still include them, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definitions. + +2. Set the `has_defined_sources` variable (scoped to the `zendesk_source` package) to `True`, like such: ```yml # dbt_project.yml vars: From 1ef02ee6603ea8b1cae05116dd3d04740f33cc1e Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 7 Feb 2024 15:56:16 -0500 Subject: [PATCH 17/17] add flags.WHICH condiitonal --- models/ticket_history/int_zendesk__field_history_pivot.sql | 4 ++-- models/utils/int_zendesk__calendar_spine.sql | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/models/ticket_history/int_zendesk__field_history_pivot.sql b/models/ticket_history/int_zendesk__field_history_pivot.sql index 6406ab23..bc9efc7f 100644 --- a/models/ticket_history/int_zendesk__field_history_pivot.sql +++ b/models/ticket_history/int_zendesk__field_history_pivot.sql @@ -10,9 +10,9 @@ ) }} -{% if execute -%} +{% if execute and flags.WHICH in ('run', 'build') -%} -- if you are unioning multiple connectors, you may not be able to `dbt compile` before `dbt run`ing on a new schema. - {% set results = run_query('select distinct field_name from ' ~ source('zendesk', 'field_history') if var('zendesk_union_schemas', []) == [] and var('zendesk_union_databases', []) == [] else var('field_history') ) %} + {% set results = run_query('select distinct field_name from ' ~ var('field_history') ) %} {% set results_list = results.columns[0].values() %} {% endif -%} diff --git a/models/utils/int_zendesk__calendar_spine.sql b/models/utils/int_zendesk__calendar_spine.sql index 75a4e53c..bb169a57 100644 --- a/models/utils/int_zendesk__calendar_spine.sql +++ b/models/utils/int_zendesk__calendar_spine.sql @@ -1,12 +1,12 @@ --- depends_on: {{ ref('stg_zendesk__ticket') }} and {{ source('zendesk', 'ticket') }} +-- depends_on: {{ ref('stg_zendesk__ticket') }} with spine as ( - {% if execute %} + {% if execute and flags.WHICH in ('run', 'build') %} {% set current_ts = dbt.current_timestamp_backcompat() %} {% set first_date_query %} -- if you are unioning multiple connectors, you may not be able to `dbt compile` before `dbt run`ing on a new schema. - select min( created_at ) as min_date from {{ source('zendesk', 'ticket') if var('zendesk_union_schemas', []) == [] and var('zendesk_union_databases', []) == [] else var('ticket') }} + select min( created_at ) as min_date from {{ var('ticket') }} -- by default take all the data where cast(created_at as date) >= {{ dbt.dateadd('year', - var('ticket_field_history_timeframe_years', 50), current_ts ) }} {% endset %}