From c9f9264c870ddf72170a7c736ea88ff5f86b04a0 Mon Sep 17 00:00:00 2001 From: Angelos Alexopoulos Date: Mon, 21 Oct 2019 17:11:41 +0300 Subject: [PATCH 1/8] Support ticket metric events and satisfaction ratings --- build.gradle | 2 +- .../input/zendesk/ZendeskInputPlugin.java | 6 +- .../embulk/input/zendesk/models/Target.java | 1 + .../services/ZendeskNormalServices.java | 81 ++++++++++++++++--- .../services/ZendeskSupportAPIService.java | 21 +++-- .../input/zendesk/utils/ZendeskConstants.java | 1 + .../input/zendesk/utils/ZendeskUtils.java | 4 + 7 files changed, 99 insertions(+), 17 deletions(-) diff --git a/build.gradle b/build.gradle index 65dca31..5197096 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ configurations { provided } -version = "0.3.6" +version = "0.3.7" sourceCompatibility = 1.8 targetCompatibility = 1.8 diff --git a/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java b/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java index ed36708..9358bfe 100644 --- a/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java +++ b/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java @@ -23,6 +23,7 @@ import org.embulk.input.zendesk.services.ZendeskService; import org.embulk.input.zendesk.services.ZendeskSupportAPIService; import org.embulk.input.zendesk.services.ZendeskUserEventService; +import org.embulk.input.zendesk.services.ZendeskNormalServices; import org.embulk.input.zendesk.utils.ZendeskConstants; import org.embulk.input.zendesk.utils.ZendeskDateUtils; import org.embulk.input.zendesk.utils.ZendeskUtils; @@ -173,7 +174,8 @@ public ConfigDiff transaction(final ConfigSource config, final Control control) // For non-incremental target, we will split records based on number of pages. 100 records per page // In preview, run with taskCount = 1 - if (!Exec.isPreview() && !getZendeskService(task).isSupportIncremental() && getZendeskService(task) instanceof ZendeskSupportAPIService) { + if (!Exec.isPreview() && !getZendeskService(task).isSupportIncremental() && getZendeskService(task) instanceof ZendeskSupportAPIService && !Target.SATISFACTION_RATINGS.equals(task.getTarget())) { + final JsonNode result = getZendeskService(task).getDataFromPath("", 0, false, 0); if (result.has(ZendeskConstants.Field.COUNT) && !result.get(ZendeskConstants.Field.COUNT).isNull() && result.get(ZendeskConstants.Field.COUNT).isInt()) { @@ -360,6 +362,8 @@ protected ZendeskService dispatchPerTarget(ZendeskInputPlugin.PluginTask task) case TICKET_EVENTS: case TICKET_FORMS: case TICKET_FIELDS: + case TICKET_METRIC_EVENTS: + case SATISFACTION_RATINGS: return new ZendeskSupportAPIService(task); case RECIPIENTS: case SCORES: diff --git a/src/main/java/org/embulk/input/zendesk/models/Target.java b/src/main/java/org/embulk/input/zendesk/models/Target.java index 5610dd4..f121d31 100644 --- a/src/main/java/org/embulk/input/zendesk/models/Target.java +++ b/src/main/java/org/embulk/input/zendesk/models/Target.java @@ -12,6 +12,7 @@ public enum Target */ TICKETS("tickets"), USERS("users"), ORGANIZATIONS("organizations"), TICKET_EVENTS("ticket_events"), TICKET_METRICS("metric_sets"), TICKET_FIELDS("ticket_fields"), TICKET_FORMS("ticket_forms"), + TICKET_METRIC_EVENTS("ticket_metric_events"), SATISFACTION_RATINGS("satisfaction_ratings"), RECIPIENTS("recipients"), SCORES("responses"), OBJECT_RECORDS("data"), RELATIONSHIP_RECORDS("data"), USER_EVENTS("data"); String jsonName; diff --git a/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java b/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java index 03cb544..21ef631 100644 --- a/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java +++ b/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java @@ -49,7 +49,12 @@ public TaskReport addRecordToImporter(final int taskIndex, final RecordImporter importDataForIncremental(task, recordImporter, taskReport); } else { - importDataForNonIncremental(task, taskIndex, recordImporter); + if(task.getTarget().equals(Target.SATISFACTION_RATINGS)){ + importDataForNonIncremental(task, taskIndex, recordImporter, true); + } + else{ + importDataForNonIncremental(task, taskIndex, recordImporter, false); + } } return taskReport; @@ -101,14 +106,24 @@ private void importDataForIncremental(final ZendeskInputPlugin.PluginTask task, ); long apiEndTime = 0; + int afterStartTimeIndex = 11; while (true) { int recordCount = 0; // Page argument isn't used in incremental API so we just set it to 0 final JsonNode result = getDataFromPath("", 0, false, startTime); final Iterator iterator = ZendeskUtils.getListRecords(result, task.getTarget().getJsonName()); - apiEndTime = result.get(ZendeskConstants.Field.END_TIME).asLong(); + if (result.has(ZendeskConstants.Field.END_TIME)){ + apiEndTime = result.get(ZendeskConstants.Field.END_TIME).asLong(); + } + else{ + if(result.has(ZendeskConstants.Field.NEXT_PAGE)){ + String next_page = result.get(ZendeskConstants.Field.NEXT_PAGE).textValue(); + apiEndTime = Long.parseLong(next_page.substring(next_page.indexOf(ZendeskConstants.Field.START_TIME)+afterStartTimeIndex)); + } + } + logger.info("Api END Time = '{}'", apiEndTime); int numberOfRecords = 0; if (result.has(ZendeskConstants.Field.COUNT)) { numberOfRecords = result.get(ZendeskConstants.Field.COUNT).asInt(); @@ -269,18 +284,64 @@ private boolean isUpdatedBySystem(final JsonNode recordJsonNode, final long star return false; } - private void importDataForNonIncremental(final ZendeskInputPlugin.PluginTask task, final int taskIndex, RecordImporter recordImporter) + private void importDataForNonIncremental(final ZendeskInputPlugin.PluginTask task, final int taskIndex, RecordImporter recordImporter, final boolean getStartDate) { - // Page start from 1 => page = taskIndex + 1 - final JsonNode result = getDataFromPath("", taskIndex + 1, false, 0); - final Iterator iterator = ZendeskUtils.getListRecords(result, task.getTarget().getJsonName()); + long startTime = 0; - while (iterator.hasNext()) { - fetchSubResourceAndAddToImporter(iterator.next(), task, recordImporter); + if (Target.SATISFACTION_RATINGS.equals(task.getTarget())){ + if (getStartDate && task.getStartTime().isPresent()) { + startTime = ZendeskDateUtils.getStartTime(task.getStartTime().get()); + } + logger.info("Start time = {}", startTime); - if (Exec.isPreview()) { - break; + int i = 1; + final Set knownIds = ConcurrentHashMap.newKeySet(); + while (true) { + final JsonNode result = getDataFromPath("", taskIndex +i, false, startTime); + final Iterator iterator = ZendeskUtils.getListRecords(result, task.getTarget().getJsonName()); + while (iterator.hasNext()) { + final JsonNode recordJsonNode = iterator.next(); + + if (task.getDedup()) { + final String recordID = recordJsonNode.get(ZendeskConstants.Field.ID).asText(); + + // add success -> no duplicate + if (!knownIds.add(recordID)) { + continue; + } + } + fetchSubResourceAndAddToImporter(recordJsonNode, task, recordImporter); + + if (Exec.isPreview()) { + break; + } + } + + i++; + if(result.has(ZendeskConstants.Field.NEXT_PAGE)){ + String next_page = result.get(ZendeskConstants.Field.NEXT_PAGE).textValue(); + if (next_page == null){ + logger.info("No next page exists. Exiting..."); + return; + } + logger.info("Next page = {}", next_page); + } + } + + } + else{ + + // Page start from 1 => page = taskIndex + 1 + final JsonNode result = getDataFromPath("", taskIndex + 1 , false, 0); + final Iterator iterator = ZendeskUtils.getListRecords(result, task.getTarget().getJsonName()); + while (iterator.hasNext()) { + fetchSubResourceAndAddToImporter(iterator.next(), task, recordImporter); + + if (Exec.isPreview()) { + break; + } } } + } } diff --git a/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java b/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java index b768a7a..bf973f8 100644 --- a/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java +++ b/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java @@ -15,7 +15,9 @@ public ZendeskSupportAPIService(final PluginTask task) public boolean isSupportIncremental() { - return !(task.getTarget().equals(Target.TICKET_FORMS) || task.getTarget().equals(Target.TICKET_FIELDS)); + return !(task.getTarget().equals(Target.TICKET_FORMS) + || task.getTarget().equals(Target.TICKET_FIELDS) + || task.getTarget().equals(Target.SATISFACTION_RATINGS)); } @Override @@ -30,9 +32,18 @@ protected String buildURI(final int page, long startTime) } } else { - uriBuilder.setParameter("sort_by", "id") - .setParameter("per_page", String.valueOf(100)) - .setParameter("page", String.valueOf(page)); + if (Target.SATISFACTION_RATINGS.equals(task.getTarget())){ + uriBuilder.setParameter(ZendeskConstants.Field.START_TIME, String.valueOf(startTime)) + .setParameter("sort_by", "id") + .setParameter("per_page", String.valueOf(100)) + .setParameter("page", String.valueOf(page)); + + } + else{ + uriBuilder.setParameter("sort_by", "id") + .setParameter("per_page", String.valueOf(100)) + .setParameter("page", String.valueOf(page)); + } } return uriBuilder.toString(); @@ -40,7 +51,7 @@ protected String buildURI(final int page, long startTime) private String buildPath() { - return (isSupportIncremental() + return (isSupportIncremental() && !(Target.SATISFACTION_RATINGS.equals(task.getTarget())) ? ZendeskConstants.Url.API_INCREMENTAL : ZendeskConstants.Url.API) + "/" + diff --git a/src/main/java/org/embulk/input/zendesk/utils/ZendeskConstants.java b/src/main/java/org/embulk/input/zendesk/utils/ZendeskConstants.java index 8000c59..d9255f9 100644 --- a/src/main/java/org/embulk/input/zendesk/utils/ZendeskConstants.java +++ b/src/main/java/org/embulk/input/zendesk/utils/ZendeskConstants.java @@ -28,6 +28,7 @@ public static class Field public static final String GENERATED_TIMESTAMP = "generated_timestamp"; public static final String UPDATED_AT = "updated_at"; public static final String ID = "id"; + public static final String NEXT_PAGE="next_page"; } public static class Url diff --git a/src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java b/src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java index 6fb5181..4d6afab 100644 --- a/src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java +++ b/src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java @@ -15,10 +15,13 @@ import java.net.URISyntaxException; import java.util.Base64; import java.util.Iterator; +import org.embulk.spi.Exec; +import org.slf4j.Logger; public class ZendeskUtils { private static final ObjectMapper mapper = new ObjectMapper(); + private static final Logger logger = Exec.getLogger(ZendeskUtils.class); static { mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); @@ -79,6 +82,7 @@ public static URIBuilder getURIBuilder(final String urlString) private static JsonNode parseJsonNode(final String jsonText) { try { + logger.info("Text {}", jsonText); return ZendeskUtils.mapper.readTree(jsonText); } catch (final IOException e) { From b9542de86bd11bc93e917d36357b3c1bf463386a Mon Sep 17 00:00:00 2001 From: Angelos Alexopoulos Date: Mon, 21 Oct 2019 17:14:29 +0300 Subject: [PATCH 2/8] reduce logs --- src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java b/src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java index 4d6afab..ecf73d5 100644 --- a/src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java +++ b/src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java @@ -82,7 +82,7 @@ public static URIBuilder getURIBuilder(final String urlString) private static JsonNode parseJsonNode(final String jsonText) { try { - logger.info("Text {}", jsonText); + // logger.info("Text {}", jsonText); return ZendeskUtils.mapper.readTree(jsonText); } catch (final IOException e) { From 40add380cf2f53380a0136b382b7838c90f58ab4 Mon Sep 17 00:00:00 2001 From: Angelos Alexopoulos Date: Mon, 21 Oct 2019 17:16:46 +0300 Subject: [PATCH 3/8] Increase version to 0.3.8 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 5197096..cee80b5 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ configurations { provided } -version = "0.3.7" +version = "0.3.8" sourceCompatibility = 1.8 targetCompatibility = 1.8 From dd309f346782f8d1ae7e5e5ab0a556e04c710868 Mon Sep 17 00:00:00 2001 From: Angelos Alexopoulos Date: Thu, 31 Oct 2019 11:17:33 +0200 Subject: [PATCH 4/8] Support SLA Polices --- README.md | 7 +++++- build.gradle | 2 +- .../input/zendesk/ZendeskInputPlugin.java | 1 + .../embulk/input/zendesk/models/Target.java | 3 ++- .../services/ZendeskSupportAPIService.java | 23 ++++++++++++------- 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index a76f23b..b5f4aa5 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,12 @@ Embulk input plugin for loading [Zendesk](https://www.zendesk.com/) records. Required Embulk version >= 0.9.6. -**NOTE** This plugin don't support JSON type columns e.g. custom fields, tags, etc for now. But they will be supported soon. +**NOTE** This plugin differs from original plugin to following: +* DOES support JSON type columns +* Supports satisfaction_ratings API +* Supports ticket_metric_events API +* Supports SLA Policies API + * **Plugin type**: input * **Resume supported**: no diff --git a/build.gradle b/build.gradle index cee80b5..7b6c5f8 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ configurations { provided } -version = "0.3.8" +version = "0.3.9" sourceCompatibility = 1.8 targetCompatibility = 1.8 diff --git a/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java b/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java index 9358bfe..6f18d4b 100644 --- a/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java +++ b/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java @@ -364,6 +364,7 @@ protected ZendeskService dispatchPerTarget(ZendeskInputPlugin.PluginTask task) case TICKET_FIELDS: case TICKET_METRIC_EVENTS: case SATISFACTION_RATINGS: + case SLA_POLICIES: return new ZendeskSupportAPIService(task); case RECIPIENTS: case SCORES: diff --git a/src/main/java/org/embulk/input/zendesk/models/Target.java b/src/main/java/org/embulk/input/zendesk/models/Target.java index f121d31..6eb38fe 100644 --- a/src/main/java/org/embulk/input/zendesk/models/Target.java +++ b/src/main/java/org/embulk/input/zendesk/models/Target.java @@ -13,7 +13,8 @@ public enum Target TICKETS("tickets"), USERS("users"), ORGANIZATIONS("organizations"), TICKET_EVENTS("ticket_events"), TICKET_METRICS("metric_sets"), TICKET_FIELDS("ticket_fields"), TICKET_FORMS("ticket_forms"), TICKET_METRIC_EVENTS("ticket_metric_events"), SATISFACTION_RATINGS("satisfaction_ratings"), - RECIPIENTS("recipients"), SCORES("responses"), OBJECT_RECORDS("data"), RELATIONSHIP_RECORDS("data"), USER_EVENTS("data"); + SLA_POLICIES("sla_policies"), SCORES("responses"), OBJECT_RECORDS("data"), + RECIPIENTS("recipients"), RELATIONSHIP_RECORDS("data"), USER_EVENTS("data"); String jsonName; diff --git a/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java b/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java index bf973f8..fa35c47 100644 --- a/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java +++ b/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java @@ -16,6 +16,7 @@ public ZendeskSupportAPIService(final PluginTask task) public boolean isSupportIncremental() { return !(task.getTarget().equals(Target.TICKET_FORMS) + || task.getTarget().equals(Target.SLA_POLICIES) || task.getTarget().equals(Target.TICKET_FIELDS) || task.getTarget().equals(Target.SATISFACTION_RATINGS)); } @@ -51,13 +52,19 @@ protected String buildURI(final int page, long startTime) private String buildPath() { - return (isSupportIncremental() && !(Target.SATISFACTION_RATINGS.equals(task.getTarget())) - ? ZendeskConstants.Url.API_INCREMENTAL - : ZendeskConstants.Url.API) + - "/" + - (Target.TICKET_METRICS.equals(task.getTarget()) - ? Target.TICKETS.toString() - : task.getTarget().toString()) - + ".json"; + if(Target.SLA_POLICIES.equals(task.getTarget())) + { + return ZendeskConstants.Url.API+"/slas/policies"; + } + else{ + return (isSupportIncremental() && !(Target.SATISFACTION_RATINGS.equals(task.getTarget())) + ? ZendeskConstants.Url.API_INCREMENTAL + : ZendeskConstants.Url.API) + + "/" + + (Target.TICKET_METRICS.equals(task.getTarget()) + ? Target.TICKETS.toString() + : task.getTarget().toString()) + + ".json"; + } } } From 490b859cbb06aa3fdb4e2ca5ffcd532a060491b9 Mon Sep 17 00:00:00 2001 From: Angelos Alexopoulos Date: Mon, 17 Feb 2020 18:08:22 +0200 Subject: [PATCH 5/8] Remove the check for system updated tickets --- build.gradle | 2 +- gradlew | 2 +- .../zendesk/clients/ZendeskRestClient.java | 2 + .../zendesk/services/ZendeskNPSService.java | 5 +++ .../services/ZendeskNormalServices.java | 37 +++++++++++++++---- .../services/ZendeskSupportAPIService.java | 24 +++++++++--- 6 files changed, 57 insertions(+), 15 deletions(-) diff --git a/build.gradle b/build.gradle index 7b6c5f8..e53213a 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ configurations { provided } -version = "0.3.9" +version = "0.4.4" sourceCompatibility = 1.8 targetCompatibility = 1.8 diff --git a/gradlew b/gradlew index cccdd3d..e09f44e 100755 --- a/gradlew +++ b/gradlew @@ -168,5 +168,5 @@ eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$A if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then cd "$(dirname "$0")" fi - +echo "$JAVACMD" "$@" exec "$JAVACMD" "$@" diff --git a/src/main/java/org/embulk/input/zendesk/clients/ZendeskRestClient.java b/src/main/java/org/embulk/input/zendesk/clients/ZendeskRestClient.java index cd93b56..b02c31c 100644 --- a/src/main/java/org/embulk/input/zendesk/clients/ZendeskRestClient.java +++ b/src/main/java/org/embulk/input/zendesk/clients/ZendeskRestClient.java @@ -235,6 +235,8 @@ private HttpRequestBase createGetRequest(final String url, final PluginTask task final HttpGet request = new HttpGet(url); final ImmutableMap headers = buildAuthHeader(task); headers.forEach(request::setHeader); + +// headers.values().forEach(logger::info); return request; } diff --git a/src/main/java/org/embulk/input/zendesk/services/ZendeskNPSService.java b/src/main/java/org/embulk/input/zendesk/services/ZendeskNPSService.java index d369528..3711a74 100644 --- a/src/main/java/org/embulk/input/zendesk/services/ZendeskNPSService.java +++ b/src/main/java/org/embulk/input/zendesk/services/ZendeskNPSService.java @@ -15,6 +15,11 @@ public boolean isSupportIncremental() { return true; } + @Override + protected String buildURI(final int page, final long startTime, final long endTime) + { + return buildURI(page, startTime); + } @Override protected String buildURI(final int page, final long startTime) diff --git a/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java b/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java index 21ef631..5a52d8d 100644 --- a/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java +++ b/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java @@ -59,11 +59,15 @@ public TaskReport addRecordToImporter(final int taskIndex, final RecordImporter return taskReport; } - public JsonNode getDataFromPath(String path, final int page, final boolean isPreview, final long startTime) + { + return getDataFromPath(path, page,isPreview, startTime, 0); + } + + public JsonNode getDataFromPath(String path, final int page, final boolean isPreview, final long startTime, final long endTime) { if (path.isEmpty()) { - path = buildURI(page, startTime); + path = buildURI(page, startTime, endTime); } final String response = getZendeskRestClient().doGet(path, task, isPreview); @@ -71,6 +75,8 @@ public JsonNode getDataFromPath(String path, final int page, final boolean isPre } protected abstract String buildURI(int page, long startTime); + protected abstract String buildURI(int page, long startTime, long endTime); + @VisibleForTesting protected ZendeskRestClient getZendeskRestClient() @@ -127,14 +133,16 @@ private void importDataForIncremental(final ZendeskInputPlugin.PluginTask task, int numberOfRecords = 0; if (result.has(ZendeskConstants.Field.COUNT)) { numberOfRecords = result.get(ZendeskConstants.Field.COUNT).asInt(); + logger.info("Number of Records {}", numberOfRecords); } while (iterator.hasNext()) { final JsonNode recordJsonNode = iterator.next(); - if (isUpdatedBySystem(recordJsonNode, startTime)) { - continue; - } +// if (isUpdatedBySystem(recordJsonNode, startTime)) { +// logger.info("This record is update by system? {} IGNORED!! ", recordJsonNode.get("id")); +// continue; +// } // Contain some records that later than end_time. Checked and don't add. // Because the api already sorted by updated_at or timestamp for ticket_events, we just need to break no need to check further. @@ -142,6 +150,7 @@ private void importDataForIncremental(final ZendeskInputPlugin.PluginTask task, long checkedTime = 0; if (recordJsonNode.has(ZendeskConstants.Field.UPDATED_AT) && !recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).isNull()) { checkedTime = ZendeskDateUtils.isoToEpochSecond(recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).textValue()); +// logger.info("Checked time {}", checkedTime); } // ticket events is updated by system not user's action so it only has timestamp field @@ -155,11 +164,13 @@ private void importDataForIncremental(final ZendeskInputPlugin.PluginTask task, } if (checkedTime > endTime) { + logger.info("Lets break since checked time > endTime {}", endTime); break; } } if (task.getDedup()) { +// logger.info("Do we dedup??"); final String recordID = recordJsonNode.get(ZendeskConstants.Field.ID).asText(); // add success -> no duplicate @@ -170,6 +181,7 @@ private void importDataForIncremental(final ZendeskInputPlugin.PluginTask task, pool.submit(() -> fetchSubResourceAndAddToImporter(recordJsonNode, task, recordImporter)); recordCount++; +// logger.info("Record count {}", recordCount); if (Exec.isPreview()) { return; } @@ -184,6 +196,8 @@ private void importDataForIncremental(final ZendeskInputPlugin.PluginTask task, : apiEndTime; if (numberOfRecords < ZendeskConstants.Misc.MAXIMUM_RECORDS_INCREMENTAL || startTime > endTime) { + logger.info("Break because number of Records < MAXIMUM RECORDS {}", numberOfRecords); + logger.info("or start time {}> end Time {}", startTime, endTime); break; } } @@ -239,6 +253,7 @@ private void storeStartTimeForConfigDiff(final TaskReport taskReport, final long private void fetchSubResourceAndAddToImporter(final JsonNode jsonNode, final ZendeskInputPlugin.PluginTask task, final RecordImporter recordImporter) { +// logger.info("Fetch Sub Resource {}", jsonNode.get("id")); task.getIncludes().forEach(include -> { final String relatedObjectName = include.trim(); @@ -254,13 +269,14 @@ private void fetchSubResourceAndAddToImporter(final JsonNode jsonNode, final Zen } } catch (final ConfigException e) { + logger.info("Some exception for id {}", jsonNode.get("id")); // Sometimes we get 404 when having invalid endpoint, so ignore when we get 404 InvalidEndpoint if (!(e.getCause() instanceof ZendeskException && ((ZendeskException) e.getCause()).getStatusCode() == HttpStatus.SC_NOT_FOUND)) { throw e; } } }); - +// logger.info("Add record {}", jsonNode.get("id")); recordImporter.addRecord(jsonNode); } @@ -287,6 +303,7 @@ private boolean isUpdatedBySystem(final JsonNode recordJsonNode, final long star private void importDataForNonIncremental(final ZendeskInputPlugin.PluginTask task, final int taskIndex, RecordImporter recordImporter, final boolean getStartDate) { long startTime = 0; + long endTime = 0; if (Target.SATISFACTION_RATINGS.equals(task.getTarget())){ if (getStartDate && task.getStartTime().isPresent()) { @@ -294,10 +311,16 @@ private void importDataForNonIncremental(final ZendeskInputPlugin.PluginTask tas } logger.info("Start time = {}", startTime); + if (task.getEndTime().isPresent()) { + endTime = ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get()); + } + + logger.info("End time = {}", endTime); + int i = 1; final Set knownIds = ConcurrentHashMap.newKeySet(); while (true) { - final JsonNode result = getDataFromPath("", taskIndex +i, false, startTime); + final JsonNode result = getDataFromPath("", taskIndex +i, false, startTime, endTime); final Iterator iterator = ZendeskUtils.getListRecords(result, task.getTarget().getJsonName()); while (iterator.hasNext()) { final JsonNode recordJsonNode = iterator.next(); diff --git a/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java b/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java index fa35c47..c15e4f8 100644 --- a/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java +++ b/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java @@ -22,7 +22,12 @@ public boolean isSupportIncremental() } @Override - protected String buildURI(final int page, long startTime) + protected String buildURI(final int page, long startTime){ + return buildURI(page, startTime); + } + + @Override + protected String buildURI(final int page, long startTime, long endTime) { final URIBuilder uriBuilder = ZendeskUtils.getURIBuilder(task.getLoginUrl()).setPath(buildPath()); @@ -34,11 +39,18 @@ protected String buildURI(final int page, long startTime) } else { if (Target.SATISFACTION_RATINGS.equals(task.getTarget())){ - uriBuilder.setParameter(ZendeskConstants.Field.START_TIME, String.valueOf(startTime)) - .setParameter("sort_by", "id") - .setParameter("per_page", String.valueOf(100)) - .setParameter("page", String.valueOf(page)); - + if (endTime>0){ + uriBuilder.setParameter(ZendeskConstants.Field.START_TIME, String.valueOf(startTime)) + .setParameter(ZendeskConstants.Field.END_TIME, String.valueOf(endTime)) + .setParameter("sort_by", "id") + .setParameter("per_page", String.valueOf(100)) + .setParameter("page", String.valueOf(page)); + }else{ + uriBuilder.setParameter(ZendeskConstants.Field.START_TIME, String.valueOf(startTime)) + .setParameter("sort_by", "id") + .setParameter("per_page", String.valueOf(100)) + .setParameter("page", String.valueOf(page)); + } } else{ uriBuilder.setParameter("sort_by", "id") From 484831069052fcc7253f635c72e41dccc341b421 Mon Sep 17 00:00:00 2001 From: Angelos Alexopoulos Date: Wed, 5 May 2021 13:07:10 +0300 Subject: [PATCH 6/8] Adding groups --- README.md | 1 + build.gradle | 2 +- src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java | 2 ++ src/main/java/org/embulk/input/zendesk/models/Target.java | 2 +- .../input/zendesk/services/ZendeskSupportAPIService.java | 3 ++- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b5f4aa5..48d5679 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Required Embulk version >= 0.9.6. * Supports satisfaction_ratings API * Supports ticket_metric_events API * Supports SLA Policies API +* Supports Groups API * **Plugin type**: input diff --git a/build.gradle b/build.gradle index e53213a..fa90f9e 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ configurations { provided } -version = "0.4.4" +version = "0.4.5" sourceCompatibility = 1.8 targetCompatibility = 1.8 diff --git a/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java b/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java index 6f18d4b..eb8f9a3 100644 --- a/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java +++ b/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java @@ -374,6 +374,8 @@ protected ZendeskService dispatchPerTarget(ZendeskInputPlugin.PluginTask task) return new ZendeskCustomObjectService(task); case USER_EVENTS: return new ZendeskUserEventService(task); + case GROUPS: + return new ZendeskSupportAPIService(task); default: throw new ConfigException("Unsupported " + task.getTarget() + ", supported values: '" + Arrays.toString(Target.values()) + "'"); } diff --git a/src/main/java/org/embulk/input/zendesk/models/Target.java b/src/main/java/org/embulk/input/zendesk/models/Target.java index 6eb38fe..74d144a 100644 --- a/src/main/java/org/embulk/input/zendesk/models/Target.java +++ b/src/main/java/org/embulk/input/zendesk/models/Target.java @@ -14,7 +14,7 @@ public enum Target TICKET_METRICS("metric_sets"), TICKET_FIELDS("ticket_fields"), TICKET_FORMS("ticket_forms"), TICKET_METRIC_EVENTS("ticket_metric_events"), SATISFACTION_RATINGS("satisfaction_ratings"), SLA_POLICIES("sla_policies"), SCORES("responses"), OBJECT_RECORDS("data"), - RECIPIENTS("recipients"), RELATIONSHIP_RECORDS("data"), USER_EVENTS("data"); + RECIPIENTS("recipients"), RELATIONSHIP_RECORDS("data"), USER_EVENTS("data"), GROUPS("groups"); String jsonName; diff --git a/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java b/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java index c15e4f8..487ed9f 100644 --- a/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java +++ b/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java @@ -18,7 +18,8 @@ public boolean isSupportIncremental() return !(task.getTarget().equals(Target.TICKET_FORMS) || task.getTarget().equals(Target.SLA_POLICIES) || task.getTarget().equals(Target.TICKET_FIELDS) - || task.getTarget().equals(Target.SATISFACTION_RATINGS)); + || task.getTarget().equals(Target.SATISFACTION_RATINGS) + || task.getTarget().equals(Target.GROUPS)); } @Override From 4d73c831532b16c837632408ac1ce3a847929097 Mon Sep 17 00:00:00 2001 From: Angelos Alexopoulos Date: Wed, 5 May 2021 13:28:10 +0300 Subject: [PATCH 7/8] Removing commented code --- .../zendesk/services/ZendeskNormalServices.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java b/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java index 5a52d8d..531776a 100644 --- a/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java +++ b/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java @@ -139,18 +139,12 @@ private void importDataForIncremental(final ZendeskInputPlugin.PluginTask task, while (iterator.hasNext()) { final JsonNode recordJsonNode = iterator.next(); -// if (isUpdatedBySystem(recordJsonNode, startTime)) { -// logger.info("This record is update by system? {} IGNORED!! ", recordJsonNode.get("id")); -// continue; -// } - // Contain some records that later than end_time. Checked and don't add. // Because the api already sorted by updated_at or timestamp for ticket_events, we just need to break no need to check further. if (apiEndTime > endTime) { long checkedTime = 0; if (recordJsonNode.has(ZendeskConstants.Field.UPDATED_AT) && !recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).isNull()) { checkedTime = ZendeskDateUtils.isoToEpochSecond(recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).textValue()); -// logger.info("Checked time {}", checkedTime); } // ticket events is updated by system not user's action so it only has timestamp field @@ -170,7 +164,6 @@ private void importDataForIncremental(final ZendeskInputPlugin.PluginTask task, } if (task.getDedup()) { -// logger.info("Do we dedup??"); final String recordID = recordJsonNode.get(ZendeskConstants.Field.ID).asText(); // add success -> no duplicate @@ -181,7 +174,6 @@ private void importDataForIncremental(final ZendeskInputPlugin.PluginTask task, pool.submit(() -> fetchSubResourceAndAddToImporter(recordJsonNode, task, recordImporter)); recordCount++; -// logger.info("Record count {}", recordCount); if (Exec.isPreview()) { return; } @@ -253,7 +245,6 @@ private void storeStartTimeForConfigDiff(final TaskReport taskReport, final long private void fetchSubResourceAndAddToImporter(final JsonNode jsonNode, final ZendeskInputPlugin.PluginTask task, final RecordImporter recordImporter) { -// logger.info("Fetch Sub Resource {}", jsonNode.get("id")); task.getIncludes().forEach(include -> { final String relatedObjectName = include.trim(); @@ -276,7 +267,6 @@ private void fetchSubResourceAndAddToImporter(final JsonNode jsonNode, final Zen } } }); -// logger.info("Add record {}", jsonNode.get("id")); recordImporter.addRecord(jsonNode); } @@ -309,14 +299,11 @@ private void importDataForNonIncremental(final ZendeskInputPlugin.PluginTask tas if (getStartDate && task.getStartTime().isPresent()) { startTime = ZendeskDateUtils.getStartTime(task.getStartTime().get()); } - logger.info("Start time = {}", startTime); if (task.getEndTime().isPresent()) { endTime = ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get()); } - logger.info("End time = {}", endTime); - int i = 1; final Set knownIds = ConcurrentHashMap.newKeySet(); while (true) { From 7f33749ec04f2929b00b04c3ac1ccc27995b6e57 Mon Sep 17 00:00:00 2001 From: Angelos Alexopoulos Date: Wed, 5 May 2021 13:29:14 +0300 Subject: [PATCH 8/8] Removing commented code --- src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java b/src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java index ecf73d5..49821d4 100644 --- a/src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java +++ b/src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java @@ -82,7 +82,6 @@ public static URIBuilder getURIBuilder(final String urlString) private static JsonNode parseJsonNode(final String jsonText) { try { - // logger.info("Text {}", jsonText); return ZendeskUtils.mapper.readTree(jsonText); } catch (final IOException e) {