Merge pull request #34 from treasure-data/fix_mailchimp_api_json_pars…

…er_error Fix error parsing MailChimp API response JSON when push the large data
treasure-data · Sep 19, 2017 · c49db40 · c49db40
2 parents 40908b0 + 880e956
commit c49db40
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 4 deletions.
diff --git a/src/main/java/org/embulk/output/mailchimp/MailChimpHttpClient.java b/src/main/java/org/embulk/output/mailchimp/MailChimpHttpClient.java
@@ -1,6 +1,8 @@
 package org.embulk.output.mailchimp;
 
 import com.fasterxml.jackson.core.Base64Variants;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.MissingNode;
@@ -31,8 +33,8 @@ public class MailChimpHttpClient
 {
     private static final Logger LOG = Exec.getLogger(MailChimpHttpClient.class);
     private final ObjectMapper jsonMapper = new ObjectMapper()
-            .configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false)
-            .configure(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+            .configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false)
+            .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
 
     /**
      * Instantiates a new Mailchimp http client.

diff --git a/src/main/java/org/embulk/output/mailchimp/MailChimpOutputPluginDelegate.java b/src/main/java/org/embulk/output/mailchimp/MailChimpOutputPluginDelegate.java
@@ -103,6 +103,10 @@ public interface PluginTask
         @Config("language_column")
         @ConfigDefault("null")
         Optional<String> getLanguageColumn();
+
+        @Config("max_records_per_request")
+        @ConfigDefault("500")
+        int getMaxRecordsPerRequest();
     }
 
     /**

diff --git a/src/main/java/org/embulk/output/mailchimp/MailChimpRecordBuffer.java b/src/main/java/org/embulk/output/mailchimp/MailChimpRecordBuffer.java
@@ -46,7 +46,6 @@ public class MailChimpRecordBuffer
         extends RecordBuffer
 {
     private static final Logger LOG = Exec.getLogger(MailChimpRecordBuffer.class);
-    private static final int MAX_RECORD_PER_BATCH_REQUEST = 500;
     private final MailChimpOutputPluginDelegate.PluginTask task;
     private final MailChimpClient mailChimpClient;
     private final ObjectMapper mapper;
@@ -88,7 +87,7 @@ public void bufferRecord(ServiceRecord serviceRecord)
             totalCount++;
 
             records.add(record);
-            if (requestCount >= MAX_RECORD_PER_BATCH_REQUEST) {
+            if (requestCount >= task.getMaxRecordsPerRequest()) {
                 ObjectNode subcribers = processSubcribers(records, task);
                 ReportResponse reportResponse = mailChimpClient.push(subcribers, task);