Merge pull request #39 from treasure-data/add_pagination_interest_cat…

…egories Added pagination for interest categories
treasure-data · Oct 16, 2017 · 4a074a5 · 4a074a5
2 parents 57c9ef5 + d70dcfb
commit 4a074a5
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 8 deletions.
diff --git a/src/main/java/org/embulk/output/mailchimp/MailChimpClient.java b/src/main/java/org/embulk/output/mailchimp/MailChimpClient.java
@@ -132,12 +132,32 @@ Map<String, Map<String, InterestResponse>> extractInterestCategoriesByGroupNames
         if (task.getGroupingColumns().isPresent() && !task.getGroupingColumns().get().isEmpty()) {
             List<String> interestCategoryNames = task.getGroupingColumns().get();
 
-            String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories",
-                                                   task.getListId());
-
-            JsonNode response = client.sendRequest(endpoint, HttpMethod.GET, task);
-            InterestCategoriesResponse interestCategoriesResponse = mapper.treeToValue(response,
-                                                                                       InterestCategoriesResponse.class);
+            int count = 100;
+            int offset = 0;
+            int page = 1;
+            boolean hasMore = true;
+            JsonNode response;
+            List<CategoriesResponse> allCategoriesResponse = new ArrayList<>();
+
+            while (hasMore) {
+                String endpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories?count={1}&offset={2}",
+                                                       task.getListId(),
+                                                       count,
+                                                       offset);
+
+                response = client.sendRequest(endpoint, HttpMethod.GET, task);
+                InterestCategoriesResponse interestCategoriesResponse = mapper.treeToValue(response,
+                                                                                           InterestCategoriesResponse.class);
+
+                allCategoriesResponse.addAll(interestCategoriesResponse.getCategories());
+                if (hasMorePage(interestCategoriesResponse.getTotalItems(), count, page)) {
+                    offset = count;
+                    page++;
+                }
+                else {
+                    hasMore = false;
+                }
+            }
 
             Function<CategoriesResponse, String> function = new Function<CategoriesResponse, String>()
             {
@@ -150,7 +170,7 @@ public String apply(CategoriesResponse input)
 
             // Transform to a list of available category names and validate with data that user input
             ImmutableList<String> availableCategories = FluentIterable
-                    .from(interestCategoriesResponse.getCategories())
+                    .from(allCategoriesResponse)
                     .transform(function)
                     .toList();
 
@@ -160,7 +180,7 @@ public String apply(CategoriesResponse input)
                 }
             }
 
-            for (CategoriesResponse categoriesResponse : interestCategoriesResponse.getCategories()) {
+            for (CategoriesResponse categoriesResponse : allCategoriesResponse) {
                 String detailEndpoint = MessageFormat.format(mailchimpEndpoint + "/lists/{0}/interest-categories/{1}/interests",
                                                              task.getListId(),
                                                              categoriesResponse.getId());

diff --git a/src/main/java/org/embulk/output/mailchimp/model/InterestCategoriesResponse.java b/src/main/java/org/embulk/output/mailchimp/model/InterestCategoriesResponse.java
@@ -12,6 +12,9 @@ public class InterestCategoriesResponse
     @JsonProperty("categories")
     private List<CategoriesResponse> categories;
 
+    @JsonProperty("total_items")
+    private int totalItems;
+
     public List<CategoriesResponse> getCategories()
     {
         return categories;
@@ -21,4 +24,14 @@ public void setCategories(List<CategoriesResponse> categories)
     {
         this.categories = categories;
     }
+
+    public int getTotalItems()
+    {
+        return totalItems;
+    }
+
+    public void setTotalItems(int totalItems)
+    {
+        this.totalItems = totalItems;
+    }
 }