Make multiple paginated requests to support 50+ issues

sebaherrera07 · Jan 12, 2024 · b92be8c · b92be8c
1 parent 9b21141
commit b92be8c
Show file tree

Hide file tree

Showing 5 changed files with 89 additions and 18 deletions.
diff --git a/.rubocop.yml b/.rubocop.yml
@@ -18,6 +18,7 @@ Metrics/BlockLength:
     - lib/tasks/auto_annotate_models.rake
 
 Metrics/ClassLength:
+  Max: 150
   Exclude:
     - spec/support/mocks/jira_api_responses.rb
 
@@ -65,3 +66,6 @@ Style/Documentation:
 
 Style/HashSyntax:
   Enabled: false
+
+Style/WhileUntilModifier:
+  Enabled: false
diff --git a/app/services/jira_api_client_service.rb b/app/services/jira_api_client_service.rb
@@ -10,12 +10,17 @@ def query_projects
   end
 
   def query_project_epics(project_key)
-    request_query_params = { query: { jql: "project = #{project_key} AND issuetype = Epic" } }
-    response = HTTParty.get("#{BASE_URL}/search", request_params.merge(request_query_params))
-    json_response = JSON.parse(response.body)
-    json_response['issues'].map do |epic_hash|
-      epic(epic_hash)
+    jql_string = "project = #{project_key} AND issuetype = Epic"
+
+    epics = []
+    start_at = 0
+    total = 1
+
+    while start_at < total
+      start_at, total, epics = query_epics_page(jql_string, start_at, epics)
     end
+
+    epics
   end
 
   def query_project_epic(project_key, epic_key)
@@ -29,12 +34,16 @@ def query_project_epic(project_key, epic_key)
   def query_epic_issues(project_key, epic_key, labels = [])
     jql_string = "project = #{project_key} AND (parent = #{epic_key} OR parentepic = #{epic_key}) AND issuetype != Epic"
     jql_string += " AND labels in (#{labels.join(',')})" if labels.present?
-    request_query_params = { query: { jql: jql_string } }
-    response = HTTParty.get("#{BASE_URL}/search", request_params.merge(request_query_params))
-    json_response = JSON.parse(response.body)
-    json_response['issues'].map do |issue_hash|
-      issue(issue_hash)
+
+    issues = []
+    start_at = 0
+    total = 1
+
+    while start_at < total
+      start_at, total, issues = query_issues_page(jql_string, start_at, issues)
     end
+
+    issues
   end
 
   private
@@ -103,4 +112,32 @@ def start_date(epic_hash)
     end
     nil
   end
+
+  def query_epics_page(jql_string, start_at, epics)
+    request_query_params = { query: { jql: jql_string, startAt: start_at, maxResults: 100 } }
+
+    response = HTTParty.get("#{BASE_URL}/search", request_params.merge(request_query_params))
+    json_response = JSON.parse(response.body)
+
+    total = json_response['total']
+    epics += json_response['issues'].map { |epic_hash| epic(epic_hash) }
+
+    start_at += json_response['maxResults']
+
+    [start_at, total, epics]
+  end
+
+  def query_issues_page(jql_string, start_at, issues)
+    request_query_params = { query: { jql: jql_string, startAt: start_at, maxResults: 100 } }
+
+    response = HTTParty.get("#{BASE_URL}/search", request_params.merge(request_query_params))
+    json_response = JSON.parse(response.body)
+
+    total = json_response['total']
+    issues += json_response['issues'].map { |issue_hash| issue(issue_hash) }
+
+    start_at += json_response['maxResults']
+
+    [start_at, total, issues]
+  end
 end
diff --git a/spec/services/jira_api_client_service_spec.rb b/spec/services/jira_api_client_service_spec.rb
@@ -44,7 +44,9 @@
         headers: { 'Accept' => 'application/json' },
         basic_auth: [ENV.fetch('JIRA_USERNAME'), ENV.fetch('JIRA_API_TOKEN')],
         query: {
-          jql: "project = #{project_key} AND issuetype = Epic"
+          jql: "project = #{project_key} AND issuetype = Epic",
+          startAt: 0,
+          maxResults: 100
         }
       )
     end
@@ -105,7 +107,10 @@
           headers: { 'Accept' => 'application/json' },
           basic_auth: [ENV.fetch('JIRA_USERNAME'), ENV.fetch('JIRA_API_TOKEN')],
           query: {
-            jql: "project = #{project_key} AND (parent = #{epic_key} OR parentepic = #{epic_key}) AND issuetype != Epic"
+            jql: "project = #{project_key} AND (parent = #{epic_key} OR parentepic = #{epic_key}) " \
+                 'AND issuetype != Epic',
+            startAt: 0,
+            maxResults: 100
           }
         )
       end
@@ -130,7 +135,9 @@
           basic_auth: [ENV.fetch('JIRA_USERNAME'), ENV.fetch('JIRA_API_TOKEN')],
           query: {
             jql: "project = #{project_key} AND (parent = #{epic_key} OR parentepic = #{epic_key}) " \
-                 "AND issuetype != Epic AND labels in (#{labels.join(',')})"
+                 "AND issuetype != Epic AND labels in (#{labels.join(',')})",
+            startAt: 0,
+            maxResults: 100
           }
         )
       end
@@ -158,7 +165,10 @@
           headers: { 'Accept' => 'application/json' },
           basic_auth: [ENV.fetch('JIRA_USERNAME'), ENV.fetch('JIRA_API_TOKEN')],
           query: {
-            jql: "project = #{project_key} AND (parent = #{epic_key} OR parentepic = #{epic_key}) AND issuetype != Epic"
+            jql: "project = #{project_key} AND (parent = #{epic_key} OR parentepic = #{epic_key}) " \
+                 'AND issuetype != Epic',
+            startAt: 0,
+            maxResults: 100
           }
         )
       end

diff --git a/spec/support/mocks/jira_api_mocker.rb b/spec/support/mocks/jira_api_mocker.rb
@@ -11,7 +11,13 @@ def stub_query_projects
 
   def stub_query_project_epics(project_key)
     url = "#{BASE_URL}/search"
-    query_params = { query: { jql: "project = #{project_key} AND issuetype = Epic" } }
+    query_params = {
+      query: {
+        jql: "project = #{project_key} AND issuetype = Epic",
+        startAt: 0,
+        maxResults: 100
+      }
+    }
     stub_request(url, request_params(query_params)).to_return(
       status: 200,
       body: JiraApiResponses.query_project_epics_response_body(project_key)
@@ -31,7 +37,9 @@ def stub_query_epic_issues(project_key, epic_key)
     url = "#{BASE_URL}/search"
     query_params = {
       query: {
-        jql: "project = #{project_key} AND (parent = #{epic_key} OR parentepic = #{epic_key}) AND issuetype != Epic"
+        jql: "project = #{project_key} AND (parent = #{epic_key} OR parentepic = #{epic_key}) AND issuetype != Epic",
+        startAt: 0,
+        maxResults: 100
       }
     }
     stub_request(url, request_params(query_params)).to_return(
@@ -44,7 +52,9 @@ def stub_query_epic_issues_with_labels(project_key, epic_key, labels)
     url = "#{BASE_URL}/search"
     query_params = { query: {
       jql: "project = #{project_key} AND (parent = #{epic_key} OR parentepic = #{epic_key}) AND issuetype != Epic " \
-           "AND labels in (#{labels.join(',')})"
+           "AND labels in (#{labels.join(',')})",
+      startAt: 0,
+      maxResults: 100
     } }
     stub_request(url, request_params(query_params)).to_return(
       status: 200,
@@ -56,7 +66,9 @@ def stub_query_epic_issues_with_custom_points_field(project_key, epic_key)
     url = "#{BASE_URL}/search"
     query_params = {
       query: {
-        jql: "project = #{project_key} AND (parent = #{epic_key} OR parentepic = #{epic_key}) AND issuetype != Epic"
+        jql: "project = #{project_key} AND (parent = #{epic_key} OR parentepic = #{epic_key}) AND issuetype != Epic",
+        startAt: 0,
+        maxResults: 100
       }
     }
     stub_request(url, request_params(query_params)).to_return(

diff --git a/spec/support/mocks/jira_api_responses.rb b/spec/support/mocks/jira_api_responses.rb
@@ -18,6 +18,8 @@ def self.query_projects_response_body
 
   def self.query_project_epics_response_body(project_key)
     {
+      'total' => 2,
+      'maxResults' => 100,
       'issues' => [
         {
           'key' => 'EPIC-1',
@@ -62,6 +64,8 @@ def self.query_project_epic_response_body(project_key, epic_key)
 
   def self.query_epic_issues_response_body(project_key, epic_key)
     {
+      'total' => 2,
+      'maxResults' => 100,
       'issues' => [
         {
           'key' => 'ISSUE-1',
@@ -107,6 +111,8 @@ def self.query_epic_issues_response_body(project_key, epic_key)
 
   def self.query_epic_issues_with_labels_response_body(project_key, epic_key, labels)
     {
+      'total' => 2,
+      'maxResults' => 100,
       'issues' => [
         {
           'key' => 'ISSUE-1',
@@ -152,6 +158,8 @@ def self.query_epic_issues_with_labels_response_body(project_key, epic_key, labe
 
   def self.query_epic_issues_with_custom_points_field_response_body(project_key, epic_key)
     {
+      'total' => 2,
+      'maxResults' => 100,
       'issues' => [
         {
           'key' => 'ISSUE-1',