From 614c4dc321429d05df2c6463bed1289ab718f444 Mon Sep 17 00:00:00 2001
From: Leena Gupte <leena.gupte@digital.cabinet-office.gov.uk>
Date: Mon, 20 Feb 2017 17:24:58 +0000
Subject: [PATCH] Request number of page views from GA in batches

We were sending a list of all of the content items in an organisation
to Google Analytics and requesting the number of page views for them all.

This is ok for organisations with a smaller number of content items, but
for organisations with content items in the tens of thousands, the
request fails.

To fix this, the request to GA has been wrapped in a batch. The batch
size has been hard-coded to 1 to get the test to pass, but will made to
be configurable in the next commit.

Remove unnecessary assignment in the existing test. Assigning content items
to the organisation does affect the outcome of the test, so it has been
removed.
---
 .../importers/number_of_views_by_organisation.rb    | 13 ++++++++-----
 .../number_of_views_by_organisation_spec.rb         | 13 +++++++------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/app/models/importers/number_of_views_by_organisation.rb b/app/models/importers/number_of_views_by_organisation.rb
index 51d9182da..a8c4c19f1 100644
--- a/app/models/importers/number_of_views_by_organisation.rb
+++ b/app/models/importers/number_of_views_by_organisation.rb
@@ -2,13 +2,16 @@ module Importers
   class NumberOfViewsByOrganisation
     def run(slug)
       organisation = Organisation.find_by(slug: slug)
+      google_analytics_service = GoogleAnalyticsService.new
 
-      base_paths = organisation.content_items.pluck(:base_path)
+      organisation.content_items.find_in_batches(batch_size: 1) do |content_items|
+        base_paths = content_items.pluck(:base_path)
 
-      results = GoogleAnalyticsService.new.page_views(base_paths)
-      results.each do |result|
-        content_item = ContentItem.find_by(base_path: result[:base_path])
-        content_item.update!(unique_page_views: result[:page_views])
+        results = google_analytics_service.page_views(base_paths)
+        results.each do |result|
+          content_item = ContentItem.find_by(base_path: result[:base_path])
+          content_item.update!(unique_page_views: result[:page_views])
+        end
       end
     end
   end
diff --git a/spec/models/importers/number_of_views_by_organisation_spec.rb b/spec/models/importers/number_of_views_by_organisation_spec.rb
index 62cf42ac9..fce08e52c 100644
--- a/spec/models/importers/number_of_views_by_organisation_spec.rb
+++ b/spec/models/importers/number_of_views_by_organisation_spec.rb
@@ -6,7 +6,7 @@
     let!(:content_item_first) { create(:content_item, base_path: 'the-link/first', organisations: [organisation]) }
     let!(:content_item_second) { create(:content_item, base_path: 'the-link/second', organisations: [organisation]) }
 
-    before do
+    it "updates the number of views for all content items" do
       allow_any_instance_of(GoogleAnalyticsService).to receive(:page_views).and_return(
         [
           {
@@ -19,11 +19,6 @@
           },
         ]
       )
-    end
-
-    it "updates the number of views for all content items" do
-      organisation.content_items = [content_item_first, content_item_second]
-
       subject.run('the-slug')
 
       content_item_one = ContentItem.find_by(base_path: 'the-link/first')
@@ -32,5 +27,11 @@
       expect(content_item_one.unique_page_views).to eq(3)
       expect(content_item_two.unique_page_views).to eq(2)
     end
+
+    it "perform the requests in batches" do
+      expect_any_instance_of(GoogleAnalyticsService).to receive(:page_views).twice.and_return([])
+
+      subject.run('the-slug')
+    end
   end
 end