Merge pull request #76 from alphagov/numviews-for-all

Import Google Analytics data in batches

app/models/importers/number_of_views_by_organisation.rb

@@ -1,14 +1,19 @@
 module Importers
   class NumberOfViewsByOrganisation
+    BATCH_SIZE = 3000
+
     def run(slug)
       organisation = Organisation.find_by(slug: slug)
+      google_analytics_service = GoogleAnalyticsService.new
 
-      base_paths = organisation.content_items.pluck(:base_path)
+      organisation.content_items.find_in_batches(batch_size: BATCH_SIZE) do |content_items|
+        base_paths = content_items.pluck(:base_path)
 
-      results = GoogleAnalyticsService.new.page_views(base_paths)
-      results.each do |result|
-        content_item = ContentItem.find_by(base_path: result[:base_path])
-        content_item.update!(unique_page_views: result[:page_views])
+        results = google_analytics_service.page_views(base_paths)
+        results.each do |result|
+          content_item = ContentItem.find_by(base_path: result[:base_path])
+          content_item.update!(unique_page_views: result[:page_views])
+        end
       end
     end
   end

spec/models/importers/number_of_views_by_organisation_spec.rb

@@ -6,7 +6,7 @@
     let!(:content_item_first) { create(:content_item, base_path: 'the-link/first', organisations: [organisation]) }
     let!(:content_item_second) { create(:content_item, base_path: 'the-link/second', organisations: [organisation]) }
 
-    before do
+    it "updates the number of views for all content items" do
       allow_any_instance_of(GoogleAnalyticsService).to receive(:page_views).and_return(
         [
           {
@@ -19,10 +19,8 @@
           },
         ]
       )
-    end
 
-    it "updates the number of views for all content items" do
-      organisation.content_items = [content_item_first, content_item_second]
+      stub_const("Importers::NumberOfViewsByOrganisation::BATCH_SIZE", 1)
 
       subject.run('the-slug')
 
@@ -32,5 +30,23 @@
       expect(content_item_one.unique_page_views).to eq(3)
       expect(content_item_two.unique_page_views).to eq(2)
     end
+
+    context "performs the request to google api in batches" do
+      it "makes two requests when the batch size is one" do
+        expect_any_instance_of(GoogleAnalyticsService).to receive(:page_views).twice.and_return([])
+
+        stub_const("Importers::NumberOfViewsByOrganisation::BATCH_SIZE", 1)
+
+        subject.run('the-slug')
+      end
+
+      it "makes one request when the batch size is two" do
+        expect_any_instance_of(GoogleAnalyticsService).to receive(:page_views).once.and_return([])
+
+        stub_const("Importers::NumberOfViewsByOrganisation::BATCH_SIZE", 2)
+
+        subject.run('the-slug')
+      end
+    end
   end
 end