Polish on books (#916)

* backgrounding jobs * support 5000 queries * better ux interactions
o19s · Jan 17, 2024 · 56c1c48 · 56c1c48
1 parent 9ea456d
commit 56c1c48
Show file tree

Hide file tree

Showing 68 changed files with 1,138 additions and 142 deletions.
diff --git a/Gemfile b/Gemfile
@@ -5,6 +5,7 @@ source 'https://rubygems.org'
 ruby '3.2.2'
 
 gem 'activerecord-import', '>= 1.0.7'
+gem 'active_storage_db'
 gem 'acts_as_list', '>= 1.0.1'
 gem 'ancestry'
 gem 'angular-rails-templates', '>= 1.0.0.beta'
@@ -41,8 +42,10 @@ gem 'rails', '~> 7.1.2'
 gem 'rails-healthcheck', '~> 1.4'
 gem 'rails-html-sanitizer'
 gem 'rack-cors', '~> 2.0'
+gem 'rapidjson'
 gem 'redis', '~> 5.0.6'
 gem 'responders'
+gem 'rubyzip'
 gem 'sassc-rails', '~> 2.1'
 gem 'sidekiq'
 gem 'terser'

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -50,6 +50,9 @@ GEM
       erubi (~> 1.11)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
+    active_storage_db (1.3.0)
+      activestorage (>= 6.0)
+      rails (>= 6.0)
     activejob (7.1.2)
       activesupport (= 7.1.2)
       globalid (>= 0.3.6)
@@ -359,6 +362,7 @@ GEM
       zeitwerk (~> 2.6)
     rainbow (3.1.1)
     rake (13.1.0)
+    rapidjson (0.2.3)
     rb-fsevent (0.11.2)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
@@ -476,6 +480,7 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  active_storage_db
   activerecord-import (>= 1.0.7)
   acts_as_list (>= 1.0.1)
   ancestry
@@ -521,11 +526,13 @@ DEPENDENCIES
   rails-erd (~> 1.6)
   rails-healthcheck (~> 1.4)
   rails-html-sanitizer
+  rapidjson
   redis (~> 5.0.6)
   responders
   rubocop
   rubocop-capybara
   rubocop-rails
+  rubyzip
   sassc-rails (~> 2.1)
   selenium-webdriver
   sidekiq

diff --git a/README.md b/README.md
@@ -619,13 +619,16 @@ Check out the [App Structure](docs/app_structure.md) file for more info on how Q
 
 Check out the [Operating Documentation](docs/operating_documentation.md) file for more informations how Quepid can be operated and configured for your company.
 
-# Thank You's
+# 🙏 Thank You's
 
 Quepid would not be possible without the contributions from many individuals and organizations.
 
 Specifically we would like to thank Erik Bugge and the folks at Kobler for funding the Only Rated feature released in Quepid [6.4.0](https://github.com/o19s/quepid/releases/tag/v6.4.0).
 
 Quepid wasn't always open source!  Check out the [credits](docs/credits.md) for a list of contributors to the project.
 
-
 If you would like to fund development of a new feature for Quepid do [get in touch](http://www.opensourceconnections.com/contact/)!
+
+## 🌟 Contributors
+
+[![quepid  contributors](https://contrib.rocks/image?repo=o19s/quepid&max=2000)](https://github.com/o19s/quepid/graphs/contributors)
diff --git a/app/assets/javascripts/components/add_query/add_query_controller.js b/app/assets/javascripts/components/add_query/add_query_controller.js
@@ -111,10 +111,10 @@ angular.module('QuepidApp')
       }
 
       function message() {
-        if (settingsSvc.isTrySelected() && settingsSvc.applicableSettings().searchEngine == 'static'){
+        if (settingsSvc.isTrySelected() && settingsSvc.applicableSettings().searchEngine === 'static'){
           ctrl.canAddQueries = false;
         }
-        if (ctrl.canAddQueries == true) {
+        if (ctrl.canAddQueries === true) {
           return 'Add a query to this case';
         }
         else {

diff --git a/app/assets/javascripts/components/frog_report/frog_report_controller.js b/app/assets/javascripts/components/frog_report/frog_report_controller.js
@@ -37,18 +37,15 @@ angular.module('QuepidApp')
         });
 
         modalInstance.result.then(
-          function(error) {
-            if ( !error ) {
-              queriesSvc.reset();
-              queriesSvc.bootstrapQueries(caseSvc.getSelectedCase().caseNo)
-                .then(function() {
-                  queriesSvc.searchAll();
-                });
-
-              flash.success = 'Ratings refreshed successfully!';
-            } else {
-              flash.error = error;
-            }
+          function() {
+            queriesSvc.reset();
+            queriesSvc.bootstrapQueries(caseSvc.getSelectedCase().caseNo)
+              .then(function() {
+                queriesSvc.searchAll();
+              });
+
+            flash.success = 'Ratings refreshed successfully!';
+
           }, function() { }
         );
 

diff --git a/app/assets/javascripts/components/frog_report/frog_report_modal_instance_controller.js b/app/assets/javascripts/components/frog_report/frog_report_modal_instance_controller.js
@@ -249,7 +249,7 @@ angular.module('QuepidApp')
 
 
       ctrl.ok = function () {
-        $uibModalInstance.close(ctrl.options);
+        $uibModalInstance.close();
       };
 
       ctrl.cancel = function () {

diff --git a/app/assets/javascripts/components/judgements/_modal.html b/app/assets/javascripts/components/judgements/_modal.html
@@ -47,7 +47,7 @@ <h3 class="modal-title">Judgements</h3>
       You have not created any Books of Judgements yet, so go ahead and create a new Book!
     </p>
 
-    <a class="btn btn-primary btn-lg" href="books/new" target="_self">
+    <a class="btn btn-primary btn-lg" ng-href="{{ctrl.createBookLink()}}" target="_self">
       <i class="fa fa-plus"></i>
       Create a book
     </a>
@@ -83,7 +83,7 @@ <h3 class="modal-title">Judgements</h3>
   <br/>
   <label>
     <input id="sync-queries" type="checkbox" ng-model='ctrl.createMissingQueries' ng-disabled="ctrl.share.books.length === 0 || ctrl.share.teams.length === 0"> Create missing Queries
-      <span class="glyphicon glyphicon-question-sign" aria-hidden="true" popover-trigger="'mouseenter'" popover-placement="right" uib-popover="Create queries in the Case thare are defined in the Book."></span>
+      <span class="glyphicon glyphicon-question-sign" aria-hidden="true" popover-trigger="'mouseenter'" popover-placement="right" uib-popover="Create queries in the Case that are defined in the Book."></span>
     </input>  
   </label>
 
@@ -110,8 +110,7 @@ <h3 class="modal-title">Judgements</h3>
 </div>
 
 <div class="modal-footer">
-
-  <a class="btn btn-primary pull-left" href="books/new?book[scorer_id]={{ctrl.share.acase.scorerId}}" target="_self" ng-disabled="processingPrompt.inProgress">
+  <a class="btn btn-primary pull-left" ng-href="{{ctrl.createBookLink()}}" target="_self" ng-disabled="processingPrompt.inProgress">
     <i class="glyphicon glyphicon-plus" style="color: #FFF"></i>
     Create a book
   </a>

diff --git a/app/assets/javascripts/components/judgements/judgements_controller.js b/app/assets/javascripts/components/judgements/judgements_controller.js
@@ -31,17 +31,15 @@ angular.module('QuepidApp')
         });
 
         modalInstance.result.then(
-          function(error) {
-            if ( !error ) {
+          function(bootstrapQueries) {
+            if ( bootstrapQueries ) {
               queriesSvc.reset();
               queriesSvc.bootstrapQueries(ctrl.acase.caseNo)
                 .then(function() {
                   queriesSvc.searchAll();
                 });
 
               flash.success = 'Ratings refreshed successfully!';
-            } else {
-              flash.error = error;
             }
           }, function() { }
         );

diff --git a/app/assets/javascripts/components/judgements/judgements_modal_instance_controller.js b/app/assets/javascripts/components/judgements/judgements_modal_instance_controller.js
@@ -133,7 +133,7 @@ angular.module('QuepidApp')
         bookSvc.refreshCaseRatingsFromBook(ctrl.share.acase.caseNo, ctrl.activeBookId, ctrl.createMissingQueries)
         .then(function() {
           $scope.processingPrompt.inProgress  = false;
-          $uibModalInstance.close();
+          $uibModalInstance.close(true);
 
           flash.success = 'Ratings have been refreshed.';
         }, function(response) {
@@ -154,26 +154,21 @@ angular.module('QuepidApp')
           bookSvc.updateQueryDocPairs(ctrl.activeBookId,ctrl.share.acase.caseNo, queriesSvc.queryArray(), ctrl.populateJudgements)
           .then(function() {
             $scope.processingPrompt.inProgress = false;
-            $uibModalInstance.close();
+            $uibModalInstance.close(false);
 
             flash.success = 'Book of judgements updated.';
           }, function(response) {
             $scope.processingPrompt.inProgress  = false;
             $scope.processingPrompt.error       = response.data.statusText;
 
-            // we could take the error and pass it out via
-            //$uibModalInstance.close($scope.processingPrompt.error);
+
           });
         }
         else {
-          $uibModalInstance.close();
+          $uibModalInstance.close(false);
         }
       };
 
-      //ctrl.ok = function () {
-        //$uibModalInstance.close(ctrl.share);
-      //};
-
       ctrl.cancel = function () {
         $uibModalInstance.dismiss('cancel');
       };
@@ -182,6 +177,13 @@ angular.module('QuepidApp')
         $uibModalInstance.dismiss('cancel');
         $location.path('/teams');
       };
+
+      ctrl.createBookLink = function() {
+        const teamIds = ctrl.share.acase.teams.map(function(team) {
+          return `&team_ids[]=${team.id}`;
+        });
+        return `books/new?book[scorer_id]=${ctrl.share.acase.scorerId}${teamIds}&origin_case_id=${ctrl.share.acase.caseNo}`;
+      };
 
     }
   ]);
diff --git a/app/controllers/api/v1/books/populate_controller.rb b/app/controllers/api/v1/books/populate_controller.rb
@@ -1,9 +1,12 @@
 # frozen_string_literal: true
 
+require 'action_view'
+
 module Api
   module V1
     module Books
       class PopulateController < Api::ApiController
+        include ActionView::Helpers::NumberHelper
         before_action :find_book, only: [ :update ]
         before_action :check_book, only: [ :update ]
         before_action :find_case
@@ -12,46 +15,30 @@ class PopulateController < Api::ApiController
         # We get a messy set of params in this method, so we don't use the normal
         # approach of strong parameter validation.  We hardcode the only params
         # we care about.
-        # rubocop:disable Metrics/AbcSize
-        # rubocop:disable Metrics/MethodLength
         # rubocop:disable Layout/LineLength
         def update
-          # down the road we should be using ActiveRecord-import and first_or_initialize instead.
-          # See how snapshots are managed.
-
-          is_book_empty = @book.query_doc_pairs.empty?
-
-          params[:query_doc_pairs].each do |pair|
-            query_doc_pair = @book.query_doc_pairs.find_or_create_by query_text: pair[:query_text],
-                                                                     doc_id:     pair[:doc_id]
-            query_doc_pair.position = pair[:position]
-            query_doc_pair.document_fields = pair[:document_fields].to_json
-
-            query = @case.queries.find_by(query_text: query_doc_pair.query_text)
-
-            query_doc_pair.information_need = query.information_need
-            query_doc_pair.notes = query.notes
-            query_doc_pair.options = query.options
+          puts "[PopulateController] Request Size is #{number_to_human_size(query_doc_pairs_params.to_s.bytesize)}"
 
-            if pair[:rating]
-              rating = query.ratings.find_by(doc_id: query_doc_pair.doc_id)
+          serialized_data = Marshal.dump(query_doc_pairs_params)
 
-              # we are smart and just look up the correct user id from rating.user_id via the database, no API data needed.
-              judgement = query_doc_pair.judgements.find_or_create_by user_id: rating.user_id
-              judgement.rating = pair[:rating]
-              judgement.user = rating.user
-              judgement.save!
-            end
-
-            query_doc_pair.save!
-          end
-
-          Analytics::Tracker.track_query_doc_pairs_bulk_updated_event current_user, @book, is_book_empty
+          puts "[PopulateController] the size of the serialized data is #{number_to_human_size(serialized_data.bytesize)}"
+          compressed_data = Zlib::Deflate.deflate(serialized_data)
+          puts "[PopulateController] the size of the compressed data is #{number_to_human_size(compressed_data.bytesize)}"
+          @book.populate_file.attach(io: StringIO.new(compressed_data), filename: "book_populate_#{@book.id}.bin.zip",
+                                     content_type: 'application/zip')
+          PopulateBookJob.perform_later current_user, @book, @case
           head :no_content
         end
-        # rubocop:enable Metrics/AbcSize
-        # rubocop:enable Metrics/MethodLength
         # rubocop:enable Layout/LineLength
+
+        private
+
+        def query_doc_pairs_params
+          # avoid StrongParameters ;-( to faciliate sending params as
+          # hash to ActiveJob via ActiveStorage by directly getting parameters from request
+          # object
+          request.parameters
+        end
       end
     end
   end

diff --git a/app/controllers/api/v1/books_controller.rb b/app/controllers/api/v1/books_controller.rb
@@ -5,19 +5,34 @@
 module Api
   module V1
     class BooksController < Api::ApiController
-      api!
       before_action :find_book, only: [ :show, :update, :destroy ]
       before_action :check_book, only: [ :show, :update, :destroy ]
 
+      def_param_group :book do
+        param :name, String
+        param :show_rank, [ true, false ]
+        param :support_implicit_judgements, [ true, false ]
+        param :owner_id, Integer
+        param :scorer_id, Integer
+        param :selection_strategy_id, Integer
+      end
+
+      api :GET, '/api/books',
+          'List all books to which the user has access.'
+      def index
+        @books = current_user.books_involved_with
+        respond_with @books
+      end
+
       # rubocop:disable Metrics/MethodLength
       # rubocop:disable Metrics/AbcSize
       # rubocop:disable Metrics/CyclomaticComplexity
       # rubocop:disable Metrics/PerceivedComplexity
+      api :GET, '/api/books/:book_id',
+          'Show the book with the given ID.'
+      param :id, :number,
+            desc: 'The ID of the requested book.', required: true
       def show
-        ActiveRecord::Type::Boolean.new
-        # @export  = bool.deserialize params[:export] || false
-        @export = 'true' == params[:export]
-
         respond_to do |format|
           format.json
           format.csv do
@@ -37,7 +52,7 @@ def show
             end
 
             @csv_array << csv_headers
-            @book.query_doc_pairs.order(:query_text).each do |qdp|
+            @book.query_doc_pairs.each do |qdp|
               row = [ make_csv_safe(qdp.query_text), qdp.doc_id ]
               unique_raters.each do |rater|
                 judgement = qdp.judgements.detect { |j| j.user == rater }
@@ -57,7 +72,8 @@ def show
       # rubocop:enable Metrics/AbcSize
       # rubocop:enable Metrics/CyclomaticComplexity
       # rubocop:enable Metrics/PerceivedComplexity
-
+      api :POST, '/api/books', 'Create a new book.'
+      param_group :book
       def create
         @book = Book.new(book_params)
         team = Team.find_by(id: params[:book][:team_id])
@@ -69,6 +85,10 @@ def create
         end
       end
 
+      api :PUT, '/api/books/:book_id', 'Update a given book.'
+      param :id, :number,
+            desc: 'The ID of the requested book.', required: true
+      param_group :book
       def update
         update_params = book_params
         if @book.update update_params
@@ -81,6 +101,9 @@ def update
         # render json: { error: 'Invalid id' }, status: :bad_request
       end
 
+      api :DELETE, '/api/books/:book_id', 'Delete a given book.'
+      param :id, :number,
+            desc: 'The ID of the requested book.', required: true
       def destroy
         @book.destroy
         # Analytics::Tracker.track_case_deleted_event current_user, @case
@@ -96,8 +119,11 @@ def book_params
       end
 
       # rubocop:disable Layout/LineLength
+      # def find_book
+      #  @book = current_user.books_involved_with.where(id: params[:id]).includes(:query_doc_pairs).preload([ query_doc_pairs: [ :judgements ] ]).first
+      # end
       def find_book
-        @book = current_user.books_involved_with.where(id: params[:id]).includes(:query_doc_pairs).preload([ query_doc_pairs: [ :judgements ] ]).first
+        @book = current_user.books_involved_with.where(id: params[:id]).first
       end
       # rubocop:enable Layout/LineLength