Set up good_job for ActiveJob backend (#1097)

As part of slimming down the production cluster (edgi-govdata-archiving/web-monitoring-ops#45), we can use good_job to do our job queuing in Postgres rather than running a Redis instance (we still use Redis as a cache, but that is outside the cluster).

.env.example

@@ -6,9 +6,6 @@ HOST_URL='web-monitoring-db.dev'
 # specified in config/database.yml
 # DATABASE_URL=postgres://user:password@localhost:5432/db-name
 
-# OPTIONAL: only set this if your Redis is at a non-standard location
-# REDIS_URL=redis://user:password@localhost:6379
-
 # E-mail address to use as the "from" address
 MAIL_SENDER='[email protected]'
 

Dockerfile

@@ -35,10 +35,7 @@ FROM base as import-worker
 LABEL maintainer="[email protected]"
 WORKDIR /app
 
-ENV QUEUES=import,analysis
-ENV VERBOSE=1
-
-CMD ["bundle", "exec", "rake", "environment", "resque:work"]
+CMD ["bundle", "exec", "good_job", "start"]
 
 
 ### RAILS SERVER TARGET ###

Gemfile

@@ -25,10 +25,9 @@ gem 'google-apis-sheets_v4'
 gem 'addressable', '~> 2.8'
 
 # Workers/Queuing
-# Resque 2.3.0 is not compatible with Redis v5; if updated, see about updating
-# the redis gem as well. See: https://github.com/resque/resque/pull/1828
-gem 'resque', '~> 2.4.0'
-# gem 'resque-heroku-signals'
+gem "good_job", "~> 3.14"
+
+# Caching
 gem 'redis', '~> 5.0'
 gem 'hiredis'
 

Gemfile.lock

@@ -123,14 +123,27 @@ GEM
       dotenv (= 2.8.1)
       railties (>= 3.2)
     erubi (1.12.0)
+    et-orbi (1.2.7)
+      tzinfo
     execjs (2.8.1)
     faraday (2.7.4)
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
     faraday-net_http (3.0.2)
     ffi (1.15.5)
+    fugit (1.8.1)
+      et-orbi (~> 1, >= 1.2.7)
+      raabro (~> 1.4)
     globalid (1.1.0)
       activesupport (>= 5.0)
+    good_job (3.14.2)
+      activejob (>= 6.0.0)
+      activerecord (>= 6.0.0)
+      concurrent-ruby (>= 1.0.2)
+      fugit (>= 1.1)
+      railties (>= 6.0.0)
+      thor (>= 0.14.1)
+      webrick (>= 1.3)
     google-apis-core (0.10.0)
       addressable (~> 2.5, >= 2.5.1)
       googleauth (>= 0.16.2, < 2.a)
@@ -177,12 +190,9 @@ GEM
     method_source (1.0.0)
     mini_mime (1.1.2)
     minitest (5.17.0)
-    mono_logger (1.1.1)
     msgpack (1.6.0)
     multi_json (1.15.0)
     multi_xml (0.6.0)
-    mustermann (3.0.0)
-      ruby2_keywords (~> 0.0.1)
     net-imap (0.3.4)
       date
       net-protocol
@@ -219,15 +229,14 @@ GEM
       nio4r (~> 2.0)
     pundit (2.3.0)
       activesupport (>= 3.0.0)
+    raabro (1.4.0)
     racc (1.6.2)
     rack (2.2.6.4)
     rack-brotli (1.2.0)
       brotli (>= 0.1.7)
       rack (>= 1.4)
     rack-cors (1.1.1)
       rack (>= 2.0.0)
-    rack-protection (3.0.4)
-      rack
     rack-test (2.0.2)
       rack (>= 1.3)
     rails (7.0.4.2)
@@ -265,8 +274,6 @@ GEM
       redis-client (>= 0.9.0)
     redis-client (0.12.1)
       connection_pool
-    redis-namespace (1.9.0)
-      redis (>= 4)
     regexp_parser (2.6.2)
     representable (3.2.0)
       declarative (< 0.1.0)
@@ -275,11 +282,6 @@ GEM
     responders (3.0.1)
       actionpack (>= 5.0)
       railties (>= 5.0)
-    resque (2.4.0)
-      mono_logger (~> 1.0)
-      multi_json (~> 1.0)
-      redis-namespace (~> 1.6)
-      sinatra (>= 0.9.2)
     retriable (3.1.2)
     rexml (3.2.5)
     rubocop (1.44.1)
@@ -326,11 +328,6 @@ GEM
       faraday (>= 0.17.5, < 3.a)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
-    sinatra (3.0.4)
-      mustermann (~> 3.0)
-      rack (~> 2.2, >= 2.2.4)
-      rack-protection (= 3.0.4)
-      tilt (~> 2.0)
     spring (4.1.1)
     spring-watcher-listen (2.1.0)
       listen (>= 2.7, < 4.0)
@@ -386,6 +383,7 @@ DEPENDENCIES
   concurrent-ruby (~> 1.2)
   devise
   dotenv-rails
+  good_job (~> 3.14)
   google-apis-sheets_v4
   hiredis
   httparty
@@ -401,7 +399,6 @@ DEPENDENCIES
   rack-cors
   rails (~> 7.0.4.2)
   redis (~> 5.0)
-  resque (~> 2.4.0)
   rubocop (~> 1.44.1)
   rubocop-performance (~> 1.15.2)
   rubocop-rails (~> 2.17.4)

README.md

@@ -28,7 +28,7 @@ We maintain a publicly available *staging server* at https://api-staging.monitor
 
 2. Ensure you have PostgreSQL 9.5+. If you are on MacOS, we recommend [Postgres.app](https://postgresapp.com). It makes running multiple versions of PostgreSQL much simpler and gives you easy access to start and stop your databases.
 
-3. Ensure you have [Redis](https://redis.io)
+3. Ensure you have [Redis](https://redis.io) (used for caching).
 
     On MacOS:
 
@@ -157,24 +157,16 @@ We maintain a publicly available *staging server* at https://api-staging.monitor
 
     You should now have a server running and can visit it at http://localhost:3000/. Open that up in a browser and go to town!
 
-11. Bulk importing, automated analysis, and e-mail invitations all run as asynchronous jobs, managed by a Redis queue. If you plan to use any of these features, you must also start a Redis server and worker.
-
-    Start redis:
-
-    ```sh
-    $ redis-server
-    ```
-
-    Start a worker:
+11. Bulk importing, automated analysis, and e-mail invitations all run as asynchronous jobs (using the fantastic [good_job gem](https://github.com/bensheldon/good_job)). If you plan to use any of these features, you must also start a worker:
 
     ```sh
-    $ QUEUE=* VERBOSE=1 bundle exec rake environment resque:work
+    $ bundle exec good_job start
     ```
 
-    If you only want to run particular type of job, you can set a list of queue names in the `QUEUES` environment variable:
+    If you only want to run particular type of job, you can set a list of queue names with the `--queues` option:
 
     ```sh
-    $ QUEUES=mailers,import,analysis VERBOSE=1 bundle exec rake environment resque:work
+    $ bundle exec good_job start --queues=mailers,import,analysis
     ```
 
     Each job type runs on a different queue:

Rakefile

@@ -1,7 +1,6 @@
 # Add your own tasks in files placed in lib/tasks ending in .rake,
 # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
 
-require 'resque/tasks'
 require_relative 'config/application'
 
 Rails.application.load_tasks

app/controllers/healthcheck_controller.rb

@@ -9,13 +9,6 @@ def index
       db = error.to_s
     end
 
-    queues = 'ok'
-    begin
-      Resque.size(:not_a_real_queue_but_thats_ok)
-    rescue StandardError => error
-      queues = error.to_s
-    end
-
-    render json: { app: 'ok', db:, queues: }
+    render json: { app: 'ok', db: }
   end
 end

app/jobs/import_versions_job.rb

@@ -24,12 +24,8 @@ def perform(import)
     end
 
     if AnalyzeChangeJob.supported?
-      begin
-        @added.uniq(&:uuid).each do |version|
-          AnalyzeChangeJob.perform_later(version) if version.different?
-        end
-      rescue Redis::CannotConnectError => error
-        Rails.logger.error "Import #{import.id}: Cannot queue AnalyzeChangeJob -- #{error.message}"
+      @added.uniq(&:uuid).each do |version|
+        AnalyzeChangeJob.perform_later(version) if version.different?
       end
     else
       Rails.logger.warn "Import #{import.id}: Auto-analysis is not configured; AnalyzeChangeJobs were not scheduled for imported versions."

bin/worker_dev

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 set -e
 
-QUEUE=* VERBOSE=1 RESQUE_PRE_SHUTDOWN_TIMEOUT=10 RESQUE_TERM_TIMEOUT=10 bundle exec rake environment resque:work
+bundle exec good_job start

config/application.rb

@@ -11,7 +11,7 @@ class Application < Rails::Application
     config.load_defaults 7.0
     config.eager_load_paths << "#{Rails.root}/lib/api"
 
-    config.active_job.queue_adapter = :resque
+    config.active_job.queue_adapter = :good_job
 
     # Deliver mail on the `mailers` queue. This is the old default from
     # Rails 6.0 and earlier; I think it's useful. Keeping it also lets us

config/environments/production.rb

@@ -67,10 +67,6 @@
   # Use a different cache store in production.
   # config.cache_store = :mem_cache_store
 
-  # Use a real queuing backend for Active Job (and separate queues per environment).
-  # config.active_job.queue_adapter     = :resque
-  # config.active_job.queue_name_prefix = "webpage_versions_db_production"
-
   # Action Mailer configuration
   config.action_mailer.default_url_options = {
     host: ENV.fetch('HOST_URL', 'api.monitoring.envirodatagov.org').chomp('/')

config/initializers/good_job.rb

@@ -0,0 +1,7 @@
+# Global options
+GoodJob.active_record_parent_class = 'ApplicationRecord'
+
+# Application options
+Rails.application.configure do
+  config.good_job.on_thread_error = ->(exception) { Sentry.capture_exception(exception) }
+end

config/routes.rb

@@ -58,5 +58,10 @@
   delete 'admin/destroy_user'
   post 'admin/destroy_user'
 
+  # Show the good_job control panel at "/admin/good_job"
+  authenticate :user, ->(user) { Pundit.authorize(user, :admin, :any?) } do
+    mount GoodJob::Engine => 'admin/good_job'
+  end
+
   get 'healthcheck', to: 'healthcheck#index'
 end

db/migrate/20230328225317_create_good_jobs.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+class CreateGoodJobs < ActiveRecord::Migration[7.0]
+  def change
+    enable_extension 'pgcrypto'
+
+    create_table :good_jobs, id: :uuid do |t|
+      t.text :queue_name
+      t.integer :priority
+      t.jsonb :serialized_params
+      t.datetime :scheduled_at
+      t.datetime :performed_at
+      t.datetime :finished_at
+      t.text :error
+
+      t.timestamps
+
+      t.uuid :active_job_id
+      t.text :concurrency_key
+      t.text :cron_key
+      t.uuid :retried_good_job_id
+      t.datetime :cron_at
+
+      t.uuid :batch_id
+      t.uuid :batch_callback_id
+    end
+
+    create_table :good_job_batches, id: :uuid do |t|
+      t.timestamps
+      t.text :description
+      t.jsonb :serialized_properties
+      t.text :on_finish
+      t.text :on_success
+      t.text :on_discard
+      t.text :callback_queue_name
+      t.integer :callback_priority
+      t.datetime :enqueued_at
+      t.datetime :discarded_at
+      t.datetime :finished_at
+    end
+
+    create_table :good_job_processes, id: :uuid do |t|
+      t.timestamps
+      t.jsonb :state
+    end
+
+    create_table :good_job_settings, id: :uuid do |t|
+      t.timestamps
+      t.text :key
+      t.jsonb :value
+      t.index :key, unique: true
+    end
+
+    add_index :good_jobs, :scheduled_at, where: '(finished_at IS NULL)', name: 'index_good_jobs_on_scheduled_at'
+    add_index :good_jobs, [:queue_name, :scheduled_at], where: '(finished_at IS NULL)', name: :index_good_jobs_on_queue_name_and_scheduled_at
+    add_index :good_jobs, [:active_job_id, :created_at], name: :index_good_jobs_on_active_job_id_and_created_at
+    add_index :good_jobs, :concurrency_key, where: '(finished_at IS NULL)', name: :index_good_jobs_on_concurrency_key_when_unfinished
+    add_index :good_jobs, [:cron_key, :created_at], name: :index_good_jobs_on_cron_key_and_created_at
+    add_index :good_jobs, [:cron_key, :cron_at], name: :index_good_jobs_on_cron_key_and_cron_at, unique: true
+    add_index :good_jobs, [:active_job_id], name: :index_good_jobs_on_active_job_id
+    add_index :good_jobs, [:finished_at], where: 'retried_good_job_id IS NULL AND finished_at IS NOT NULL', name: :index_good_jobs_jobs_on_finished_at
+    add_index :good_jobs, [:priority, :created_at], order: { priority: 'DESC NULLS LAST', created_at: :asc },
+                                                    where: 'finished_at IS NULL', name: :index_good_jobs_jobs_on_priority_created_at_when_unfinished
+    add_index :good_jobs, [:batch_id], where: 'batch_id IS NOT NULL'
+    add_index :good_jobs, [:batch_callback_id], where: 'batch_callback_id IS NOT NULL'
+  end
+end