Merge remote-tracking branch 'refs/remotes/origin/master' into featur…

…e/github/stages/retry_counter

# Conflicts:
#	db/schema.rb
#	spec/app/github_app_spec.rb

.simplecov

@@ -12,11 +12,12 @@ SimpleCov.start do
   enable_coverage :branch
   primary_coverage :branch
   add_filter %r{^/spec/}
+  add_filter 'database_loader.rb'
   add_group 'Models', 'lib/models'
   add_group 'GitHub Functions', 'lib/github'
   add_group 'Bamboo CI Functions', 'lib/bamboo_ci'
   add_group 'Helpers', 'lib/helpers'
-  add_group 'Others', %w[app/ database_loader.rb]
+  add_group 'Others', %w[app/]
 
   minimum_coverage_by_file line: 90, branch: 90
 end

Gemfile

@@ -34,6 +34,9 @@ gem 'puma', '5.5.2'
 
 gem 'rake', '13.0.6'
 
+# Delayed Job
+gem 'delayed_job_active_record'
+
 # Code lint
 gem 'rubocop', '1.56.1', group: %i[development test]
 gem 'rubocop-performance', group: %i[development test]

Gemfile.lock

@@ -24,6 +24,11 @@ GEM
       activerecord (>= 5.a)
       database_cleaner-core (~> 2.0.0)
     database_cleaner-core (2.0.1)
+    delayed_job (4.1.11)
+      activesupport (>= 3.0, < 8.0)
+    delayed_job_active_record (4.1.8)
+      activerecord (>= 3.0, < 8.0)
+      delayed_job (>= 3.0, < 5)
     diff-lcs (1.5.0)
     docile (1.4.0)
     factory_bot (6.2.1)
@@ -152,6 +157,7 @@ PLATFORMS
 
 DEPENDENCIES
   database_cleaner
+  delayed_job_active_record
   factory_bot
   faker
   jwt (= 2.2.2)

Rakefile

@@ -13,6 +13,25 @@ require 'otr-activerecord'
 
 load 'tasks/otr-activerecord.rake'
 
+require_relative 'config/delayed_job'
+require_relative 'config/setup'
+
+namespace :jobs do
+  desc 'Clear the delayed_job queue.'
+  task :clear do
+    Delayed::Job.delete_all
+  end
+
+  desc 'Start a delayed_job worker.'
+  task :work do
+    puts "Starting delayed_job worker - Queues: #{ENV.fetch('QUEUES', 'default')}"
+    Delayed::Worker.new(
+      queue: ENV.fetch('QUEUES', 'default'),
+      quiet: false
+    ).start
+  end
+end
+
 namespace :db do
   # Some db tasks require your app code to be loaded; they'll expect to find it here
   task :environment do

app/github_app.rb

@@ -38,6 +38,12 @@ def sinatra_logger_level
     attr_writer :sinatra_logger_level
   end
 
+  get '/telemetry' do
+    content_type :json
+
+    Telemetry.instance.stats
+  end
+
   get '/ping' do
     halt 200, 'Pong'
   end
@@ -125,15 +131,23 @@ def sinatra_logger_level
 
       halt 200, 'OK' unless %w[rerequested].include? payload['action'].downcase
 
-      re_run = Github::Retry.new(payload, logger_level: GithubApp.sinatra_logger_level)
+      re_run = Github::Retry::Command.new(payload, logger_level: GithubApp.sinatra_logger_level)
       halt re_run.start
     when 'installation'
       logger.debug '>>> Received a new installation policy'
       halt 202, 'Updated'
     when 'issue_comment'
-      logger.debug '>>> Received a new issue comment'
-
-      halt Github::ReRun::Comment.new(payload, logger_level: GithubApp.sinatra_logger_level).start
+      halt 404, 'Action not found' if payload.nil? or payload['comment'].nil?
+
+      case payload.dig('comment', 'body').to_s.downcase
+      when /ci:retry/
+        halt Github::Retry::Comment.new(payload, logger_level: GithubApp.sinatra_logger_level).start
+      when /ci:rerun/
+        halt Github::ReRun::Comment.new(payload, logger_level: GithubApp.sinatra_logger_level).start
+      else
+        logger.debug '>>> Just a comment'
+        halt 200, 'Just a comment'
+      end
     when 'check_suite'
       logger.debug '>>> Received a new check_suite command'
       halt 200, 'OK' unless payload['action'].downcase.match?('rerequested')

bin/console.rb

@@ -16,5 +16,6 @@
 puts "Starting console: #{ENV.fetch('RAILS_ENV', nil)}"
 
 require_relative '../config/setup'
+require_relative '../config/delayed_job'
 
 IRB.start

config.ru

@@ -9,15 +9,27 @@
 #  frozen_string_literal: true
 
 require_relative 'app/github_app'
+require_relative 'config/delayed_job'
 
 require 'puma'
 require 'rack/handler/puma'
 require 'rack/session/cookie'
 
 File.write('.session.key', SecureRandom.hex(32))
 
+pids = []
+pids << spawn("RACK_ENV=#{ENV.fetch('RACK_ENV', 'development')} rake jobs:work QUEUES=0,1,2,3")
+pids << spawn("RACK_ENV=#{ENV.fetch('RACK_ENV', 'development')} rake jobs:work QUEUES=4,5,6")
+pids << spawn("RACK_ENV=#{ENV.fetch('RACK_ENV', 'development')} rake jobs:work QUEUES=7,8,9")
+
 use Rack::Session::Cookie, secret: File.read('.session.key'), same_site: true, max_age: 86_400
 
 Rack::Handler::Puma.run Rack::URLMap.new('/' => GithubApp)
 
+pids.each do |pid|
+  Process.kill('TERM', pid.to_i)
+rescue Errno::ESRCH
+  puts "Process #{pid} already dead"
+end
+
 exit 0

config/delayed_job.rb

@@ -0,0 +1,38 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  delayed_job.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2024 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+require_relative '../lib/helpers/github_logger'
+require_relative '../database_loader'
+
+require 'delayed_job'
+require 'active_support'
+
+module Rails
+  class << self
+    attr_accessor :logger
+  end
+end
+
+DELAYED_JOB_TIMER = 5
+
+Rails.logger = GithubLogger.instance.create('delayed_job.log', Logger::INFO)
+ActiveRecord::Base.logger = GithubLogger.instance.create('delayed_job.log', Logger::INFO)
+
+# this is used by DJ to guess where tmp/pids is located (default)
+RAILS_ROOT = File.expand_path(__FILE__)
+
+Delayed::Worker.backend = :active_record
+Delayed::Worker.destroy_failed_jobs = true
+Delayed::Worker.sleep_delay = 5
+Delayed::Worker.max_attempts = 5
+Delayed::Worker.max_run_time = 5.minutes
+
+config = YAML.load_file('config/database.yml')[ENV.fetch('RACK_ENV', 'development')]
+ActiveRecord::Base.establish_connection(config)

config/puma.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 #  SPDX-License-Identifier: BSD-2-Clause
 #
 #  puma.rb
@@ -8,16 +6,28 @@
 #  Copyright (c) 2023 by
 #  Network Device Education Foundation, Inc. ("NetDEF")
 #
+#  frozen_string_literal: true
 
 require_relative '../config/setup'
 require 'puma'
 
 workers 10
 
-threads_count = (ENV['RAILS_MAX_THREADS'] || 5).to_i
-threads 1, threads_count
+threads 1, (ENV['RAILS_MAX_THREADS'] || 5).to_i
+
 port GitHubApp::Configuration.instance.config['port'] || 4667
 
+activate_control_app
+
 preload_app!
 
 pidfile 'puma.pid'
+
+before_fork do
+  Thread.new do
+    loop do
+      Telemetry.instance.update_stats Puma.stats
+      sleep 30
+    end
+  end
+end

database_loader.rb

@@ -13,6 +13,7 @@
 OTR::ActiveRecord.db_dir = 'db'
 OTR::ActiveRecord.migrations_paths = ['db/migrate']
 OTR::ActiveRecord.configure_from_file! 'config/database.yml'
+ActiveRecord::Base.logger&.level = Logger::WARN
 
 Dir['lib/models/*.rb'].each { |model| require_relative model }
 

db/migrate/20240408141736_add_check_suite_cancelled_ref.rb

@@ -0,0 +1,17 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  20240408141736_add_check_suite_cancelled_ref.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2024 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+class AddCheckSuiteCancelledRef < ActiveRecord::Migration[6.0]
+  def change
+    add_column :check_suites, :cancelled_by_id, :bigint
+    add_index :check_suites, :cancelled_by_id
+    add_foreign_key :check_suites, :stages, column: :cancelled_by_id
+  end
+end

db/migrate/20240408164410_add_check_suite_cancelled_in_stage.rb

@@ -0,0 +1,15 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  20240408152636_add_check_suite_cancelled_in_stage.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2024 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+class AddCheckSuiteCancelledInStage < ActiveRecord::Migration[6.0]
+  def change
+    add_reference :stages, :cancelled_at_stage, foreign_key: { to_table: :check_suites }
+  end
+end

db/migrate/20240417094603_change_check_suite_cancelled_by_id.rb

@@ -0,0 +1,17 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  20240417094603_change_check_suite_cancelled_by_id.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2024 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+class ChangeCheckSuiteCancelledById < ActiveRecord::Migration[6.0]
+  def change
+    change_table :check_suites do |t|
+      t.references :cancelled_previous_check_suite, foreign_key: { to_table: :check_suites }
+    end
+  end
+end

db/migrate/20240417102854_remove_check_suite_columns.rb

@@ -0,0 +1,19 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  20240417102854_remove_check_suite_columns.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2024 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+class RemoveCheckSuiteColumns < ActiveRecord::Migration[6.0]
+  def change
+    if ActiveRecord::Base.connection.column_exists?(:check_suites, :cancelled_by_id)
+      remove_column :check_suites, :cancelled_by_id
+    end
+
+    remove_column :check_suites, :id_id if ActiveRecord::Base.connection.column_exists?(:check_suites, :id_id)
+  end
+end

db/migrate/20240417130601_change_check_suite_stopped_in_stage.rb

@@ -0,0 +1,30 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  202404130601_change_check_suite_stopped_in_stage.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2024 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+class ChangeCheckSuiteStoppedInStage < ActiveRecord::Migration[6.0]
+  def change
+    if ActiveRecord::Base.connection.column_exists?(:stages, :cancelled_at_stage_id)
+      remove_column :stages, :cancelled_at_stage_id
+    end
+
+    if ActiveRecord::Base.connection.column_exists?(:check_suites, :cancelled_in_stage_id)
+      remove_column :check_suites, :cancelled_in_stage_id
+    end
+
+    if ActiveRecord::Base.connection.column_exists?(:check_suites, :cancelled_previous_check_suite_id)
+      remove_column :check_suites, :cancelled_previous_check_suite_id
+    end
+
+    change_table :check_suites do |t|
+      t.references :stopped_in_stage, foreign_key: { to_table: :stages }
+      t.references :cancelled_previous_check_suite, foreign_key: { to_table: :check_suites }
+    end
+  end
+end

db/migrate/20240617121935_create_delayed_jobs.rb

@@ -0,0 +1,26 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  20240617121935_create_delayed_jobs.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2024 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+class CreateDelayedJobs < ActiveRecord::Migration[6.0]
+  def change
+    create_table :delayed_jobs do |t|
+      t.integer :priority, default: 0, null: false
+      t.integer :attempts, default: 0, null: false
+      t.text :handler, null: false
+      t.text :last_error
+      t.datetime :run_at
+      t.datetime :locked_at
+      t.datetime :failed_at
+      t.string :locked_by
+      t.string :queue
+      t.timestamps null: false
+    end
+  end
+end