From ed5393bc01f3a61c9d2b8490b3941a54de0dada1 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Tue, 9 Apr 2024 08:23:19 -0300 Subject: [PATCH 01/53] Detailed execution info Possibility of identifying when a test is interrupted by a new commit or ci:rerun and saving which stage was being executed at the time of the interruption. Signed-off-by: Rodrigo Nardi --- ...408141736_add_check_suite_cancelled_ref.rb | 17 ++++++++ ...4410_add_check_suite_cancelled_in_stage.rb | 15 +++++++ db/schema.rb | 8 +++- lib/github/build_plan.rb | 3 ++ lib/models/check_suite.rb | 2 + reports/github_user_usage.rb | 42 +++++++++++++++++++ spec/factories/ci_job.rb | 2 +- 7 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 db/migrate/20240408141736_add_check_suite_cancelled_ref.rb create mode 100644 db/migrate/20240408164410_add_check_suite_cancelled_in_stage.rb create mode 100644 reports/github_user_usage.rb diff --git a/db/migrate/20240408141736_add_check_suite_cancelled_ref.rb b/db/migrate/20240408141736_add_check_suite_cancelled_ref.rb new file mode 100644 index 0000000..a748fba --- /dev/null +++ b/db/migrate/20240408141736_add_check_suite_cancelled_ref.rb @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# 20240408141736_add_check_suite_cancelled_ref.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +class AddCheckSuiteCancelledRef < ActiveRecord::Migration[6.0] + def change + add_column :check_suites, :cancelled_by_id, :bigint + add_index :check_suites, :cancelled_by_id + add_foreign_key :check_suites, :stages, column: :cancelled_by_id + end +end diff --git a/db/migrate/20240408164410_add_check_suite_cancelled_in_stage.rb b/db/migrate/20240408164410_add_check_suite_cancelled_in_stage.rb new file mode 100644 index 0000000..40c41e9 --- /dev/null +++ b/db/migrate/20240408164410_add_check_suite_cancelled_in_stage.rb @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# 20240408152636_add_check_suite_cancelled_in_stage.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +class AddCheckSuiteCancelledInStage < ActiveRecord::Migration[6.0] + def change + add_reference :stages, :cancelled_at_stage, foreign_key: { to_table: :check_suites } + end +end diff --git a/db/schema.rb b/db/schema.rb index 5c773d5..45a3cbb 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2024_03_27_112035) do +ActiveRecord::Schema[7.0].define(version: 2024_04_08_164410) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -58,6 +58,8 @@ t.integer "retry", default: 0 t.boolean "sync", default: false t.bigint "github_user_id" + t.bigint "cancelled_by_id" + t.index ["cancelled_by_id"], name: "index_check_suites_on_cancelled_by_id" t.index ["github_user_id"], name: "index_check_suites_on_github_user_id" t.index ["pull_request_id"], name: "index_check_suites_on_pull_request_id" end @@ -142,6 +144,8 @@ t.datetime "updated_at", null: false t.bigint "check_suite_id" t.bigint "stage_configuration_id" + t.bigint "cancelled_at_stage_id" + t.index ["cancelled_at_stage_id"], name: "index_stages_on_cancelled_at_stage_id" t.index ["check_suite_id"], name: "index_stages_on_check_suite_id" t.index ["stage_configuration_id"], name: "index_stages_on_stage_configuration_id" end @@ -161,12 +165,14 @@ add_foreign_key "audit_retries", "github_users" add_foreign_key "check_suites", "github_users" add_foreign_key "check_suites", "pull_requests" + add_foreign_key "check_suites", "stages", column: "cancelled_by_id" add_foreign_key "ci_jobs", "check_suites" add_foreign_key "ci_jobs", "stages" add_foreign_key "plans", "check_suites" add_foreign_key "pull_request_subscriptions", "pull_requests" add_foreign_key "pull_requests", "github_users" add_foreign_key "stages", "check_suites" + add_foreign_key "stages", "check_suites", column: "cancelled_at_stage_id" add_foreign_key "stages", "stage_configurations" add_foreign_key "topotest_failures", "ci_jobs" end diff --git a/lib/github/build_plan.rb b/lib/github/build_plan.rb index bb3423e..7b16eee 100644 --- a/lib/github/build_plan.rb +++ b/lib/github/build_plan.rb @@ -120,6 +120,8 @@ def cancel_previous_ci_jobs ci_job.cancelled(@github_check) end + @last_check_suite.update(cancelled_in_stage: @last_check_suite.stages.where(status: :in_progress).last) + @last_check_suite.stages.where(status: %w[queued in_progress]).each do |stage| stage.cancelled(@github_check) end @@ -169,6 +171,7 @@ def ci_jobs end def stop_execution_message + @last_check_suite.update(cancelled_by_new_check_suite: @check_suite) BambooCi::StopPlan.comment(@last_check_suite, @check_suite) end diff --git a/lib/models/check_suite.rb b/lib/models/check_suite.rb index ee7e50d..9984965 100644 --- a/lib/models/check_suite.rb +++ b/lib/models/check_suite.rb @@ -15,6 +15,8 @@ class CheckSuite < ActiveRecord::Base validates :commit_sha_ref, presence: true belongs_to :pull_request + has_one :cancelled_by_new_check_suite, class_name: 'CheckSuite', foreign_key: 'cancelled_by_id' + has_one :cancelled_in_stage, class_name: 'Stage', foreign_key: 'cancelled_at_stage_id' has_many :ci_jobs, dependent: :delete_all has_many :stages, dependent: :delete_all has_many :audit_retries, dependent: :delete_all diff --git a/reports/github_user_usage.rb b/reports/github_user_usage.rb new file mode 100644 index 0000000..794a916 --- /dev/null +++ b/reports/github_user_usage.rb @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# github_user_usage.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +require_relative '../database_loader' + +ActiveRecord::Base.logger = Logger.new('/dev/null') + +user = GithubUser.find_by_github_login(ARGV[0]) + +if user.pull_requests.empty? + puts "No pull requests found for user: #{user.github_login}" +else + puts "Pull Requests for user: #{user.github_login}" + user.pull_requests.each do |pr| + puts "Pull Request: https://github.com/FRRouting/frr/pull/#{pr.github_pr_id}" + end +end + +if user.check_suites.empty? + puts "No check suites found for user: #{user.github_login}" +else + puts "Check Suites for user: #{user.github_login}" + user.check_suites.each do |cs| + puts "Check Suite: https://ci1.netdef.org/browse/#{cs.bamboo_ci_ref}" + end +end + +if user.audit_retries.empty? + puts "No audit retries found for user: #{user.github_login}" +else + puts "Audit Retries for user: #{user.github_login}" + user.audit_retries.each do |ar| + puts "Audit Retry: #{ar.retry_type} at #{ar.created_at}" + end +end diff --git a/spec/factories/ci_job.rb b/spec/factories/ci_job.rb index 7367994..2679b5c 100644 --- a/spec/factories/ci_job.rb +++ b/spec/factories/ci_job.rb @@ -16,7 +16,7 @@ check_ref { Faker::Alphanumeric.alphanumeric(number: 18, min_alpha: 3, min_numeric: 3) } check_suite - stage { create(:stage, check_suite: check_suite) } + stage { create(:stage, status: status, check_suite: check_suite) } trait :topotest_failure do after(:create) do |ci_job| From c3e853288508557aa9985c86c6a095430130e38e Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Thu, 11 Apr 2024 11:14:54 -0300 Subject: [PATCH 02/53] Pull Request - Current Execution Fixed bug that did not correctly return the current PR execution. This ended up causing some problems when updating the CI. Signed-off-by: Rodrigo Nardi --- lib/models/check_suite.rb | 2 ++ lib/models/ci_job.rb | 2 ++ lib/models/pull_request.rb | 8 ++++++-- lib/models/stage.rb | 2 ++ spec/lib/models/pull_request_spec.rb | 19 +++++++++++++++++++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/lib/models/check_suite.rb b/lib/models/check_suite.rb index ee7e50d..ae31234 100644 --- a/lib/models/check_suite.rb +++ b/lib/models/check_suite.rb @@ -19,6 +19,8 @@ class CheckSuite < ActiveRecord::Base has_many :stages, dependent: :delete_all has_many :audit_retries, dependent: :delete_all + default_scope -> { order(id: :asc) }, all_queries: true + def finished? !running? end diff --git a/lib/models/ci_job.rb b/lib/models/ci_job.rb index 9d12306..cd9fed7 100644 --- a/lib/models/ci_job.rb +++ b/lib/models/ci_job.rb @@ -30,6 +30,8 @@ class CiJob < ActiveRecord::Base scope :not_skipped, -> { where.not(status: 'skipped') } scope :failure, -> { where(status: %i[failure cancelled skipped]) } + default_scope -> { order(id: :asc) }, all_queries: true + def finished? !%w[queued in_progress].include?(status) end diff --git a/lib/models/pull_request.rb b/lib/models/pull_request.rb index a803593..0416702 100644 --- a/lib/models/pull_request.rb +++ b/lib/models/pull_request.rb @@ -26,10 +26,14 @@ def new? def finished? return true if new? - check_suites.last.finished? + current_execution&.finished? end def current_execution?(check_suite) - check_suites.last == check_suite + current_execution == check_suite + end + + def current_execution + check_suites.max_by(&:id) end end diff --git a/lib/models/stage.rb b/lib/models/stage.rb index 7b23dac..25d383b 100644 --- a/lib/models/stage.rb +++ b/lib/models/stage.rb @@ -16,6 +16,8 @@ class Stage < ActiveRecord::Base belongs_to :configuration, class_name: 'StageConfiguration', foreign_key: 'stage_configuration_id' belongs_to :check_suite + default_scope -> { order(id: :asc) }, all_queries: true + def running? jobs.where(status: %i[queued in_progress]).any? end diff --git a/spec/lib/models/pull_request_spec.rb b/spec/lib/models/pull_request_spec.rb index 369b70a..88ee1c7 100644 --- a/spec/lib/models/pull_request_spec.rb +++ b/spec/lib/models/pull_request_spec.rb @@ -40,4 +40,23 @@ expect(pull_request.finished?).to be_falsey end end + + context 'when current execution is not the last check suite' do + let(:pull_request) { create(:pull_request) } + let(:check_suite1) { create(:check_suite, pull_request: pull_request) } + let(:check_suite2) { create(:check_suite, pull_request: pull_request) } + let(:check_suite3) { create(:check_suite, pull_request: pull_request) } + + before do + check_suite1 + check_suite2 + check_suite3 + + allow(pull_request).to receive(:check_suites).and_return([check_suite2, check_suite3, check_suite1]) + end + + it 'must return true' do + expect(pull_request.current_execution?(check_suite3)).to be_truthy + end + end end From 69474a6fc53f6c4eb5a0d0e380d67055cc37db33 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Fri, 12 Apr 2024 08:04:00 -0300 Subject: [PATCH 03/53] Pull Request - Current Execution Removing unnecessary code and updating unit tests Signed-off-by: Rodrigo Nardi --- lib/models/pull_request.rb | 11 ++++------- spec/lib/models/pull_request_spec.rb | 26 ++++++++------------------ 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/lib/models/pull_request.rb b/lib/models/pull_request.rb index 0416702..5f5da5d 100644 --- a/lib/models/pull_request.rb +++ b/lib/models/pull_request.rb @@ -19,21 +19,18 @@ class PullRequest < ActiveRecord::Base has_many :check_suites, dependent: :delete_all has_many :pull_request_subscriptions, dependent: :delete_all - def new? - check_suites.nil? or check_suites.empty? - end - def finished? - return true if new? + return true if check_suites.nil? or check_suites.empty? - current_execution&.finished? + current_execution.finished? end def current_execution?(check_suite) current_execution == check_suite end + # @return [CheckSuite] def current_execution - check_suites.max_by(&:id) + check_suites.order(id: :asc).last end end diff --git a/spec/lib/models/pull_request_spec.rb b/spec/lib/models/pull_request_spec.rb index 88ee1c7..ddfb9ef 100644 --- a/spec/lib/models/pull_request_spec.rb +++ b/spec/lib/models/pull_request_spec.rb @@ -9,22 +9,6 @@ # frozen_string_literal: true describe PullRequest do - context 'when create a new PR' do - let(:pull_request) { create(:pull_request) } - - it 'must return true' do - expect(pull_request.new?).to be_truthy - end - end - - context 'when create a new PR with Check Suite' do - let(:pull_request) { create(:pull_request, :with_check_suite) } - - it 'must return true' do - expect(pull_request.new?).to be_falsey - end - end - context 'when create a new PR and check if check suite was finished' do let(:pull_request) { create(:pull_request) } @@ -51,12 +35,18 @@ check_suite1 check_suite2 check_suite3 - - allow(pull_request).to receive(:check_suites).and_return([check_suite2, check_suite3, check_suite1]) end it 'must return true' do expect(pull_request.current_execution?(check_suite3)).to be_truthy end end + + context 'when current execution is nil' do + let(:pull_request) { create(:pull_request) } + + it 'must return true in finished?' do + expect(pull_request.finished?).to be_truthy + end + end end From 01f8a298b0f66e99a0005151b654751bce75086e Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Fri, 12 Apr 2024 10:06:58 -0300 Subject: [PATCH 04/53] PlanExecution::Finished Using pull_request methods to check if it is the current execution. Signed-off-by: Rodrigo Nardi --- lib/github/plan_execution/finished.rb | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/lib/github/plan_execution/finished.rb b/lib/github/plan_execution/finished.rb index a9d78a3..2f4f0da 100644 --- a/lib/github/plan_execution/finished.rb +++ b/lib/github/plan_execution/finished.rb @@ -118,19 +118,12 @@ def build_summary(ci_job) end def finished_execution?(check_suite) - return false unless current_execution?(check_suite) + return false unless check_suite.pull_request.current_execution?(check_suite) return false unless check_suite.finished? SlackBot.instance.execution_finished_notification(check_suite) end - def current_execution?(check_suite) - pull_request = check_suite.pull_request - last_check_suite = pull_request.check_suites.reload.all.order(:created_at).last - - check_suite.id == last_check_suite.id - end - def slack_notify_success(job) SlackBot.instance.notify_success(job) end From 4afa94a54b23097bec770c22d63e72556482a020 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Mon, 15 Apr 2024 16:27:56 -0300 Subject: [PATCH 05/53] Report ReRun Adding offender Signed-off-by: Rodrigo Nardi --- reports/re_run_report.rb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/reports/re_run_report.rb b/reports/re_run_report.rb index 9d4bf34..38d0f02 100644 --- a/reports/re_run_report.rb +++ b/reports/re_run_report.rb @@ -14,8 +14,11 @@ module Reports class ReRunReport + OFFENDER_LIMIT = 4 + def report(begin_date, end_date, output: 'print', filename: 'rerun_report.json') @result = {} + @offenders = [] AuditRetry .where(created_at: [begin_date..end_date]) @@ -34,6 +37,11 @@ def generate_result(audit_retry) @result[audit_retry.check_suite.pull_request.github_pr_id][:total] += 1 + if @result[audit_retry.check_suite.pull_request.github_pr_id][:total] > OFFENDER_LIMIT and + !@offenders.include?(audit_retry.check_suite.pull_request.github_pr_id) + @offenders << audit_retry.check_suite.pull_request.github_pr_id + end + check_suite_detail(audit_retry) end @@ -95,6 +103,11 @@ def raw_output(result, file_descriptor: nil) print_test_build_retry(cs, file_descriptor) end end + + puts "\nOffenders PR (LIMIT #{OFFENDER_LIMIT}):" + @offenders.each do |offender| + print("Offender: ##{offender}", file_descriptor) + end end def print_test_build_retry(info, file_descriptor) From 97a366d668d668f6017e70e64bf8f23b86c3918d Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Wed, 17 Apr 2024 08:49:48 -0300 Subject: [PATCH 06/53] Report ReRun Fixing Rubocop issues Signed-off-by: Rodrigo Nardi --- reports/re_run_report.rb | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/reports/re_run_report.rb b/reports/re_run_report.rb index 38d0f02..06cf7d3 100644 --- a/reports/re_run_report.rb +++ b/reports/re_run_report.rb @@ -37,14 +37,18 @@ def generate_result(audit_retry) @result[audit_retry.check_suite.pull_request.github_pr_id][:total] += 1 - if @result[audit_retry.check_suite.pull_request.github_pr_id][:total] > OFFENDER_LIMIT and - !@offenders.include?(audit_retry.check_suite.pull_request.github_pr_id) - @offenders << audit_retry.check_suite.pull_request.github_pr_id - end + add_offender(audit_retry) check_suite_detail(audit_retry) end + def add_offender(audit_retry) + return if @result[audit_retry.check_suite.pull_request.github_pr_id][:total] < OFFENDER_LIMIT + return if @offenders.include?(audit_retry.check_suite.pull_request.github_pr_id) + + @offenders << audit_retry.check_suite.pull_request.github_pr_id + end + def report_initializer(audit_retry) @result[audit_retry.check_suite.pull_request.github_pr_id] ||= { total: 0, check_suites: [] } @@ -104,10 +108,7 @@ def raw_output(result, file_descriptor: nil) end end - puts "\nOffenders PR (LIMIT #{OFFENDER_LIMIT}):" - @offenders.each do |offender| - print("Offender: ##{offender}", file_descriptor) - end + ci_offenders(file_descriptor) end def print_test_build_retry(info, file_descriptor) @@ -124,6 +125,15 @@ def print(line, file_descriptor) puts line file_descriptor&.write line end + + def ci_offenders(file_descriptor) + return if @offenders.empty? + + puts "\nOffenders PR (LIMIT #{OFFENDER_LIMIT}):" + @offenders.each do |offender| + print("Offender: ##{offender}", file_descriptor) + end + end end end From e17e5b89505caddee5c5d97c2370254ab7668979 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Wed, 17 Apr 2024 11:31:21 -0300 Subject: [PATCH 07/53] Check Suite Model Changing relation and foreign key name Signed-off-by: Rodrigo Nardi --- ...4603_change_check_suite_cancelled_by_id.rb | 17 +++++ ...240417102854_remove_check_suite_columns.rb | 19 +++++ db/schema.rb | 10 ++- lib/models/check_suite.rb | 2 +- reports/chained_rerun_report.rb | 73 +++++++++++++++++++ 5 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 db/migrate/20240417094603_change_check_suite_cancelled_by_id.rb create mode 100644 db/migrate/20240417102854_remove_check_suite_columns.rb create mode 100644 reports/chained_rerun_report.rb diff --git a/db/migrate/20240417094603_change_check_suite_cancelled_by_id.rb b/db/migrate/20240417094603_change_check_suite_cancelled_by_id.rb new file mode 100644 index 0000000..64bf5db --- /dev/null +++ b/db/migrate/20240417094603_change_check_suite_cancelled_by_id.rb @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# 20240417094603_change_check_suite_cancelled_by_id.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +class ChangeCheckSuiteCancelledById < ActiveRecord::Migration[6.0] + def change + change_table :check_suites do |t| + t.references :cancelled_previous_check_suite, foreign_key: { to_table: :check_suites } + end + end +end diff --git a/db/migrate/20240417102854_remove_check_suite_columns.rb b/db/migrate/20240417102854_remove_check_suite_columns.rb new file mode 100644 index 0000000..22ed253 --- /dev/null +++ b/db/migrate/20240417102854_remove_check_suite_columns.rb @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# 20240417102854_remove_check_suite_columns.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +class RemoveCheckSuiteColumns < ActiveRecord::Migration[6.0] + def change + if ActiveRecord::Base.connection.column_exists?(:check_suites, :cancelled_by_id) + remove_column :check_suites, :cancelled_by_id + end + + remove_column :check_suites, :id_id if ActiveRecord::Base.connection.column_exists?(:check_suites, :id_id) + end +end diff --git a/db/schema.rb b/db/schema.rb index 45a3cbb..43040df 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2024_04_08_164410) do +ActiveRecord::Schema[7.0].define(version: 2024_04_17_102854) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -58,8 +58,8 @@ t.integer "retry", default: 0 t.boolean "sync", default: false t.bigint "github_user_id" - t.bigint "cancelled_by_id" - t.index ["cancelled_by_id"], name: "index_check_suites_on_cancelled_by_id" + t.bigint "cancelled_previous_check_suite_id" + t.index ["cancelled_previous_check_suite_id"], name: "index_check_suites_on_cancelled_previous_check_suite_id" t.index ["github_user_id"], name: "index_check_suites_on_github_user_id" t.index ["pull_request_id"], name: "index_check_suites_on_pull_request_id" end @@ -110,6 +110,8 @@ t.datetime "created_at", null: false t.datetime "updated_at", null: false t.bigint "pull_request_id" + t.string "channel", default: "slack" + t.string "email" t.index ["pull_request_id"], name: "index_pull_request_subscriptions_on_pull_request_id" end @@ -163,9 +165,9 @@ add_foreign_key "audit_retries", "check_suites" add_foreign_key "audit_retries", "github_users" + add_foreign_key "check_suites", "check_suites", column: "cancelled_previous_check_suite_id" add_foreign_key "check_suites", "github_users" add_foreign_key "check_suites", "pull_requests" - add_foreign_key "check_suites", "stages", column: "cancelled_by_id" add_foreign_key "ci_jobs", "check_suites" add_foreign_key "ci_jobs", "stages" add_foreign_key "plans", "check_suites" diff --git a/lib/models/check_suite.rb b/lib/models/check_suite.rb index a506301..012fda7 100644 --- a/lib/models/check_suite.rb +++ b/lib/models/check_suite.rb @@ -15,7 +15,7 @@ class CheckSuite < ActiveRecord::Base validates :commit_sha_ref, presence: true belongs_to :pull_request - has_one :cancelled_by_new_check_suite, class_name: 'CheckSuite', foreign_key: 'cancelled_by_id' + belongs_to :cancelled_previous_check_suite, class_name: 'CheckSuite', optional: true has_one :cancelled_in_stage, class_name: 'Stage', foreign_key: 'cancelled_at_stage_id' has_many :ci_jobs, dependent: :delete_all has_many :stages, dependent: :delete_all diff --git a/reports/chained_rerun_report.rb b/reports/chained_rerun_report.rb new file mode 100644 index 0000000..9de9bdc --- /dev/null +++ b/reports/chained_rerun_report.rb @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# rerun_report.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +require 'json' +require 'csv' +require_relative '../database_loader' + +module Reports + class ChainedRerun + def report(begin_date, end_date, output: 'print', filename: 'rerun_report.json') + @result = {} + @offenders = [] + + CheckSuite + .left_joins(:audit_retries) + .where.not(check_suites: { cancelled_previous_check_suite_id: nil }) + .where(created_at: [begin_date..end_date]) + .order(:created_at) + .group_by(&:pull_request_id) + .each_pair do |pull_request_id, chained_check_suites| + generate_result(pull_request_id, chained_check_suites) + end + end + + private + + def generate_result(pull_request_id, chained_check_suites) + @local_chained = {} + @chain = 1 + + create_paths(chained_check_suites) + + puts "Pull Request ID: #{pull_request_id}" + @local_chained.each_pair do |_c, path| + puts path.reverse.join(' -> ') + end + end + + def create_paths(chained_check_suites) + chained_check_suites.reverse_each do |check_suite| + initialize_or_add(check_suite) + + if chained_check_suites.map(&:id).include?(check_suite.cancelled_previous_check_suite_id) + @local_chained[@chain] << check_suite.cancelled_previous_check_suite_id + else + @local_chained[@chain] << CheckSuite.find(check_suite.cancelled_previous_check_suite_id).id + @chain += 1 + end + end + end + + def initialize_or_add(check_suite) + @local_chained[@chain] ||= [] + @local_chained[@chain] << check_suite.id unless @local_chained[@chain].include? check_suite.id + end + end +end + +return unless __FILE__ == $PROGRAM_NAME + +ActiveRecord::Base.logger = Logger.new('/dev/null') + +begin_date = ARGV[0] +end_date = ARGV[1] + +Reports::ChainedRerun.new.report(begin_date, end_date, output: ARGV[2], filename: ARGV[3]) From daeba4d34b5954a05bb9dd029840c2232c057128 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Fri, 19 Apr 2024 10:06:35 -0300 Subject: [PATCH 08/53] Check Suite Model Changing relation and foreign key name Signed-off-by: Rodrigo Nardi --- ...601_change_check_suite_stopped_in_stage.rb | 30 +++++++++++++++++++ db/schema.rb | 10 +++---- lib/github/build_plan.rb | 4 +-- lib/github/re_run/base.rb | 29 +++++++++++++++--- lib/models/check_suite.rb | 4 ++- 5 files changed, 64 insertions(+), 13 deletions(-) create mode 100644 db/migrate/20240417130601_change_check_suite_stopped_in_stage.rb diff --git a/db/migrate/20240417130601_change_check_suite_stopped_in_stage.rb b/db/migrate/20240417130601_change_check_suite_stopped_in_stage.rb new file mode 100644 index 0000000..a89bbc7 --- /dev/null +++ b/db/migrate/20240417130601_change_check_suite_stopped_in_stage.rb @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# 202404130601_change_check_suite_stopped_in_stage.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +class ChangeCheckSuiteStoppedInStage < ActiveRecord::Migration[6.0] + def change + if ActiveRecord::Base.connection.column_exists?(:stages, :cancelled_at_stage_id) + remove_column :stages, :cancelled_at_stage_id + end + + if ActiveRecord::Base.connection.column_exists?(:check_suites, :cancelled_in_stage_id) + remove_column :check_suites, :cancelled_in_stage_id + end + + if ActiveRecord::Base.connection.column_exists?(:check_suites, :cancelled_previous_check_suite_id) + remove_column :check_suites, :cancelled_previous_check_suite_id + end + + change_table :check_suites do |t| + t.references :stopped_in_stage, foreign_key: { to_table: :stages } + t.references :cancelled_previous_check_suite, foreign_key: { to_table: :check_suites } + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 43040df..ff946ad 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2024_04_17_102854) do +ActiveRecord::Schema[7.0].define(version: 2024_04_17_130601) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -58,10 +58,12 @@ t.integer "retry", default: 0 t.boolean "sync", default: false t.bigint "github_user_id" + t.bigint "stopped_in_stage_id" t.bigint "cancelled_previous_check_suite_id" t.index ["cancelled_previous_check_suite_id"], name: "index_check_suites_on_cancelled_previous_check_suite_id" t.index ["github_user_id"], name: "index_check_suites_on_github_user_id" t.index ["pull_request_id"], name: "index_check_suites_on_pull_request_id" + t.index ["stopped_in_stage_id"], name: "index_check_suites_on_stopped_in_stage_id" end create_table "ci_jobs", force: :cascade do |t| @@ -110,8 +112,6 @@ t.datetime "created_at", null: false t.datetime "updated_at", null: false t.bigint "pull_request_id" - t.string "channel", default: "slack" - t.string "email" t.index ["pull_request_id"], name: "index_pull_request_subscriptions_on_pull_request_id" end @@ -146,8 +146,6 @@ t.datetime "updated_at", null: false t.bigint "check_suite_id" t.bigint "stage_configuration_id" - t.bigint "cancelled_at_stage_id" - t.index ["cancelled_at_stage_id"], name: "index_stages_on_cancelled_at_stage_id" t.index ["check_suite_id"], name: "index_stages_on_check_suite_id" t.index ["stage_configuration_id"], name: "index_stages_on_stage_configuration_id" end @@ -168,13 +166,13 @@ add_foreign_key "check_suites", "check_suites", column: "cancelled_previous_check_suite_id" add_foreign_key "check_suites", "github_users" add_foreign_key "check_suites", "pull_requests" + add_foreign_key "check_suites", "stages", column: "stopped_in_stage_id" add_foreign_key "ci_jobs", "check_suites" add_foreign_key "ci_jobs", "stages" add_foreign_key "plans", "check_suites" add_foreign_key "pull_request_subscriptions", "pull_requests" add_foreign_key "pull_requests", "github_users" add_foreign_key "stages", "check_suites" - add_foreign_key "stages", "check_suites", column: "cancelled_at_stage_id" add_foreign_key "stages", "stage_configurations" add_foreign_key "topotest_failures", "ci_jobs" end diff --git a/lib/github/build_plan.rb b/lib/github/build_plan.rb index 7b16eee..b494acb 100644 --- a/lib/github/build_plan.rb +++ b/lib/github/build_plan.rb @@ -120,7 +120,7 @@ def cancel_previous_ci_jobs ci_job.cancelled(@github_check) end - @last_check_suite.update(cancelled_in_stage: @last_check_suite.stages.where(status: :in_progress).last) + @last_check_suite.update(stopped_in_stage: @last_check_suite.stages.where(status: :in_progress).last) @last_check_suite.stages.where(status: %w[queued in_progress]).each do |stage| stage.cancelled(@github_check) @@ -171,7 +171,7 @@ def ci_jobs end def stop_execution_message - @last_check_suite.update(cancelled_by_new_check_suite: @check_suite) + @check_suite.update(cancelled_previous_check_suite_id: @last_check_suite.id) BambooCi::StopPlan.comment(@last_check_suite, @check_suite) end diff --git a/lib/github/re_run/base.rb b/lib/github/re_run/base.rb index 94747a0..5339a58 100644 --- a/lib/github/re_run/base.rb +++ b/lib/github/re_run/base.rb @@ -46,12 +46,31 @@ def stop_previous_execution logger(Logger::INFO, 'Stopping previous execution') logger(Logger::INFO, fetch_run_ci_by_pr.inspect) + @last_check_suite = nil + fetch_run_ci_by_pr.each do |check_suite| - check_suite.ci_jobs.not_skipped.each do |ci_job| - ci_job.cancelled(@github_check) - end + stop_and_update_previous_execution(check_suite) + end + end + + def stop_and_update_previous_execution(check_suite) + if @last_check_suite.nil? + check_suite.update(stopped_in_stage: check_suite.stages.where(status: :in_progress).last) + else + check_suite.update(cancelled_previous_check_suite_id: @last_check_suite.id) + @last_check_suite.update(stopped_in_stage: check_suite.stages.where(status: :in_progress).last) + end - BambooCi::StopPlan.build(check_suite.bamboo_ci_ref) + cancel_previous_jobs(check_suite) + + @last_check_suite = check_suite + + BambooCi::StopPlan.build(check_suite.bamboo_ci_ref) + end + + def cancel_previous_jobs(check_suite) + check_suite.ci_jobs.not_skipped.each do |ci_job| + ci_job.cancelled(@github_check) end end @@ -106,6 +125,8 @@ def ci_jobs(check_suite, bamboo_plan) check_suite.update(bamboo_ci_ref: bamboo_plan.bamboo_reference, re_run: true) + check_suite.update(cancelled_previous_check_suite: @last_check_suite) + create_ci_jobs(bamboo_plan, check_suite) CheckSuite.where(commit_sha_ref: check_suite.commit_sha_ref).each do |cs| diff --git a/lib/models/check_suite.rb b/lib/models/check_suite.rb index 012fda7..8086976 100644 --- a/lib/models/check_suite.rb +++ b/lib/models/check_suite.rb @@ -15,8 +15,10 @@ class CheckSuite < ActiveRecord::Base validates :commit_sha_ref, presence: true belongs_to :pull_request + + belongs_to :stopped_in_stage, class_name: 'Stage', optional: true belongs_to :cancelled_previous_check_suite, class_name: 'CheckSuite', optional: true - has_one :cancelled_in_stage, class_name: 'Stage', foreign_key: 'cancelled_at_stage_id' + has_many :ci_jobs, dependent: :delete_all has_many :stages, dependent: :delete_all has_many :audit_retries, dependent: :delete_all From 46182c0bcee3d5bd1f1c0b77c26de8e6991bf07f Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Fri, 19 Apr 2024 10:08:09 -0300 Subject: [PATCH 09/53] Chained Rerun Updating report Signed-off-by: Rodrigo Nardi --- reports/chained_rerun_report.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reports/chained_rerun_report.rb b/reports/chained_rerun_report.rb index 9de9bdc..af781fd 100644 --- a/reports/chained_rerun_report.rb +++ b/reports/chained_rerun_report.rb @@ -14,7 +14,7 @@ module Reports class ChainedRerun - def report(begin_date, end_date, output: 'print', filename: 'rerun_report.json') + def report(begin_date, end_date) @result = {} @offenders = [] @@ -70,4 +70,4 @@ def initialize_or_add(check_suite) begin_date = ARGV[0] end_date = ARGV[1] -Reports::ChainedRerun.new.report(begin_date, end_date, output: ARGV[2], filename: ARGV[3]) +Reports::ChainedRerun.new.report(begin_date, end_date) From c11df3dd6684828a98e852bbe93488252fdc7409 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Fri, 19 Apr 2024 11:17:45 -0300 Subject: [PATCH 10/53] Build Plain Adding new test scenario Signed-off-by: Rodrigo Nardi --- spec/lib/github/build_plan_spec.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/spec/lib/github/build_plan_spec.rb b/spec/lib/github/build_plan_spec.rb index cbaa96f..a1cdbd4 100644 --- a/spec/lib/github/build_plan_spec.rb +++ b/spec/lib/github/build_plan_spec.rb @@ -173,6 +173,14 @@ expect(before_in_progress_jobs).not_to eq(after_in_progress_jobs) expect(before_success_jobs).to eq(after_success_jobs) end + + it 'must set stopped_in_stage' do + expect(previous_check_suite.reload.stopped_in_stage).not_to eq(nil) + end + + it 'must set cancelled_previous_check_suite' do + expect(check_suite.cancelled_previous_check_suite).to eq(previous_check_suite) + end end end From 51e3551f7ac15c74c5c24aae4ffb971737348732 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Mon, 22 Apr 2024 10:44:49 -0300 Subject: [PATCH 11/53] Reports Adding new reports Signed-off-by: Rodrigo Nardi --- reports/build_stage_failed.rb | 32 ++++++++++++++++++++++++++++++++ reports/summary.rb | 24 +++++++++++++----------- 2 files changed, 45 insertions(+), 11 deletions(-) create mode 100644 reports/build_stage_failed.rb diff --git a/reports/build_stage_failed.rb b/reports/build_stage_failed.rb new file mode 100644 index 0000000..2dbe897 --- /dev/null +++ b/reports/build_stage_failed.rb @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# build_stage_failed.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +require_relative '../database_loader' +require_relative '../lib/helpers/configuration' + +begin_date = ARGV[0] +end_date = ARGV[1] +author = ARGV[2] + +check_suites = [] + +Stage + .joins(:jobs, :check_suite) + .where(stages: { name: 'Build' }) + .where(jobs: { created_at: [begin_date..end_date], status: %i[failure skipped] }) + .where(check_suites: { author: author }) + .each do |stage| + message = "Check Suite ID: https://ci1.netdef.org/browse/#{stage.check_suite.bamboo_ci_ref}" + check_suites << message unless check_suites.include? message +end + +check_suites.each do |line| + puts line +end diff --git a/reports/summary.rb b/reports/summary.rb index cbc5b11..4ebaa33 100644 --- a/reports/summary.rb +++ b/reports/summary.rb @@ -30,22 +30,24 @@ .reverse .to_h -build_errors = CiJob - .where("name ILIKE '% build'") - .where(created_at: [begin_date..end_date], status: %i[failure skipped]) +build_errors = Stage + .joins(:jobs) + .where(name: 'Build') + .where(jobs: { created_at: [begin_date..end_date], status: %i[failure skipped] }) + .map(&:check_suite_id) + .uniq build_errors_count = build_errors.size build_errors_author = CheckSuite - .where(id: build_errors.map(&:check_suite_id).uniq) - .group(:author, :pull_request_id) + .unscoped + .select('check_suites.author, COUNT(check_suites.author) as count') + .where(id: build_errors).where.not(author: 'mergify[bot]') + .group('check_suites.author') + .order('count DESC') .limit(10) - .count('check_suites.author') - .sort_by { |_k, v| v } .reverse - .map do |key, value| - author, pr_id = key - pr = PullRequest.find(pr_id) - { "#{author}: PR ##{pr.github_pr_id}": value } + .map do |entry| + { "#{entry.author}": entry.count } end puts "Report from #{begin_date} to #{end_date}\n\n" From 86dfbafd5bbd8863744744f2aa635e1e876c0700 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Fri, 3 May 2024 11:03:26 -0300 Subject: [PATCH 12/53] Summary Changing logs Signed-off-by: Rodrigo Nardi --- lib/github/build/summary.rb | 9 ++++++++- lib/github/plan_execution/finished.rb | 28 +++++++++++++-------------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/lib/github/build/summary.rb b/lib/github/build/summary.rb index 15cb9d2..d432552 100644 --- a/lib/github/build/summary.rb +++ b/lib/github/build/summary.rb @@ -27,7 +27,7 @@ def initialize(job, logger_level: Logger::INFO, agent: 'Github') @loggers << GithubLogger.instance.create(filename, logger_level) end - @loggers << GithubLogger.instance.create("pr#{@check_suite.pull_request.github_pr_id}.log", logger_level) + @pr_log = GithubLogger.instance.create("pr#{@check_suite.pull_request.github_pr_id}.log", logger_level) end def build_summary @@ -36,6 +36,8 @@ def build_summary check_and_update_github_ref(current_stage) logger(Logger::INFO, "build_summary: #{current_stage.inspect}") + msg = "Github::Build::Summary - #{@job.inspect}, #{current_stage.inspect}, bamboo info: #{bamboo_info}" + @pr_log.info(msg) # Update current stage update_summary(current_stage) @@ -52,6 +54,11 @@ def build_summary private + def bamboo_info + finish = Github::PlanExecution::Finished.new({ 'bamboo_ref' => @check_suite.bamboo_ci_ref }) + finish.fetch_build_status + end + def check_and_update_github_ref(current_stage) current_refs = @github.fetch_check_runs diff --git a/lib/github/plan_execution/finished.rb b/lib/github/plan_execution/finished.rb index 2f4f0da..3b69de5 100644 --- a/lib/github/plan_execution/finished.rb +++ b/lib/github/plan_execution/finished.rb @@ -40,16 +40,8 @@ def finished [200, 'Finished'] end - private - - # This method will move all tests that no longer exist in BambooCI to the skipped state, - # because there are no executions for them. - def clear_deleted_jobs - github_check = Github::Check.new(@check_suite) - - @check_suite.ci_jobs.where(status: %w[queued in_progress]).each do |ci_job| - ci_job.skipped(github_check) - end + def fetch_build_status + get_request(URI("https://127.0.0.1/rest/api/latest/result/status/#{@check_suite.bamboo_ci_ref}")) end # Checks if CI still running @@ -63,6 +55,18 @@ def in_progress?(build_status) true end + private + + # This method will move all tests that no longer exist in BambooCI to the skipped state, + # because there are no executions for them. + def clear_deleted_jobs + github_check = Github::Check.new(@check_suite) + + @check_suite.ci_jobs.where(status: %w[queued in_progress]).each do |ci_job| + ci_job.skipped(github_check) + end + end + def ci_stopped?(build_status) build_status.key?('message') and !build_status.key?('finished') end @@ -151,10 +155,6 @@ def check_stages def fetch_ci_execution @result = get_status(@check_suite.bamboo_ci_ref) end - - def fetch_build_status - get_request(URI("https://127.0.0.1/rest/api/latest/result/status/#{@check_suite.bamboo_ci_ref}")) - end end end end From 0d70a0d4953288faa0c15ed60658b018fd6ef158 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Fri, 3 May 2024 16:24:43 -0300 Subject: [PATCH 13/53] Summary It was adding Github::PlanExecution::Finished to validate whether the execution has finished. This can prevent the CI from not updating. Signed-off-by: Rodrigo Nardi --- lib/github/update_status.rb | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/github/update_status.rb b/lib/github/update_status.rb index 3c52549..91469fc 100644 --- a/lib/github/update_status.rb +++ b/lib/github/update_status.rb @@ -79,10 +79,7 @@ def update_status return [200, 'Success'] unless @job.check_suite.pull_request.current_execution? @job.check_suite - summary = Github::Build::Summary.new(@job) - summary.build_summary - - finished_execution? + update_build_summary_or_finished [200, 'Success'] rescue StandardError => e @@ -91,14 +88,12 @@ def update_status [500, 'Internal Server Error'] end - def finished_execution? - return false unless current_execution? - return false unless @check_suite.finished? - - logger Logger::INFO, ">>> @check_suite#{@check_suite.inspect} -> finished? #{@check_suite.finished?}" - logger Logger::INFO, @check_suite.ci_jobs.last.inspect + def update_build_summary_or_finished + summary = Github::Build::Summary.new(@job) + summary.build_summary - SlackBot.instance.execution_finished_notification(@check_suite) + finished = Github::PlanExecution::Finished.new({ 'bamboo_ref' => @job.check_suite.bamboo_ci_ref }) + finished.finished end def current_execution? From 61f356b68778f0874c05e39780dd03fd50c1be35 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Tue, 14 May 2024 13:30:10 -0300 Subject: [PATCH 14/53] New command - ci:retry Implementation of a new command for partial CI execution. Rerunning only failed tests Signed-off-by: Rodrigo Nardi --- app/github_app.rb | 15 +- lib/github/build/retry.rb | 1 + lib/github/build/summary.rb | 2 + lib/github/check.rb | 4 + lib/github/retry.rb | 122 --------------- lib/github/retry/base.rb | 147 ++++++++++++++++++ lib/github/retry/command.rb | 47 ++++++ lib/github/retry/comment.rb | 70 +++++++++ lib/github/update_status.rb | 4 + lib/github_ci_app.rb | 3 +- lib/models/check_suite.rb | 4 + .../{retry_spec.rb => retry/command_spec.rb} | 2 +- spec/lib/github/retry/comment_spec.rb | 90 +++++++++++ 13 files changed, 383 insertions(+), 128 deletions(-) delete mode 100644 lib/github/retry.rb create mode 100644 lib/github/retry/base.rb create mode 100644 lib/github/retry/command.rb create mode 100644 lib/github/retry/comment.rb rename spec/lib/github/{retry_spec.rb => retry/command_spec.rb} (99%) create mode 100644 spec/lib/github/retry/comment_spec.rb diff --git a/app/github_app.rb b/app/github_app.rb index 2f450a0..9294ed2 100755 --- a/app/github_app.rb +++ b/app/github_app.rb @@ -125,15 +125,22 @@ def sinatra_logger_level halt 200, 'OK' unless %w[rerequested].include? payload['action'].downcase - re_run = Github::Retry.new(payload, logger_level: GithubApp.sinatra_logger_level) + re_run = Github::Retry::Command.new(payload, logger_level: GithubApp.sinatra_logger_level) halt re_run.start when 'installation' logger.debug '>>> Received a new installation policy' halt 202, 'Updated' when 'issue_comment' - logger.debug '>>> Received a new issue comment' - - halt Github::ReRun::Comment.new(payload, logger_level: GithubApp.sinatra_logger_level).start + halt 404, 'Action not found' if payload.nil? or payload['comment'].nil? + + case payload.dig('comment', 'body') + when /ci:retry/ + halt Github::Retry::Comment.new(payload, logger_level: GithubApp.sinatra_logger_level).start + when /ci:rerun/ + halt Github::ReRun::Comment.new(payload, logger_level: GithubApp.sinatra_logger_level).start + else + logger.debug '>>> Invalid comment command' + end when 'check_suite' logger.debug '>>> Received a new check_suite command' halt 200, 'OK' unless payload['action'].downcase.match?('rerequested') diff --git a/lib/github/build/retry.rb b/lib/github/build/retry.rb index e08f899..f6b2efc 100644 --- a/lib/github/build/retry.rb +++ b/lib/github/build/retry.rb @@ -33,6 +33,7 @@ def enqueued_stages stage = Stage.find_by(check_suite: @check_suite, name: bamboo_stage.github_check_run_name) + next if stage.nil? next if stage.success? url = "https://ci1.netdef.org/browse/#{stage.check_suite.bamboo_ci_ref}" diff --git a/lib/github/build/summary.rb b/lib/github/build/summary.rb index d432552..cffabc1 100644 --- a/lib/github/build/summary.rb +++ b/lib/github/build/summary.rb @@ -80,6 +80,8 @@ def must_update_previous_stage(current_stage) return if previous_stage.nil? or !(previous_stage.in_progress? or previous_stage.queued?) + logger(Logger::INFO, "must_update_previous_stage: #{previous_stage.inspect}") + finished_stage_summary(previous_stage) end diff --git a/lib/github/check.rb b/lib/github/check.rb index 3623de3..29081f0 100644 --- a/lib/github/check.rb +++ b/lib/github/check.rb @@ -54,6 +54,10 @@ def comment_reaction_thumb_up(repo, comment_id) @app.create_issue_comment_reaction(repo, comment_id, '+1') end + def comment_reaction_thumb_down(repo, comment_id) + @app.create_issue_comment_reaction(repo, comment_id, '-1') + end + def create(name) @app.create_check_run( @check_suite.pull_request.repository, diff --git a/lib/github/retry.rb b/lib/github/retry.rb deleted file mode 100644 index e75b34b..0000000 --- a/lib/github/retry.rb +++ /dev/null @@ -1,122 +0,0 @@ -# SPDX-License-Identifier: BSD-2-Clause -# -# retry.rb -# Part of NetDEF CI System -# -# Copyright (c) 2023 by -# Network Device Education Foundation, Inc. ("NetDEF") -# -# frozen_string_literal: true - -require 'logger' - -require_relative '../../database_loader' -require_relative '../bamboo_ci/retry' -require_relative '../bamboo_ci/stop_plan' -require_relative '../github/build/retry' - -require_relative 'check' -require_relative 'build/unavailable_jobs' - -module Github - class Retry - def initialize(payload, logger_level: Logger::INFO) - create_logger(logger_level) - - @payload = payload - end - - def start - return [422, 'Payload can not be blank'] if @payload.nil? or @payload.empty? - - stage = Stage.find_by_check_ref(@payload.dig('check_run', 'id')) - - logger(Logger::DEBUG, "Running stage #{stage.inspect}") - - return [404, 'Stage not found'] if stage.nil? - return [406, 'Already enqueued this execution'] if stage.queued? or stage.in_progress? - - check_suite = stage.check_suite - - return enqueued(stage) if check_suite.in_progress? - - normal_flow(check_suite) - end - - private - - def normal_flow(check_suite) - check_suite.update(retry: check_suite.retry + 1) - - create_ci_jobs(check_suite) - - BambooCi::Retry.restart(check_suite.bamboo_ci_ref) - Github::Build::UnavailableJobs.new(check_suite).update - - SlackBot.instance.execution_started_notification(check_suite) - - [200, 'Retrying failure jobs'] - end - - def create_ci_jobs(check_suite) - github_check = Github::Check.new(check_suite) - - audit_retry = - AuditRetry.create(check_suite: check_suite, - github_username: @payload.dig('sender', 'login'), - github_id: @payload.dig('sender', 'id'), - github_type: @payload.dig('sender', 'type'), - retry_type: 'partial') - - Github::UserInfo.new(@payload.dig('sender', 'id'), check_suite: check_suite, audit_retry: audit_retry) - - build_retry = Github::Build::Retry.new(check_suite, github_check, audit_retry) - - build_retry.enqueued_stages - build_retry.enqueued_failure_tests - - BambooCi::StopPlan.build(check_suite.bamboo_ci_ref) - end - - def enqueued(stage) - github_check = Github::Check.new(stage.check_suite) - previous_stage = github_check.get_check_run(stage.check_ref) - - reason = slack_notification(stage) - - output = { title: previous_stage.dig(:output, :title).to_s, summary: previous_stage.dig(:output, :summary).to_s } - - stage.enqueue(github_check) - stage.failure(github_check, output: output) - - [406, reason] - end - - def slack_notification(job) - reason = SlackBot.instance.invalid_rerun_group(job) - - logger(Logger::WARN, ">>> #{job.inspect} #{reason}") - - pull_request = job.check_suite.pull_request - - PullRequestSubscription - .where(target: [pull_request.github_pr_id, pull_request.author], notification: %w[all errors]) - .uniq(&:slack_user_id) - .each { |subscription| SlackBot.instance.invalid_rerun_dm(job, subscription) } - - reason - end - - def create_logger(logger_level) - @logger_manager = [] - @logger_manager << GithubLogger.instance.create('github_app.log', logger_level) - @logger_manager << GithubLogger.instance.create('github_retry.log', logger_level) - end - - def logger(severity, message) - @logger_manager.each do |logger_object| - logger_object.add(severity, message) - end - end - end -end diff --git a/lib/github/retry/base.rb b/lib/github/retry/base.rb new file mode 100644 index 0000000..5b2a036 --- /dev/null +++ b/lib/github/retry/base.rb @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# base.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +# SPDX-License-Identifier: BSD-2-Clause +# +# base.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +require 'logger' + +require_relative '../../../database_loader' + +require_relative '../../bamboo_ci/retry' +require_relative '../../bamboo_ci/stop_plan' +require_relative '../../github/build/retry' + +require_relative '../check' +require_relative '../build/unavailable_jobs' + +module Github + module Retry + class Base + def initialize(payload) + @payload = payload + end + + def start + return [422, 'Payload can not be blank'] if @payload.nil? or @payload.empty? + + logger(Logger::DEBUG, "Running stage #{@stage.inspect}") + + return [404, 'Stage not found'] if @stage.nil? + return [406, 'Already enqueued this execution'] if @stage.queued? or @stage.in_progress? + + @check_suite = @stage.check_suite + + return enqueued if @check_suite.in_progress? + + normal_flow + end + + private + + def github_reaction_feedback(comment_id) + return if comment_id.nil? + + github_check = Github::Check.new(@stage.check_suite) + github_check.comment_reaction_thumb_up(@stage.check_suite.pull_request.repository, comment_id) + end + + def github_reaction_feedback_down(comment_id) + return if comment_id.nil? + + github_check = Github::Check.new(@stage.check_suite) + github_check.comment_reaction_thumb_down(@stage.check_suite.pull_request.repository, comment_id) + end + + def normal_flow + @check_suite.update(retry: @check_suite.retry + 1) + + create_ci_jobs(@check_suite) + + BambooCi::Retry.restart(@check_suite.bamboo_ci_ref) + Github::Build::UnavailableJobs.new(@check_suite).update + + SlackBot.instance.execution_started_notification(@check_suite) + + github_reaction_feedback(@comment_id) if @comment_id + + [200, 'Retrying failure jobs'] + end + + def create_ci_jobs(check_suite) + github_check = Github::Check.new(check_suite) + + audit_retry = + AuditRetry.create(check_suite: check_suite, + github_username: @payload.dig('sender', 'login'), + github_id: @payload.dig('sender', 'id'), + github_type: @payload.dig('sender', 'type'), + retry_type: 'partial') + + Github::UserInfo.new(@payload.dig('sender', 'id'), check_suite: check_suite, audit_retry: audit_retry) + + build_retry = Github::Build::Retry.new(check_suite, github_check, audit_retry) + + build_retry.enqueued_stages + build_retry.enqueued_failure_tests + + BambooCi::StopPlan.build(check_suite.bamboo_ci_ref) + end + + def enqueued + github_check = Github::Check.new(@stage.check_suite) + previous_stage = github_check.get_check_run(@stage.check_ref) + + reason = slack_notification + + output = { title: previous_stage.dig(:output, :title).to_s, summary: previous_stage.dig(:output, :summary).to_s } + + @stage.enqueue(github_check) + @stage.failure(github_check, output: output) + + [406, reason] + end + + def slack_notification + reason = SlackBot.instance.invalid_rerun_group(@stage) + + logger(Logger::WARN, ">>> #{@stage.inspect} #{reason}") + + pull_request = @stage.check_suite.pull_request + + PullRequestSubscription + .where(target: [pull_request.github_pr_id, pull_request.author], notification: %w[all errors]) + .uniq(&:slack_user_id) + .each { |subscription| SlackBot.instance.invalid_rerun_dm(@stage, subscription) } + + reason + end + + def create_logger(logger_level) + @logger_manager = [] + @logger_manager << GithubLogger.instance.create('github_app.log', logger_level) + @logger_manager << GithubLogger.instance.create('github_retry.log', logger_level) + end + + def logger(severity, message) + @logger_manager.each do |logger_object| + logger_object.add(severity, message) + end + end + end + end +end diff --git a/lib/github/retry/command.rb b/lib/github/retry/command.rb new file mode 100644 index 0000000..284b0a4 --- /dev/null +++ b/lib/github/retry/command.rb @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# retry.rb +# Part of NetDEF CI System +# +# Copyright (c) 2023 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +# SPDX-License-Identifier: BSD-2-Clause +# +# command.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +# SPDX-License-Identifier: BSD-2-Clause +# +# command.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +require_relative 'base' + +module Github + module Retry + class Command < Base + def initialize(payload, logger_level: Logger::INFO) + super(payload) + + create_logger(logger_level) + + @payload = payload + + @stage = Stage.find_by_check_ref(@payload.dig('check_run', 'id')) + end + end + end +end diff --git a/lib/github/retry/comment.rb b/lib/github/retry/comment.rb new file mode 100644 index 0000000..08508ba --- /dev/null +++ b/lib/github/retry/comment.rb @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# comment.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +# SPDX-License-Identifier: BSD-2-Clause +# +# comment.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +require_relative 'base' + +module Github + module Retry + class Comment < Base + def initialize(payload, logger_level: Logger::INFO) + super(payload) + + create_logger(logger_level) + + @payload = payload + + fetch_stage + @comment_id = comment_id + end + + def start + output = super + + logger(Logger::INFO, "Github::Retry::Comment - response #{output.inspect}") + if [404, 406, 422].include?(output.first.to_i) + github_reaction_feedback_down(@comment_id) + + return output + end + + github_reaction_feedback(@comment_id) + + output + end + + private + + def fetch_stage + pull_request = PullRequest.find_by(github_pr_id: pr_id) + check_suite = pull_request.check_suites.last + @stage = check_suite.stages_failure.min_by(&:id) + end + + def pr_id + @payload.dig('issue', 'number') || @payload.dig('check_suite', 'pull_requests')&.last&.[]('number') + end + + def comment_id + @payload.dig('comment', 'id') + end + end + end +end + diff --git a/lib/github/update_status.rb b/lib/github/update_status.rb index 91469fc..f597b15 100644 --- a/lib/github/update_status.rb +++ b/lib/github/update_status.rb @@ -92,6 +92,10 @@ def update_build_summary_or_finished summary = Github::Build::Summary.new(@job) summary.build_summary + return unless @job.finished? + + logger(Logger::INFO, "Github::PlanExecution::Finished: '#{@job.check_suite.bamboo_ci_ref}'") + finished = Github::PlanExecution::Finished.new({ 'bamboo_ref' => @job.check_suite.bamboo_ci_ref }) finished.finished end diff --git a/lib/github_ci_app.rb b/lib/github_ci_app.rb index 80ec374..8579b38 100644 --- a/lib/github_ci_app.rb +++ b/lib/github_ci_app.rb @@ -23,7 +23,8 @@ require_relative 'github/re_run/comment' require_relative 'github/build_plan' require_relative 'github/check' -require_relative 'github/retry' +require_relative 'github/retry/command' +require_relative 'github/retry/comment' require_relative 'github/update_status' require_relative 'github/plan_execution/finished' require_relative 'github/user_info' diff --git a/lib/models/check_suite.rb b/lib/models/check_suite.rb index 8086976..71688ea 100644 --- a/lib/models/check_suite.rb +++ b/lib/models/check_suite.rb @@ -25,6 +25,10 @@ class CheckSuite < ActiveRecord::Base default_scope -> { order(id: :asc) }, all_queries: true + def stages_failure + stages.joins(:jobs).where(jobs: { status: %w[cancelled failure] }).all.uniq + end + def finished? !running? end diff --git a/spec/lib/github/retry_spec.rb b/spec/lib/github/retry/command_spec.rb similarity index 99% rename from spec/lib/github/retry_spec.rb rename to spec/lib/github/retry/command_spec.rb index 9b5f7f3..a5a844f 100644 --- a/spec/lib/github/retry_spec.rb +++ b/spec/lib/github/retry/command_spec.rb @@ -8,7 +8,7 @@ # # frozen_string_literal: true -describe Github::Retry do +describe Github::Retry::Command do let(:github_retry) { described_class.new(payload) } let(:fake_unavailable) { Github::Build::UnavailableJobs.new(nil) } diff --git a/spec/lib/github/retry/comment_spec.rb b/spec/lib/github/retry/comment_spec.rb new file mode 100644 index 0000000..5457ae7 --- /dev/null +++ b/spec/lib/github/retry/comment_spec.rb @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# retry_spec.rb +# Part of NetDEF CI System +# +# Copyright (c) 2023 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +# SPDX-License-Identifier: BSD-2-Clause +# +# comment_spec.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +describe Github::Retry::Comment do + let(:github_retry) { described_class.new(payload) } + let(:fake_client) { Octokit::Client.new } + let(:fake_github_check) { Github::Check.new(nil) } + let(:fake_plan_run) { BambooCi::PlanRun.new(nil) } + let(:fake_unavailable) { Github::Build::UnavailableJobs.new(nil) } + + before do + allow(File).to receive(:read).and_return('') + allow(OpenSSL::PKey::RSA).to receive(:new).and_return(OpenSSL::PKey::RSA.new(2048)) + allow(Github::Build::UnavailableJobs).to receive(:new).and_return(fake_unavailable) + end + + describe 'Valid payload' do + let(:fake_client) { Octokit::Client.new } + let(:fake_github_check) { Github::Check.new(nil) } + let(:fake_translation) { create(:stage_configuration) } + + context 'when receives a valid command' do + let(:check_suite) { create(:check_suite, :with_running_ci_jobs) } + let(:ci_jobs) do + [ + { name: 'First Test', job_ref: 'UNIT-TEST-FIRST-1', stage: fake_translation.bamboo_stage_name }, + { name: 'Checkout', job_ref: 'CHK-01', stage: fake_translation.bamboo_stage_name } + ] + end + let(:payload) do + { + 'action' => 'created', + 'comment' => { 'body' => "ci:retry ##{check_suite.commit_sha_ref}", 'id' => 1 }, + 'repository' => { 'full_name' => check_suite.pull_request.repository }, + 'issue' => { 'number' => check_suite.pull_request.github_pr_id } + } + end + let(:check_suites) { CheckSuite.where(commit_sha_ref: check_suite.commit_sha_ref) } + + before do + allow(Octokit::Client).to receive(:new).and_return(fake_client) + allow(fake_client).to receive(:find_app_installations).and_return([{ 'id' => 1 }]) + allow(fake_client).to receive(:create_app_installation_access_token).and_return({ 'token' => 1 }) + + allow(Github::Check).to receive(:new).and_return(fake_github_check) + allow(fake_github_check).to receive(:create).and_return(check_suite) + allow(fake_github_check).to receive(:add_comment) + allow(fake_github_check).to receive(:cancelled) + allow(fake_github_check).to receive(:in_progress) + allow(fake_github_check).to receive(:queued) + allow(fake_github_check).to receive(:comment_reaction_thumb_up) + allow(fake_github_check).to receive(:fetch_username).and_return({}) + + allow(BambooCi::PlanRun).to receive(:new).and_return(fake_plan_run) + allow(fake_plan_run).to receive(:start_plan).and_return(200) + allow(fake_plan_run).to receive(:bamboo_reference).and_return('UNIT-TEST-1') + allow(fake_plan_run).to receive(:bamboo_reference).and_return('CHK-01') + + allow(BambooCi::Retry).to receive(:restart) + + allow(BambooCi::StopPlan).to receive(:build) + allow(BambooCi::RunningPlan).to receive(:fetch).with(fake_plan_run.bamboo_reference).and_return(ci_jobs) + + check_suite.stages.last.update(status: :failure) + end + + it 'must returns success' do + expect(github_retry.start).to eq([200, 'Retrying failure jobs']) + expect(check_suite.stages.last.reload.status).to eq('queued') + end + end + end +end From adb3d19fa6ebe5a7e397f232fa3d2672b6b22653 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Wed, 15 May 2024 09:06:27 -0300 Subject: [PATCH 15/53] New command - ci:retry Fixing unit tests Signed-off-by: Rodrigo Nardi --- lib/github/retry/base.rb | 1 + spec/lib/github/build/summary_spec.rb | 4 ++++ spec/lib/github/update_status_spec.rb | 6 ++++++ 3 files changed, 11 insertions(+) diff --git a/lib/github/retry/base.rb b/lib/github/retry/base.rb index 5b2a036..e39a378 100644 --- a/lib/github/retry/base.rb +++ b/lib/github/retry/base.rb @@ -62,6 +62,7 @@ def github_reaction_feedback(comment_id) def github_reaction_feedback_down(comment_id) return if comment_id.nil? + return if @stage.check_suite.nil? github_check = Github::Check.new(@stage.check_suite) github_check.comment_reaction_thumb_down(@stage.check_suite.pull_request.repository, comment_id) diff --git a/spec/lib/github/build/summary_spec.rb b/spec/lib/github/build/summary_spec.rb index ab95dc6..33ffa09 100644 --- a/spec/lib/github/build/summary_spec.rb +++ b/spec/lib/github/build/summary_spec.rb @@ -12,6 +12,7 @@ let(:summary) { described_class.new(ci_job) } let(:fake_client) { Octokit::Client.new } let(:fake_github_check) { Github::Check.new(nil) } + let(:fake_finish_plan) { Github::PlanExecution::Finished.new({ 'bamboo_ref' => 'UBUNTU-1' }) } let(:pull_request) { create(:pull_request) } let(:check_suite) { create(:check_suite, pull_request: pull_request) } let(:position1) { BambooStageTranslation.find_by_position(1) } @@ -24,6 +25,9 @@ allow(fake_client).to receive(:find_app_installations).and_return([{ 'id' => 1 }]) allow(fake_client).to receive(:create_app_installation_access_token).and_return({ 'token' => 1 }) + allow(Github::PlanExecution::Finished).to receive(:new).and_return(fake_finish_plan) + allow(fake_finish_plan).to receive(:fetch_build_status) + allow(File).to receive(:read).and_return('') allow(OpenSSL::PKey::RSA).to receive(:new).and_return(OpenSSL::PKey::RSA.new(2048)) diff --git a/spec/lib/github/update_status_spec.rb b/spec/lib/github/update_status_spec.rb index 551558c..51e378f 100644 --- a/spec/lib/github/update_status_spec.rb +++ b/spec/lib/github/update_status_spec.rb @@ -11,6 +11,12 @@ describe Github::UpdateStatus do let(:update_status) { described_class.new(payload) } let(:fake_unavailable) { Github::Build::UnavailableJobs.new(nil) } + let(:fake_finish_plan) { Github::PlanExecution::Finished.new({ 'bamboo_ref' => 'UBUNTU-1' }) } + + before do + allow(Github::PlanExecution::Finished).to receive(:new).and_return(fake_finish_plan) + allow(fake_finish_plan).to receive(:fetch_build_status) + end describe 'Validates different Ci Job status' do let(:payload) do From 755bc2f42de690e85a6f9b15b96c792429a2e724 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Wed, 15 May 2024 13:13:42 -0300 Subject: [PATCH 16/53] New command - ci:retry Adding new tests and increasing code coverage Signed-off-by: Rodrigo Nardi --- app/github_app.rb | 3 +- lib/github/re_run/comment.rb | 4 +- lib/github/retry/base.rb | 2 +- lib/github/retry/comment.rb | 4 +- spec/app/github_app_spec.rb | 72 ++++++++++++++++++++++- spec/lib/github/retry/comment_spec.rb | 84 ++++++++++++++++----------- 6 files changed, 126 insertions(+), 43 deletions(-) diff --git a/app/github_app.rb b/app/github_app.rb index 9294ed2..4197eff 100755 --- a/app/github_app.rb +++ b/app/github_app.rb @@ -139,7 +139,8 @@ def sinatra_logger_level when /ci:rerun/ halt Github::ReRun::Comment.new(payload, logger_level: GithubApp.sinatra_logger_level).start else - logger.debug '>>> Invalid comment command' + logger.debug '>>> Just a comment' + halt 200, 'Just a comment' end when 'check_suite' logger.debug '>>> Received a new check_suite command' diff --git a/lib/github/re_run/comment.rb b/lib/github/re_run/comment.rb index b26ebe3..630817f 100644 --- a/lib/github/re_run/comment.rb +++ b/lib/github/re_run/comment.rb @@ -164,9 +164,7 @@ def sha256 end def action? - return false if action.nil? - - action.downcase.match? 'ci:rerun' and @payload['action'] == 'created' + action.to_s.downcase.match? 'ci:rerun' and @payload['action'] == 'created' end end end diff --git a/lib/github/retry/base.rb b/lib/github/retry/base.rb index e39a378..f17a6db 100644 --- a/lib/github/retry/base.rb +++ b/lib/github/retry/base.rb @@ -62,7 +62,7 @@ def github_reaction_feedback(comment_id) def github_reaction_feedback_down(comment_id) return if comment_id.nil? - return if @stage.check_suite.nil? + return if @stage.nil? github_check = Github::Check.new(@stage.check_suite) github_check.comment_reaction_thumb_down(@stage.check_suite.pull_request.repository, comment_id) diff --git a/lib/github/retry/comment.rb b/lib/github/retry/comment.rb index 08508ba..9d20c4e 100644 --- a/lib/github/retry/comment.rb +++ b/lib/github/retry/comment.rb @@ -53,12 +53,14 @@ def start def fetch_stage pull_request = PullRequest.find_by(github_pr_id: pr_id) + return unless pull_request + check_suite = pull_request.check_suites.last @stage = check_suite.stages_failure.min_by(&:id) end def pr_id - @payload.dig('issue', 'number') || @payload.dig('check_suite', 'pull_requests')&.last&.[]('number') + @payload.dig('issue', 'number') end def comment_id diff --git a/spec/app/github_app_spec.rb b/spec/app/github_app_spec.rb index 365aed9..9f45632 100644 --- a/spec/app/github_app_spec.rb +++ b/spec/app/github_app_spec.rb @@ -195,7 +195,7 @@ end end - context 'when receive HTTP_X_GITHUB_EVENT issue_comment' do + context 'when receive HTTP_X_GITHUB_EVENT issue_comment - rerun' do let(:headers) do { 'HTTP_ACCEPT' => 'application/json', @@ -204,7 +204,73 @@ } end - let(:payload) { { x: 1, 'action' => 'rerequested' } } + let(:payload) { { x: 1, 'action' => 'rerequested', 'comment' => { 'body' => 'ci:rerun' } } } + + before do + allow(GitHubApp::Configuration.instance).to receive(:debug?).and_return(false) + end + + it 'must returns error' do + post '/', payload.to_json, headers + + expect(last_response.status).to eq 404 + end + end + + context 'when receive HTTP_X_GITHUB_EVENT issue_comment - retry' do + let(:headers) do + { + 'HTTP_ACCEPT' => 'application/json', + 'HTTP_SIGNATURE' => header, + 'HTTP_X_GITHUB_EVENT' => 'issue_comment' + } + end + + let(:payload) { { x: 1, 'action' => 'rerequested', 'comment' => { 'body' => 'ci:retry' } } } + + before do + allow(GitHubApp::Configuration.instance).to receive(:debug?).and_return(false) + end + + it 'must returns error' do + post '/', payload.to_json, headers + + expect(last_response.status).to eq 404 + end + end + + context 'when receive HTTP_X_GITHUB_EVENT issue_comment - potato' do + let(:headers) do + { + 'HTTP_ACCEPT' => 'application/json', + 'HTTP_SIGNATURE' => header, + 'HTTP_X_GITHUB_EVENT' => 'issue_comment' + } + end + + let(:payload) { { x: 1, 'action' => 'rerequested', 'comment' => { 'body' => 'just a potato' } } } + + before do + allow(GitHubApp::Configuration.instance).to receive(:debug?).and_return(false) + end + + it 'must returns success' do + post '/', payload.to_json, headers + + expect(last_response.status).to eq 200 + end + end + + context 'when receive HTTP_X_GITHUB_EVENT issue_comment - comment null' do + let(:headers) do + { + 'HTTP_ACCEPT' => 'application/json', + 'HTTP_SIGNATURE' => header, + 'HTTP_X_GITHUB_EVENT' => 'issue_comment' + } + end + + let(:payload) { { x: 1, 'action' => 'rerequested', 'comment' => nil } } before do allow(GitHubApp::Configuration.instance).to receive(:debug?).and_return(false) @@ -226,7 +292,7 @@ } end - let(:payload) { { x: 1, 'action' => 'rerequested' } } + let(:payload) { { x: 1, 'action' => 'rerequested', 'comment' => nil } } before do allow(GitHubApp::Configuration.instance).to receive(:debug?).and_return(false) diff --git a/spec/lib/github/retry/comment_spec.rb b/spec/lib/github/retry/comment_spec.rb index 5457ae7..8037ed3 100644 --- a/spec/lib/github/retry/comment_spec.rb +++ b/spec/lib/github/retry/comment_spec.rb @@ -1,13 +1,3 @@ -# SPDX-License-Identifier: BSD-2-Clause -# -# retry_spec.rb -# Part of NetDEF CI System -# -# Copyright (c) 2023 by -# Network Device Education Foundation, Inc. ("NetDEF") -# -# frozen_string_literal: true - # SPDX-License-Identifier: BSD-2-Clause # # comment_spec.rb @@ -36,6 +26,32 @@ let(:fake_github_check) { Github::Check.new(nil) } let(:fake_translation) { create(:stage_configuration) } + before do + allow(Octokit::Client).to receive(:new).and_return(fake_client) + allow(fake_client).to receive(:find_app_installations).and_return([{ 'id' => 1 }]) + allow(fake_client).to receive(:create_app_installation_access_token).and_return({ 'token' => 1 }) + + allow(Github::Check).to receive(:new).and_return(fake_github_check) + allow(fake_github_check).to receive(:create).and_return(check_suite) + allow(fake_github_check).to receive(:add_comment) + allow(fake_github_check).to receive(:cancelled) + allow(fake_github_check).to receive(:in_progress) + allow(fake_github_check).to receive(:queued) + allow(fake_github_check).to receive(:comment_reaction_thumb_up) + allow(fake_github_check).to receive(:comment_reaction_thumb_down) + allow(fake_github_check).to receive(:fetch_username).and_return({}) + + allow(BambooCi::PlanRun).to receive(:new).and_return(fake_plan_run) + allow(fake_plan_run).to receive(:start_plan).and_return(200) + allow(fake_plan_run).to receive(:bamboo_reference).and_return('UNIT-TEST-1') + allow(fake_plan_run).to receive(:bamboo_reference).and_return('CHK-01') + + allow(BambooCi::Retry).to receive(:restart) + + allow(BambooCi::StopPlan).to receive(:build) + allow(BambooCi::RunningPlan).to receive(:fetch).with(fake_plan_run.bamboo_reference).and_return(ci_jobs) + end + context 'when receives a valid command' do let(:check_suite) { create(:check_suite, :with_running_ci_jobs) } let(:ci_jobs) do @@ -52,33 +68,10 @@ 'issue' => { 'number' => check_suite.pull_request.github_pr_id } } end - let(:check_suites) { CheckSuite.where(commit_sha_ref: check_suite.commit_sha_ref) } before do - allow(Octokit::Client).to receive(:new).and_return(fake_client) - allow(fake_client).to receive(:find_app_installations).and_return([{ 'id' => 1 }]) - allow(fake_client).to receive(:create_app_installation_access_token).and_return({ 'token' => 1 }) - - allow(Github::Check).to receive(:new).and_return(fake_github_check) - allow(fake_github_check).to receive(:create).and_return(check_suite) - allow(fake_github_check).to receive(:add_comment) - allow(fake_github_check).to receive(:cancelled) - allow(fake_github_check).to receive(:in_progress) - allow(fake_github_check).to receive(:queued) - allow(fake_github_check).to receive(:comment_reaction_thumb_up) - allow(fake_github_check).to receive(:fetch_username).and_return({}) - - allow(BambooCi::PlanRun).to receive(:new).and_return(fake_plan_run) - allow(fake_plan_run).to receive(:start_plan).and_return(200) - allow(fake_plan_run).to receive(:bamboo_reference).and_return('UNIT-TEST-1') - allow(fake_plan_run).to receive(:bamboo_reference).and_return('CHK-01') - - allow(BambooCi::Retry).to receive(:restart) - - allow(BambooCi::StopPlan).to receive(:build) - allow(BambooCi::RunningPlan).to receive(:fetch).with(fake_plan_run.bamboo_reference).and_return(ci_jobs) - check_suite.stages.last.update(status: :failure) + check_suite.stages.last.jobs.each { |job| job.update(status: :failure) } end it 'must returns success' do @@ -86,5 +79,28 @@ expect(check_suite.stages.last.reload.status).to eq('queued') end end + + context 'when receives a valid command - but still running' do + let(:check_suite) { create(:check_suite, :with_running_ci_jobs) } + let(:ci_jobs) do + [ + { name: 'First Test', job_ref: 'UNIT-TEST-FIRST-1', stage: fake_translation.bamboo_stage_name }, + { name: 'Checkout', job_ref: 'CHK-01', stage: fake_translation.bamboo_stage_name } + ] + end + let(:payload) do + { + 'action' => 'created', + 'comment' => { 'body' => "ci:retry ##{check_suite.commit_sha_ref}", 'id' => 1 }, + 'repository' => { 'full_name' => check_suite.pull_request.repository }, + 'issue' => { 'number' => check_suite.pull_request.github_pr_id } + } + end + + it 'must returns stage not found' do + expect(github_retry.start).to eq([404, 'Stage not found']) + expect(check_suite.stages.last.reload.status).to eq('in_progress') + end + end end end From 29465d1d12458ff9e6e4db3b6bcadbd8233e4dc7 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Thu, 16 May 2024 08:22:06 -0300 Subject: [PATCH 17/53] New command - ci:retry Adding new tests Signed-off-by: Rodrigo Nardi --- lib/github/retry/base.rb | 17 +++-------------- lib/github/retry/comment.rb | 14 ++------------ spec/lib/github/retry/comment_spec.rb | 27 +++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 26 deletions(-) diff --git a/lib/github/retry/base.rb b/lib/github/retry/base.rb index f17a6db..6a56115 100644 --- a/lib/github/retry/base.rb +++ b/lib/github/retry/base.rb @@ -8,16 +8,6 @@ # # frozen_string_literal: true -# SPDX-License-Identifier: BSD-2-Clause -# -# base.rb -# Part of NetDEF CI System -# -# Copyright (c) 2024 by -# Network Device Education Foundation, Inc. ("NetDEF") -# -# frozen_string_literal: true - require 'logger' require_relative '../../../database_loader' @@ -61,11 +51,10 @@ def github_reaction_feedback(comment_id) end def github_reaction_feedback_down(comment_id) - return if comment_id.nil? - return if @stage.nil? + return if comment_id.nil? or @check_suite.nil? - github_check = Github::Check.new(@stage.check_suite) - github_check.comment_reaction_thumb_down(@stage.check_suite.pull_request.repository, comment_id) + github_check = Github::Check.new(@check_suite) + github_check.comment_reaction_thumb_down(@check_suite.pull_request.repository, comment_id) end def normal_flow diff --git a/lib/github/retry/comment.rb b/lib/github/retry/comment.rb index 9d20c4e..dd84d5c 100644 --- a/lib/github/retry/comment.rb +++ b/lib/github/retry/comment.rb @@ -8,16 +8,6 @@ # # frozen_string_literal: true -# SPDX-License-Identifier: BSD-2-Clause -# -# comment.rb -# Part of NetDEF CI System -# -# Copyright (c) 2024 by -# Network Device Education Foundation, Inc. ("NetDEF") -# -# frozen_string_literal: true - require_relative 'base' module Github @@ -55,8 +45,8 @@ def fetch_stage pull_request = PullRequest.find_by(github_pr_id: pr_id) return unless pull_request - check_suite = pull_request.check_suites.last - @stage = check_suite.stages_failure.min_by(&:id) + @check_suite = pull_request.check_suites.last + @stage = @check_suite.stages_failure.min_by(&:id) end def pr_id diff --git a/spec/lib/github/retry/comment_spec.rb b/spec/lib/github/retry/comment_spec.rb index 8037ed3..545c267 100644 --- a/spec/lib/github/retry/comment_spec.rb +++ b/spec/lib/github/retry/comment_spec.rb @@ -103,4 +103,31 @@ end end end + + describe 'Invalid payload' do + let(:fake_client) { Octokit::Client.new } + let(:fake_github_check) { Github::Check.new(nil) } + let(:fake_translation) { create(:stage_configuration) } + + before do + allow(Octokit::Client).to receive(:new).and_return(fake_client) + allow(fake_client).to receive(:find_app_installations).and_return([{ 'id' => 1 }]) + allow(fake_client).to receive(:create_app_installation_access_token).and_return({ 'token' => 1 }) + end + + context 'when receives an invalid PR' do + let(:payload) do + { + 'action' => 'created', + 'comment' => { 'body' => "ci:retry", 'id' => 1 }, + 'repository' => { 'full_name' => 'test' }, + 'issue' => { 'number' => 0 } + } + end + + it 'must returns stage not found' do + expect(github_retry.start).to eq([404, 'Stage not found']) + end + end + end end From b5ef57eb93c7aa421afccfef93730f214364439b Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Thu, 16 May 2024 08:39:57 -0300 Subject: [PATCH 18/53] New command - ci:retry Code lint Signed-off-by: Rodrigo Nardi --- lib/github/retry/base.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/github/retry/base.rb b/lib/github/retry/base.rb index 6a56115..36a570c 100644 --- a/lib/github/retry/base.rb +++ b/lib/github/retry/base.rb @@ -98,7 +98,10 @@ def enqueued reason = slack_notification - output = { title: previous_stage.dig(:output, :title).to_s, summary: previous_stage.dig(:output, :summary).to_s } + output = { + title: previous_stage.dig(:output, :title).to_s, + summary: previous_stage.dig(:output, :summary).to_s + } @stage.enqueue(github_check) @stage.failure(github_check, output: output) From 0e0230ca391e1f6308011f60597e1a4abe4a5fec Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Thu, 16 May 2024 09:32:19 -0300 Subject: [PATCH 19/53] New command - ci:retry Code lint Signed-off-by: Rodrigo Nardi --- lib/github/retry/command.rb | 21 --------------------- lib/github/retry/comment.rb | 1 - spec/lib/github/retry/comment_spec.rb | 2 +- 3 files changed, 1 insertion(+), 23 deletions(-) diff --git a/lib/github/retry/command.rb b/lib/github/retry/command.rb index 284b0a4..93b8082 100644 --- a/lib/github/retry/command.rb +++ b/lib/github/retry/command.rb @@ -1,23 +1,3 @@ -# SPDX-License-Identifier: BSD-2-Clause -# -# retry.rb -# Part of NetDEF CI System -# -# Copyright (c) 2023 by -# Network Device Education Foundation, Inc. ("NetDEF") -# -# frozen_string_literal: true - -# SPDX-License-Identifier: BSD-2-Clause -# -# command.rb -# Part of NetDEF CI System -# -# Copyright (c) 2024 by -# Network Device Education Foundation, Inc. ("NetDEF") -# -# frozen_string_literal: true - # SPDX-License-Identifier: BSD-2-Clause # # command.rb @@ -26,7 +6,6 @@ # Copyright (c) 2024 by # Network Device Education Foundation, Inc. ("NetDEF") # -# frozen_string_literal: true require_relative 'base' diff --git a/lib/github/retry/comment.rb b/lib/github/retry/comment.rb index dd84d5c..f044743 100644 --- a/lib/github/retry/comment.rb +++ b/lib/github/retry/comment.rb @@ -59,4 +59,3 @@ def comment_id end end end - diff --git a/spec/lib/github/retry/comment_spec.rb b/spec/lib/github/retry/comment_spec.rb index 545c267..2347551 100644 --- a/spec/lib/github/retry/comment_spec.rb +++ b/spec/lib/github/retry/comment_spec.rb @@ -119,7 +119,7 @@ let(:payload) do { 'action' => 'created', - 'comment' => { 'body' => "ci:retry", 'id' => 1 }, + 'comment' => { 'body' => 'ci:retry', 'id' => 1 }, 'repository' => { 'full_name' => 'test' }, 'issue' => { 'number' => 0 } } From 0cabd091ce4402e558571b243280f6fe1c16e902 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Thu, 16 May 2024 10:31:56 -0300 Subject: [PATCH 20/53] New command - ci:retry Code lint Signed-off-by: Rodrigo Nardi --- lib/github/retry/command.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/github/retry/command.rb b/lib/github/retry/command.rb index 93b8082..1faf8a6 100644 --- a/lib/github/retry/command.rb +++ b/lib/github/retry/command.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true # SPDX-License-Identifier: BSD-2-Clause # # command.rb From cd372a02c7ad3c077e873e2f1e67e07496379559 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Thu, 16 May 2024 10:36:18 -0300 Subject: [PATCH 21/53] New command - ci:retry Code lint Signed-off-by: Rodrigo Nardi --- lib/github/retry/command.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/github/retry/command.rb b/lib/github/retry/command.rb index 1faf8a6..79645c3 100644 --- a/lib/github/retry/command.rb +++ b/lib/github/retry/command.rb @@ -1,4 +1,3 @@ -# frozen_string_literal: true # SPDX-License-Identifier: BSD-2-Clause # # command.rb @@ -7,6 +6,7 @@ # Copyright (c) 2024 by # Network Device Education Foundation, Inc. ("NetDEF") # +# frozen_string_literal: true require_relative 'base' From bdc2cbc7de68bc799666405b2970268f49d0bc71 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Tue, 28 May 2024 00:06:51 -0300 Subject: [PATCH 22/53] Watchdog Fixing in the watchdog to validate pending stages instead of tests. It may be that the tests are up to date, but Github is not. Signed-off-by: Rodrigo Nardi --- lib/github/build/summary.rb | 1 + lib/github/plan_execution/finished.rb | 7 +++++++ workers/watch_dog.rb | 4 ++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/github/build/summary.rb b/lib/github/build/summary.rb index 15cb9d2..db2b1ef 100644 --- a/lib/github/build/summary.rb +++ b/lib/github/build/summary.rb @@ -12,6 +12,7 @@ require_relative '../../bamboo_ci/download' require_relative '../../bamboo_ci/running_plan' require_relative '../../helpers/github_logger' +require_relative '../../bamboo_ci/result' module Github module Build diff --git a/lib/github/plan_execution/finished.rb b/lib/github/plan_execution/finished.rb index 2f4f0da..faf146d 100644 --- a/lib/github/plan_execution/finished.rb +++ b/lib/github/plan_execution/finished.rb @@ -36,12 +36,19 @@ def finished check_stages clear_deleted_jobs + update_all_stages [200, 'Finished'] end private + def update_all_stages + @check_suite.stages.each do |stage| + build_summary(stage.jobs.last) + end + end + # This method will move all tests that no longer exist in BambooCI to the skipped state, # because there are no executions for them. def clear_deleted_jobs diff --git a/workers/watch_dog.rb b/workers/watch_dog.rb index 63a2acb..80dd82e 100644 --- a/workers/watch_dog.rb +++ b/workers/watch_dog.rb @@ -40,8 +40,8 @@ def check_suites def check_suites_fetch_map CheckSuite - .joins(:ci_jobs) - .where(ci_jobs: { status: %w[queued in_progress] }, created_at: [..Time.now]) + .joins(:stages) + .where(stages: { status: %w[queued in_progress] }, created_at: [..Time.now]) .map(&:id) .uniq end From 9413fa9f6c9eed5bbbb5bd44947c0454e3f7524a Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Tue, 28 May 2024 10:57:01 -0300 Subject: [PATCH 23/53] Github APP Adding downcase when reading comment body to make case insensitive the command execution Signed-off-by: Rodrigo Nardi --- app/github_app.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/github_app.rb b/app/github_app.rb index 4197eff..4235066 100755 --- a/app/github_app.rb +++ b/app/github_app.rb @@ -133,7 +133,7 @@ def sinatra_logger_level when 'issue_comment' halt 404, 'Action not found' if payload.nil? or payload['comment'].nil? - case payload.dig('comment', 'body') + case payload.dig('comment', 'body').to_s.downcase when /ci:retry/ halt Github::Retry::Comment.new(payload, logger_level: GithubApp.sinatra_logger_level).start when /ci:rerun/ From 19b995471ce94bcf7327db763c79e2b40cecc3b6 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Wed, 29 May 2024 23:27:19 -0300 Subject: [PATCH 24/53] Database Loader Changing ActiveRecord logger level Signed-off-by: Rodrigo Nardi --- database_loader.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/database_loader.rb b/database_loader.rb index caabddf..4f4f8ac 100644 --- a/database_loader.rb +++ b/database_loader.rb @@ -13,6 +13,7 @@ OTR::ActiveRecord.db_dir = 'db' OTR::ActiveRecord.migrations_paths = ['db/migrate'] OTR::ActiveRecord.configure_from_file! 'config/database.yml' +ActiveRecord::Base.logger.level = Logger::WARN Dir['lib/models/*.rb'].each { |model| require_relative model } From c628d7d849bb4dcbefa0c5fa1b5709ff7f31d279 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Fri, 31 May 2024 10:16:41 -0300 Subject: [PATCH 25/53] Check class Fixing warning message Signed-off-by: Rodrigo Nardi --- lib/github/check.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/github/check.rb b/lib/github/check.rb index 29081f0..19e0a2c 100644 --- a/lib/github/check.rb +++ b/lib/github/check.rb @@ -111,7 +111,7 @@ def fetch_check_runs end def installation_id - @authenticate_app.find_app_installations.first['id'].to_i + @authenticate_app.find_app_installations(accept: 'application/vnd.github.v3+json').first['id'].to_i end def signature @@ -198,7 +198,10 @@ def authenticate(jwt) return if installation_id.zero? - token = @authenticate_app.create_app_installation_access_token(installation_id)[:token] + token = + @authenticate_app + .create_app_installation_access_token(installation_id, accept: 'application/vnd.github.v3+json')[:token] + @app = Octokit::Client.new(bearer_token: token) end end From 859797408e62e1c3320660006bfc7563cc653d07 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Fri, 31 May 2024 10:21:09 -0300 Subject: [PATCH 26/53] Finished class Fixing update last test Signed-off-by: Rodrigo Nardi --- lib/github/plan_execution/finished.rb | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/github/plan_execution/finished.rb b/lib/github/plan_execution/finished.rb index e72a1bd..7a202a6 100644 --- a/lib/github/plan_execution/finished.rb +++ b/lib/github/plan_execution/finished.rb @@ -59,9 +59,15 @@ def in_progress?(build_status) private def update_all_stages - @check_suite.stages.each do |stage| - build_summary(stage.jobs.last) - end + last_stage = + Stage + .joins(:configuration) + .where(check_suite: @check_suite) + .max_by { |stage| stage.configuration.position } + + return if last_stage.nil? or last_stage.jobs.last.nil? + + build_summary(last_stage.jobs.last) end # This method will move all tests that no longer exist in BambooCI to the skipped state, @@ -150,6 +156,8 @@ def slack_notify_cancelled(job) def check_stages github_check = Github::Check.new(@check_suite) @logger.info ">>> @result: #{@result.inspect}" + return if @result.nil? or @result.empty? or @result['status-code'] == 404 + @result.dig('stages', 'stage').each do |stage| stage.dig('results', 'result').each do |result| ci_job = CiJob.find_by(job_ref: result['buildResultKey'], check_suite_id: @check_suite.id) From fbc2289c3b51e61addccb0d6d5d3b5120509a2a4 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Fri, 31 May 2024 10:34:40 -0300 Subject: [PATCH 27/53] Database Loader Adding safe navigation Signed-off-by: Rodrigo Nardi --- Gemfile.lock | 3 +++ database_loader.rb | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 7530fd2..96d2263 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,6 +6,8 @@ GEM activerecord (7.0.7.2) activemodel (= 7.0.7.2) activesupport (= 7.0.7.2) + activerecord-postgresql-adapter (0.0.1) + pg activesupport (7.0.7.2) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) @@ -151,6 +153,7 @@ PLATFORMS x86_64-linux DEPENDENCIES + activerecord-postgresql-adapter database_cleaner factory_bot faker diff --git a/database_loader.rb b/database_loader.rb index 4f4f8ac..a75a2cf 100644 --- a/database_loader.rb +++ b/database_loader.rb @@ -13,7 +13,7 @@ OTR::ActiveRecord.db_dir = 'db' OTR::ActiveRecord.migrations_paths = ['db/migrate'] OTR::ActiveRecord.configure_from_file! 'config/database.yml' -ActiveRecord::Base.logger.level = Logger::WARN +ActiveRecord::Base.logger&.level = Logger::WARN Dir['lib/models/*.rb'].each { |model| require_relative model } From b4029fce33c1a7c50a037762abf546aa0839b3de Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Fri, 31 May 2024 10:40:54 -0300 Subject: [PATCH 28/53] Gemfile.lock Fixing current version Signed-off-by: Rodrigo Nardi --- Gemfile.lock | 3 --- 1 file changed, 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 96d2263..7530fd2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,8 +6,6 @@ GEM activerecord (7.0.7.2) activemodel (= 7.0.7.2) activesupport (= 7.0.7.2) - activerecord-postgresql-adapter (0.0.1) - pg activesupport (7.0.7.2) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) @@ -153,7 +151,6 @@ PLATFORMS x86_64-linux DEPENDENCIES - activerecord-postgresql-adapter database_cleaner factory_bot faker From 8bc98d87314494edb54e9cb9678e9aa213004643 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Fri, 31 May 2024 11:08:19 -0300 Subject: [PATCH 29/53] Finished Increasing code coverage Signed-off-by: Rodrigo Nardi --- .simplecov | 3 +- lib/github/plan_execution/finished.rb | 2 +- spec/factories/check_suite.rb | 6 +++ .../github/plan_execution/finished_spec.rb | 50 +++++++++++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/.simplecov b/.simplecov index e4056ec..d461293 100644 --- a/.simplecov +++ b/.simplecov @@ -12,11 +12,12 @@ SimpleCov.start do enable_coverage :branch primary_coverage :branch add_filter %r{^/spec/} + add_filter 'database_loader.rb' add_group 'Models', 'lib/models' add_group 'GitHub Functions', 'lib/github' add_group 'Bamboo CI Functions', 'lib/bamboo_ci' add_group 'Helpers', 'lib/helpers' - add_group 'Others', %w[app/ database_loader.rb] + add_group 'Others', %w[app/] minimum_coverage_by_file line: 90, branch: 90 end diff --git a/lib/github/plan_execution/finished.rb b/lib/github/plan_execution/finished.rb index 7a202a6..3ec3940 100644 --- a/lib/github/plan_execution/finished.rb +++ b/lib/github/plan_execution/finished.rb @@ -156,7 +156,7 @@ def slack_notify_cancelled(job) def check_stages github_check = Github::Check.new(@check_suite) @logger.info ">>> @result: #{@result.inspect}" - return if @result.nil? or @result.empty? or @result['status-code'] == 404 + return if @result.nil? or @result.empty? or @result['status-code']&.between?(400, 500) @result.dig('stages', 'stage').each do |stage| stage.dig('results', 'result').each do |result| diff --git a/spec/factories/check_suite.rb b/spec/factories/check_suite.rb index e5d8fbf..b368e2b 100644 --- a/spec/factories/check_suite.rb +++ b/spec/factories/check_suite.rb @@ -39,5 +39,11 @@ create_list(:ci_job, 5, check_suite: check_suite, stage: stage, status: 1) end end + + trait :with_stages do + after(:create) do |check_suite| + create(:stage, check_suite: check_suite) + end + end end end diff --git a/spec/lib/github/plan_execution/finished_spec.rb b/spec/lib/github/plan_execution/finished_spec.rb index 6f6eebe..12aad0a 100644 --- a/spec/lib/github/plan_execution/finished_spec.rb +++ b/spec/lib/github/plan_execution/finished_spec.rb @@ -77,6 +77,56 @@ end describe 'Finishing execution' do + context 'when receives a valid payload - but stages does not have tests' do + let(:check_suite) { create(:check_suite, :with_stages) } + let(:status) { 200 } + let(:body) do + { + 'stages' => { + 'stage' => [ + 'results' => { + 'result' => + check_suite.ci_jobs.map { |job| { 'buildResultKey' => job.job_ref, 'state' => 'Successful' } } + } + ] + } + } + end + let(:payload) { { 'bamboo_ref' => check_suite.bamboo_ci_ref } } + + it 'must returns error' do + expect(pla_exec.finished).to eq([200, 'Finished']) + end + end + + context 'when receives a valid payload - but result has error' do + let(:status) { 200 } + let(:body) do + { + 'status-code' => 400 + } + end + let(:payload) { { 'bamboo_ref' => check_suite.bamboo_ci_ref } } + + before do + stub_request(:get, url) + .with( + headers: { + 'Accept' => %w[*/* application/json], + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Authorization' => 'Basic dXNlcjpwYXNzd29yZA==', + 'Host' => '127.0.0.1', + 'User-Agent' => 'Ruby' + } + ) + .to_return(status: status, body: body.to_json, headers: {}) + end + + it 'must returns error' do + expect(pla_exec.finished).to eq([200, 'Finished']) + end + end + context 'when receives a valid payload - Successful' do let(:status) { 200 } let(:body) do From 400418edfd5bdea825f2b1072fc0f8ef26b20375 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Sat, 1 Jun 2024 11:26:20 -0300 Subject: [PATCH 30/53] Github::Check class GitHub's accept header fixed to avoid warnings and make logs cleaner. Signed-off-by: Rodrigo Nardi --- lib/github/check.rb | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/github/check.rb b/lib/github/check.rb index 19e0a2c..84120b6 100644 --- a/lib/github/check.rb +++ b/lib/github/check.rb @@ -63,7 +63,7 @@ def create(name) @check_suite.pull_request.repository, name, @check_suite.commit_sha_ref, - accept: 'application/vnd.github+json' + accept: 'application/vnd.github.antiope-preview+json' ) end @@ -92,14 +92,20 @@ def skipped(check_ref, output = {}) end def get_check_run(check_ref) - @app.check_run(@check_suite.pull_request.repository, check_ref).to_h + @app.check_run(@check_suite.pull_request.repository, + check_ref, + accept: 'application/vnd.github.antiope-preview+json').to_h end def fetch_check_runs return [] if @check_suite.nil? @app - .check_runs_for_ref(@check_suite.pull_request.repository, @check_suite.pull_request.branch_name) + .check_runs_for_ref( + @check_suite.pull_request.repository, + @check_suite.pull_request.branch_name, + accept: 'application/vnd.github.antiope-preview+json' + ) .to_h[:check_runs] .map do |check_run| check_run[:id] @@ -129,7 +135,7 @@ def fetch_username(username) def basic_status(check_ref, status, output) opts = { status: status, - accept: 'application/vnd.github+json' + accept: 'application/vnd.github.antiope-preview+json' } opts[:output] = output unless output.empty? From 9324918c6a1807a34921fbdab77fa4782d9237ed Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Sat, 1 Jun 2024 11:36:56 -0300 Subject: [PATCH 31/53] Github::Check class Updating unit tests Signed-off-by: Rodrigo Nardi --- spec/lib/github/check_spec.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/lib/github/check_spec.rb b/spec/lib/github/check_spec.rb index 6627ddc..a2ad915 100644 --- a/spec/lib/github/check_spec.rb +++ b/spec/lib/github/check_spec.rb @@ -90,7 +90,7 @@ before do allow(fake_client).to receive(:create_check_run) .with(check_suite.pull_request.repository, name, - check_suite.commit_sha_ref, accept: 'application/vnd.github+json') + check_suite.commit_sha_ref, accept: 'application/vnd.github.antiope-preview+json') .and_return(pr_info) end @@ -110,7 +110,7 @@ id, { status: status, - accept: 'application/vnd.github+json' + accept: 'application/vnd.github.antiope-preview+json' }) .and_return(pr_info) end @@ -133,7 +133,7 @@ { status: status, output: output, - accept: 'application/vnd.github+json' + accept: 'application/vnd.github.antiope-preview+json' }) .and_return(pr_info) end @@ -199,7 +199,7 @@ { status: status, conclusion: conclusion, - accept: 'application/vnd.github+json' + accept: 'application/vnd.github.antiope-preview+json' }) .and_return(true) end From e0a21014f9c70ef793c29eaecdbd1dc0d5a5b935 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Mon, 3 Jun 2024 13:41:34 -0300 Subject: [PATCH 32/53] Rerun Report Filter PRs when is offender Signed-off-by: Rodrigo Nardi --- reports/re_run_report.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/reports/re_run_report.rb b/reports/re_run_report.rb index 06cf7d3..a022885 100644 --- a/reports/re_run_report.rb +++ b/reports/re_run_report.rb @@ -96,6 +96,8 @@ def create_csv(filename) def raw_output(result, file_descriptor: nil) result.each_pair do |pull_request, info| + next if info[:total] <= OFFENDER_LIMIT + print("\nPull Request: ##{pull_request} - Reruns: #{info[:total]}", file_descriptor) info[:check_suites].each do |cs| From 5804278699b3037152ac0d3b239d096c337406b7 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Thu, 6 Jun 2024 11:04:02 -0300 Subject: [PATCH 33/53] CI Retry Old PRs that listed all the tests that are run can cause problems when running ci:retry. Because, the old tests have no relationship with a Stage and break when searching for the configuration Signed-off-by: Rodrigo Nardi --- lib/github/build/retry.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/github/build/retry.rb b/lib/github/build/retry.rb index f6b2efc..ba0f74c 100644 --- a/lib/github/build/retry.rb +++ b/lib/github/build/retry.rb @@ -45,7 +45,7 @@ def enqueued_stages def enqueued_failure_tests @check_suite.ci_jobs.where.not(status: :success).each do |ci_job| - next unless ci_job.stage.configuration.can_retry? + next if must_skip_retry?(ci_job) logger(Logger::WARN, "Enqueue CiJob: #{ci_job.inspect}") ci_job.enqueue(@github) @@ -57,6 +57,10 @@ def enqueued_failure_tests private + def must_skip_retry?(ci_job) + ci_job.stage.nil? || !ci_job.stage.configuration.can_retry? + end + def logger(severity, message) @loggers.each do |logger_object| logger_object.add(severity, message) From bf15b2f4304ce4dcfa99e2b76770453bd2e5a821 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Tue, 11 Jun 2024 08:45:06 -0300 Subject: [PATCH 34/53] Puma Telemetry Adding puma-metrics to check status Signed-off-by: Rodrigo Nardi --- config/puma.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/config/puma.rb b/config/puma.rb index df9de0f..530b58a 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - # SPDX-License-Identifier: BSD-2-Clause # # puma.rb @@ -8,14 +6,20 @@ # Copyright (c) 2023 by # Network Device Education Foundation, Inc. ("NetDEF") # +# frozen_string_literal: true require_relative '../config/setup' require 'puma' +require 'puma/metrics/app' workers 10 -threads_count = (ENV['RAILS_MAX_THREADS'] || 5).to_i -threads 1, threads_count +plugin :metrics + +threads 1, (ENV['RAILS_MAX_THREADS'] || 5).to_i + +metrics_url 'tcp://0.0.0.0:9090' + port GitHubApp::Configuration.instance.config['port'] || 4667 preload_app! From cbfe18048d173d6942d3974a5c831562389fa78a Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Tue, 11 Jun 2024 08:48:21 -0300 Subject: [PATCH 35/53] Puma Telemetry Running in different port Signed-off-by: Rodrigo Nardi --- config/puma.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/puma.rb b/config/puma.rb index 530b58a..d76c8d2 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -18,7 +18,9 @@ threads 1, (ENV['RAILS_MAX_THREADS'] || 5).to_i -metrics_url 'tcp://0.0.0.0:9090' +metrics_port = ENV['RACK_ENV'] == 'production' ? 9393 : 9394 + +metrics_url "tcp://0.0.0.0:#{metrics_port}" port GitHubApp::Configuration.instance.config['port'] || 4667 From ae13c022eb5ebb6b48a3ac7ff46bc5e6f022b45d Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Tue, 11 Jun 2024 09:16:25 -0300 Subject: [PATCH 36/53] Puma Telemetry Adding gem 'puma-metrics' Signed-off-by: Rodrigo Nardi --- Gemfile | 1 + Gemfile.lock | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/Gemfile b/Gemfile index f7cda4c..1a4902e 100644 --- a/Gemfile +++ b/Gemfile @@ -27,6 +27,7 @@ gem 'otr-activerecord', '2.0.3' # PostgreSQL adapter gem 'pg', '1.2.3' +gem 'puma-metrics', '1.2.5' gem 'netrc' diff --git a/Gemfile.lock b/Gemfile.lock index 7530fd2..7af1a86 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -79,9 +79,13 @@ GEM ast (~> 2.4.1) racc pg (1.2.3) + prometheus-client (4.2.2) public_suffix (5.0.3) puma (5.5.2) nio4r (~> 2.0) + puma-metrics (1.2.5) + prometheus-client (>= 0.10) + puma (>= 5.0) racc (1.7.1) rack (2.1.4) rack-protection (2.0.8.1) @@ -160,6 +164,7 @@ DEPENDENCIES otr-activerecord (= 2.0.3) pg (= 1.2.3) puma (= 5.5.2) + puma-metrics (= 1.2.5) rack (= 2.1.4) rack-test rake (= 13.0.6) From 879a3f5bfd4e5ff7a1fbb9683581c7557a41f098 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Wed, 12 Jun 2024 11:15:38 -0300 Subject: [PATCH 37/53] Puma Telemetry Using Puma.stats lib to get telemetry Signed-off-by: Rodrigo Nardi --- app/github_app.rb | 6 ++++++ config/puma.rb | 18 +++++++++++------- lib/github_ci_app.rb | 1 + lib/helpers/telemetry.rb | 24 ++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 7 deletions(-) create mode 100644 lib/helpers/telemetry.rb diff --git a/app/github_app.rb b/app/github_app.rb index 4235066..beb6bbd 100755 --- a/app/github_app.rb +++ b/app/github_app.rb @@ -38,6 +38,12 @@ def sinatra_logger_level attr_writer :sinatra_logger_level end + get '/telemetry' do + content_type :json + + Telemetry.instance.stats + end + get '/ping' do halt 200, 'Pong' end diff --git a/config/puma.rb b/config/puma.rb index d76c8d2..7403331 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -10,20 +10,24 @@ require_relative '../config/setup' require 'puma' -require 'puma/metrics/app' workers 10 -plugin :metrics - threads 1, (ENV['RAILS_MAX_THREADS'] || 5).to_i -metrics_port = ENV['RACK_ENV'] == 'production' ? 9393 : 9394 - -metrics_url "tcp://0.0.0.0:#{metrics_port}" - port GitHubApp::Configuration.instance.config['port'] || 4667 +activate_control_app + preload_app! pidfile 'puma.pid' + +before_fork do + Thread.new do + loop do + Telemetry.instance.update_stats Puma.stats + sleep 30 + end + end +end diff --git a/lib/github_ci_app.rb b/lib/github_ci_app.rb index 8579b38..b09faf4 100644 --- a/lib/github_ci_app.rb +++ b/lib/github_ci_app.rb @@ -34,6 +34,7 @@ require_relative 'helpers/github_logger' require_relative 'helpers/request' require_relative 'helpers/sinatra_payload' +require_relative 'helpers/telemetry' # Slack libs require_relative 'slack/slack' diff --git a/lib/helpers/telemetry.rb b/lib/helpers/telemetry.rb new file mode 100644 index 0000000..9d8898e --- /dev/null +++ b/lib/helpers/telemetry.rb @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# telemetry.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +require 'singleton' +require 'json' + +class Telemetry + include Singleton + + def update_stats(stats) + File.write('telemetry.json', stats.to_json) + end + + def stats + JSON.parse(File.read('telemetry.json')) + end +end From 0588c7686ae07cfab2f48b6a96b1d9f9e02a2dfb Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Wed, 12 Jun 2024 11:17:45 -0300 Subject: [PATCH 38/53] Puma Telemetry Removing useless gem Signed-off-by: Rodrigo Nardi --- Gemfile | 1 - Gemfile.lock | 5 ----- 2 files changed, 6 deletions(-) diff --git a/Gemfile b/Gemfile index 1a4902e..f7cda4c 100644 --- a/Gemfile +++ b/Gemfile @@ -27,7 +27,6 @@ gem 'otr-activerecord', '2.0.3' # PostgreSQL adapter gem 'pg', '1.2.3' -gem 'puma-metrics', '1.2.5' gem 'netrc' diff --git a/Gemfile.lock b/Gemfile.lock index 7af1a86..7530fd2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -79,13 +79,9 @@ GEM ast (~> 2.4.1) racc pg (1.2.3) - prometheus-client (4.2.2) public_suffix (5.0.3) puma (5.5.2) nio4r (~> 2.0) - puma-metrics (1.2.5) - prometheus-client (>= 0.10) - puma (>= 5.0) racc (1.7.1) rack (2.1.4) rack-protection (2.0.8.1) @@ -164,7 +160,6 @@ DEPENDENCIES otr-activerecord (= 2.0.3) pg (= 1.2.3) puma (= 5.5.2) - puma-metrics (= 1.2.5) rack (= 2.1.4) rack-test rake (= 13.0.6) From 02838d25292452b841cc190b16c2c31145455f6f Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Thu, 13 Jun 2024 11:12:58 -0300 Subject: [PATCH 39/53] Puma Telemetry Code smells Signed-off-by: Rodrigo Nardi --- lib/helpers/telemetry.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/helpers/telemetry.rb b/lib/helpers/telemetry.rb index 9d8898e..d647105 100644 --- a/lib/helpers/telemetry.rb +++ b/lib/helpers/telemetry.rb @@ -17,7 +17,7 @@ class Telemetry def update_stats(stats) File.write('telemetry.json', stats.to_json) end - + def stats JSON.parse(File.read('telemetry.json')) end From 769335479cb4f0f1cb29eea024f1ee72ec724ae9 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Thu, 13 Jun 2024 14:33:44 -0300 Subject: [PATCH 40/53] Puma Telemetry Code coverage Signed-off-by: Rodrigo Nardi --- lib/helpers/telemetry.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/helpers/telemetry.rb b/lib/helpers/telemetry.rb index d647105..d21e67b 100644 --- a/lib/helpers/telemetry.rb +++ b/lib/helpers/telemetry.rb @@ -14,6 +14,7 @@ class Telemetry include Singleton + # :nocov: def update_stats(stats) File.write('telemetry.json', stats.to_json) end @@ -21,4 +22,5 @@ def update_stats(stats) def stats JSON.parse(File.read('telemetry.json')) end + # :nocov: end From 5b247e8989393e8b784ae28d8d4907764f988381 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Sat, 15 Jun 2024 08:41:03 -0300 Subject: [PATCH 41/53] GitHub Warnings Adding accept type to reactions. Signed-off-by: Rodrigo Nardi --- lib/github/check.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/github/check.rb b/lib/github/check.rb index 84120b6..a09fc62 100644 --- a/lib/github/check.rb +++ b/lib/github/check.rb @@ -51,11 +51,13 @@ def add_comment(pr_id, comment, repo) end def comment_reaction_thumb_up(repo, comment_id) - @app.create_issue_comment_reaction(repo, comment_id, '+1') + @app.create_issue_comment_reaction(repo, comment_id, '+1', + accept: Octokit::Preview::PREVIEW_TYPES[:reactions]) end def comment_reaction_thumb_down(repo, comment_id) - @app.create_issue_comment_reaction(repo, comment_id, '-1') + @app.create_issue_comment_reaction(repo, comment_id, '-1', + accept: Octokit::Preview::PREVIEW_TYPES[:reactions]) end def create(name) @@ -63,7 +65,7 @@ def create(name) @check_suite.pull_request.repository, name, @check_suite.commit_sha_ref, - accept: 'application/vnd.github.antiope-preview+json' + accept: Octokit::Preview::PREVIEW_TYPES[:checks] ) end @@ -94,7 +96,7 @@ def skipped(check_ref, output = {}) def get_check_run(check_ref) @app.check_run(@check_suite.pull_request.repository, check_ref, - accept: 'application/vnd.github.antiope-preview+json').to_h + accept: Octokit::Preview::PREVIEW_TYPES[:checks]).to_h end def fetch_check_runs @@ -104,7 +106,7 @@ def fetch_check_runs .check_runs_for_ref( @check_suite.pull_request.repository, @check_suite.pull_request.branch_name, - accept: 'application/vnd.github.antiope-preview+json' + accept: Octokit::Preview::PREVIEW_TYPES[:checks] ) .to_h[:check_runs] .map do |check_run| @@ -135,7 +137,7 @@ def fetch_username(username) def basic_status(check_ref, status, output) opts = { status: status, - accept: 'application/vnd.github.antiope-preview+json' + accept: Octokit::Preview::PREVIEW_TYPES[:checks] } opts[:output] = output unless output.empty? From f2cd69f10b24c6fa85fefa6cbfeca85168ba68ab Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Mon, 17 Jun 2024 10:06:18 -0300 Subject: [PATCH 42/53] GitHub Warnings Updating unit tests Signed-off-by: Rodrigo Nardi --- spec/lib/github/check_spec.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/lib/github/check_spec.rb b/spec/lib/github/check_spec.rb index a2ad915..89d5b97 100644 --- a/spec/lib/github/check_spec.rb +++ b/spec/lib/github/check_spec.rb @@ -74,7 +74,9 @@ let(:pr_info) { { comment_id: comment_id } } before do - allow(fake_client).to receive(:create_issue_comment_reaction).with(repo, comment_id, '+1').and_return(pr_info) + allow(fake_client).to receive(:create_issue_comment_reaction) + .with(repo, comment_id, '+1', accept: Octokit::Preview::PREVIEW_TYPES[:reactions]) + .and_return(pr_info) end it 'must returns pull request info' do From 1ba0f5cf61ff528819be4c4e55832018c48d1071 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Mon, 17 Jun 2024 20:03:41 -0300 Subject: [PATCH 43/53] Delayed Job This commit introduces the use of DelayedJob to avoid spamming the GitHub CI and simplify its output. Two workers were added to update the output and status of each PR. Signed-off-by: Rodrigo Nardi --- Rakefile | 16 +++++++++ bin/console.rb | 1 + config.ru | 5 +++ config/delayed_job.rb | 36 +++++++++++++++++++ .../20240617121935_create_delayed_jobs.rb | 26 ++++++++++++++ db/schema.rb | 16 ++++++++- lib/delayed_job_ctrl/delayed_job_ctrl.rb | 34 ++++++++++++++++++ lib/github/update_status.rb | 27 +++++++++----- lib/github_ci_app.rb | 2 ++ workers/ci_job_status.rb | 30 ++++++++++++++++ 10 files changed, 184 insertions(+), 9 deletions(-) create mode 100644 config/delayed_job.rb create mode 100644 db/migrate/20240617121935_create_delayed_jobs.rb create mode 100644 lib/delayed_job_ctrl/delayed_job_ctrl.rb create mode 100644 workers/ci_job_status.rb diff --git a/Rakefile b/Rakefile index 8d3e9f4..252df08 100644 --- a/Rakefile +++ b/Rakefile @@ -13,6 +13,22 @@ require 'otr-activerecord' load 'tasks/otr-activerecord.rake' +task :environment do + require_relative 'config/delayed_job' +end + +namespace :jobs do + desc 'Clear the delayed_job queue.' + task clear: :environment do + Delayed::Job.delete_all + end + + desc 'Start a delayed_job worker.' + task work: :environment do + Delayed::Worker.new(min_priority: ENV.fetch('MIN_PRIORITY', 1), max_priority: ENV.fetch('MAX_PRIORITY', 10)).start + end +end + namespace :db do # Some db tasks require your app code to be loaded; they'll expect to find it here task :environment do diff --git a/bin/console.rb b/bin/console.rb index b6424af..4633219 100755 --- a/bin/console.rb +++ b/bin/console.rb @@ -16,5 +16,6 @@ puts "Starting console: #{ENV.fetch('RAILS_ENV', nil)}" require_relative '../config/setup' +require_relative '../config/delayed_job' IRB.start diff --git a/config.ru b/config.ru index b8d98ff..79724d7 100755 --- a/config.ru +++ b/config.ru @@ -9,6 +9,8 @@ # frozen_string_literal: true require_relative 'app/github_app' +require_relative 'config/delayed_job' +require_relative 'lib/delayed_job_ctrl/delayed_job_ctrl' require 'puma' require 'rack/handler/puma' @@ -16,6 +18,9 @@ require 'rack/session/cookie' File.write('.session.key', SecureRandom.hex(32)) +DelayedJobCtrl.instance.create_worker(0, 5) +DelayedJobCtrl.instance.create_worker(6, 9) + use Rack::Session::Cookie, secret: File.read('.session.key'), same_site: true, max_age: 86_400 Rack::Handler::Puma.run Rack::URLMap.new('/' => GithubApp) diff --git a/config/delayed_job.rb b/config/delayed_job.rb new file mode 100644 index 0000000..e780e9c --- /dev/null +++ b/config/delayed_job.rb @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# delayed_job.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +require_relative '../lib/helpers/github_logger' +require_relative '../database_loader' + +require 'delayed_job' +require 'active_support' + +module Rails + class << self + attr_accessor :logger + end +end + +Rails.logger = GithubLogger.instance.create('delayed_job.log', Logger::INFO) +ActiveRecord::Base.logger = GithubLogger.instance.create('delayed_job.log', Logger::INFO) + +# this is used by DJ to guess where tmp/pids is located (default) +RAILS_ROOT = File.expand_path(__FILE__) + +Delayed::Worker.backend = :active_record +Delayed::Worker.destroy_failed_jobs = true +Delayed::Worker.sleep_delay = 5 +Delayed::Worker.max_attempts = 5 +Delayed::Worker.max_run_time = 5.minutes + +config = YAML.load_file('config/database.yml')[ENV.fetch('RACK_ENV', 'development')] +ActiveRecord::Base.establish_connection(config) diff --git a/db/migrate/20240617121935_create_delayed_jobs.rb b/db/migrate/20240617121935_create_delayed_jobs.rb new file mode 100644 index 0000000..d048d92 --- /dev/null +++ b/db/migrate/20240617121935_create_delayed_jobs.rb @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# 20240617121935_create_delayed_jobs.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +class CreateDelayedJobs < ActiveRecord::Migration[6.0] + def change + create_table :delayed_jobs do |t| + t.integer :priority, default: 0, null: false + t.integer :attempts, default: 0, null: false + t.text :handler, null: false + t.text :last_error + t.datetime :run_at + t.datetime :locked_at + t.datetime :failed_at + t.string :locked_by + t.string :queue + t.timestamps null: false + end + end +end diff --git a/db/schema.rb b/db/schema.rb index ff946ad..5dafe7f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2024_04_17_130601) do +ActiveRecord::Schema[7.0].define(version: 2024_06_17_121935) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -81,6 +81,20 @@ t.index ["stage_id"], name: "index_ci_jobs_on_stage_id" end + create_table "delayed_jobs", force: :cascade do |t| + t.integer "priority", default: 0, null: false + t.integer "attempts", default: 0, null: false + t.text "handler", null: false + t.text "last_error" + t.datetime "run_at", precision: nil + t.datetime "locked_at", precision: nil + t.datetime "failed_at", precision: nil + t.string "locked_by" + t.string "queue" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "github_users", force: :cascade do |t| t.string "github_login" t.string "github_username" diff --git a/lib/delayed_job_ctrl/delayed_job_ctrl.rb b/lib/delayed_job_ctrl/delayed_job_ctrl.rb new file mode 100644 index 0000000..88990d4 --- /dev/null +++ b/lib/delayed_job_ctrl/delayed_job_ctrl.rb @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# delayed_job_ctrl.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +require 'singleton' +require_relative '../../config/delayed_job' + +class DelayedJobCtrl + include Singleton + + def initialize + @threads = [] + @stop_requested = false + end + + def create_worker(min_priority, max_priority) + @threads << + Thread.new do + worker = Delayed::Worker.new(min_priority: min_priority, max_priority: max_priority, quiet: false) + + Thread.exit if @stop_requested + + worker.start + rescue StandardError + Thread.exit + end + end +end diff --git a/lib/github/update_status.rb b/lib/github/update_status.rb index f597b15..9acdbf0 100644 --- a/lib/github/update_status.rb +++ b/lib/github/update_status.rb @@ -79,7 +79,7 @@ def update_status return [200, 'Success'] unless @job.check_suite.pull_request.current_execution? @job.check_suite - update_build_summary_or_finished + insert_new_delayed_job [200, 'Success'] rescue StandardError => e @@ -88,16 +88,27 @@ def update_status [500, 'Internal Server Error'] end - def update_build_summary_or_finished - summary = Github::Build::Summary.new(@job) - summary.build_summary + def insert_new_delayed_job + queue = @job.check_suite.pull_request.github_pr_id % 10 - return unless @job.finished? + if can_add_new_job? + return CiJobStatus.delay(run_at: 5.seconds.from_now, priority: queue).update(@job.check_suite.id, @job.id) + end + + delete_and_create_delayed_job(queue) + end - logger(Logger::INFO, "Github::PlanExecution::Finished: '#{@job.check_suite.bamboo_ci_ref}'") + def delete_and_create_delayed_job(queue) + fetch_delayed_job.destroy_all + CiJobStatus.delay(run_at: 5.seconds.from_now, priority: queue).update(@job.check_suite.id, @job.id) + end + + def can_add_new_job? + fetch_delayed_job.empty? + end - finished = Github::PlanExecution::Finished.new({ 'bamboo_ref' => @job.check_suite.bamboo_ci_ref }) - finished.finished + def fetch_delayed_job + Delayed::Job.where('handler LIKE ?', "%method_name: :update\nargs:\n- #{@job.check_suite.id}%") end def current_execution? diff --git a/lib/github_ci_app.rb b/lib/github_ci_app.rb index b09faf4..58cf6b3 100644 --- a/lib/github_ci_app.rb +++ b/lib/github_ci_app.rb @@ -36,6 +36,8 @@ require_relative 'helpers/sinatra_payload' require_relative 'helpers/telemetry' +require_relative '../workers/ci_job_status' + # Slack libs require_relative 'slack/slack' diff --git a/workers/ci_job_status.rb b/workers/ci_job_status.rb new file mode 100644 index 0000000..970244c --- /dev/null +++ b/workers/ci_job_status.rb @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# ci_job_status.rb +# Part of NetDEF CI System +# +# Copyright (c) 2024 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +require_relative '../config/setup' + +class CiJobStatus + def self.update(check_suite_id, ci_job_id) + @logger = GithubLogger.instance.create('ci_job_status.log', Logger::INFO) + @logger.info("CiJobStatus::Update: Checksuite #{check_suite_id} -> '#{ci_job_id}'") + + job = CiJob.find(ci_job_id) + + summary = Github::Build::Summary.new(job) + summary.build_summary + + return unless job.finished? + + @logger.info("Github::PlanExecution::Finished: '#{job.check_suite.bamboo_ci_ref}'") + + finished = Github::PlanExecution::Finished.new({ 'bamboo_ref' => job.check_suite.bamboo_ci_ref }) + finished.finished + end +end From 437b3adab54edc964b926b575e018ecc223b0cda Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Mon, 17 Jun 2024 20:06:47 -0300 Subject: [PATCH 44/53] Delayed Job Adding no code coverage to this class Signed-off-by: Rodrigo Nardi --- lib/delayed_job_ctrl/delayed_job_ctrl.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/delayed_job_ctrl/delayed_job_ctrl.rb b/lib/delayed_job_ctrl/delayed_job_ctrl.rb index 88990d4..1e36835 100644 --- a/lib/delayed_job_ctrl/delayed_job_ctrl.rb +++ b/lib/delayed_job_ctrl/delayed_job_ctrl.rb @@ -14,6 +14,7 @@ class DelayedJobCtrl include Singleton + # :nocov: def initialize @threads = [] @stop_requested = false @@ -31,4 +32,5 @@ def create_worker(min_priority, max_priority) Thread.exit end end + # :nocov: end From 5c8e248fcd55d264a2eee4dc479d96e5a5ba8359 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Mon, 17 Jun 2024 20:31:03 -0300 Subject: [PATCH 45/53] Delayed Job Adding DelayedJob at unit tests Signed-off-by: Rodrigo Nardi --- config.ru | 2 ++ lib/delayed_job_ctrl/delayed_job_ctrl.rb | 6 ++++++ lib/github/update_status.rb | 8 ++++++-- spec/spec_helper.rb | 10 ++++++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/config.ru b/config.ru index 79724d7..6afbff4 100755 --- a/config.ru +++ b/config.ru @@ -25,4 +25,6 @@ use Rack::Session::Cookie, secret: File.read('.session.key'), same_site: true, m Rack::Handler::Puma.run Rack::URLMap.new('/' => GithubApp) +DelayedJobCtrl.instance.stop_workers + exit 0 diff --git a/lib/delayed_job_ctrl/delayed_job_ctrl.rb b/lib/delayed_job_ctrl/delayed_job_ctrl.rb index 1e36835..f4ef16e 100644 --- a/lib/delayed_job_ctrl/delayed_job_ctrl.rb +++ b/lib/delayed_job_ctrl/delayed_job_ctrl.rb @@ -14,6 +14,8 @@ class DelayedJobCtrl include Singleton + DELAY = 5 + # :nocov: def initialize @threads = [] @@ -32,5 +34,9 @@ def create_worker(min_priority, max_priority) Thread.exit end end + + def stop_workers + @stop_requested = true + end # :nocov: end diff --git a/lib/github/update_status.rb b/lib/github/update_status.rb index 9acdbf0..969361f 100644 --- a/lib/github/update_status.rb +++ b/lib/github/update_status.rb @@ -92,7 +92,9 @@ def insert_new_delayed_job queue = @job.check_suite.pull_request.github_pr_id % 10 if can_add_new_job? - return CiJobStatus.delay(run_at: 5.seconds.from_now, priority: queue).update(@job.check_suite.id, @job.id) + return CiJobStatus + .delay(run_at: DelayedJobCtrl::DELAY.seconds.from_now, priority: queue) + .update(@job.check_suite.id, @job.id) end delete_and_create_delayed_job(queue) @@ -100,7 +102,9 @@ def insert_new_delayed_job def delete_and_create_delayed_job(queue) fetch_delayed_job.destroy_all - CiJobStatus.delay(run_at: 5.seconds.from_now, priority: queue).update(@job.check_suite.id, @job.id) + CiJobStatus + .delay(run_at: DelayedJobCtrl::DELAY.seconds.from_now, priority: queue) + .update(@job.check_suite.id, @job.id) end def can_add_new_job? diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index be86bd0..30769a6 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -26,6 +26,7 @@ SimpleCov.start require_relative '../app/github_app' +require_relative '../lib/delayed_job_ctrl/delayed_job_ctrl' def app GithubApp @@ -40,6 +41,15 @@ def app config.before(:all) do DatabaseCleaner.clean + DelayedJobCtrl.instance.create_worker(0, 10) + end + + config.after(:all) do + DelayedJobCtrl.instance.stop_workers + end + + config.before(:each) do + Delayed::Worker.delay_jobs = false end config.after(:each) do From ce6a15ec2badb131f59058579793a0a52e9ab8cc Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Tue, 18 Jun 2024 06:08:21 -0300 Subject: [PATCH 46/53] Delayed Job Adding Gem Signed-off-by: Rodrigo Nardi --- Gemfile | 3 +++ Gemfile.lock | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/Gemfile b/Gemfile index f7cda4c..a7db1a5 100644 --- a/Gemfile +++ b/Gemfile @@ -34,6 +34,9 @@ gem 'puma', '5.5.2' gem 'rake', '13.0.6' +# Delayed Job +gem 'delayed_job_active_record' + # Code lint gem 'rubocop', '1.56.1', group: %i[development test] gem 'rubocop-performance', group: %i[development test] diff --git a/Gemfile.lock b/Gemfile.lock index 7530fd2..088517d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -24,6 +24,11 @@ GEM activerecord (>= 5.a) database_cleaner-core (~> 2.0.0) database_cleaner-core (2.0.1) + delayed_job (4.1.11) + activesupport (>= 3.0, < 8.0) + delayed_job_active_record (4.1.8) + activerecord (>= 3.0, < 8.0) + delayed_job (>= 3.0, < 5) diff-lcs (1.5.0) docile (1.4.0) factory_bot (6.2.1) @@ -152,6 +157,7 @@ PLATFORMS DEPENDENCIES database_cleaner + delayed_job_active_record factory_bot faker jwt (= 2.2.2) From f092a6cbdee6d03d1edd343ff3efd98d7bc4646d Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Thu, 20 Jun 2024 10:17:17 -0300 Subject: [PATCH 47/53] Delayed Job Running DelayedJob in a separate process from Puma. Signed-off-by: Rodrigo Nardi --- Rakefile | 15 +++++---- config.ru | 9 ++--- config/delayed_job.rb | 2 ++ lib/delayed_job_ctrl/delayed_job_ctrl.rb | 42 ------------------------ lib/github/update_status.rb | 4 +-- spec/spec_helper.rb | 15 +++++++-- 6 files changed, 30 insertions(+), 57 deletions(-) delete mode 100644 lib/delayed_job_ctrl/delayed_job_ctrl.rb diff --git a/Rakefile b/Rakefile index 252df08..6bfb59b 100644 --- a/Rakefile +++ b/Rakefile @@ -13,19 +13,22 @@ require 'otr-activerecord' load 'tasks/otr-activerecord.rake' -task :environment do - require_relative 'config/delayed_job' -end +require_relative 'config/delayed_job' +require_relative 'config/setup' namespace :jobs do desc 'Clear the delayed_job queue.' - task clear: :environment do + task :clear do Delayed::Job.delete_all end desc 'Start a delayed_job worker.' - task work: :environment do - Delayed::Worker.new(min_priority: ENV.fetch('MIN_PRIORITY', 1), max_priority: ENV.fetch('MAX_PRIORITY', 10)).start + task :work do + Delayed::Worker.new( + min_priority: ENV.fetch('MIN_PRIORITY', 0), + max_priority: ENV.fetch('MAX_PRIORITY', 10), + quiet: false + ).start end end diff --git a/config.ru b/config.ru index 6afbff4..2f0c02b 100755 --- a/config.ru +++ b/config.ru @@ -10,7 +10,6 @@ require_relative 'app/github_app' require_relative 'config/delayed_job' -require_relative 'lib/delayed_job_ctrl/delayed_job_ctrl' require 'puma' require 'rack/handler/puma' @@ -18,13 +17,15 @@ require 'rack/session/cookie' File.write('.session.key', SecureRandom.hex(32)) -DelayedJobCtrl.instance.create_worker(0, 5) -DelayedJobCtrl.instance.create_worker(6, 9) +pids = [] +pids << spawn("RACK_ENV=#{ENV.fetch('RACK_ENV', 'development')} rake jobs:work MIN_PRIORITY=0 MAX_PRIORITY=3") +pids << spawn("RACK_ENV=#{ENV.fetch('RACK_ENV', 'development')} rake jobs:work MIN_PRIORITY=4 MAX_PRIORITY=6") +pids << spawn("RACK_ENV=#{ENV.fetch('RACK_ENV', 'development')} rake jobs:work MIN_PRIORITY=7 MAX_PRIORITY=9") use Rack::Session::Cookie, secret: File.read('.session.key'), same_site: true, max_age: 86_400 Rack::Handler::Puma.run Rack::URLMap.new('/' => GithubApp) -DelayedJobCtrl.instance.stop_workers +pids.each { |pid| Process.kill('TERM', pid.to_i) } exit 0 diff --git a/config/delayed_job.rb b/config/delayed_job.rb index e780e9c..eb33989 100644 --- a/config/delayed_job.rb +++ b/config/delayed_job.rb @@ -20,6 +20,8 @@ class << self end end +DELAYED_JOB_TIMER = 5 + Rails.logger = GithubLogger.instance.create('delayed_job.log', Logger::INFO) ActiveRecord::Base.logger = GithubLogger.instance.create('delayed_job.log', Logger::INFO) diff --git a/lib/delayed_job_ctrl/delayed_job_ctrl.rb b/lib/delayed_job_ctrl/delayed_job_ctrl.rb deleted file mode 100644 index f4ef16e..0000000 --- a/lib/delayed_job_ctrl/delayed_job_ctrl.rb +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-License-Identifier: BSD-2-Clause -# -# delayed_job_ctrl.rb -# Part of NetDEF CI System -# -# Copyright (c) 2024 by -# Network Device Education Foundation, Inc. ("NetDEF") -# -# frozen_string_literal: true - -require 'singleton' -require_relative '../../config/delayed_job' - -class DelayedJobCtrl - include Singleton - - DELAY = 5 - - # :nocov: - def initialize - @threads = [] - @stop_requested = false - end - - def create_worker(min_priority, max_priority) - @threads << - Thread.new do - worker = Delayed::Worker.new(min_priority: min_priority, max_priority: max_priority, quiet: false) - - Thread.exit if @stop_requested - - worker.start - rescue StandardError - Thread.exit - end - end - - def stop_workers - @stop_requested = true - end - # :nocov: -end diff --git a/lib/github/update_status.rb b/lib/github/update_status.rb index 969361f..c189bb5 100644 --- a/lib/github/update_status.rb +++ b/lib/github/update_status.rb @@ -93,7 +93,7 @@ def insert_new_delayed_job if can_add_new_job? return CiJobStatus - .delay(run_at: DelayedJobCtrl::DELAY.seconds.from_now, priority: queue) + .delay(run_at: DELAYED_JOB_TIMER.seconds.from_now, priority: queue) .update(@job.check_suite.id, @job.id) end @@ -103,7 +103,7 @@ def insert_new_delayed_job def delete_and_create_delayed_job(queue) fetch_delayed_job.destroy_all CiJobStatus - .delay(run_at: DelayedJobCtrl::DELAY.seconds.from_now, priority: queue) + .delay(run_at: DELAYED_JOB_TIMER.seconds.from_now, priority: queue) .update(@job.check_suite.id, @job.id) end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 30769a6..7a2393f 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -26,7 +26,7 @@ SimpleCov.start require_relative '../app/github_app' -require_relative '../lib/delayed_job_ctrl/delayed_job_ctrl' +require_relative '../config/delayed_job' def app GithubApp @@ -39,13 +39,22 @@ def app config.include FactoryBot::Syntax::Methods config.include WebMock::API + pid = nil + config.before(:all) do DatabaseCleaner.clean - DelayedJobCtrl.instance.create_worker(0, 10) + + pid = Thread.new do + Delayed::Worker.new( + min_priority: 0, + max_priority: 10, + quiet: true + ).start + end end config.after(:all) do - DelayedJobCtrl.instance.stop_workers + pid&.exit end config.before(:each) do From 4d77ac4081a738a5e3c7e35d8177f63be4ee8d94 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Thu, 20 Jun 2024 10:46:26 -0300 Subject: [PATCH 48/53] Delayed Job Changing DelyaedJob to use queue instead priority Signed-off-by: Rodrigo Nardi --- Rakefile | 4 ++-- config.ru | 6 +++--- lib/github/update_status.rb | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Rakefile b/Rakefile index 6bfb59b..62e12e2 100644 --- a/Rakefile +++ b/Rakefile @@ -24,9 +24,9 @@ namespace :jobs do desc 'Start a delayed_job worker.' task :work do + puts "Starting delayed_job worker - Queues: #{ENV.fetch('QUEUES', 'default')}" Delayed::Worker.new( - min_priority: ENV.fetch('MIN_PRIORITY', 0), - max_priority: ENV.fetch('MAX_PRIORITY', 10), + queue: ENV.fetch('QUEUES', 'default'), quiet: false ).start end diff --git a/config.ru b/config.ru index 2f0c02b..0d34e4c 100755 --- a/config.ru +++ b/config.ru @@ -18,9 +18,9 @@ require 'rack/session/cookie' File.write('.session.key', SecureRandom.hex(32)) pids = [] -pids << spawn("RACK_ENV=#{ENV.fetch('RACK_ENV', 'development')} rake jobs:work MIN_PRIORITY=0 MAX_PRIORITY=3") -pids << spawn("RACK_ENV=#{ENV.fetch('RACK_ENV', 'development')} rake jobs:work MIN_PRIORITY=4 MAX_PRIORITY=6") -pids << spawn("RACK_ENV=#{ENV.fetch('RACK_ENV', 'development')} rake jobs:work MIN_PRIORITY=7 MAX_PRIORITY=9") +pids << spawn("RACK_ENV=#{ENV.fetch('RACK_ENV', 'development')} rake jobs:work QUEUES=0,1,2,3") +pids << spawn("RACK_ENV=#{ENV.fetch('RACK_ENV', 'development')} rake jobs:work QUEUES=4,5,6") +pids << spawn("RACK_ENV=#{ENV.fetch('RACK_ENV', 'development')} rake jobs:work QUEUES=7,8,9") use Rack::Session::Cookie, secret: File.read('.session.key'), same_site: true, max_age: 86_400 diff --git a/lib/github/update_status.rb b/lib/github/update_status.rb index c189bb5..a9ec4ba 100644 --- a/lib/github/update_status.rb +++ b/lib/github/update_status.rb @@ -93,7 +93,7 @@ def insert_new_delayed_job if can_add_new_job? return CiJobStatus - .delay(run_at: DELAYED_JOB_TIMER.seconds.from_now, priority: queue) + .delay(run_at: DELAYED_JOB_TIMER.seconds.from_now, queue: queue) .update(@job.check_suite.id, @job.id) end @@ -103,7 +103,7 @@ def insert_new_delayed_job def delete_and_create_delayed_job(queue) fetch_delayed_job.destroy_all CiJobStatus - .delay(run_at: DELAYED_JOB_TIMER.seconds.from_now, priority: queue) + .delay(run_at: DELAYED_JOB_TIMER.seconds.from_now, queue: queue) .update(@job.check_suite.id, @job.id) end From 587dee9ceba624c0ae072f72f833376f1156b3e0 Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Thu, 20 Jun 2024 10:50:21 -0300 Subject: [PATCH 49/53] Delayed Job Changing Process kill handler Signed-off-by: Rodrigo Nardi --- config.ru | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config.ru b/config.ru index 0d34e4c..bfc2438 100755 --- a/config.ru +++ b/config.ru @@ -26,6 +26,10 @@ use Rack::Session::Cookie, secret: File.read('.session.key'), same_site: true, m Rack::Handler::Puma.run Rack::URLMap.new('/' => GithubApp) -pids.each { |pid| Process.kill('TERM', pid.to_i) } +pids.each do |pid| + Process.kill('TERM', pid.to_i) +rescue Errno::ESRCH + puts "Process #{pid} already dead" +end exit 0 From 954a613893bcb23546db83638d0daf3fc4664fdb Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Fri, 21 Jun 2024 21:45:55 -0300 Subject: [PATCH 50/53] Github::Check May receive a null object when searching for all check runs referring to Pull Request. Signed-off-by: Rodrigo Nardi --- lib/github/check.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/github/check.rb b/lib/github/check.rb index a09fc62..ec7528c 100644 --- a/lib/github/check.rb +++ b/lib/github/check.rb @@ -102,14 +102,17 @@ def get_check_run(check_ref) def fetch_check_runs return [] if @check_suite.nil? - @app + check_runs = + @app .check_runs_for_ref( @check_suite.pull_request.repository, @check_suite.pull_request.branch_name, accept: Octokit::Preview::PREVIEW_TYPES[:checks] - ) - .to_h[:check_runs] - .map do |check_run| + ).to_h[:check_runs] + + return [] if check_runs.nil? + + check_runs.map do |check_run| check_run[:id] end rescue Octokit::UnprocessableEntity => e From 704c37518f90de398592b91f7c2dda96b794df4d Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Fri, 21 Jun 2024 22:10:59 -0300 Subject: [PATCH 51/53] Github::Check Removing unnecessary command Signed-off-by: Rodrigo Nardi --- lib/github/build/summary.rb | 18 +----------------- lib/github/check.rb | 22 ---------------------- spec/lib/github/check_spec.rb | 17 +++++++++++++++++ 3 files changed, 18 insertions(+), 39 deletions(-) diff --git a/lib/github/build/summary.rb b/lib/github/build/summary.rb index 4a6f191..80f1d5d 100644 --- a/lib/github/build/summary.rb +++ b/lib/github/build/summary.rb @@ -34,7 +34,7 @@ def initialize(job, logger_level: Logger::INFO, agent: 'Github') def build_summary current_stage = @job.stage current_stage = fetch_parent_stage if current_stage.nil? - check_and_update_github_ref(current_stage) + current_stage.refresh_reference(@github) logger(Logger::INFO, "build_summary: #{current_stage.inspect}") msg = "Github::Build::Summary - #{@job.inspect}, #{current_stage.inspect}, bamboo info: #{bamboo_info}" @@ -60,22 +60,6 @@ def bamboo_info finish.fetch_build_status end - def check_and_update_github_ref(current_stage) - current_refs = @github.fetch_check_runs - - logger(Logger::INFO, - 'check_and_update_github_ref - current_stage: ' \ - "#{current_stage.inspect} current_refs: #{current_refs.inspect}") - - return if current_refs.include? current_stage.check_ref.to_i - - logger(Logger::INFO, - 'check_and_update_github_ref - current_stage: ' \ - "#{current_stage.inspect} current_refs: #{current_refs.inspect} - Refreshing reference.") - - current_stage.refresh_reference(@github) - end - def must_update_previous_stage(current_stage) previous_stage = current_stage.previous_stage diff --git a/lib/github/check.rb b/lib/github/check.rb index ec7528c..a5f6b20 100644 --- a/lib/github/check.rb +++ b/lib/github/check.rb @@ -99,28 +99,6 @@ def get_check_run(check_ref) accept: Octokit::Preview::PREVIEW_TYPES[:checks]).to_h end - def fetch_check_runs - return [] if @check_suite.nil? - - check_runs = - @app - .check_runs_for_ref( - @check_suite.pull_request.repository, - @check_suite.pull_request.branch_name, - accept: Octokit::Preview::PREVIEW_TYPES[:checks] - ).to_h[:check_runs] - - return [] if check_runs.nil? - - check_runs.map do |check_run| - check_run[:id] - end - rescue Octokit::UnprocessableEntity => e - @logger.error "fetch_check_runs: #{e.class} #{e.message}" - - [] - end - def installation_id @authenticate_app.find_app_installations(accept: 'application/vnd.github.v3+json').first['id'].to_i end diff --git a/spec/lib/github/check_spec.rb b/spec/lib/github/check_spec.rb index 89d5b97..61daa87 100644 --- a/spec/lib/github/check_spec.rb +++ b/spec/lib/github/check_spec.rb @@ -84,6 +84,23 @@ end end + context 'when call comment_reaction_thumb_down' do + let(:pr_id) { 1 } + let(:repo) { 'test' } + let(:comment_id) { 1 } + let(:pr_info) { { comment_id: comment_id } } + + before do + allow(fake_client).to receive(:create_issue_comment_reaction) + .with(repo, comment_id, '-1', accept: Octokit::Preview::PREVIEW_TYPES[:reactions]) + .and_return(pr_info) + end + + it 'must returns pull request info' do + expect(check.comment_reaction_thumb_down(repo, comment_id)).to eq(pr_info) + end + end + context 'when call create' do let(:pr_id) { 1 } let(:name) { 'test' } From a27dbf754a759e6dca2d82b4d9d0890f86b5652a Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Fri, 21 Jun 2024 22:12:19 -0300 Subject: [PATCH 52/53] Github::Check Code lint Signed-off-by: Rodrigo Nardi --- spec/lib/github/check_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/lib/github/check_spec.rb b/spec/lib/github/check_spec.rb index 61daa87..97438d9 100644 --- a/spec/lib/github/check_spec.rb +++ b/spec/lib/github/check_spec.rb @@ -92,8 +92,8 @@ before do allow(fake_client).to receive(:create_issue_comment_reaction) - .with(repo, comment_id, '-1', accept: Octokit::Preview::PREVIEW_TYPES[:reactions]) - .and_return(pr_info) + .with(repo, comment_id, '-1', accept: Octokit::Preview::PREVIEW_TYPES[:reactions]) + .and_return(pr_info) end it 'must returns pull request info' do From b28980dfcc557428110e666706dd3cae199b344a Mon Sep 17 00:00:00 2001 From: Rodrigo Nardi Date: Wed, 26 Jun 2024 09:03:18 -0300 Subject: [PATCH 53/53] Summary Adding queued status output Signed-off-by: Rodrigo Nardi --- lib/github/build/summary.rb | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/github/build/summary.rb b/lib/github/build/summary.rb index 80f1d5d..4ab7170 100644 --- a/lib/github/build/summary.rb +++ b/lib/github/build/summary.rb @@ -167,9 +167,8 @@ def update_summary(stage) def summary_basic_output(stage) jobs = stage.jobs.reload - in_progress = jobs.where(status: :in_progress) - header = ":arrow_right: Jobs in progress: #{in_progress.size}/#{jobs.size}\n\n" + header = queued_message(jobs) header += in_progress_message(jobs) header += generate_success_failure_info(stage.name, jobs) @@ -200,11 +199,25 @@ def generate_success_failure_info(name, jobs) end def in_progress_message(jobs) - jobs.where(status: %i[in_progress queued]).map do |job| + in_progress = jobs.where(status: :in_progress) + + message = "\n\n:arrow_right: Jobs in progress: #{in_progress.size}/#{jobs.size}\n\n" + + message + jobs.where(status: %i[in_progress]).map do |job| "- **#{job.name}** -> https://ci1.netdef.org/browse/#{job.job_ref}\n" end.join("\n") end + def queued_message(jobs) + queued = jobs.where(status: :queued) + + message = ":arrow_right: Jobs queued: #{queued.size}/#{jobs.size}\n\n" + message + + queued.map do |job| + "- **#{job.name}** -> https://ci1.netdef.org/browse/#{job.job_ref}\n" + end.join("\n") + end + def success_message(jobs) jobs.where(status: :success).map do |job| "- **#{job.name}** -> https://ci1.netdef.org/browse/#{job.job_ref}\n"