diff --git a/app/github_app.rb b/app/github_app.rb index e89af4d..beebad1 100755 --- a/app/github_app.rb +++ b/app/github_app.rb @@ -118,7 +118,7 @@ def sinatra_logger_level when 'check_run' logger.debug "Check Run #{payload.dig('check_run', 'id')} - #{payload['action']}" - halt 200, 'OK' unless %w[created rerequested].include? payload['action'].downcase + halt 200, 'OK' unless %w[rerequested].include? payload['action'].downcase re_run = Github::Retry.new(payload, logger_level: GithubApp.sinatra_logger_level) halt re_run.start diff --git a/db/migrate/20231108100757_add_ci_job_stage.rb b/db/migrate/20231108100757_add_ci_job_stage.rb new file mode 100644 index 0000000..a4dd112 --- /dev/null +++ b/db/migrate/20231108100757_add_ci_job_stage.rb @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# 20231023090822_create_pull_request_subscribe.rb +# Part of NetDEF CI System +# +# Copyright (c) 2023 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +class AddCiJobStage < ActiveRecord::Migration[6.0] + def change + add_column :ci_jobs, :stage, :boolean, default: false + end +end diff --git a/db/schema.rb b/db/schema.rb index d990214..b8ff5c4 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2023_10_23_090823) do +ActiveRecord::Schema[7.0].define(version: 2023_11_08_100757) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -40,6 +40,7 @@ t.datetime "updated_at", null: false t.bigint "check_suite_id" t.integer "retry", default: 0 + t.boolean "stage", default: false t.index ["check_suite_id"], name: "index_ci_jobs_on_check_suite_id" end diff --git a/lib/bamboo_ci/download.rb b/lib/bamboo_ci/download.rb new file mode 100644 index 0000000..da9f242 --- /dev/null +++ b/lib/bamboo_ci/download.rb @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# download.rb +# Part of NetDEF CI System +# +# Copyright (c) 2023 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +require 'logger' + +require_relative 'api' + +module BambooCi + class Download + extend BambooCi::Api + + def self.build_log(url) + count = 0 + uri = URI(url) + + begin + body = download(uri).split("\n").last(10).join("\n") + + raise 'Must try' if body.empty? + + body + rescue StandardError + count += 1 + + sleep 5 + retry if count <= 3 + + '' + end + end + end +end diff --git a/lib/bamboo_ci/result.rb b/lib/bamboo_ci/result.rb index f3f322d..280dee5 100644 --- a/lib/bamboo_ci/result.rb +++ b/lib/bamboo_ci/result.rb @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-2-Clause # -# stop_plan.rb +# result.rb # Part of NetDEF CI System # # Copyright (c) 2023 by @@ -16,8 +16,8 @@ module BambooCi class Result extend BambooCi::Api - def self.fetch(job_key) - uri = URI("https://127.0.0.1/rest/api/latest/result/#{job_key}?expand=testResults.failedTests.testResult.errors") + def self.fetch(job_key, expand: 'testResults.failedTests.testResult.errors') + uri = URI("https://127.0.0.1/rest/api/latest/result/#{job_key}?expand=#{expand}") get_request(uri) end end diff --git a/lib/github/build/action.rb b/lib/github/build/action.rb new file mode 100644 index 0000000..eea17ea --- /dev/null +++ b/lib/github/build/action.rb @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# action.rb +# Part of NetDEF CI System +# +# Copyright (c) 2023 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +module Github + module Build + class Action + BUILD_STAGE = 'Build' + TESTS_STAGE = 'Tests' + SOURCE_CODE = 'Verify Source' + SUMMARY = [BUILD_STAGE, TESTS_STAGE].freeze + STAGE_POSITION = { SOURCE_CODE => '01', BUILD_STAGE => '02', TESTS_STAGE => '03' }.freeze + + def initialize(check_suite, github, logger_level: Logger::INFO) + @check_suite = check_suite + @github = github + @loggers = [] + + %w[github_app.log github_build_action.log].each do |filename| + logger_app = Logger.new(filename, 1, 1_024_000) + logger_app.level = logger_level + + @loggers << logger_app + end + + logger(Logger::INFO, "Building action to CheckSuite @#{@check_suite.inspect}") + end + + def create_summary + logger(Logger::INFO, "SUMMARY #{SUMMARY.inspect}") + + SUMMARY.each do |name| + create_check_run_stage(name) + end + rescue StandardError => e + logger(Logger::Error, "#{e.class} - #{e.message}") + end + + def create_stage(name) + bamboo_ci = @check_suite.bamboo_ci_ref.split('-').last + + stage = + CiJob.create(check_suite: @check_suite, name: name, job_ref: "#{name}-#{bamboo_ci}", stage: true) + + return stage if stage.persisted? + + logger(Logger::ERROR, "Failed to created: #{stage.inspect} -> #{stage.errors.inspect}") + + nil + end + + def create_jobs(jobs, rerun: false) + jobs.each do |job| + ci_job = CiJob.create(check_suite: @check_suite, name: job[:name], job_ref: job[:job_ref]) + + next unless ci_job.persisted? + + if rerun + next if ci_job.checkout_code? + + url = "https://ci1.netdef.org/browse/#{ci_job.job_ref}" + ci_job.enqueue(@github, { title: ci_job.name, summary: "Details at [#{url}](#{url})" }) + else + ci_job.create_check_run + end + + next unless ci_job.checkout_code? + + ci_job.update(stage: true) + url = "https://ci1.netdef.org/browse/#{ci_job.job_ref}" + ci_job.in_progress(@github, { title: ci_job.name, summary: "Details at [#{url}](#{url})" }) + end + end + + private + + def create_check_run_stage(name) + stage = CiJob.find_by(name: name, check_suite_id: @check_suite.id) + + logger(Logger::INFO, "STAGE #{name} #{stage.inspect} - @#{@check_suite.inspect}") + + stage = create_stage(name) if stage.nil? + + return if stage.nil? or stage.checkout_code? or stage.success? + + logger(Logger::INFO, ">>> Enqueued #{stage.inspect}") + + stage.enqueue(@github, initial_output(stage)) + end + + def initial_output(ci_job) + output = { title: '', summary: '' } + url = "https://ci1.netdef.org/browse/#{ci_job.check_suite.bamboo_ci_ref}" + + output[:title] = "#{ci_job.name} summary" + output[:summary] = "Details at [#{url}](#{url})" + + output + end + + def logger(severity, message) + @loggers.each do |logger_object| + logger_object.add(severity, message) + end + end + end + end +end diff --git a/lib/github/build/retry.rb b/lib/github/build/retry.rb new file mode 100644 index 0000000..bf6bf12 --- /dev/null +++ b/lib/github/build/retry.rb @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# retry.rb +# Part of NetDEF CI System +# +# Copyright (c) 2023 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +require_relative 'action' + +module Github + module Build + class Retry < Action + def initialize(check_suite, github, logger_level: Logger::INFO) + super(check_suite, github) + + @loggers = [] + + %w[github_app.log github_build_retry.log].each do |filename| + logger_app = Logger.new(filename, 1, 1_024_000) + logger_app.level = logger_level + + @loggers << logger_app + end + end + + def enqueued_stages + @check_suite.ci_jobs.stages.where.not(status: :success).each do |ci_job| + logger(Logger::WARN, "Enqueue stages: #{ci_job.inspect}") + + next if ci_job.success? or ci_job.checkout_code? + + ci_job.enqueue(@github, initial_output(ci_job)) + ci_job.update(retry: ci_job.retry + 1) + end + end + + def enqueued_failure_tests + @check_suite.ci_jobs.skip_stages.where.not(status: :success).each do |ci_job| + next if ci_job.checkout_code? + + logger(Logger::WARN, "Enqueue CiJob: #{ci_job.inspect}") + ci_job.enqueue(@github) + ci_job.update(retry: ci_job.retry + 1) + end + end + end + end +end diff --git a/lib/github/build/summary.rb b/lib/github/build/summary.rb new file mode 100644 index 0000000..38fefed --- /dev/null +++ b/lib/github/build/summary.rb @@ -0,0 +1,237 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# summary.rb +# Part of NetDEF CI System +# +# Copyright (c) 2023 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +require_relative '../../github/check' +require_relative '../../bamboo_ci/download' + +module Github + module Build + class Summary + def initialize(job, logger_level: Logger::INFO) + @job = job.reload + @check_suite = @job.check_suite + @github = Github::Check.new(@check_suite) + @loggers = [] + + %w[github_app.log github_build_summary.log].each do |filename| + logger_app = Logger.new(filename, 1, 1_024_000) + logger_app.level = logger_level + + @loggers << logger_app + end + end + + def build_summary(name) + stage = @check_suite.ci_jobs.find_by(name: name) + + logger(Logger::INFO, "build_summary: #{name} -> #{stage.inspect}") + + return if stage.nil? + + update_summary(stage, name) + finished_summary(stage) + missing_stage(stage) + end + + def missing_stage(stage) + missing_test_stage(stage) + missing_build_stage + end + + def missing_test_stage(stage) + tests_stage = @check_suite.ci_jobs.find_by(name: Github::Build::Action::TESTS_STAGE) + url = "https://ci1.netdef.org/browse/#{stage.check_suite.bamboo_ci_ref}" + tests_failure = { + title: "#{Github::Build::Action::TESTS_STAGE} summary", + summary: "Build Stage failed so it will not be possible to run the tests.\nDetails at [#{url}](#{url})." + } + + return tests_stage.cancelled(@github, tests_failure) if stage.build? and stage.failure? + return tests_stage.in_progress(@github) if stage.build? and stage.success? + + return unless stage.test? + return unless @check_suite.finished? + + update_tests_stage(tests_stage) + end + + def missing_build_stage + build_stage = @check_suite.ci_jobs.find_by(name: Github::Build::Action::BUILD_STAGE) + + return if build_stage.nil? + return if build_stage.success? or build_stage.failure? + return unless @check_suite.build_stage_finished? + + url = "https://ci1.netdef.org/browse/#{build_stage.check_suite.bamboo_ci_ref}" + output = { + title: "#{Github::Build::Action::BUILD_STAGE} summary", + summary: "Build stage failure. Please check Bamboo CI.\nDetails at [#{url}](#{url})." + } + + success = @check_suite.build_stage_success? + logger(Logger::INFO, "missing_build_stage: #{build_stage.inspect}, success: #{success}") + + success ? build_stage.success(@github, output) : build_stage.failure(@github, output) + end + + def update_tests_stage(stage) + success = @check_suite.ci_jobs.skip_checkout_code.where(status: %w[failure cancelled]).empty? + + output = { title: "#{stage.name} summary", summary: summary_basic_output(stage.name) } + + logger(Logger::INFO, "update_tests_stage: #{stage.inspect}, success: #{success}") + + success ? stage.success(@github, output) : stage.failure(@github, output) + end + + def finished_summary(stage) + logger(Logger::INFO, "Finished stage: #{stage.inspect}, CiJob status: #{@job.status}") + return if @job.in_progress? + + finished_build_summary(stage) + finished_tests_summary(stage) + end + + def finished_build_summary(stage) + return unless stage.build? + return unless @check_suite.build_stage_finished? + + logger(Logger::INFO, "finished_build_summary: #{stage.inspect}. Reason Job: #{@job.inspect}") + + name = Github::Build::Action::BUILD_STAGE + url = "https://ci1.netdef.org/browse/#{stage.check_suite.bamboo_ci_ref}" + output = { + title: "#{name} summary", + summary: "#{summary_basic_output(name)}\nDetails at [#{url}](#{url})." + } + + logger(Logger::DEBUG, output) + + @check_suite.build_stage_success? ? stage.success(@github, output) : stage.failure(@github, output) + end + + def finished_tests_summary(stage) + return unless stage.test? + return unless @check_suite.finished? + + logger(Logger::INFO, "finished_tests_summary: #{stage.inspect}. Reason Job: #{@job.inspect}") + + name = Github::Build::Action::TESTS_STAGE + url = "https://ci1.netdef.org/browse/#{stage.check_suite.bamboo_ci_ref}" + + output = { + title: "#{name} summary", + summary: "#{summary_basic_output(name)}\nDetails at [#{url}](#{url})." + } + + @check_suite.success? ? stage.success(@github, output) : stage.failure(@github, output) + end + + def update_summary(stage, name) + logger(Logger::INFO, "Updating summary status #{stage.inspect} -> @job.status: #{@job.status}") + + url = "https://ci1.netdef.org/browse/#{@check_suite.bamboo_ci_ref}" + output = { + title: "#{name} summary", + summary: "#{summary_basic_output(name)}\nDetails at [#{url}](#{url})." + } + + stage.in_progress(@github, output) + end + + def summary_basic_output(name) + filter = Github::Build::Action::BUILD_STAGE.include?(name) ? '.* (B|b)uild' : '(Address|TopoTest|Check|Static)' + + jobs = @check_suite.ci_jobs.skip_checkout_code.filter_by(filter).reload + in_progress = jobs.where(status: :in_progress) + + header = ":arrow_right: Jobs in progress: #{in_progress.size}/#{jobs.size}\n\n" + header += in_progress_message(jobs) + header += generate_success_failure_info(name, jobs) + + header[0..65_535] + end + + def generate_success_failure_info(name, jobs) + header = '' + + [ + { + title: ':heavy_multiplication_x: Jobs Failure', + queue: other_message(name, jobs), + size: jobs.where.not(status: %i[in_progress queued success]).size + }, + { + title: ':heavy_check_mark: Jobs Success', + queue: success_message(jobs), + size: jobs.where(status: :success).size + } + ].each do |info| + next if info[:queue].nil? or info[:queue].empty? + + header += "\n#{info[:title]}: #{info[:size]}/#{jobs.size}\n\n#{info[:queue]}" + end + + header + end + + private + + def in_progress_message(jobs) + jobs.where(status: :in_progress).map do |job| + "- **#{job.name}** -> https://ci1.netdef.org/browse/#{job.job_ref}\n" + end.join("\n") + end + + def success_message(jobs) + jobs.where(status: :success).map do |job| + "- **#{job.name}** -> https://ci1.netdef.org/browse/#{job.job_ref}\n" + end.join("\n") + end + + def other_message(name, jobs) + jobs.where.not(status: %i[in_progress queued success]).map do |job| + generate_message(name, job) + end.join("\n") + end + + def generate_message(name, job) + failures = name.downcase.match?('build') ? build_message(job) : tests_message(job) + + "- #{job.name} -> https://ci1.netdef.org/browse/#{job.job_ref}\n#{failures}" + end + + def tests_message(job) + failure = job.topotest_failures.first + + return '' if failure.nil? + + "\t :no_entry_sign: #{failure.test_suite} #{failure.test_case}\n ```\n#{failure.message}\n```\n" + end + + def build_message(job) + output = BambooCi::Result.fetch(job.job_ref, expand: 'testResults.failedTests.testResult.errors,artifacts') + entry = output.dig('artifacts', 'artifact')&.find { |elem| elem['name'] == 'ErrorLog' } + + return '' if entry.nil? or entry.empty? + + body = BambooCi::Download.build_log(entry.dig('link', 'href')) + + "```\n#{body}\n```\n" + end + + def logger(severity, message) + @loggers.each do |logger_object| + logger_object.add(severity, message) + end + end + end + end +end diff --git a/lib/github/build/unavailable_jobs.rb b/lib/github/build/unavailable_jobs.rb index 6294754..62c9d37 100644 --- a/lib/github/build/unavailable_jobs.rb +++ b/lib/github/build/unavailable_jobs.rb @@ -30,8 +30,7 @@ def update(new_check_suite: nil) running_jobs = BambooCi::RunningPlan.fetch(@check_suite.bamboo_ci_ref).map { |entry| entry[:job_ref] } - @check_suite.ci_jobs.where.not(job_ref: running_jobs).each do |unavailable_job| - @logger.warn ">>> Unavailable Job: #{unavailable_job.inspect}" + @check_suite.ci_jobs.skip_stages.where.not(job_ref: running_jobs).each do |unavailable_job| unavailable_job.skipped(@github, output(unavailable_job)) unavailable_job.update(check_suite: new_check_suite) unless new_check_suite.nil? end diff --git a/lib/github/build_plan.rb b/lib/github/build_plan.rb index 3a78615..7bed621 100644 --- a/lib/github/build_plan.rb +++ b/lib/github/build_plan.rb @@ -15,6 +15,7 @@ require_relative '../bamboo_ci/running_plan' require_relative '../bamboo_ci/plan_run' require_relative 'check' +require_relative 'build/action' module Github class BuildPlan @@ -109,7 +110,7 @@ def stop_previous_execution end def cancel_previous_ci_jobs - @last_check_suite.ci_jobs.where(status: %w[queued in_progress]).each do |ci_job| + @last_check_suite.ci_jobs.skip_stages.where(status: %w[queued in_progress]).each do |ci_job| @logger.warn("Cancelling Job #{ci_job.inspect}") ci_job.cancelled(@github_check) end @@ -147,7 +148,9 @@ def ci_jobs return [422, 'Failed to fetch RunningPlan'] if jobs.nil? or jobs.empty? - create_ci_jobs(jobs) + action = Github::Build::Action.new(@check_suite, @github_check) + action.create_jobs(jobs) + action.create_summary @logger.info ">>> @has_previous_exec: #{@has_previous_exec}" stop_execution_message if @has_previous_exec @@ -160,21 +163,6 @@ def stop_execution_message BambooCi::StopPlan.comment(@last_check_suite, @check_suite) end - def create_ci_jobs(jobs) - jobs.each do |job| - ci_job = CiJob.create(check_suite: @check_suite, name: job[:name], job_ref: job[:job_ref]) - - next unless ci_job.persisted? - - ci_job.create_check_run - - next unless ci_job.checkout_code? - - url = "https://ci1.netdef.org/browse/#{ci_job.job_ref}" - ci_job.in_progress(@github_check, { title: ci_job.name, summary: "Details at [#{url}](#{url})" }) - end - end - def ci_vars ci_vars = [] ci_vars << { value: @github_check.signature, name: 'signature_secret' } diff --git a/lib/github/check.rb b/lib/github/check.rb index afd0896..85ef5f3 100644 --- a/lib/github/check.rb +++ b/lib/github/check.rb @@ -22,7 +22,7 @@ class Check def initialize(check_suite) @check_suite = check_suite @config = GitHubApp::Configuration.instance.config - @logger = Logger.new($stdout) + @logger = Logger.new('github_check_api.log', 2, 1_024_000) authenticate_app end @@ -105,7 +105,7 @@ def fetch_username(username) private - def basic_status(id, status, output) + def basic_status(check_ref, status, output) opts = { status: status, accept: 'application/vnd.github+json' @@ -113,11 +113,16 @@ def basic_status(id, status, output) opts[:output] = output unless output.empty? - @app.update_check_run( - @check_suite.pull_request.repository, - id.to_i, - opts - ) + resp = + @app.update_check_run( + @check_suite.pull_request.repository, + check_ref.to_i, + opts + ).to_h + + @logger.info("basic_status: #{check_ref}, status: #{status} -> resp: #{resp}") + + resp end # PS: Conclusion and status are the same name from GitHub Check doc. @@ -133,11 +138,18 @@ def completed(check_ref, status, conclusion, output) opts[:output] = output unless output.empty? - @logger.info @app.update_check_run( - @check_suite.pull_request.repository, - check_ref, - opts - ) + resp = + @app.update_check_run( + @check_suite.pull_request.repository, + check_ref, + opts + ).to_h + + @logger.info("completed: #{check_ref}, status: #{status}, conclusion: #{conclusion} -> resp: #{resp}") + + resp + rescue Octokit::NotFound + @logger.error "#{check_ref} not found at GitHub" end def authenticate_app diff --git a/lib/github/re_run/base.rb b/lib/github/re_run/base.rb index 2125ea9..c45a056 100644 --- a/lib/github/re_run/base.rb +++ b/lib/github/re_run/base.rb @@ -14,6 +14,7 @@ require_relative '../parsers/pull_request_commit' require_relative '../check' +require_relative '../build/action' require_relative '../build/unavailable_jobs' module Github @@ -48,8 +49,7 @@ def stop_previous_execution logger(Logger::INFO, fetch_run_ci_by_pr.inspect) fetch_run_ci_by_pr.each do |check_suite| - check_suite.ci_jobs.not_skipped.each do |ci_job| - logger(Logger::WARN, "Cancelling Job #{ci_job.inspect}") + check_suite.ci_jobs.skip_stages.not_skipped.each do |ci_job| ci_job.cancelled(@github_check) end @@ -60,24 +60,9 @@ def stop_previous_execution def create_ci_jobs(bamboo_plan, check_suite) jobs = BambooCi::RunningPlan.fetch(bamboo_plan.bamboo_reference) - jobs.each do |job| - ci_job = CiJob.create( - check_suite: check_suite, - name: job[:name], - job_ref: job[:job_ref] - ) - - logger(Logger::DEBUG, ">>> CI Job: #{ci_job.inspect}") - next unless ci_job.persisted? - - url = "https://ci1.netdef.org/browse/#{ci_job.job_ref}" - - ci_job.enqueue(@github_check, { title: ci_job.name, summary: "Details at [#{url}](#{url})" }) - - next unless ci_job.checkout_code? - - ci_job.in_progress(@github_check, { title: ci_job.name, summary: "Details at [#{url}](#{url})" }) - end + action = Github::Build::Action.new(check_suite, @github_check) + action.create_summary + action.create_jobs(jobs, rerun: true) end def fetch_plan diff --git a/lib/github/retry.rb b/lib/github/retry.rb index 93baffb..f655619 100644 --- a/lib/github/retry.rb +++ b/lib/github/retry.rb @@ -13,6 +13,7 @@ require_relative '../../database_loader' require_relative '../bamboo_ci/retry' require_relative '../bamboo_ci/stop_plan' +require_relative '../github/build/retry' require_relative 'check' require_relative 'build/unavailable_jobs' @@ -60,14 +61,10 @@ def normal_flow(check_suite) def create_ci_jobs(check_suite) github_check = Github::Check.new(check_suite) - check_suite.ci_jobs.where.not(status: :success).each do |ci_job| - next if ci_job.checkout_code? + build_retry = Github::Build::Retry.new(check_suite, github_check) - ci_job.enqueue(github_check) - ci_job.update(retry: ci_job.retry + 1) - - logger(Logger::WARN, "Stopping Job: #{ci_job.name} - #{ci_job.job_ref}") - end + build_retry.enqueued_stages + build_retry.enqueued_failure_tests BambooCi::StopPlan.build(check_suite.bamboo_ci_ref) end @@ -89,6 +86,8 @@ def enqueued(job) def slack_notification(job) reason = SlackBot.instance.invalid_rerun_group(job) + logger(Logger::WARN, ">>> #{job.inspect} #{reason}") + pull_request = job.check_suite.pull_request PullRequestSubscription diff --git a/lib/github/update_status.rb b/lib/github/update_status.rb index ea70d40..943a6bb 100644 --- a/lib/github/update_status.rb +++ b/lib/github/update_status.rb @@ -13,6 +13,7 @@ require_relative '../../database_loader' require_relative '../../lib/bamboo_ci/result' require_relative '../slack_bot/slack_bot' +require_relative 'build/summary' module Github class UpdateStatus @@ -70,6 +71,10 @@ def update_status slack_notify_failure end + summary = Github::Build::Summary.new(@job) + summary.build_summary(Github::Build::Action::BUILD_STAGE) if @job.build? + summary.build_summary(Github::Build::Action::TESTS_STAGE) if @job.test? + finished_execution? end @@ -104,10 +109,18 @@ def failure end def fetch_and_update_failures(to_be_replaced) - output = BambooCi::Result.fetch(@job.job_ref) - return if output.nil? or output.empty? - - @output[:summary] = @output[:summary].sub(to_be_replaced, fetch_failures(output))[0..65_535] + count = 0 + begin + output = BambooCi::Result.fetch(@job.job_ref) + return if output.nil? or output.empty? + + @output[:summary] = @output[:summary].sub(to_be_replaced, fetch_failures(output))[0..65_535] + rescue NoMethodError => e + @logger.error "#{e.class} #{e.message}" + count += 1 + sleep 5 + retry if count <= 10 + end end def fetch_failures(output) @@ -131,11 +144,10 @@ def fetch_failures(output) end def skipping_jobs - return if @job.checkout_code? - return unless @job.name.downcase.match?(/(code|build)/) and @status == 'failure' + return if @job.checkout_code? or @job.test? + return unless @job.build? and @status == 'failure' @job.check_suite.ci_jobs.where(status: :queued).each do |job| - @logger.info ">>> Skipping job: #{job.inspect}" job.cancelled(@github_check) end end @@ -143,25 +155,13 @@ def skipping_jobs def slack_notify_success return unless current_execution? - fetch_subscriptions(%w[all pass]).each do |subscription| - SlackBot.instance.notify_success(@job, subscription) - end + SlackBot.instance.notify_success(@job) end def slack_notify_failure return unless current_execution? - fetch_subscriptions(%w[all errors]).each do |subscription| - SlackBot.instance.notify_errors(@job, subscription) - end - end - - def fetch_subscriptions(notification) - pull_request = @job.check_suite.pull_request - - PullRequestSubscription - .where(target: [pull_request.github_pr_id, pull_request.author], notification: notification) - .uniq(&:slack_user_id) + SlackBot.instance.notify_errors(@job) end end end diff --git a/lib/helpers/request.rb b/lib/helpers/request.rb index fb68af9..e45862d 100644 --- a/lib/helpers/request.rb +++ b/lib/helpers/request.rb @@ -16,6 +16,18 @@ module GitHubApp module Request + def download(uri, machine: 'ci1.netdef.org') + user, passwd = fetch_user_pass(machine) + http = create_http(uri) + + # Create Request + req = Net::HTTP::Get.new(uri) + # Add authorization headers + req.basic_auth user, passwd + + http.request(req).body + end + def get_request(uri, machine: 'ci1.netdef.org') user, passwd = fetch_user_pass(machine) http = create_http(uri) diff --git a/lib/models/check_suite.rb b/lib/models/check_suite.rb index 849e53a..1edce5d 100644 --- a/lib/models/check_suite.rb +++ b/lib/models/check_suite.rb @@ -18,7 +18,30 @@ class CheckSuite < ActiveRecord::Base has_many :ci_jobs, dependent: :delete_all def finished? - ci_jobs.where(status: %i[queued in_progress]).empty? + ci_jobs + .skip_stages + .where(status: %i[queued in_progress]) + .empty? + end + + def build_stage_finished? + ci_jobs + .skip_stages + .where("name ILIKE '% build'") + .where(status: %i[queued in_progress]) + .empty? + end + + def build_stage_success? + ci_jobs + .skip_stages + .where("name ILIKE '% build'") + .where(status: %i[failure cancelled skipped]) + .empty? + end + + def success? + ci_jobs.skip_stages.where(status: %i[failure cancelled skipped]).empty? end def in_progress? diff --git a/lib/models/ci_job.rb b/lib/models/ci_job.rb index 469f1fa..6b6dfff 100644 --- a/lib/models/ci_job.rb +++ b/lib/models/ci_job.rb @@ -20,12 +20,24 @@ class CiJob < ActiveRecord::Base has_many :topotest_failures, dependent: :delete_all scope :sha256, ->(sha) { joins(:check_suite).where(check_suite: { commit_sha_ref: sha }) } + scope :filter_by, ->(filter) { where('name ~ ?', filter) } + scope :skip_stages, -> { where(stage: false) } + scope :stages, -> { where(stage: true) } + scope :skip_checkout_code, -> { where.not(name: 'Checkout Code') } scope :not_skipped, -> { where.not(status: 'skipped') } def checkout_code? name.downcase.match? 'checkout' end + def build? + name.downcase.match? 'build' + end + + def test? + !build? and !checkout_code? + end + def finished? !%w[queued in_progress].include?(status.to_s) end @@ -35,45 +47,80 @@ def create_check_run end def enqueue(github, output = {}) - check_run = github.create(name) - github.queued(check_run.id, output) - update(check_ref: check_run.id, status: :queued) + return update(status: :queued) unless stage + + github_check_run_name = checkout_code? ? Github::Build::Action::SOURCE_CODE : name + + count = 0 + begin + check_run = github.create(github_stage_full_name(github_check_run_name)) + github.queued(check_run.id, output) + update(check_ref: check_run.id, status: :queued) + rescue StandardError + count += 1 + sleep 1 + + retry if count <= 5 + end end def in_progress(github, output = {}) - check_run = save_check_run(github) - github.in_progress(check_run.id, output) + if stage or !check_ref.nil? + create_github_check(github) + github.in_progress(check_ref, output) + end - update(check_ref: check_run.id, status: :in_progress) + update(status: :in_progress) end def cancelled(github, output = {}) - github.cancelled(check_ref, output) + if stage or !check_ref.nil? + create_github_check(github) + github.cancelled(check_ref, output) + end update(status: :cancelled) end def failure(github, output = {}) - github.failure(check_ref, output) + if stage or !check_ref.nil? + create_github_check(github) + github.failure(check_ref, output) + end update(status: :failure) end def success(github, output = {}) - github.success(check_ref, output) + if stage or !check_ref.nil? + create_github_check(github) + github.success(check_ref, output) + end update(status: :success) end def skipped(github, output = {}) - github.skipped(check_ref, output) + if stage or !check_ref.nil? + create_github_check(github) + github.skipped(check_ref, output) + end update(status: :skipped) end private - def save_check_run(github) - github.create(name) + def create_github_check(github) + return unless check_ref.nil? + + github_check_run_name = checkout_code? ? Github::Build::Action::SOURCE_CODE : name + + check_run = github.create(github_stage_full_name(github_check_run_name)) + update(check_ref: check_run.id) + end + + def github_stage_full_name(name) + "[CI] #{name}" end end diff --git a/lib/slack_bot/slack_bot.rb b/lib/slack_bot/slack_bot.rb index 4e3ba01..efc56b0 100644 --- a/lib/slack_bot/slack_bot.rb +++ b/lib/slack_bot/slack_bot.rb @@ -45,13 +45,19 @@ def invalid_rerun_dm(job, subscription) body: { message: reason, slack_user_id: subscription.slack_user_id }.to_json) end - def notify_errors(job, subscription) + def notify_errors(job) message = generate_notification_message(job, 'Failed') - url = "#{GitHubApp::Configuration.instance.config['slack_bot_url']}/github/user" - post_request(URI(url), - machine: 'slack_bot.netdef.org', - body: { message: message, slack_user_id: subscription.slack_user_id }.to_json) + pull_request = job.check_suite.pull_request + + PullRequestSubscription + .where(target: [pull_request.github_pr_id, pull_request.author], notification: %w[all errors]) + .uniq(&:slack_user_id).each do |subscription| + url = "#{GitHubApp::Configuration.instance.config['slack_bot_url']}/github/user" + post_request(URI(url), + machine: 'slack_bot.netdef.org', + body: { message: message, slack_user_id: subscription.slack_user_id }.to_json) + end end def notify_cancelled(job, subscription) @@ -63,13 +69,19 @@ def notify_cancelled(job, subscription) body: { message: message, slack_user_id: subscription.slack_user_id }.to_json) end - def notify_success(job, subscription) - message = generate_notification_message(job, 'Success') + def notify_success(job) + pull_request = job.check_suite.pull_request - url = "#{GitHubApp::Configuration.instance.config['slack_bot_url']}/github/user" - post_request(URI(url), - machine: 'slack_bot.netdef.org', - body: { message: message, slack_user_id: subscription.slack_user_id }.to_json) + PullRequestSubscription + .where(target: [pull_request.github_pr_id, pull_request.author], notification: %w[all pass]) + .uniq(&:slack_user_id).each do |subscription| + message = generate_notification_message(job, 'Success') + + url = "#{GitHubApp::Configuration.instance.config['slack_bot_url']}/github/user" + post_request(URI(url), + machine: 'slack_bot.netdef.org', + body: { message: message, slack_user_id: subscription.slack_user_id }.to_json) + end end def execution_started_notification(check_suite) diff --git a/spec/factories/ci_job.rb b/spec/factories/ci_job.rb index 0531475..f033c05 100644 --- a/spec/factories/ci_job.rb +++ b/spec/factories/ci_job.rb @@ -13,12 +13,36 @@ name { Faker::App.name } status { 0 } job_ref { Faker::Alphanumeric.alphanumeric(number: 18, min_alpha: 3, min_numeric: 3) } - check_ref { rand(1_000_000) } check_suite trait :checkout_code do name { 'Checkout Code' } + stage { true } + end + + trait :build_stage do + name { Github::Build::Action::BUILD_STAGE } + stage { true } + end + + trait :tests_stage do + name { Github::Build::Action::TESTS_STAGE } + stage { true } + end + + trait :build do + name { 'UnitTest build' } + end + + trait :test do + name { 'TopoTests Ubuntu 18.04 amd64' } + end + + trait :topotest_failure do + after(:create) do |ci_job| + create(:topotest_failure, ci_job: ci_job) + end end trait :in_progress do @@ -28,5 +52,9 @@ trait :failure do status { 'failure' } end + + trait :success do + status { 'success' } + end end end diff --git a/spec/lib/bamboo_ci/download_spec.rb b/spec/lib/bamboo_ci/download_spec.rb new file mode 100644 index 0000000..69ffa6b --- /dev/null +++ b/spec/lib/bamboo_ci/download_spec.rb @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# download_spec.rb +# Part of NetDEF CI System +# +# Copyright (c) 2023 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +describe BambooCi::Download do + context 'when download a file' do + let(:service) { described_class.build_log(url) } + let(:plan_key) { 1 } + let(:url) { 'https://127.0.0.1/rest/api/latest/queue/' } + + before do + allow(Netrc).to receive(:read).and_return({ 'ci1.netdef.org' => %w[user password] }) + + stub_request(:get, url) + .with( + headers: { + 'Accept' => '*/*', + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Authorization' => 'Basic dXNlcjpwYXNzd29yZA==', + 'Host' => '127.0.0.1', + 'User-Agent' => 'Ruby' + } + ) + .to_return([{ status: 200, body: '', headers: {} }, { status: 200, body: 'ok', headers: {} }]) + end + + it 'must returns success' do + expect(service).to eq('ok') + end + end + + context 'when download a file, but never return success' do + let(:service) { described_class.build_log(url) } + let(:plan_key) { 1 } + let(:url) { 'https://127.0.0.1/rest/api/latest/queue/' } + + before do + allow(Netrc).to receive(:read).and_return({ 'ci1.netdef.org' => %w[user password] }) + + stub_request(:get, url) + .with( + headers: { + 'Accept' => '*/*', + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Authorization' => 'Basic dXNlcjpwYXNzd29yZA==', + 'Host' => '127.0.0.1', + 'User-Agent' => 'Ruby' + } + ) + .to_return({ status: 200, body: '', headers: {} }) + end + + it 'must returns an empty string' do + expect(service).to eq('') + end + end +end diff --git a/spec/lib/github/build/summary_spec.rb b/spec/lib/github/build/summary_spec.rb new file mode 100644 index 0000000..3c90ba3 --- /dev/null +++ b/spec/lib/github/build/summary_spec.rb @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# summary_spec.rb +# Part of NetDEF CI System +# +# Copyright (c) 2023 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +describe Github::Build::Summary do + let(:summary) { described_class.new(ci_job) } + let(:fake_client) { Octokit::Client.new } + let(:fake_github_check) { Github::Check.new(nil) } + let(:check_suite) { create(:check_suite) } + + before do + allow(Octokit::Client).to receive(:new).and_return(fake_client) + allow(fake_client).to receive(:find_app_installations).and_return([{ 'id' => 1 }]) + allow(fake_client).to receive(:create_app_installation_access_token).and_return({ 'token' => 1 }) + + allow(File).to receive(:read).and_return('') + allow(OpenSSL::PKey::RSA).to receive(:new).and_return(OpenSSL::PKey::RSA.new(2048)) + + allow(Octokit::Client).to receive(:new).and_return(fake_client) + allow(fake_client).to receive(:find_app_installations).and_return([{ 'id' => 1 }]) + allow(fake_client).to receive(:create_app_installation_access_token).and_return({ 'token' => 1 }) + + allow(Github::Check).to receive(:new).and_return(fake_github_check) + allow(fake_github_check).to receive(:create).and_return(ci_job.check_suite) + allow(fake_github_check).to receive(:failure).and_return(ci_job.check_suite) + allow(fake_github_check).to receive(:in_progress).and_return(ci_job.check_suite) + allow(fake_github_check).to receive(:skipped).and_return(ci_job.check_suite) + allow(fake_github_check).to receive(:success).and_return(ci_job.check_suite) + allow(fake_github_check).to receive(:cancelled).and_return(ci_job.check_suite) + allow(fake_github_check).to receive(:queued).and_return(ci_job.check_suite) + allow(BambooCi::Result).to receive(:fetch).and_return({}) + end + + context 'when the build stage finished successfully' do + let(:ci_job) { create(:ci_job, :build, :success, check_suite: check_suite) } + let(:build_stage) { create(:ci_job, :build_stage, check_suite: check_suite) } + let(:tests_stage) { create(:ci_job, :tests_stage, check_suite: check_suite) } + + before do + build_stage + tests_stage + end + + it 'must update stage' do + summary.build_summary(Github::Build::Action::BUILD_STAGE) + expect(build_stage.reload.status).to eq('success') + expect(tests_stage.reload.status).to eq('in_progress') + end + end + + context 'when the build stage finished unsuccessfully' do + let(:ci_job) { create(:ci_job, :build, :failure, check_suite: check_suite) } + let(:build_stage) { create(:ci_job, :build_stage, check_suite: check_suite) } + let(:tests_stage) { create(:ci_job, :tests_stage, check_suite: check_suite) } + + before do + build_stage + tests_stage + end + + it 'must update stage' do + summary.build_summary(Github::Build::Action::BUILD_STAGE) + expect(build_stage.reload.status).to eq('failure') + expect(tests_stage.reload.status).to eq('cancelled') + end + end + + context 'when the build stage still running' do + let(:ci_job) { create(:ci_job, :build, :success, check_suite: check_suite) } + let(:ci_job_running) { create(:ci_job, :build, :in_progress, check_suite: check_suite) } + let(:build_stage) { create(:ci_job, :build_stage, check_suite: check_suite) } + let(:tests_stage) { create(:ci_job, :tests_stage, check_suite: check_suite) } + + before do + ci_job_running + build_stage + tests_stage + end + + it 'must update stage' do + summary.build_summary(Github::Build::Action::BUILD_STAGE) + expect(build_stage.reload.status).to eq('in_progress') + expect(tests_stage.reload.status).to eq('queued') + end + end + + context 'when the tests stage finished successfully' do + let(:ci_job) { create(:ci_job, :test, :success, check_suite: check_suite) } + let(:build_stage) { create(:ci_job, :build_stage, status: :success, check_suite: check_suite) } + let(:tests_stage) { create(:ci_job, :tests_stage, check_suite: check_suite) } + + before do + build_stage + tests_stage + end + + it 'must update stage' do + summary.build_summary(Github::Build::Action::TESTS_STAGE) + expect(build_stage.reload.status).to eq('success') + expect(tests_stage.reload.status).to eq('success') + end + end + + context 'when the tests stage finished unsuccessfully' do + let(:ci_job) { create(:ci_job, :test, :failure, check_suite: check_suite) } + let(:build_stage) { create(:ci_job, :build_stage, status: :success, check_suite: check_suite) } + let(:tests_stage) { create(:ci_job, :tests_stage, check_suite: check_suite) } + + before do + build_stage + tests_stage + end + + it 'must update stage' do + summary.build_summary(Github::Build::Action::TESTS_STAGE) + expect(build_stage.reload.status).to eq('success') + expect(tests_stage.reload.status).to eq('failure') + end + end + + context 'when the tests stage still running' do + let(:ci_job) { create(:ci_job, :test, :success, check_suite: check_suite) } + let(:ci_job_running) { create(:ci_job, :test, :in_progress, check_suite: check_suite) } + let(:build_stage) { create(:ci_job, :build_stage, status: :success, check_suite: check_suite) } + let(:tests_stage) { create(:ci_job, :tests_stage, check_suite: check_suite) } + + before do + ci_job_running + build_stage + tests_stage + end + + it 'must update stage' do + summary.build_summary(Github::Build::Action::TESTS_STAGE) + expect(build_stage.reload.status).to eq('success') + expect(tests_stage.reload.status).to eq('in_progress') + end + end +end diff --git a/spec/lib/github/build_plan_spec.rb b/spec/lib/github/build_plan_spec.rb index 2fba7f4..9de655a 100644 --- a/spec/lib/github/build_plan_spec.rb +++ b/spec/lib/github/build_plan_spec.rb @@ -61,6 +61,7 @@ allow(Github::Check).to receive(:new).and_return(fake_github_check) allow(fake_github_check).to receive(:create).and_return(fake_check_run) allow(fake_github_check).to receive(:in_progress).and_return(fake_check_run) + allow(fake_github_check).to receive(:queued).and_return(fake_check_run) allow(BambooCi::RunningPlan).to receive(:fetch).with(fake_plan_run.bamboo_reference).and_return(ci_jobs) end @@ -82,7 +83,7 @@ let(:author) { 'Johnny Silverhand' } let(:pull_request) { PullRequest.last } let(:check_suite) { pull_request.check_suites.last } - let(:ci_job) { check_suite.ci_jobs.last } + let(:ci_job) { check_suite.ci_jobs.find_by(name: 'First Test') } let(:ci_jobs) { [{ name: 'First Test', job_ref: 'UNIT-TEST-FIRST-1' }] } let(:plan) { create(:plan, github_repo_name: repo) } diff --git a/spec/lib/github/check_spec.rb b/spec/lib/github/check_spec.rb index 47039b1..6627ddc 100644 --- a/spec/lib/github/check_spec.rb +++ b/spec/lib/github/check_spec.rb @@ -157,11 +157,11 @@ conclusion: conclusion, accept: 'application/vnd.github+json' }) - .and_return(true) + .and_return({}) end it 'must returns success' do - expect(check.cancelled(id)).to be_truthy + expect(check.cancelled(id)).to eq({}) end end @@ -179,11 +179,11 @@ conclusion: conclusion, accept: 'application/vnd.github+json' }) - .and_return(true) + .and_return({}) end it 'must returns success' do - expect(check.success(id)).to be_truthy + expect(check.success(id)).to eq({}) end end @@ -225,11 +225,11 @@ output: output, accept: 'application/vnd.github+json' }) - .and_return(true) + .and_return({}) end it 'must returns success' do - expect(check.failure(id, output)).to be_truthy + expect(check.failure(id, output)).to eq({}) end end @@ -247,11 +247,11 @@ conclusion: conclusion, accept: 'application/vnd.github+json' }) - .and_return(true) + .and_return({}) end it 'must returns success' do - expect(check.skipped(id)).to be_truthy + expect(check.skipped(id)).to eq({}) end end diff --git a/spec/lib/github/retry_spec.rb b/spec/lib/github/retry_spec.rb index 1ff3f8c..d363db0 100644 --- a/spec/lib/github/retry_spec.rb +++ b/spec/lib/github/retry_spec.rb @@ -54,9 +54,12 @@ context 'when Ci Job is failure' do let(:check_suite) { create(:check_suite) } let(:ci_job) { create(:ci_job, check_suite: check_suite, status: 'failure') } - let(:ci_job_checkout_code) { create(:ci_job, :checkout_code, check_suite: check_suite, status: 'failure') } + let(:ci_job_build_stage) { create(:ci_job, :build_stage, check_suite: check_suite, status: 'failure') } let(:fake_client) { Octokit::Client.new } let(:fake_github_check) { Github::Check.new(nil) } + let(:ci_job_checkout_code) do + create(:ci_job, :checkout_code, stage: false, check_suite: check_suite, status: 'failure') + end before do allow(Octokit::Client).to receive(:new).and_return(fake_client) @@ -73,6 +76,43 @@ allow(BambooCi::RunningPlan).to receive(:fetch).and_return([]) ci_job_checkout_code + ci_job_build_stage + end + + it 'must returns success' do + expect(github_retry.start).to eq([200, 'Retrying failure jobs']) + expect(ci_job.reload.status).to eq('queued') + end + + it 'must still have its previous status' do + expect(ci_job_checkout_code.reload.status).to eq('failure') + end + end + + context 'when Ci Job is failure and checkout code is stage' do + let(:check_suite) { create(:check_suite) } + let(:ci_job) { create(:ci_job, check_suite: check_suite, status: 'failure') } + let(:ci_job_build_stage) { create(:ci_job, :build_stage, check_suite: check_suite, status: 'failure') } + let(:fake_client) { Octokit::Client.new } + let(:fake_github_check) { Github::Check.new(nil) } + let(:ci_job_checkout_code) do + create(:ci_job, :checkout_code, check_suite: check_suite, status: 'failure') + end + + before do + allow(Octokit::Client).to receive(:new).and_return(fake_client) + allow(fake_client).to receive(:find_app_installations).and_return([{ 'id' => 1 }]) + allow(fake_client).to receive(:create_app_installation_access_token).and_return({ 'token' => 1 }) + + allow(Github::Check).to receive(:new).and_return(fake_github_check) + allow(fake_github_check).to receive(:create).and_return(ci_job.check_suite) + allow(fake_github_check).to receive(:queued) + + allow(BambooCi::StopPlan).to receive(:build) + allow(BambooCi::Retry).to receive(:restart) + + ci_job_checkout_code + ci_job_build_stage end it 'must returns success' do diff --git a/spec/lib/github/update_status_spec.rb b/spec/lib/github/update_status_spec.rb index 355072f..4832620 100644 --- a/spec/lib/github/update_status_spec.rb +++ b/spec/lib/github/update_status_spec.rb @@ -24,6 +24,8 @@ let(:fake_github_check) { Github::Check.new(nil) } before do + allow(SlackBot.instance).to receive(:notify_success) + allow(SlackBot.instance).to receive(:notify_success) allow(File).to receive(:read).and_return('') allow(OpenSSL::PKey::RSA).to receive(:new).and_return(OpenSSL::PKey::RSA.new(2048)) @@ -38,6 +40,7 @@ allow(fake_github_check).to receive(:skipped).and_return(ci_job.check_suite) allow(fake_github_check).to receive(:success).and_return(ci_job.check_suite) allow(fake_github_check).to receive(:cancelled).and_return(ci_job.check_suite) + allow(fake_github_check).to receive(:queued).and_return(ci_job.check_suite) allow(Github::Build::UnavailableJobs).to receive(:new).and_return(fake_unavailable) end @@ -372,28 +375,256 @@ expect(update_status.update).to eq([200, 'Success']) end end + end - context 'when update CI Job to failure but it is an old execution' do - let(:pull_request) { create(:pull_request) } - let(:check_suite) { create(:check_suite, pull_request: pull_request) } - let(:check_suite_new) { create(:check_suite, pull_request: pull_request) } - let(:ci_job) { create(:ci_job, name: 'AMD Build', status: 'in_progress', check_suite: check_suite) } - let(:ci_job_new) { create(:ci_job, name: 'AMD Build', status: 'in_progress', check_suite: check_suite_new) } - let(:subscription) { create(:pull_request_subscription, target: ci_job.check_suite.pull_request.github_pr_id) } + describe 'Build Stage' do + let(:payload) do + { + 'status' => status, + 'bamboo_ref' => ci_job.job_ref + } + end + + context 'when Ci Job AMD Build update from queued -> in_progress' do + let(:ci_job) { create(:ci_job, name: 'AMD Build', status: 'queued') } + let(:build) { create(:ci_job, :build_stage, status: 'queued', check_suite: ci_job.check_suite) } + let(:tests) { create(:ci_job, :tests_stage, status: 'queued', check_suite: ci_job.check_suite) } + let(:status) { 'in_progress' } + + before do + ci_job + tests + build + end + + it 'must returns success' do + expect(update_status.update).to eq([200, 'Success']) + end + + it 'must update Build Job' do + update_status.update + expect(build.reload.status).to eq(status) + end + + it 'must keep Tests enqueued' do + update_status.update + expect(tests.reload.status).to eq('queued') + end + end + + context 'when Ci Job TopoTest Part 0 update from queued -> in_progress' do + let(:ci_job) { create(:ci_job, name: 'TopoTest Part 0', status: 'queued') } + let(:status) { 'in_progress' } + let(:build) { create(:ci_job, :build_stage, status: 'queued', check_suite: ci_job.check_suite) } + let(:tests) { create(:ci_job, :tests_stage, status: 'queued', check_suite: ci_job.check_suite) } + + before do + ci_job + tests + build + end + + it 'must returns success' do + expect(update_status.update).to eq([200, 'Success']) + end + + it 'must update Build Job' do + update_status.update + expect(build.reload.status).to eq('success') + end + + it 'must keep Tests enqueued' do + update_status.update + expect(tests.reload.status).to eq(status) + end + end + + context 'when Ci Job TopoTest Part 0 update from in_progress -> success' do + let(:ci_job) { create(:ci_job, name: 'TopoTest Part 0', status: 'in_progress') } + let(:status) { 'success' } + let(:tests) { create(:ci_job, :tests_stage, status: 'in_progress', check_suite: ci_job.check_suite) } + + before do + ci_job + tests + end + + it 'must returns success' do + expect(update_status.update).to eq([200, 'Success']) + end + + it 'must change Tests to success' do + update_status.update + expect(tests.reload.status).to eq(status) + end + end + + context 'when Ci Job TopoTest Part 0 update from in_progress -> failure' do + let(:ci_job) { create(:ci_job, :topotest_failure, name: 'TopoTest Part 0', status: 'in_progress') } let(:status) { 'failure' } + let(:tests) { create(:ci_job, :tests_stage, status: 'in_progress', check_suite: ci_job.check_suite) } + + let(:test_failure) do + create(:ci_job, + :topotest_failure, + name: 'TopoTest Part 1', + status: 'failure', + check_suite: ci_job.check_suite) + end before do - check_suite - ci_job_new - subscription + ci_job + test_failure + tests + end - stub_request(:post, "#{GitHubApp::Configuration.instance.config['slack_bot_url']}/github/user") - .to_return(status: 200, body: '', headers: {}) + it 'must returns success' do + expect(update_status.update).to eq([200, 'Success']) + end + + it 'must change Tests to success' do + update_status.update + expect(tests.reload.status).to eq(status) + end + end + + context 'when Ci Job AMD Build update from in_progress -> failure' do + let(:ci_job) { create(:ci_job, name: 'AMD Build', status: 'in_progress') } + let(:arm) { create(:ci_job, name: 'ARM8 Build', status: 'failure') } + let(:test) { create(:ci_job, name: 'TopoTest Part 0', status: 'in_progress', check_suite: ci_job.check_suite) } + let(:build) { create(:ci_job, :build_stage, status: 'in_progress', check_suite: ci_job.check_suite) } + let(:tests) { create(:ci_job, :tests_stage, status: 'queued', check_suite: ci_job.check_suite) } + let(:status) { 'failure' } + let(:url) do + "https://127.0.0.1/rest/api/latest/result/#{ci_job.job_ref}?" \ + 'expand=testResults.failedTests.testResult.errors,artifacts' + end + + let(:response) do + { + 'artifacts' => { + 'artifact' => [ + { + 'name' => 'ErrorLog', + 'link' => { + 'href' => 'https://127.0.0.1/ok.log' + } + } + ] + } + } + end + + before do + stub_request(:get, url) + .with( + headers: { + 'Accept' => %w[*/* application/json], + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Authorization' => 'Basic Og==', + 'Host' => '127.0.0.1', + 'User-Agent' => 'Ruby' + } + ) + .to_return(status: 200, body: response.to_json, headers: {}) + + stub_request(:get, 'https://127.0.0.1/ok.log') + .with( + headers: { + 'Accept' => '*/*', + 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Authorization' => 'Basic Og==', + 'Host' => '127.0.0.1', + 'User-Agent' => 'Ruby' + } + ) + .to_return(status: 200, body: 'make: *** [all] Error 2', headers: {}) + + ci_job + test + tests + build + arm end it 'must returns success' do expect(update_status.update).to eq([200, 'Success']) end + + it 'must update Build Job' do + update_status.update + expect(build.reload.status).to eq(status) + end + + it 'must keep Tests skipped' do + update_status.update + expect(tests.reload.status).to eq('cancelled') + end + end + + context 'when Ci Job AMD Build update from in_progress -> success' do + let(:ci_job) { create(:ci_job, name: 'AMD Build', status: 'in_progress') } + let(:test) { create(:ci_job, name: 'TopoTest Part 0', status: 'queued', check_suite: ci_job.check_suite) } + let(:build) { create(:ci_job, :build_stage, status: 'in_progress', check_suite: ci_job.check_suite) } + let(:tests) { create(:ci_job, :tests_stage, status: 'queued', check_suite: ci_job.check_suite) } + let(:status) { 'success' } + + before do + ci_job + test + tests + build + end + + it 'must returns success' do + expect(update_status.update).to eq([200, 'Success']) + end + + it 'must update Build Job' do + update_status.update + expect(build.reload.status).to eq(status) + end + + it 'must keep Tests enqueued' do + update_status.update + expect(tests.reload.status).to eq('in_progress') + end + end + end + + describe '#current_execution' do + let(:pull_request) { create(:pull_request) } + let(:check_suite1) { create(:check_suite, pull_request: pull_request) } + let(:check_suite2) { create(:check_suite, pull_request: pull_request) } + let(:ci_job) { create(:ci_job, name: 'AMD Build', status: 'in_progress', check_suite: check_suite1) } + let(:ci_job_new) { create(:ci_job, name: 'AMD Build', status: 'in_progress', check_suite: check_suite2) } + + context 'when old execution fails' do + let(:status) { 'failure' } + + before do + ci_job + ci_job_new + end + + it 'must not generate slack message' do + update_status.update + expect(ci_job.reload.status).to eq(status) + end + end + + context 'when old execution passes' do + let(:status) { 'success' } + + before do + ci_job + ci_job_new + end + + it 'must not generate slack message' do + update_status.update + expect(ci_job.reload.status).to eq(status) + end end end end diff --git a/spec/lib/models/ci_job_spec.rb b/spec/lib/models/ci_job_spec.rb new file mode 100644 index 0000000..728d9d9 --- /dev/null +++ b/spec/lib/models/ci_job_spec.rb @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# ci_job_spec.rb +# Part of NetDEF CI System +# +# Copyright (c) 2023 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# frozen_string_literal: true + +describe CiJob do + describe '#enqueue' do + let(:stage) { create(:ci_job, stage: true) } + let(:fake_client) { Octokit::Client.new } + let(:github_fail) { Github::Check.new(nil) } + let(:github_success) { Github::Check.new(nil) } + + before do + allow(File).to receive(:read).and_return('') + allow(OpenSSL::PKey::RSA).to receive(:new).and_return(OpenSSL::PKey::RSA.new(2048)) + allow(Octokit::Client).to receive(:new).and_return(fake_client) + allow(fake_client).to receive(:find_app_installations).and_return([{ 'id' => 1 }]) + allow(fake_client).to receive(:create_app_installation_access_token).and_return({ 'token' => 1 }) + + allow(github_fail).to receive(:create).and_raise + allow(github_success).to receive(:create).and_return(stage) + allow(github_success).to receive(:queued).and_return(stage) + end + + it 'must handle the error' do + expect { stage.enqueue(github_fail) }.not_to raise_error + end + + it 'must update status' do + expect { stage.enqueue(github_success) }.not_to raise_error + expect(stage.reload.status).to eq('queued') + end + end +end diff --git a/spec/lib/slack_bot/slack_bot_spec.rb b/spec/lib/slack_bot/slack_bot_spec.rb index 0e7826e..e2ec84c 100644 --- a/spec/lib/slack_bot/slack_bot_spec.rb +++ b/spec/lib/slack_bot/slack_bot_spec.rb @@ -37,8 +37,8 @@ }.to_json end - it { expect { slack_bot.notify_success(job, subscription) }.not_to raise_error } - it { expect { slack_bot.notify_errors(job, subscription) }.not_to raise_error } + it { expect { slack_bot.notify_success(job) }.not_to raise_error } + it { expect { slack_bot.notify_errors(job) }.not_to raise_error } it { expect { slack_bot.notify_cancelled(job, subscription) }.not_to raise_error } it { expect { slack_bot.execution_started_notification(job.check_suite) }.not_to raise_error } it { expect { slack_bot.execution_finished_notification(job.check_suite) }.not_to raise_error } diff --git a/workers/watch_dog.rb b/workers/watch_dog.rb index be42e47..2840182 100644 --- a/workers/watch_dog.rb +++ b/workers/watch_dog.rb @@ -11,6 +11,8 @@ require 'logger' require_relative 'base' require_relative '../lib/slack_bot/slack_bot' +require_relative '../lib/github/build/action' +require_relative '../lib/github/build/summary' class WatchDog < Base def perform @@ -42,7 +44,13 @@ def check(suites) @logger.info ">>> Updating suite: #{check_suite.inspect}" check_stages(check_suite) clear_deleted_jobs(check_suite) - SlackBot.instance.execution_finished_notification(check_suite) + end + end + + def finish_stages(check_suite) + check_suite.ci_jobs.stages.each do |stage| + summary = Github::Build::Summary.new(stage) + summary.missing_stage(stage) end end @@ -80,7 +88,7 @@ def check_suites_fetch_map def clear_deleted_jobs(check_suite) github_check = Github::Check.new(check_suite) - check_suite.ci_jobs.where(status: %w[queued in_progress]).each do |ci_job| + check_suite.ci_jobs.skip_stages.where(status: %w[queued in_progress]).each do |ci_job| ci_job.skipped(github_check) end end @@ -91,22 +99,23 @@ def check_stages(check_suite) stage.dig('results', 'result').each do |result| ci_job = CiJob.find_by(job_ref: result['buildResultKey'], check_suite_id: check_suite.id) - @logger.info ">>> CiJob: #{ci_job.inspect}}" - next if ci_job.finished? && !ci_job.job_ref.nil? - - update_ci_job_status(github_check, ci_job, result['state']) + update_stage_status(ci_job, result, github_check) end end end + def update_stage_status(ci_job, result, github) + @logger.info ">>> CiJob: #{ci_job.inspect}}" + return if ci_job.nil? + return if ci_job.finished? && !ci_job.job_ref.nil? + + update_ci_job_status(github, ci_job, result['state']) + end + def update_ci_job_status(github_check, ci_job, state) ci_job.enqueue(github_check) if ci_job.job_ref.nil? - url = "https://ci1.netdef.org/browse/#{ci_job.job_ref}" - output = { - title: ci_job.name, - summary: "Details at [#{url}](#{url})\nUnfortunately we were unable to access the execution results." - } + output = create_output_message(ci_job) @logger.info ">>> CiJob: #{ci_job.inspect} updating status" case state @@ -122,10 +131,43 @@ def update_ci_job_status(github_check, ci_job, state) else puts 'Ignored' end + + build_summary(ci_job) + end + + def create_output_message(ci_job) + url = "https://ci1.netdef.org/browse/#{ci_job.job_ref}" + + { + title: ci_job.name, + summary: "Details at [#{url}](#{url})\nUnfortunately we were unable to access the execution results." + } + end + + def build_summary(ci_job) + summary = Github::Build::Summary.new(ci_job) + summary.build_summary(Github::Build::Action::BUILD_STAGE) if ci_job.build? + summary.build_summary(Github::Build::Action::TESTS_STAGE) if ci_job.test? + + finished_execution?(ci_job.check_suite) + end + + def finished_execution?(check_suite) + return false unless current_execution?(check_suite) + return false unless check_suite.finished? + + SlackBot.instance.execution_finished_notification(check_suite) + end + + def current_execution?(check_suite) + pull_request = check_suite.pull_request + last_check_suite = pull_request.check_suites.reload.all.order(:created_at).last + + check_suite.id == last_check_suite.id end - def fetch_subscriptions(notification) - pull_request = @job.check_suite.pull_request + def fetch_subscriptions(notification, job) + pull_request = job.check_suite.pull_request PullRequestSubscription .where(target: [pull_request.github_pr_id, pull_request.author], notification: notification) @@ -135,19 +177,19 @@ def fetch_subscriptions(notification) end def slack_notify_success(job) - fetch_subscriptions(%w[all pass]).each do |subscription| + fetch_subscriptions(%w[all pass], job).each do |subscription| SlackBot.instance.notify_success(job, subscription) end end def slack_notify_failure(job) - fetch_subscriptions(%w[all errors]).each do |subscription| + fetch_subscriptions(%w[all errors], job).each do |subscription| SlackBot.instance.notify_errors(job, subscription) end end def slack_notify_cancelled(job) - fetch_subscriptions(%w[all errors]).each do |subscription| + fetch_subscriptions(%w[all errors], job).each do |subscription| SlackBot.instance.notify_cancelled(job, subscription) end end