CI Stages

app/github_app.rb

@@ -118,7 +118,7 @@ def sinatra_logger_level
     when 'check_run'
       logger.debug "Check Run #{payload.dig('check_run', 'id')} - #{payload['action']}"
 
-      halt 200, 'OK' unless %w[created rerequested].include? payload['action'].downcase
+      halt 200, 'OK' unless %w[rerequested].include? payload['action'].downcase
 
       re_run = Github::Retry.new(payload, logger_level: GithubApp.sinatra_logger_level)
       halt re_run.start

db/migrate/20231108100757_add_ci_job_stage.rb

@@ -0,0 +1,15 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  20231023090822_create_pull_request_subscribe.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2023 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+class AddCiJobStage < ActiveRecord::Migration[6.0]
+  def change
+    add_column :ci_jobs, :stage, :boolean, default: false
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_10_23_090823) do
+ActiveRecord::Schema[7.0].define(version: 2023_11_08_100757) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -40,6 +40,7 @@
     t.datetime "updated_at", null: false
     t.bigint "check_suite_id"
     t.integer "retry", default: 0
+    t.boolean "stage", default: false
     t.index ["check_suite_id"], name: "index_ci_jobs_on_check_suite_id"
   end
 

lib/bamboo_ci/download.rb

@@ -0,0 +1,39 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  download.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2023 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+require 'logger'
+
+require_relative 'api'
+
+module BambooCi
+  class Download
+    extend BambooCi::Api
+
+    def self.build_log(url)
+      count = 0
+      uri = URI(url)
+
+      begin
+        body = download(uri).split("\n").last(10).join("\n")
+
+        raise 'Must try' if body.empty?
+
+        body
+      rescue StandardError
+        count += 1
+
+        sleep 5
+        retry if count <= 3
+
+        ''
+      end
+    end
+  end
+end

lib/bamboo_ci/result.rb

@@ -1,6 +1,6 @@
 #  SPDX-License-Identifier: BSD-2-Clause
 #
-#  stop_plan.rb
+#  result.rb
 #  Part of NetDEF CI System
 #
 #  Copyright (c) 2023 by
@@ -16,8 +16,8 @@ module BambooCi
   class Result
     extend BambooCi::Api
 
-    def self.fetch(job_key)
-      uri = URI("https://127.0.0.1/rest/api/latest/result/#{job_key}?expand=testResults.failedTests.testResult.errors")
+    def self.fetch(job_key, expand: 'testResults.failedTests.testResult.errors')
+      uri = URI("https://127.0.0.1/rest/api/latest/result/#{job_key}?expand=#{expand}")
       get_request(uri)
     end
   end

lib/github/build/action.rb

@@ -0,0 +1,114 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  action.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2023 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+module Github
+  module Build
+    class Action
+      BUILD_STAGE = 'Build'
+      TESTS_STAGE = 'Tests'
+      SOURCE_CODE = 'Verify Source'
+      SUMMARY = [BUILD_STAGE, TESTS_STAGE].freeze
+      STAGE_POSITION = { SOURCE_CODE => '01', BUILD_STAGE => '02', TESTS_STAGE => '03' }.freeze
+
+      def initialize(check_suite, github, logger_level: Logger::INFO)
+        @check_suite = check_suite
+        @github = github
+        @loggers = []
+
+        %w[github_app.log github_build_action.log].each do |filename|
+          logger_app = Logger.new(filename, 1, 1_024_000)
+          logger_app.level = logger_level
+
+          @loggers << logger_app
+        end
+
+        logger(Logger::INFO, "Building action to CheckSuite @#{@check_suite.inspect}")
+      end
+
+      def create_summary
+        logger(Logger::INFO, "SUMMARY #{SUMMARY.inspect}")
+
+        SUMMARY.each do |name|
+          create_check_run_stage(name)
+        end
+      rescue StandardError => e
+        logger(Logger::Error, "#{e.class} - #{e.message}")
+      end
+
+      def create_stage(name)
+        bamboo_ci = @check_suite.bamboo_ci_ref.split('-').last
+
+        stage =
+          CiJob.create(check_suite: @check_suite, name: name, job_ref: "#{name}-#{bamboo_ci}", stage: true)
+
+        return stage if stage.persisted?
+
+        logger(Logger::ERROR, "Failed to created: #{stage.inspect} -> #{stage.errors.inspect}")
+
+        nil
+      end
+
+      def create_jobs(jobs, rerun: false)
+        jobs.each do |job|
+          ci_job = CiJob.create(check_suite: @check_suite, name: job[:name], job_ref: job[:job_ref])
+
+          next unless ci_job.persisted?
+
+          if rerun
+            next if ci_job.checkout_code?
+
+            url = "https://ci1.netdef.org/browse/#{ci_job.job_ref}"
+            ci_job.enqueue(@github, { title: ci_job.name, summary: "Details at [#{url}](#{url})" })
+          else
+            ci_job.create_check_run
+          end
+
+          next unless ci_job.checkout_code?
+
+          ci_job.update(stage: true)
+          url = "https://ci1.netdef.org/browse/#{ci_job.job_ref}"
+          ci_job.in_progress(@github, { title: ci_job.name, summary: "Details at [#{url}](#{url})" })
+        end
+      end
+
+      private
+
+      def create_check_run_stage(name)
+        stage = CiJob.find_by(name: name, check_suite_id: @check_suite.id)
+
+        logger(Logger::INFO, "STAGE #{name} #{stage.inspect} - @#{@check_suite.inspect}")
+
+        stage = create_stage(name) if stage.nil?
+
+        return if stage.nil? or stage.checkout_code? or stage.success?
+
+        logger(Logger::INFO, ">>> Enqueued #{stage.inspect}")
+
+        stage.enqueue(@github, initial_output(stage))
+      end
+
+      def initial_output(ci_job)
+        output = { title: '', summary: '' }
+        url = "https://ci1.netdef.org/browse/#{ci_job.check_suite.bamboo_ci_ref}"
+
+        output[:title] = "#{ci_job.name} summary"
+        output[:summary] = "Details at [#{url}](#{url})"
+
+        output
+      end
+
+      def logger(severity, message)
+        @loggers.each do |logger_object|
+          logger_object.add(severity, message)
+        end
+      end
+    end
+  end
+end

lib/github/build/retry.rb

@@ -0,0 +1,51 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  retry.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2023 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+require_relative 'action'
+
+module Github
+  module Build
+    class Retry < Action
+      def initialize(check_suite, github, logger_level: Logger::INFO)
+        super(check_suite, github)
+
+        @loggers = []
+
+        %w[github_app.log github_build_retry.log].each do |filename|
+          logger_app = Logger.new(filename, 1, 1_024_000)
+          logger_app.level = logger_level
+
+          @loggers << logger_app
+        end
+      end
+
+      def enqueued_stages
+        @check_suite.ci_jobs.stages.where.not(status: :success).each do |ci_job|
+          logger(Logger::WARN, "Enqueue stages: #{ci_job.inspect}")
+
+          next if ci_job.success? or ci_job.checkout_code?
+
+          ci_job.enqueue(@github, initial_output(ci_job))
+          ci_job.update(retry: ci_job.retry + 1)
+        end
+      end
+
+      def enqueued_failure_tests
+        @check_suite.ci_jobs.skip_stages.where.not(status: :success).each do |ci_job|
+          next if ci_job.checkout_code?
+
+          logger(Logger::WARN, "Enqueue CiJob: #{ci_job.inspect}")
+          ci_job.enqueue(@github)
+          ci_job.update(retry: ci_job.retry + 1)
+        end
+      end
+    end
+  end
+end