Merge pull request #105 from RodrigoMNardi/bug/github/watchdog

WatchDog

config/delayed_job.rb

@@ -34,8 +34,6 @@ class << self
 Delayed::Worker.max_attempts = 5
 Delayed::Worker.max_run_time = 5.minutes
 
-Delayed::Job.delete_all unless ENV.fetch('RAILS_ENV', 'test') == 'test'
-
 # Load the database configuration
 
 config = YAML.load_file('config/database.yml')[ENV.fetch('RACK_ENV', 'development')]

lib/github/build/action.rb

@@ -3,6 +3,17 @@
 #  action.rb
 #  Part of NetDEF CI System
 #
+#  This class handles the build action for a given CheckSuite.
+#  It creates summaries, jobs, and timeout workers for the CheckSuite.
+#
+#  Methods:
+#  - initialize(check_suite, github, jobs, logger_level: Logger::INFO): Initializes the Action class with the
+#    given parameters.
+#  - create_summary(rerun: false): Creates a summary for the CheckSuite, including jobs and timeout workers.
+#
+#  Example usage:
+#    Github::Build::Action.new(check_suite, github, jobs).create_summary
+#
 #  Copyright (c) 2023 by
 #  Network Device Education Foundation, Inc. ("NetDEF")
 #
@@ -11,6 +22,13 @@
 module Github
   module Build
     class Action
+      ##
+      # Initializes the Action class with the given parameters.
+      #
+      # @param [CheckSuite] check_suite The CheckSuite to handle.
+      # @param [Github] github The Github instance to use.
+      # @param [Array] jobs The jobs to create for the CheckSuite.
+      # @param [Integer] logger_level The logging level to use (default: Logger::INFO).
       def initialize(check_suite, github, jobs, logger_level: Logger::INFO)
         @check_suite = check_suite
         @github = github
@@ -26,6 +44,10 @@ def initialize(check_suite, github, jobs, logger_level: Logger::INFO)
         logger(Logger::WARN, ">>>> Building action to CheckSuite: #{@check_suite.inspect}")
       end
 
+      ##
+      # Creates a summary for the CheckSuite, including jobs and timeout workers.
+      #
+      # @param [Boolean] rerun Indicates if the jobs should be rerun (default: false).
       def create_summary(rerun: false)
         logger(Logger::INFO, "SUMMARY #{@stages.inspect}")
 
@@ -37,10 +59,15 @@ def create_summary(rerun: false)
 
         logger(Logger::INFO, "@jobs - #{@jobs.inspect}")
         create_jobs(rerun)
+        create_timeout_worker
       end
 
       private
 
+      ##
+      # Creates jobs for the CheckSuite.
+      #
+      # @param [Boolean] rerun Indicates if the jobs should be rerun.
       def create_jobs(rerun)
         @jobs.each do |job|
           ci_job = create_ci_job(job)
@@ -60,13 +87,32 @@ def create_jobs(rerun)
         end
       end
 
+      ##
+      # Creates a timeout worker for the CheckSuite.
+      def create_timeout_worker
+        logger(Logger::INFO, "CiJobStatus::Update: TimeoutExecution for '#{@check_suite.id}'")
+
+        TimeoutExecution
+          .delay(run_at: 30.minute.from_now.utc, queue: 'timeout_execution')
+          .timeout(@check_suite.id)
+      end
+
+      ##
+      # Starts the stage in progress if configured to do so.
+      #
+      # @param [CiJob] ci_job The CI job to start in progress.
       def stage_with_start_in_progress(ci_job)
         return unless !ci_job.stage.nil? and ci_job.stage.configuration.start_in_progress?
 
         ci_job.in_progress(@github)
         ci_job.stage.in_progress(@github, output: {})
       end
 
+      ##
+      # Creates a CI job for the given job parameters.
+      #
+      # @param [Hash] job The job parameters.
+      # @return [CiJob, nil] The created CI job or nil if the stage configuration is not found.
       def create_ci_job(job)
         stage_config = StageConfiguration.find_by(bamboo_stage_name: job[:stage])
 
@@ -79,6 +125,10 @@ def create_ci_job(job)
         CiJob.create(check_suite: @check_suite, name: job[:name], job_ref: job[:job_ref], stage: stage)
       end
 
+      ##
+      # Creates a check run stage for the given stage configuration.
+      #
+      # @param [StageConfiguration] stage_config The stage configuration.
       def create_check_run_stage(stage_config)
         stage = Stage.find_by(name: stage_config.github_check_run_name, check_suite_id: @check_suite.id)
 
@@ -92,6 +142,11 @@ def create_check_run_stage(stage_config)
         stage.enqueue(@github, output: initial_output(stage))
       end
 
+      ##
+      # Creates a new stage for the given stage configuration.
+      #
+      # @param [StageConfiguration] stage_config The stage configuration.
+      # @return [Stage] The created stage.
       def create_stage(stage_config)
         name = stage_config.github_check_run_name
 
@@ -110,6 +165,11 @@ def create_stage(stage_config)
         stage
       end
 
+      ##
+      # Generates the initial output for a CI job.
+      #
+      # @param [CiJob] ci_job The CI job.
+      # @return [Hash] The initial output.
       def initial_output(ci_job)
         output = { title: '', summary: '' }
         url = "https://ci1.netdef.org/browse/#{ci_job.check_suite.bamboo_ci_ref}"
@@ -120,6 +180,11 @@ def initial_output(ci_job)
         output
       end
 
+      ##
+      # Logs a message with the given severity.
+      #
+      # @param [Integer] severity The severity level.
+      # @param [String] message The message to log.
       def logger(severity, message)
         @loggers.each do |logger_object|
           logger_object.add(severity, message)

lib/github/plan_execution/finished.rb

@@ -3,6 +3,18 @@
 #  check_suite_finished.rb
 #  Part of NetDEF CI System
 #
+#  This class handles the logic for determining if a CheckSuite has finished execution.
+#  It interacts with the Bamboo CI system to fetch the build status and updates the CheckSuite accordingly.
+#
+#  Methods:
+#  - initialize(payload): Initializes the Finished class with the given payload.
+#  - finished: Main method to handle the completion logic for a CheckSuite.
+#  - fetch_build_status: Fetches the build status from Bamboo CI.
+#  - in_progress?(build_status): Checks if the CI build is still in progress.
+#
+#  Example usage:
+#    Github::PlanExecution::Finished.new(payload).finished
+#
 #  Copyright (c) 2024 by
 #  Network Device Education Foundation, Inc. ("NetDEF")
 #
@@ -17,12 +29,22 @@ module PlanExecution
     class Finished
       include BambooCi::Api
 
+      ##
+      # Initializes the Finished class with the given payload.
+      #
+      # @param [Hash] payload The payload containing information about the CheckSuite.
       def initialize(payload)
-        @check_suite = CheckSuite.find_by(bamboo_ci_ref: payload['bamboo_ref'])
+        @check_suite = CheckSuite.find_by(bamboo_ci_ref: payload['bamboo_ref']) if payload['bamboo_ref']
+        @check_suite = CheckSuite.find(payload['check_suite_id']) if payload['check_suite_id']
         @logger = GithubLogger.instance.create('github_plan_execution_finished.log', Logger::INFO)
         @hanged = payload['hanged'] || false
       end
 
+      ##
+      # Main method to handle the completion logic for a CheckSuite.
+      # Fetches the CI execution status and updates the CheckSuite accordingly.
+      #
+      # @return [Array] An array containing the status code and message.
       def finished
         @logger.info ">>> Check Suite: #{@check_suite.inspect}"
 
@@ -31,9 +53,9 @@ def finished
         fetch_ci_execution
         build_status = fetch_build_status
 
-        @logger.info ">>> build_status: #{build_status.inspect}"
+        @logger.info ">>> build_status: #{build_status.inspect}. Hanged? #{@hanged}"
 
-        return [200, 'Still running'] if in_progress?(build_status)
+        return [200, 'Still running'] if in_progress?(build_status) and !@hanged
 
         check_stages
         clear_deleted_jobs
@@ -42,11 +64,19 @@ def finished
         [200, 'Finished']
       end
 
+      ##
+      # Fetches the build status from Bamboo CI.
+      #
+      # @return [Hash] The build status.
       def fetch_build_status
         get_request(URI("https://127.0.0.1/rest/api/latest/result/status/#{@check_suite.bamboo_ci_ref}"))
       end
 
-      # Checks if CI still running
+      ##
+      # Checks if the CI build is still in progress.
+      #
+      # @param [Hash] build_status The build status.
+      # @return [Boolean] Returns true if the build is still in progress, false otherwise.
       def in_progress?(build_status)
         @logger.info ">>> ci_stopped?: #{ci_stopped?(build_status)}"
         @logger.info ">>> ci_hanged?: #{ci_hanged?(build_status)}"
@@ -59,6 +89,9 @@ def in_progress?(build_status)
 
       private
 
+      ##
+      # Updates the status of all stages for the CheckSuite.
+      # Builds a summary for the last stage's last job.
       def update_all_stages
         last_stage =
           Stage
@@ -71,8 +104,8 @@ def update_all_stages
         build_summary(last_stage.jobs.last)
       end
 
-      # This method will move all tests that no longer exist in BambooCI to the skipped state,
-      # because there are no executions for them.
+      ##
+      # Moves all tests that no longer exist in BambooCI to the skipped state.
       def clear_deleted_jobs
         github_check = Github::Check.new(@check_suite)
 
@@ -81,22 +114,44 @@ def clear_deleted_jobs
         end
       end
 
+      ##
+      # Checks if the CI build has stopped.
+      #
+      # @param [Hash] build_status The build status.
+      # @return [Boolean] Returns true if the build has stopped, false otherwise.
       def ci_stopped?(build_status)
         build_status.key?('message') and !build_status.key?('finished')
       end
 
+      ##
+      # Checks if the CI build has hanged.
+      #
+      # @param [Hash] build_status The build status.
+      # @return [Boolean] Returns true if the build has hanged, false otherwise.
       def ci_hanged?(build_status)
         return true if ci_stopped?(build_status)
 
         build_status.dig('progress', 'percentageCompleted').to_f >= 2.0
       end
 
+      ##
+      # Updates the status of a stage based on the CI job result.
+      #
+      # @param [CiJob] ci_job The CI job to update.
+      # @param [Hash] result The result of the CI job.
+      # @param [Github::Check] github The Github check instance.
       def update_stage_status(ci_job, result, github)
         return if ci_job.nil? || (ci_job.finished? && !ci_job.job_ref.nil?)
 
         update_ci_job_status(github, ci_job, result['state'])
       end
 
+      ##
+      # Updates the status of a CI job based on the state.
+      #
+      # @param [Github::Check] github_check The Github check instance.
+      # @param [CiJob] ci_job The CI job to update.
+      # @param [String] state The state of the CI job.
       def update_ci_job_status(github_check, ci_job, state)
         ci_job.enqueue(github_check) if ci_job.job_ref.nil?
 
@@ -119,6 +174,11 @@ def update_ci_job_status(github_check, ci_job, state)
         build_summary(ci_job)
       end
 
+      ##
+      # Creates an output message for a CI job.
+      #
+      # @param [CiJob] ci_job The CI job to create the message for.
+      # @return [Hash] The output message.
       def create_output_message(ci_job)
         url = "https://ci1.netdef.org/browse/#{ci_job.job_ref}"
 
@@ -128,13 +188,22 @@ def create_output_message(ci_job)
         }
       end
 
+      ##
+      # Builds a summary for a CI job.
+      #
+      # @param [CiJob] ci_job The CI job to build the summary for.
       def build_summary(ci_job)
         summary = Github::Build::Summary.new(ci_job, agent: 'WatchDog')
         summary.build_summary
 
         finished_execution?(ci_job.check_suite)
       end
 
+      ##
+      # Checks if the execution of the CheckSuite has finished.
+      #
+      # @param [CheckSuite] check_suite The CheckSuite to check.
+      # @return [Boolean] Returns true if the execution has finished, false otherwise.
       def finished_execution?(check_suite)
         return false unless check_suite.pull_request.current_execution?(check_suite)
         return false unless check_suite.finished?
@@ -154,6 +223,8 @@ def slack_notify_cancelled(job)
         SlackBot.instance.notify_cancelled(job)
       end
 
+      ##
+      # Checks the stages of the CheckSuite and updates their status.
       def check_stages
         github_check = Github::Check.new(@check_suite)
         @logger.info ">>> @result: #{@result.inspect}"
@@ -168,6 +239,8 @@ def check_stages
         end
       end
 
+      ##
+      # Fetches the CI execution status for the CheckSuite.
       def fetch_ci_execution
         @result = get_status(@check_suite.bamboo_ci_ref)
       end