Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve worker load threshold detection #6010

Merged
merged 1 commit into from
Oct 22, 2024
Merged

Improve worker load threshold detection #6010

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 18 additions & 11 deletions lib/OpenQA/Worker.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ BEGIN {
use Fcntl;
use File::Path qw(make_path remove_tree);
use File::Spec::Functions 'catdir';
use List::Util qw(all max min);
use Mojo::IOLoop;
use Mojo::File 'path';
use POSIX;
Expand Down Expand Up @@ -750,18 +751,24 @@ sub _handle_job_status_changed ($self, $job, $event_data) {
}
}

sub _load_avg ($field = 2) {
my $value = eval { (split(' ', path($ENV{OPENQA_LOAD_AVG_FILE} // '/proc/loadavg')->slurp))[$field] };
sub _load_avg ($path = $ENV{OPENQA_LOAD_AVG_FILE} // '/proc/loadavg') {
my @load = eval { split(' ', path($path)->slurp) };
log_warning "Unable to determine average load: $@" if $@;
okurz marked this conversation as resolved.
Show resolved Hide resolved
return looks_like_number($value) ? $value : undef;
}

sub _check_system_utilization ($self) {
my $settings = $self->settings->global_settings;
return undef unless my $threshold = $settings->{CRITICAL_LOAD_AVG_THRESHOLD};
my $load_avg = _load_avg;
return "The average load $load_avg is exceeding the configured threshold of $threshold."
if defined $load_avg && $load_avg >= $threshold;
splice @load, 3; # remove non-load numbers
log_error "Unable to parse system load from file '$path'" and return [] unless all { looks_like_number $_ } @load;
return \@load;
}

sub _check_system_utilization (
$self,
$threshold = $self->settings->global_settings->{CRITICAL_LOAD_AVG_THRESHOLD},
$load = _load_avg())
{
okurz marked this conversation as resolved.
Show resolved Hide resolved
return undef unless $threshold && @$load >= 3;
# look at the load evolution over time to react quick enough if the load
# rises but accept a falling edge
return "The average load (@$load) is exceeding the configured threshold of $threshold."
if max(@$load) > $threshold && ($load->[0] > $load->[1] || $load->[0] > $load->[2] || min(@$load) > $threshold);
return undef;
}

Expand Down
20 changes: 17 additions & 3 deletions t/24-worker-overall.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@ $ENV{OPENQA_CONFIG} = "$FindBin::Bin/data/24-worker-overall";
# file specified via OPENQA_LOGFILE instead of stdout/stderr.
$ENV{OPENQA_LOGFILE} = undef;

my $load_avg_file = simulate_load('0.93 0.95 10.25 2/2207 1212', 'worker-overall-load-avg');

# define fake isotovideo
{
package Test::FakeProcess; # uncoverable statement count:1
Expand Down Expand Up @@ -90,6 +88,7 @@ my $dbus_mock = Test::MockModule->new('Net::DBus', no_auto => 1);
$dbus_mock->define(system => sub (@) { Test::FakeDBus->new });
my $cache_service_client_mock = Test::MockModule->new('OpenQA::CacheService::Client');
$cache_service_client_mock->redefine(info => sub { Test::FakeCacheServiceClientInfo->new });
my $load_avg_file = simulate_load('10.93 10.91 10.25 2/2207 1212', 'worker-overall-load-avg');

like(
exception {
Expand Down Expand Up @@ -120,6 +119,21 @@ combined_like { $worker->log_setup_info }
qr/.*http:\/\/localhost:9527,https:\/\/remotehost.*qemu_i386,qemu_x86_64.*Errors occurred.*foo.*bar.*/s,
'setup info with parse errors';

subtest 'worker load' => sub {
my $load = OpenQA::Worker::_load_avg();
is scalar @$load, 3, 'expected number of load values';
is $load->[0], 10.93, 'expected load';
is_deeply $load, [10.93, 10.91, 10.25], 'expected computed system load, rising flank';
is_deeply OpenQA::Worker::_load_avg(path($ENV{OPENQA_CONFIG}, 'invalid_loadavg')), [], 'error on invalid load';
ok !$worker->_check_system_utilization, 'default threshold not exceeded';
ok $worker->_check_system_utilization(10), 'stricter threshold exceeded by load';
ok !$worker->_check_system_utilization(10, [3, 9, 11]), 'load ok on falling flank';
ok $worker->_check_system_utilization(10, [12, 9, 3]), 'load exceeded on rising flank';
ok $worker->_check_system_utilization(10, [12, 3, 9]), 'load exceeded on rising flank and old load';
ok $worker->_check_system_utilization(10, [11, 13, 12]), 'load still exceeded on short load dip';
ok $worker->_check_system_utilization(10, [11, 12, 13]), 'load still exceeded on falling flank but high';
};

subtest 'delay and exec' => sub {
my $worker_mock = Test::MockModule->new('OpenQA::Worker');
$worker_mock->redefine(init => 42);
Expand Down Expand Up @@ -854,7 +868,7 @@ qr/Job 42 from some-host finished - reason: done.*A QEMU instance using.*Skippin
$worker_mock->unmock('is_qemu_running');
$worker->settings->global_settings->{CRITICAL_LOAD_AVG_THRESHOLD} = '10';
is $worker->status->{status}, 'broken', 'worker considered broken when average load exceeds threshold';
like $worker->current_error, qr/load 10\.25.*exceeding.*10/, 'error shows current load and threshold';
like $worker->current_error, qr/load \(.*10\.25.*exceeding.*10/, 'error shows current load and threshold';

# assume the error is gone
$load_avg_file->remove;
Expand Down
Loading