Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@perlpunk
perlpunk committed Dec 21, 2023
1 parent 434781e commit 2eb1a6d
Show file tree
Hide file tree
Showing 2 changed files with 100 additions and 28 deletions.
73 changes: 51 additions & 22 deletions lib/OpenQA/Schema/Result/Jobs.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1955,6 +1955,29 @@ sub descendants ($self, $limit = -1) {
$self->{_descendants} = $sth->fetchrow_array;
}

sub incomplete_ancestors ($self, $limit = -1) {
my $ancestors = $self->{_incomplete_ancestors};
return $ancestors if defined $ancestors;
my $sth = $self->result_source->schema->storage->dbh->prepare(<<~'EOM');
with recursive orig_id as (
select ? as orig_id, 0 as level, ? as orig_result, ? as orig_reason
union all
select id as orig_id, orig_id.level + 1 as level, result as orig_result, reason as orig_reason
from jobs join orig_id on orig_id.orig_id = jobs.clone_id
where result = 'incomplete' and (? < 0 or level < ?))
select orig_id, level, orig_reason from orig_id order by level asc;
EOM
$sth->bind_param(1, $self->id, SQL_BIGINT);
$sth->bind_param(2, $self->result, SQL_VARCHAR);
$sth->bind_param(3, $self->reason, SQL_VARCHAR);
$sth->bind_param(4, $limit, SQL_BIGINT);
$sth->bind_param(5, $limit, SQL_BIGINT);
$sth->execute;
$ancestors = $sth->fetchall_arrayref;
shift @$ancestors;
$self->{_incomplete_ancestors} = $ancestors;
}

sub latest_job ($self) {
return $self unless my $clone = $self->clone;
return $clone->latest_job;
Expand Down Expand Up @@ -2065,32 +2088,12 @@ sub done ($self, %args) {
$worker->update({job_id => undef});
}

my %new_val = (state => DONE);
# update result unless already known (it is already known for CANCELLED jobs)
# update the reason if updating the result or if there is no reason yet
my $reason = $args{reason};
my $result_unknown = $self->result eq NONE;
my $reason_unknown = !$self->reason;
my %new_val = (state => DONE);
my $restart = 0;
$new_val{result} = $result if $result_unknown;
if (($result_unknown || $reason_unknown) && defined $reason) {
# restart incompletes when the reason matches the configured regex
my $auto_clone_regex = OpenQA::App->singleton->config->{global}->{auto_clone_regex};
if ($result eq INCOMPLETE and $auto_clone_regex and $reason =~ $auto_clone_regex) {
my $limit = OpenQA::App->singleton->config->{global}{auto_clone_limit};
my $ancestors = $self->ancestors($limit + 1);
$restart = 1 if $ancestors < $limit;
}

# limit length of the reason
# note: The reason can be anything the worker picked up as useful information so better cut it at a
# reasonable, human-readable length. This also avoids growing the database too big.
$reason = substr($reason, 0, 300) . '…' if defined $reason && length $reason > 300;
$new_val{reason} = $reason;
}
elsif ($reason_unknown && !defined $reason && $result eq INCOMPLETE) {
$new_val{reason} = 'no test modules scheduled/uploaded';
}
$self->_reason_and_result(\%new_val, $result, $reason, \$restart);
$self->update(\%new_val);
$self->unblock;
my %finalize_opts = (lax => 1);
Expand All @@ -2113,6 +2116,32 @@ sub done ($self, %args) {
return $new_val{result} // $self->result;
}

sub _reason_and_result ($self, $new_val, $result, $reason, $restart) {
my $result_unknown = $self->result eq NONE;
my $reason_unknown = !$self->reason;
$new_val->{result} = $result if $result_unknown;
if (($result_unknown || $reason_unknown) && defined $reason) {
# restart incompletes when the reason matches the configured regex
my $auto_clone_regex = OpenQA::App->singleton->config->{global}->{auto_clone_regex};
if ($result eq INCOMPLETE and $auto_clone_regex and $reason =~ $auto_clone_regex) {
my $limit = OpenQA::App->singleton->config->{global}{auto_clone_limit};
my $ancestors = $self->incomplete_ancestors($limit + 1);
# how many of those incomplete ancestors had a reason not matching auto_clone?
my $unrelated = grep { $_->[2] !~ m/$auto_clone_regex/ } @$ancestors;
$$restart = 1 if @$ancestors < $limit || $unrelated > 0;
}

# limit length of the reason
# note: The reason can be anything the worker picked up as useful information so better cut it at a
# reasonable, human-readable length. This also avoids growing the database too big.
$reason = substr($reason, 0, 300) . '…' if defined $reason && length $reason > 300;
$new_val->{reason} = $reason;
}
elsif ($reason_unknown && !defined $reason && $result eq INCOMPLETE) {
$new_val->{reason} = 'no test modules scheduled/uploaded';
}
}

sub cancel ($self, $result, $reason = undef) {
return undef if $self->result ne NONE;
my %data = (state => CANCELLED, result => $result);
Expand Down
55 changes: 49 additions & 6 deletions t/api/04-jobs.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1484,12 +1484,55 @@ subtest 'marking job as done' => sub {
$t->json_is('/job/state' => DONE, 'state set');
$t->json_like('/job/clone_id' => qr/\d+/, 'job cloned when reason does match configured regex');

local $t->app->config->{global}{auto_clone_limit} = 1;
my $clone_id = $t->tx->res->json->{job}->{clone_id};
$t->post_ok(Mojo::URL->new("/api/v1/jobs/$clone_id/set_done")->query(\%cache_failure_params));
perform_minion_jobs($t->app->minion);
$t->get_ok("/api/v1/jobs/$clone_id")->status_is(200);
$t->json_is('/job/clone_id' => undef, 'job not cloned if ancestors is already above the limit');
subtest 'auto_clone limits' => sub {
$schema->txn_begin;
local $t->app->config->{global}{auto_clone_limit} = 5;
my @jobs;
my $backend_reason = 'backend died: VNC Connection refused';
for my $i (reverse 10 .. 20) {
my $clone_id = $i < 20 ? $i + 1 : undef;
my $new = $jobs->create(
{
id => $i,
clone_id => $clone_id,
state => DONE,
result => INCOMPLETE,
TEST => 'foo',
reason => $backend_reason,
});
push @jobs, $new;
}
my $last = $jobs[0];
$last->update({reason => undef});
my $last_incompletes = $last->incomplete_ancestors(5);
is scalar @$last_incompletes, 5, 'incomplete_ancestors returns a row of 5 incompletes with the same reason';

my $restart = 0;
my %new;
$last->_reason_and_result(\%new, 'incomplete', $backend_reason, \$restart);
is $restart, 0, '_reason_and_result: more than 5 auto_clone incompletes, no restart';

$jobs[3]->update({reason => 'unrelated'});
delete $last->{_incomplete_ancestors};
$last_incompletes = $last->incomplete_ancestors(5);
is scalar @$last_incompletes, 5,
'incomplete_ancestors returns a row of 5 incompletes with different reasons';
$restart = 0;
%new = ();
$last->_reason_and_result(\%new, 'incomplete', 'backend died: VNC Connection refused', \$restart);
is $restart, 1,
'_reason_and_result: more than 5 incompletes, but with non auto_clone reason in between, restart';

$jobs[3]->update({result => 'failed'});
delete $last->{_incomplete_ancestors};
$last_incompletes = $last->incomplete_ancestors(5);
is scalar @$last_incompletes, 2, 'incomplete_ancestors returns a row of 2 incompletes';
$restart = 0;
%new = ();
$last->_reason_and_result(\%new, 'incomplete', 'backend died: VNC Connection refused', \$restart);
is $restart, 1, '_reason_and_result: more than 5 ancestors, but with non failed result in between, restart';
$schema->txn_rollback;
};
};
subtest 'job is already done with reason, not overriding existing result and reason' => sub {
$t->post_ok('/api/v1/jobs/99961/set_done?result=passed&reason=foo')->status_is(200);
Expand Down

0 comments on commit 2eb1a6d

Please sign in to comment.