Skip to content

Commit

Permalink
Merge pull request os-autoinst#5096 from Martchus/partial-retrigger
Browse files Browse the repository at this point in the history 
Allow partial retrigger
  • Loading branch information
@mergify
mergify[bot] authored Jun 9, 2023
2 parents 69ffe8a + 2336070 commit 57b586f
Show file tree
Hide file tree
Showing 2 changed files with 102 additions and 78 deletions.
153 changes: 77 additions & 76 deletions lib/OpenQA/Schema/Result/ScheduledProducts.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,7 @@ sub _schedule_iso {
my $obsolete = delete $args->{_OBSOLETE} // 0;
my $onlysame = delete $args->{_ONLY_OBSOLETE_SAME_BUILD} // 0;
my $skip_chained_deps = delete $args->{_SKIP_CHAINED_DEPS} // 0;
my $include_children = delete $args->{_INCLUDE_CHILDREN} // 0;
my $force = delete $args->{_FORCE_DEPRIORITIZEBUILD};
$force = delete $args->{_FORCE_OBSOLETE} || $force;
if (($deprioritize || $obsolete) && $args->{TEST} && !$force) {
Expand All @@ -278,13 +279,13 @@ sub _schedule_iso {
my $yaml = delete $args->{SCENARIO_DEFINITIONS_YAML};
my $yaml_file = delete $args->{SCENARIO_DEFINITIONS_YAML_FILE};
if (defined $yaml) {
$result = $self->_schedule_from_yaml($args, $skip_chained_deps, string => $yaml);
$result = $self->_schedule_from_yaml($args, $skip_chained_deps, $include_children, string => $yaml);
}
elsif (defined $yaml_file) {
$result = $self->_schedule_from_yaml($args, $skip_chained_deps, file => $yaml_file);
$result = $self->_schedule_from_yaml($args, $skip_chained_deps, $include_children, file => $yaml_file);
}
else {
$result = $self->_generate_jobs($args, \@notes, $skip_chained_deps);
$result = $self->_generate_jobs($args, \@notes, $skip_chained_deps, $include_children);
}
return {error => $result->{error_message}, error_code => $result->{error_code} // 400}
if defined $result->{error_message};
Expand Down Expand Up @@ -342,8 +343,8 @@ sub _schedule_iso {
my $job = $jobs_resultset->create_from_settings($settings, $self->id);
push @created_jobs, $job;
my $j_id = $job->id;
$job_ids_by_test_machine{_settings_key($settings)} //= [];
push @{$job_ids_by_test_machine{_settings_key($settings)}}, $j_id;
$job_ids_by_test_machine{_job_ref($settings)} //= [];
push @{$job_ids_by_test_machine{_job_ref($settings)}}, $j_id;

# set prio if defined explicitly (otherwise default prio is used)
$job->update({priority => $prio}) if (defined($prio));
Expand Down Expand Up @@ -429,16 +430,17 @@ sub _schedule_iso {

=over 4
=item _settings_key()
=item _job_ref()
Return settings key for given job settings. Internal method.
Return the "job reference" for the specified job settings. It is used internally as a key for the job in various
hash maps. It is also used to refer to a job in dependency specifications.
=back
=cut

sub _settings_key ($settings) {
my ($test, $machine) = ($settings->{TEST}, $settings->{MACHINE});
sub _job_ref ($job_settings) {
my ($test, $machine) = ($job_settings->{TEST}, $job_settings->{MACHINE});
return $machine ? "$test\@$machine" : $test;
}

Expand All @@ -455,23 +457,23 @@ is used.
=cut

sub _parse_dep_variable {
my ($value, $settings) = @_;

sub _parse_dep_variable ($value, $job_settings) {
return unless defined $value;
return map {
if ($_ =~ /^(.+)\@([^@]+)$/) {
[$1, $2];
}
elsif ($_ =~ /^(.+):([^:]+)$/) {
[$1, $2]; # for backwards compatibility
}
else {
[$_, $settings->{MACHINE}];
}
if ($_ =~ /^(.+)\@([^@]+)$/) { [$1, $2] }
elsif ($_ =~ /^(.+):([^:]+)$/) { [$1, $2] } # for backwards compatibility
else { [$_, $job_settings->{MACHINE}] }
} split(/\s*,\s*/, $value);
}

sub _chained_parents ($job) {
[_parse_dep_variable($job->{START_AFTER_TEST}, $job), _parse_dep_variable($job->{START_DIRECTLY_AFTER_TEST}, $job)];
}

sub _parallel_parents ($job) {
[_parse_dep_variable($job->{PARALLEL_WITH}, $job)];
}

=over 4
=item _sort_dep()
Expand All @@ -484,46 +486,31 @@ used in B<_populate_wanted_jobs_for_parent_dependencies>.
=cut

sub _sort_dep ($list) {
my %done;
my %count;
my @out;

for my $job (@$list) {
$count{_settings_key($job)} //= 0;
$count{_settings_key($job)}++;
}
my (%done, %count, @out);
++$count{_job_ref($_)} for @$list;

my $added;
do {
$added = 0;
for (my $added;; $added = 0) {
for my $job (@$list) {
next if $done{$job};
my @parents;
push @parents, _parse_dep_variable($job->{START_AFTER_TEST}, $job),
_parse_dep_variable($job->{START_DIRECTLY_AFTER_TEST}, $job),
_parse_dep_variable($job->{PARALLEL_WITH}, $job);

my $c = 0; # number of parents that must go to @out before this job
foreach my $parent (@parents) {
my $parent_test_machine = join('@', @$parent);
$c += $count{$parent_test_machine} if defined $count{$parent_test_machine};
}

if ($c == 0) { # no parents, we can do this job
push @out, $job;
$done{$job} = 1;
$count{_settings_key($job)}--;
$added = 1;
my $has_parents_to_go_before;
for my $parent (@{_chained_parents($job)}, @{_parallel_parents($job)}) {
if ($count{join('@', @$parent)}) {
$has_parents_to_go_before = 1;
last;
}
}
next if $has_parents_to_go_before;
push @out, $job; # no parents go before, we can do this job
$done{$job} = $added = 1;
$count{_job_ref($job)}--;
}
} while ($added);
last unless $added;
}

#cycles, broken dep, put at the end of the list
# put cycles and broken dependencies at the end of the list
for my $job (@$list) {
next if $done{$job};
push @out, $job;
push @out, $job unless $done{$job};
}

return \@out;
}

Expand All @@ -540,7 +527,7 @@ method used in the B<schedule_iso()> method.
=cut

sub _generate_jobs {
my ($self, $args, $notes, $skip_chained_deps) = @_;
my ($self, $args, $notes, $skip_chained_deps, $include_children) = @_;

my $ret = [];
my $schema = $self->result_source->schema;
Expand Down Expand Up @@ -620,7 +607,7 @@ sub _generate_jobs {
push @$ret, \%settings;
}
}
$ret = _populate_wanted_jobs_for_parent_dependencies($ret, \%wanted, $skip_chained_deps);
$ret = _populate_wanted_jobs_for_parent_dependencies($ret, \%wanted, $skip_chained_deps, $include_children);
return {error_message => $error_message, settings_result => $ret};
}

Expand Down Expand Up @@ -769,7 +756,7 @@ sub _create_download_lists {
}
}

sub _schedule_from_yaml ($self, $args, $skip_chained_deps, @load_yaml_args) {
sub _schedule_from_yaml ($self, $args, $skip_chained_deps, $include_children, @load_yaml_args) {
my $data = eval { load_yaml(@load_yaml_args) };
if (my $error = $@) { return {error_message => "Unable to load YAML: $error"} }
my $app = OpenQA::App->singleton;
Expand Down Expand Up @@ -830,7 +817,9 @@ sub _schedule_from_yaml ($self, $args, $skip_chained_deps, @load_yaml_args) {
}

return {
settings_result => _populate_wanted_jobs_for_parent_dependencies(\@job_templates, \%wanted, $skip_chained_deps),
settings_result => _populate_wanted_jobs_for_parent_dependencies(
\@job_templates, \%wanted, $skip_chained_deps, $include_children
),
error_message => $error_msg,
};
}
Expand All @@ -854,42 +843,54 @@ sub _merge_settings_uppercase ($source_settings, $destination_settings, $excepti
sub _populate_wanted_jobs_for_test_arg ($args, $settings, $wanted) {
return undef if $args->{MACHINE} && $args->{MACHINE} ne $settings->{MACHINE}; # skip if machine does not match
my @tests = $args->{TEST} ? split(/\s*,\s*/, $args->{TEST}) : (); # allow multiple, comma-separated TEST values
return $wanted->{_settings_key($settings)} = 1 unless @tests;
return $wanted->{_job_ref($settings)} = 1 unless @tests;
my $settings_test = $settings->{TEST};
for my $test (@tests) {
if ($test eq $settings_test) {
$wanted->{_settings_key($settings)} = 1;
$wanted->{_job_ref($settings)} = 1;
last;
}
}
}

sub _populate_wanted_jobs_for_parent_dependencies ($jobs, $wanted, $skip_chained_deps) {
sub _is_any_parent_wanted ($jobs, $parents, $wanted_list, $visited = {}) {
for my $parent (@$parents) {
my $parent_job_ref = join('@', @$parent);
next if $visited->{$parent_job_ref}++; # prevent deep recursion if there are dependency cycles
for my $job (@$jobs) {
my $job_ref = _job_ref($job);
next unless $job_ref eq $parent_job_ref;
return 1 if $wanted_list->{$job_ref};
return 1 if _is_any_parent_wanted($jobs, _chained_parents($job), $wanted_list, $visited);
}
}
return 0;
}

sub _populate_wanted_jobs_for_parent_dependencies ($jobs, $wanted, $skip_chained_deps, $include_children) {
# sort $jobs so parents are first
$jobs = _sort_dep($jobs);

# iterate in reverse order to go though children first and being able easily delete from $jobs
# iterate in reverse order to go though children first and being able to easily delete from $jobs
for (my $i = $#{$jobs}; $i >= 0; --$i) {
# delete unwanted jobs
my $job = $jobs->[$i];
if (!$wanted->{_settings_key($job)}) {
splice @$jobs, $i, 1;
next;
}

# parse relevant parents from job settings
my @parents;
push @parents, _parse_dep_variable($job->{START_AFTER_TEST}, $job),
_parse_dep_variable($job->{START_DIRECTLY_AFTER_TEST}, $job)
unless $skip_chained_deps;
push @parents, _parse_dep_variable($job->{PARALLEL_WITH}, $job);
my $chained_parents = !$skip_chained_deps || $include_children ? _chained_parents($job) : [];
my $parents = _parallel_parents($job);
push @$parents, @$chained_parents unless $skip_chained_deps;

# delete unwanted jobs unless the parent is wanted and we include children
my $unwanted = !$wanted->{_job_ref($job)};
splice @$jobs, $i, 1 and next
if $unwanted && (!$include_children || !_is_any_parent_wanted($jobs, $chained_parents, $wanted));

# add parents to wanted list
for my $parent (@parents) {
my $parent_test_machine = join('@', @$parent);
my @parents_job_template = grep { join('@', $_->{TEST}, $_->{MACHINE}) eq $parent_test_machine } @$jobs;
for my $parent_job_template (@parents_job_template) {
$wanted->{join('@', $parent_job_template->{TEST}, $parent_job_template->{MACHINE})} = 1;
for my $parent (@$parents) {
my $parent_job_ref = join('@', @$parent);
for my $job (@$jobs) {
my $job_ref = _job_ref($job);
$wanted->{$job_ref} = 1 if $job_ref eq $parent_job_ref;
}
}
}
Expand Down
27 changes: 25 additions & 2 deletions t/api/02-iso.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -831,11 +831,34 @@ subtest '_SKIP_CHAINED_DEPS prevents scheduling parent tests' => sub {
add_opensuse_test('child_test_2', START_AFTER_TEST => 'parent_test', @machine);
# add nested child; not supposed to be included regardless of _SKIP_CHAINED_DEPS
add_opensuse_test('child_test_3', START_AFTER_TEST => 'child_test_1', @machine);

# add another level of chained nesting; not supposed to be included regardless of _SKIP_CHAINED_DEPS
add_opensuse_test('child_test_4', START_AFTER_TEST => 'child_test_3', @machine);
# add dependency cycle; it should not lead to deep recursion
add_opensuse_test('child_test_5', START_AFTER_TEST => 'child_test_6,child_test_3', @machine);
# add another level of chained nesting; not supposed to be included regardless of _SKIP_CHAINED_DEPS
my @deps = (START_AFTER_TEST => 'child_test_5,child_test_8', PARALLEL_WITH => 'child_test_7');
add_opensuse_test('child_test_6', @deps, @machine);
# add parallel parent for nested job; supposed to be pulled in as usual
add_opensuse_test('child_test_7', @machine);
# add chained parent for nested job; not supposed to be pulled in, even with _INCLUDE_CHILDREN=1
add_opensuse_test('child_test_8', @machine);

# schedule with _SKIP_CHAINED_DEPS; no chained parents should be scheduled
my $res = schedule_iso($t, {%iso, _GROUP => 'opensuse test', TEST => 'child_test_1', _SKIP_CHAINED_DEPS => 1});
is $res->json->{count}, 2, '2 jobs scheduled';
my %created_jobs = map { $jobs->find($_)->settings_hash->{'TEST'} => 1 } @{$res->json->{ids}};
is_deeply \%created_jobs, {child_test_1 => 1, child_test_2 => 1}, 'parent jobs not scheduled';
is_deeply \%created_jobs, {child_test_1 => 1, child_test_2 => 1}, 'only parallel parent scheduled'
or diag explain \%created_jobs;

# schedule with _INCLULDE_CHILDREN as well; now also all nested chained children should be included
my %expected_jobs = map { ("child_test_$_" => 1) } 1 .. 7;
$res = schedule_iso($t,
{%iso, _GROUP => 'opensuse test', TEST => 'child_test_1', _SKIP_CHAINED_DEPS => 1, _INCLUDE_CHILDREN => 1});
is $res->json->{count}, 7, '7 jobs scheduled';
%created_jobs = map { $jobs->find($_)->settings_hash->{'TEST'} => 1 } @{$res->json->{ids}};
is_deeply \%created_jobs, \%expected_jobs, 'only parallel parent and (nested) children scheduled'
or diag explain \%created_jobs;

$schema->txn_rollback;
};

Expand Down

0 comments on commit 57b586f

Please sign in to comment.