From b08b00c2ac83b667e9c73dbbda64617748b49b33 Mon Sep 17 00:00:00 2001 From: "Jose D. Gomez R" <1josegomezr@gmail.com> Date: Thu, 16 Nov 2023 14:36:37 +0100 Subject: [PATCH] ObsRsync Plugin support for HTTP authentication in Build Service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Heavily inspired by: - https://github.com/openSUSE/obs-build/blob/master/PBuild/SigAuth.pm - openSUSE/cavil@583009d - https://www.suse.com/c/multi-factor-authentication-on-suses-build-service/ Addresses poo#139073 - Add openssh to devel dependencies for OBSRsync. Co-authored-by: Martchus Co-authored-by: Tina Müller (tinita) Co-authored-by: Oliver Kurz --- dependencies.yaml | 1 + dist/rpm/openQA.spec | 4 +- etc/openqa/openqa.ini | 11 +++ lib/OpenQA/Setup.pm | 2 + lib/OpenQA/WebAPI/Plugin/ObsRsync.pm | 48 +++++++++-- t/config.t | 2 + t/ui/27-plugin_obs_rsync_obs_status.t | 116 +++++++++++++++++++++++--- tools/ci/ci-packages.txt | 1 + 8 files changed, 168 insertions(+), 17 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 1753ec92916..9b6d5c03d36 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -175,6 +175,7 @@ test_requires: '%main_requires': '%python_scripts_requires': '%worker_requires': + openssh-common: curl: jq: ShellCheck: diff --git a/dist/rpm/openQA.spec b/dist/rpm/openQA.spec index 0e76bb28d1c..fb3eeff256f 100644 --- a/dist/rpm/openQA.spec +++ b/dist/rpm/openQA.spec @@ -64,7 +64,7 @@ # Do not require on this in individual sub-packages except for the devel # package. # The following line is generated from dependencies.yaml -%define test_requires %common_requires %main_requires %python_scripts_requires %worker_requires ShellCheck curl jq os-autoinst-devel perl(App::cpanminus) perl(Perl::Critic) perl(Perl::Critic::Freenode) perl(Selenium::Remote::Driver) >= 1.23 perl(Selenium::Remote::WDKeys) perl(Test::Exception) perl(Test::Fatal) perl(Test::MockModule) perl(Test::MockObject) perl(Test::Mojo) perl(Test::Most) perl(Test::Output) perl(Test::Pod) perl(Test::Strict) perl(Test::Warnings) >= 0.029 postgresql-server python3-setuptools python3-yamllint +%define test_requires %common_requires %main_requires %python_scripts_requires %worker_requires ShellCheck curl jq openssh-common os-autoinst-devel perl(App::cpanminus) perl(Perl::Critic) perl(Perl::Critic::Freenode) perl(Selenium::Remote::Driver) >= 1.23 perl(Selenium::Remote::WDKeys) perl(Test::Exception) perl(Test::Fatal) perl(Test::MockModule) perl(Test::MockObject) perl(Test::Mojo) perl(Test::Most) perl(Test::Output) perl(Test::Pod) perl(Test::Strict) perl(Test::Warnings) >= 0.029 postgresql-server python3-setuptools python3-yamllint %ifarch x86_64 %define qemu qemu qemu-kvm %else @@ -199,6 +199,8 @@ Recommends: qemu Recommends: rsync # Optionally enabled with USE_PNGQUANT=1 Recommends: pngquant +# for Build Service Authentication +Recommends: openssh-common %if 0%{?suse_version} >= 1330 Requires(pre): group(nogroup) %endif diff --git a/etc/openqa/openqa.ini b/etc/openqa/openqa.ini index 69d23008b9b..7b52a90f1b4 100644 --- a/etc/openqa/openqa.ini +++ b/etc/openqa/openqa.ini @@ -296,3 +296,14 @@ concurrent = 0 # lookup_depth = 10 # Specify at how many state changes the search will be aborted (state = combination of failed/softfailed/skipped modules): # state_changes_limit = 3 + +# Configuration for the OBS rsync plugin +[obs_rsync] +# project_status_url = %obs_instance%/build/%%PROJECT/_result; +# concurrency = 2 +# queue_limit = 200 +# retry_interval = 60 +# retry_max_count = 2 +# home = +# username = openqa-user +# ssh_key_file = ~/.ssh/id_rsa diff --git a/lib/OpenQA/Setup.pm b/lib/OpenQA/Setup.pm index 41bff36dfe5..8dfb4f6071b 100644 --- a/lib/OpenQA/Setup.pm +++ b/lib/OpenQA/Setup.pm @@ -132,6 +132,8 @@ sub read_config ($app) { queue_limit => 200, concurrency => 2, project_status_url => '', + username => '', + ssh_key_file => '' }, cleanup => { concurrent => 0, diff --git a/lib/OpenQA/WebAPI/Plugin/ObsRsync.pm b/lib/OpenQA/WebAPI/Plugin/ObsRsync.pm index 6ac8edd5317..86c86b4457e 100644 --- a/lib/OpenQA/WebAPI/Plugin/ObsRsync.pm +++ b/lib/OpenQA/WebAPI/Plugin/ObsRsync.pm @@ -2,13 +2,13 @@ # SPDX-License-Identifier: GPL-2.0-or-later package OpenQA::WebAPI::Plugin::ObsRsync; -use Mojo::Base 'Mojolicious::Plugin'; +use Mojo::Base 'Mojolicious::Plugin', -signatures; use Mojo::File; use Mojo::URL; use Mojo::UserAgent; use POSIX 'strftime'; - +use File::Which qw(which); use OpenQA::Log qw(log_error); my $dirty_status_filename = '.dirty_status'; @@ -40,11 +40,16 @@ sub register { my $plugin_r = $app->routes->find('ensure_operator'); my $plugin_api_r = $app->routes->find('api_ensure_operator'); + # ssh-keygen is needed for the Build Service authentication. + die("ssh-keygen is not availabe. Aborting.\n") unless which('ssh-keygen'); + if (!$plugin_r) { $app->log->error('Routes not configured, plugin ObsRsync will be disabled') unless $plugin_r; } else { $app->helper('obs_rsync.home' => sub { shift->app->config->{obs_rsync}->{home} }); + $app->helper('obs_rsync.username' => sub { shift->app->config->{obs_rsync}->{username} }); + $app->helper('obs_rsync.ssh_key_file' => sub { shift->app->config->{obs_rsync}->{ssh_key_file} }); $app->helper('obs_rsync.concurrency' => sub { shift->app->config->{obs_rsync}->{concurrency} }); $app->helper('obs_rsync.retry_interval' => sub { shift->app->config->{obs_rsync}->{retry_interval} }); $app->helper('obs_rsync.retry_max_count' => sub { shift->app->config->{obs_rsync}->{retry_max_count} }); @@ -58,7 +63,7 @@ sub register { my $repo = $helper->get_api_repo($alias); my $url = $helper->get_api_dirty_status_url($project); return undef unless $url; - my @res = $self->_is_obs_project_status_dirty($url, $project, $repo); + my @res = $self->_is_obs_project_status_dirty($url, $project, $repo, $helper); if ($trace && scalar @res > 1 && $res[1]) { # ignore potential errors because we use this only for cosmetic rendering open(my $fh, '>', Mojo::File->new($c->obs_rsync->home, $project, $dirty_status_filename)) @@ -165,11 +170,24 @@ sub register { # try to determine whether project is dirty # undef means status is unknown sub _is_obs_project_status_dirty { - my ($self, $url, $project, $repo) = @_; + my ($self, $url, $project, $repo, $helper) = @_; return undef unless $url; + # Use only one UserAgent my $ua = $self->{ua} ||= Mojo::UserAgent->new; - my $res = $ua->get($url)->result; + my $tx = $ua->get($url); + my $res = $tx->result; + # Retry if authentication is required + if ($res->code == 401) { + my $username = $helper->username; + my $ssh_key_file = $helper->ssh_key_file; + my $auth_header = _bs_ssh_auth($res->headers->www_authenticate, $username, $ssh_key_file); + + # Reassign the results + $tx = $ua->get($url, {Authorization => $auth_header}); + $res = $tx->result; + } + return undef unless $res->is_success; return _parse_obs_response_dirty($res, $repo); } @@ -515,4 +533,24 @@ sub _for_every_batch { return @ret; } +# Based on https://www.suse.com/c/multi-factor-authentication-on-suses-build-service/ +sub _bs_ssh_sign ($key, $realm, $value) { + die "SSH key file not found at $key" unless -s $key; + # This needs to be a bit portable for CI testing + my $tmp = Mojo::File::tempfile('obs-rsync-ssh-keyfile-XXXXX')->spew($value); + my @lines = split "\n", qx/ssh-keygen -Y sign -f "$key" -q -n "$realm" < $tmp/; + shift @lines; + pop @lines; + return join '', @lines; +} + +sub _bs_ssh_auth ($challenge, $user, $key) { + die "Unexpected OBS challenge: $challenge" unless $challenge =~ /realm="([^"]+)".*headers="\(created\)"/; + my $realm = $1; + + my $now = time(); + my $signature = _bs_ssh_sign($key, $realm, "(created): $now"); + return qq{Signature keyId="$user",algorithm="ssh",signature="$signature",headers="(created)",created="$now"}; +} + 1; diff --git a/t/config.t b/t/config.t index 15d3f9dab6c..a81c2cdaa5c 100644 --- a/t/config.t +++ b/t/config.t @@ -115,6 +115,8 @@ subtest 'Test configuration default modes' => sub { queue_limit => 200, concurrency => 2, project_status_url => '', + username => '', + ssh_key_file => '', }, cleanup => { concurrent => 0, diff --git a/t/ui/27-plugin_obs_rsync_obs_status.t b/t/ui/27-plugin_obs_rsync_obs_status.t index d9e47c076b1..1dc0de4a0b4 100644 --- a/t/ui/27-plugin_obs_rsync_obs_status.t +++ b/t/ui/27-plugin_obs_rsync_obs_status.t @@ -8,20 +8,30 @@ use lib "$FindBin::Bin/../lib", "$FindBin::Bin/../../external/os-autoinst-common use OpenQA::Test::TimeLimit '30'; use OpenQA::Test::Utils qw(perform_minion_jobs wait_for_or_bail_out); use OpenQA::Test::ObsRsync 'setup_obs_rsync_test'; - use Mojolicious; use IO::Socket::INET; use Mojo::Server::Daemon; use Mojo::IOLoop::Server; use Mojo::IOLoop::ReadWriteProcess 'process'; use Mojo::IOLoop::ReadWriteProcess::Session 'session'; +use Mojo::File qw(path tempfile); + +my $mocked_time = 0; + +BEGIN { + *CORE::GLOBAL::time = sub { + return $mocked_time if $mocked_time; + return time(); + }; +} + +$SIG{INT} = sub { session->clean }; # uncoverable statement count:2 -$SIG{INT} = sub { session->clean }; END { session->clean } my $port = Mojo::IOLoop::Server->generate_port; my $host = "http://127.0.0.1:$port"; -my $url = "$host/public/build/%%PROJECT/_result"; +my $url = "$host/build/%%PROJECT/_result"; my %fake_response_by_project = ( Proj3 => ' @@ -71,14 +81,42 @@ my %fake_response_by_project = ( Proj0 => 'invalid XML', ); +my $auth_header_exact + = qq(Signature keyId="dummy-username",algorithm="ssh",) + . qq(signature="U1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgSKpcECPm8Vjo9UznZS+) + . qq(M/QLjmXXmLzoBxkIbZ8Z/oPkAAAAaVXNlIHlvdXIgZGV2ZWxvcGVyIGFjY291bnQAAAAAAAAABn) + . qq(NoYTUxMgAAAFMAAAALc3NoLWVkMjU1MTkAAABA8cmvTy1PgpW2XhHWxQ1yw/wPGAfT2M3CGRJ3II) + . qq(7uT5Orqn1a0bWlo/lEV0WiqP+pPcQdajQ4a2YGJvpfzT1uBA==",) + . qq (headers="(created)",created="1664187470"); + note 'Starting fake API server'; -my $server_instance = process sub { + +my $server_process = sub { + use experimental 'signatures'; my $mock = Mojolicious->new; $mock->mode('test'); + + my $www_authenticate = qq(Signature realm="Use your developer account",headers="(created)"); + $mock->routes->get( + '/build/ProjWithAuth/_result' => sub ($c) { + return $c->render(status => 200, text => $fake_response_by_project{Proj1}) + if $c->req->headers->authorization; + $c->res->headers->www_authenticate($www_authenticate); + return $c->render(status => 401, text => 'login'); + }); + + $mock->routes->get( + '/build/ProjTestingSignature/_result' => sub ($c) { + my $client_auth_header = $c->req->headers->authorization // ''; + return $c->render(status => 200, text => $fake_response_by_project{Proj1}) + if $auth_header_exact eq $client_auth_header; + $c->res->headers->www_authenticate($www_authenticate); + return $c->render(status => 401, text => 'login'); + }); + for my $project (sort keys %fake_response_by_project) { $mock->routes->get( - "/public/build/$project/_result" => sub { - my $c = shift; + "/build/$project/_result" => sub ($c) { my $pkg = $c->param('package'); return $c->render(status => 404) if !$pkg and $project ne 'Proj1'; return $c->render(status => 200, text => $fake_response_by_project{$project}); @@ -87,12 +125,40 @@ my $server_instance = process sub { my $daemon = Mojo::Server::Daemon->new(app => $mock, listen => [$host]); $daemon->run; note 'Fake API server stopped'; - _exit(0); + Devel::Cover::report() if Devel::Cover->can('report'); + _exit(0); # uncoverable statement }; + +my $server_instance = process( + $server_process, + max_kill_attempts => 0, + blocking_stop => 1, + _default_blocking_signal => POSIX::SIGTERM, + kill_sleeptime => 0 +); + $server_instance->set_pipes(0)->start; wait_for_or_bail_out { IO::Socket::INET->new(PeerAddr => '127.0.0.1', PeerPort => $port) } 'API'; -my ($t, $tempdir, $home, $params) = setup_obs_rsync_test(url => $url); +my $ssh_keyfile = tempfile("$FindBin::Script-sshkey-XXXXX"); +# using the key from [0] to have a reproduceable output. +$ssh_keyfile->spew(< $url, + config => { + username => 'dummy-username', + ssh_key_file => path($ssh_keyfile), + }); my $app = $t->app; my $helper = $app->obs_rsync; @@ -110,9 +176,9 @@ subtest 'test api package helper' => sub { }; subtest 'test api url helper' => sub { - is($helper->get_api_dirty_status_url('Proj1'), "$host/public/build/Proj1/_result"); - is($helper->get_api_dirty_status_url('Proj2'), "$host/public/build/Proj2/_result?package=0product"); - is($helper->get_api_dirty_status_url('BatchedProj'), "$host/public/build/BatchedProj/_result?package=000product"); + is($helper->get_api_dirty_status_url('Proj1'), "$host/build/Proj1/_result"); + is($helper->get_api_dirty_status_url('Proj2'), "$host/build/Proj2/_result?package=0product"); + is($helper->get_api_dirty_status_url('BatchedProj'), "$host/build/BatchedProj/_result?package=000product"); }; subtest 'test builds_text helper' => sub { @@ -153,5 +219,33 @@ $t->get_ok('/admin/obs_rsync/queue')->status_is(200, 'jobs list')->content_like( $t->get_ok('/admin/obs_rsync/')->status_is(200, 'project list')->content_like(qr/published/)->content_like(qr/dirty/) ->content_like(qr/publishing/); +subtest 'build service ssh authentication' => sub { + is($helper->is_status_dirty('ProjWithAuth'), 1, 're-authenticate with ssh auth'); +}; + +subtest 'build service authentication: signature generation' => sub { + $mocked_time = 1664187470; + note 'time right now: ' . time(); + is(time(), $mocked_time, 'Time is not frozen!'); + is($helper->is_status_dirty('ProjTestingSignature'), 1, 'signature matches fixture'); + $mocked_time = undef; +}; + +subtest 'build service authentication: error handling' => sub { + $ssh_keyfile->remove(); + throws_ok { + $helper->is_status_dirty('ProjTestingSignature') + } + qr/SSH key file not found at/, 'Key detection logic failed (not existing key file)'; + + path($ssh_keyfile)->touch(); + throws_ok { + $helper->is_status_dirty('ProjTestingSignature') + } + qr/SSH key file not found at/, 'Key detection logic failed (empty key file)'; +}; + $server_instance->stop; done_testing(); + +# [0]: https://www.suse.com/c/multi-factor-authentication-on-suses-build-service/ diff --git a/tools/ci/ci-packages.txt b/tools/ci/ci-packages.txt index a28b449eb1d..44c8a6f5a3a 100644 --- a/tools/ci/ci-packages.txt +++ b/tools/ci/ci-packages.txt @@ -58,6 +58,7 @@ libXt6-1.1.5 libXvnc1-1.12.0 lsof-4.91 luit-20150706 +openssh-common-8.4p1 optipng-0.7.7 pciutils-3.5.6 perl-Algorithm-C3-0.11