From 5dee071f8ccb1fb9e3404c182beb7e0c2c1e5726 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Fri, 15 Oct 2021 15:42:07 -0700 Subject: [PATCH 1/4] Add a GitLab manifest provider --- src/rosdistro/distribution.py | 5 +- src/rosdistro/manifest_provider/gitlab.py | 137 ++++++++++++++++++++++ src/rosdistro/release.py | 3 +- test/test_manifest_providers.py | 32 +++++ 4 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 src/rosdistro/manifest_provider/gitlab.py diff --git a/src/rosdistro/distribution.py b/src/rosdistro/distribution.py index 84055c6a6..742cc638b 100644 --- a/src/rosdistro/distribution.py +++ b/src/rosdistro/distribution.py @@ -34,14 +34,15 @@ from .manifest_provider.bitbucket import bitbucket_manifest_provider from .manifest_provider.git import git_manifest_provider, git_source_manifest_provider from .manifest_provider.github import github_manifest_provider, github_source_manifest_provider +from .manifest_provider.gitlab import gitlab_manifest_provider, gitlab_source_manifest_provider from .manifest_provider.tar import tar_manifest_provider, tar_source_manifest_provider from .package import Package class Distribution(object): - default_manifest_providers = [github_manifest_provider, bitbucket_manifest_provider, git_manifest_provider, tar_manifest_provider] - default_source_manifest_providers = [github_source_manifest_provider, git_source_manifest_provider, tar_source_manifest_provider] + default_manifest_providers = [github_manifest_provider, gitlab_manifest_provider, bitbucket_manifest_provider, git_manifest_provider, tar_manifest_provider] + default_source_manifest_providers = [github_source_manifest_provider, gitlab_source_manifest_provider, git_source_manifest_provider, tar_source_manifest_provider] def __init__(self, distribution_file, manifest_providers=None, source_manifest_providers=None): self._distribution_file = distribution_file diff --git a/src/rosdistro/manifest_provider/gitlab.py b/src/rosdistro/manifest_provider/gitlab.py new file mode 100644 index 000000000..88bd7bd99 --- /dev/null +++ b/src/rosdistro/manifest_provider/gitlab.py @@ -0,0 +1,137 @@ +# Software License Agreement (BSD License) +# +# Copyright (c) 2021, Open Source Robotics Foundation, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Open Source Robotics Foundation, Inc. nor +# the names of its contributors may be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +try: + from urllib.request import urlopen, Request + from urllib.error import URLError + from urllib.parse import quote as urlquote +except ImportError: + from urllib2 import urlopen, Request + from urllib2 import URLError + from urllib2.parse import quote as urlquote + +import json +import os +import re + +from catkin_pkg.package import parse_package_string + +from rosdistro.source_repository_cache import SourceRepositoryCache +from rosdistro import logger + + +def _gitlab_paged_api_query(path, resource, attrs): + _attrs = {'per_page': 50} + _attrs.update(attrs) + _attrs.pop('pagination', None) + _attrs.pop('page', None) + + url = 'https://gitlab.com/api/v4/projects/%s/%s?pagination=keyset' % (urlquote(path, safe=''), resource) + for k, v in _attrs.items(): + url += '&%s=%s' % (k, urlquote(str(v), safe='')) + + while True: + with urlopen(url) as res: + for result in json.loads(res.read().decode('utf-8')): + yield result + + # Get the URL to the next page + links = res.getheader('Link') + if not links: + break + match = re.match(r'.*<([^>]*)>; rel="next"', links) + if not match: + break + url = match.group(1) + + +def gitlab_manifest_provider(_dist_name, repo, pkg_name): + assert repo.version + server, path = repo.get_url_parts() + if not server.endswith('gitlab.com'): + logger.debug('Skip non-gitlab url "%s"' % repo.url) + raise RuntimeError('can not handle non gitlab urls') + + release_tag = repo.get_release_tag(pkg_name) + + if not repo.has_remote_tag(release_tag): + raise RuntimeError('specified tag "%s" is not a git tag' % release_tag) + + url = 'https://gitlab.com/%s/-/raw/%s/package.xml' % (path, release_tag) + try: + logger.debug('Load package.xml file from url "%s"' % url) + return urlopen(url).read().decode('utf-8') + except URLError as e: + logger.debug('- failed (%s), trying "%s"' % (e, url)) + raise RuntimeError() + + +def gitlab_source_manifest_provider(repo): + assert repo.version + server, path = repo.get_url_parts() + if not server.endswith('gitlab.com'): + logger.debug('Skip non-gitlab url "%s"' % repo.url) + raise RuntimeError('can not handle non gitlab urls') + + # Resolve the version ref to a sha + sha = next(_gitlab_paged_api_query(path, 'repository/commits', {'per_page': 1, 'ref_name': repo.version}))['id'] + + # Look for package.xml files in the tree + package_xml_paths = set() + for obj in _gitlab_paged_api_query(path, 'repository/tree', {'recursive': 'true', 'ref': sha}): + if obj['path'].split('/')[-1] == 'package.xml': + package_xml_paths.add(os.path.dirname(obj['path'])) + + # Filter out ones that are inside other packages (eg, part of tests) + def package_xml_in_parent(path): + if path == '': + return True + parent = path + while True: + parent = os.path.dirname(parent) + if parent in package_xml_paths: + return False + if parent == '': + return True + package_xml_paths = list(filter(package_xml_in_parent, package_xml_paths)) + + cache = SourceRepositoryCache.from_ref(sha) + for package_xml_path in package_xml_paths: + url = 'https://gitlab.com/%s/-/raw/%s/%s' % \ + (path, sha, package_xml_path + '/package.xml' if package_xml_path else 'package.xml') + logger.debug('- load package.xml from %s' % url) + package_xml = urlopen(url).read().decode('utf-8') + name = parse_package_string(package_xml).name + cache.add(name, package_xml_path, package_xml) + + return cache diff --git a/src/rosdistro/release.py b/src/rosdistro/release.py index 1ceb251d8..687b06919 100644 --- a/src/rosdistro/release.py +++ b/src/rosdistro/release.py @@ -34,11 +34,12 @@ from .manifest_provider.bitbucket import bitbucket_manifest_provider from .manifest_provider.git import git_manifest_provider from .manifest_provider.github import github_manifest_provider +from .manifest_provider.gitlab import gitlab_manifest_provider class Release(object): - default_manifest_providers = [github_manifest_provider, bitbucket_manifest_provider, git_manifest_provider] + default_manifest_providers = [github_manifest_provider, gitlab_manifest_provider, bitbucket_manifest_provider, git_manifest_provider] def __init__(self, rel_file, manifest_providers=None): self._rel_file = rel_file diff --git a/test/test_manifest_providers.py b/test/test_manifest_providers.py index ef84e4f89..7675ab857 100644 --- a/test/test_manifest_providers.py +++ b/test/test_manifest_providers.py @@ -8,6 +8,7 @@ from rosdistro.manifest_provider.bitbucket import bitbucket_manifest_provider from rosdistro.manifest_provider.cache import CachedManifestProvider, sanitize_xml from rosdistro.manifest_provider.git import git_manifest_provider, git_source_manifest_provider +from rosdistro.manifest_provider.gitlab import gitlab_manifest_provider, gitlab_source_manifest_provider from rosdistro.release_repository_specification import ReleaseRepositorySpecification from rosdistro.source_repository_specification import SourceRepositorySpecification @@ -16,6 +17,10 @@ def test_bitbucket(): assert '' in bitbucket_manifest_provider('indigo', _rospeex_release_repo(), 'rospeex_msgs') +def test_gitlab(): + assert '' in gitlab_manifest_provider('foxy', _tracetools_analysis_release_repo(), 'tracetools_analysis') + + def test_cached(): class FakeDistributionCache(object): def __init__(self): @@ -102,6 +107,17 @@ def test_github_source(): assert '0.5.11' in package_xml +def test_gitlab_source(): + repo_cache = gitlab_source_manifest_provider(_tracetools_analysis_source_repo()) + + # This hash corresponds to the 1.0.3 tag. + assert repo_cache.ref() == 'cd30853005ef3a591cb8594b4aa49f9ef400d30f' + + package_path, package_xml = repo_cache['ros2trace_analysis'] + assert 'ros2trace_analysis' == package_path + assert '1.0.3' in package_xml + + def test_git_source_multi(): repo_cache = git_source_manifest_provider(_ros_source_repo()) assert repo_cache.ref() @@ -175,3 +191,19 @@ def _rospeex_release_repo(): 'url': 'https://bitbucket.org/rospeex/rospeex-release.git', 'version': '2.14.7-0' }) + + +def _tracetools_analysis_release_repo(): + return ReleaseRepositorySpecification('tracetools_analysis', { + 'packages': ['ros2trace_analysis', 'tracetools_analysis'], + 'tags': {'release': 'release/foxy/{package}/{version}'}, + 'url': 'https://gitlab.com/ros-tracing/tracetools_analysis-release.git', + 'version': '1.0.3-1' + }) + + +def _tracetools_analysis_source_repo(): + return SourceRepositorySpecification('tracetools_analysis', { + 'url': 'https://gitlab.com/ros-tracing/tracetools_analysis.git', + 'version': '1.0.3' + }) From 4fa1a3c6e0a43329ff59e5a7766ad3d647390795 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 26 Nov 2024 13:55:59 -0600 Subject: [PATCH 2/4] Update GitLab manifest provider to support auth, self hosting --- src/rosdistro/manifest_provider/gitlab.py | 93 ++++++++++++++--------- 1 file changed, 56 insertions(+), 37 deletions(-) diff --git a/src/rosdistro/manifest_provider/gitlab.py b/src/rosdistro/manifest_provider/gitlab.py index 88bd7bd99..e2bbfd125 100644 --- a/src/rosdistro/manifest_provider/gitlab.py +++ b/src/rosdistro/manifest_provider/gitlab.py @@ -31,37 +31,49 @@ # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -try: - from urllib.request import urlopen, Request - from urllib.error import URLError - from urllib.parse import quote as urlquote -except ImportError: - from urllib2 import urlopen, Request - from urllib2 import URLError - from urllib2.parse import quote as urlquote - import json import os import re +from urllib.request import urlopen, Request +from urllib.error import URLError +from urllib.parse import quote as urlquote from catkin_pkg.package import parse_package_string from rosdistro.source_repository_cache import SourceRepositoryCache from rosdistro import logger +GITLAB_PRIVATE_TOKEN = os.getenv('GITLAB_PRIVATE_TOKEN', None) + +def _gitlab_urlopen(url): + req = Request(url) + if GITLAB_PRIVATE_TOKEN: + req.add_header('Private-Token', GITLAB_PRIVATE_TOKEN) + logger.warn('Performing GitLab API query "%s"' % (url,)) + return urlopen(req) + + +def _gitlab_api_query(server, path, resource, attrs): + url = 'https://%s/api/v4/projects/%s/%s' % (server, urlquote(path, safe=''), resource) + if attrs: + url += '?' + '&'.join(k + '=' + urlquote(str(v), safe='') for k, v in attrs.items()) + return _gitlab_urlopen(url) -def _gitlab_paged_api_query(path, resource, attrs): - _attrs = {'per_page': 50} - _attrs.update(attrs) - _attrs.pop('pagination', None) - _attrs.pop('page', None) - url = 'https://gitlab.com/api/v4/projects/%s/%s?pagination=keyset' % (urlquote(path, safe=''), resource) - for k, v in _attrs.items(): - url += '&%s=%s' % (k, urlquote(str(v), safe='')) +def _gitlab_paged_api_query(server, path, resource, attrs): + _attrs = { + 'per_page': 50, + **attrs, + 'pagination': 'keyset', + 'page': '1', + } + + url = 'https://%s/api/v4/projects/%s/%s' % (server, urlquote(path, safe=''), resource) + if _attrs: + url += '?' + '&'.join(k + '=' + urlquote(str(v), safe='') for k, v in _attrs.items()) while True: - with urlopen(url) as res: + with _gitlab_urlopen(url) as res: for result in json.loads(res.read().decode('utf-8')): yield result @@ -78,37 +90,43 @@ def _gitlab_paged_api_query(path, resource, attrs): def gitlab_manifest_provider(_dist_name, repo, pkg_name): assert repo.version server, path = repo.get_url_parts() - if not server.endswith('gitlab.com'): + if not server.startswith('gitlab.'): logger.debug('Skip non-gitlab url "%s"' % repo.url) raise RuntimeError('can not handle non gitlab urls') - release_tag = repo.get_release_tag(pkg_name) - - if not repo.has_remote_tag(release_tag): - raise RuntimeError('specified tag "%s" is not a git tag' % release_tag) - - url = 'https://gitlab.com/%s/-/raw/%s/package.xml' % (path, release_tag) + resource = 'repository/files/package.xml/raw' + attrs = { + 'ref': repo.get_release_tag(pkg_name), + } try: - logger.debug('Load package.xml file from url "%s"' % url) - return urlopen(url).read().decode('utf-8') + with _gitlab_api_query(server, path, resource, attrs) as res: + return res.read().decode('utf-8') except URLError as e: - logger.debug('- failed (%s), trying "%s"' % (e, url)) - raise RuntimeError() + logger.debug('- failed (%s), trying "%s"' % (e, e.filename)) + raise def gitlab_source_manifest_provider(repo): assert repo.version server, path = repo.get_url_parts() - if not server.endswith('gitlab.com'): + if not server.startswith('gitlab.'): logger.debug('Skip non-gitlab url "%s"' % repo.url) raise RuntimeError('can not handle non gitlab urls') - # Resolve the version ref to a sha - sha = next(_gitlab_paged_api_query(path, 'repository/commits', {'per_page': 1, 'ref_name': repo.version}))['id'] + # Resolve the version ref to a sha since we need to make multiple queries + attrs = { + 'per_page': 1, + 'ref_name': repo.version, + } + sha = next(_gitlab_paged_api_query(server, path, 'repository/commits', attrs))['id'] # Look for package.xml files in the tree + attrs = { + 'recursive': 'true', + 'ref': sha, + } package_xml_paths = set() - for obj in _gitlab_paged_api_query(path, 'repository/tree', {'recursive': 'true', 'ref': sha}): + for obj in _gitlab_paged_api_query(server, path, 'repository/tree', attrs): if obj['path'].split('/')[-1] == 'package.xml': package_xml_paths.add(os.path.dirname(obj['path'])) @@ -127,10 +145,11 @@ def package_xml_in_parent(path): cache = SourceRepositoryCache.from_ref(sha) for package_xml_path in package_xml_paths: - url = 'https://gitlab.com/%s/-/raw/%s/%s' % \ - (path, sha, package_xml_path + '/package.xml' if package_xml_path else 'package.xml') - logger.debug('- load package.xml from %s' % url) - package_xml = urlopen(url).read().decode('utf-8') + resource_path = urlquote( + package_xml_path + '/package.xml' if package_xml_path else 'package.xml', safe='') + resource = 'repository/files/' + resource_path + '/raw' + with _gitlab_api_query(server, path, resource, {'ref': sha}) as res: + package_xml = res.read().decode('utf-8') name = parse_package_string(package_xml).name cache.add(name, package_xml_path, package_xml) From 1b46c7b79f978e1e24f6c5631dda2c041aeb1cb0 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 27 Nov 2024 21:57:56 -0600 Subject: [PATCH 3/4] Use urlencode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Steven! Ragnarök --- src/rosdistro/manifest_provider/gitlab.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/rosdistro/manifest_provider/gitlab.py b/src/rosdistro/manifest_provider/gitlab.py index e2bbfd125..ecef679e3 100644 --- a/src/rosdistro/manifest_provider/gitlab.py +++ b/src/rosdistro/manifest_provider/gitlab.py @@ -37,6 +37,7 @@ from urllib.request import urlopen, Request from urllib.error import URLError from urllib.parse import quote as urlquote +from urllib.parse import urlencode from catkin_pkg.package import parse_package_string @@ -56,7 +57,7 @@ def _gitlab_urlopen(url): def _gitlab_api_query(server, path, resource, attrs): url = 'https://%s/api/v4/projects/%s/%s' % (server, urlquote(path, safe=''), resource) if attrs: - url += '?' + '&'.join(k + '=' + urlquote(str(v), safe='') for k, v in attrs.items()) + url += '?' + urlencode(attrs) return _gitlab_urlopen(url) @@ -70,7 +71,7 @@ def _gitlab_paged_api_query(server, path, resource, attrs): url = 'https://%s/api/v4/projects/%s/%s' % (server, urlquote(path, safe=''), resource) if _attrs: - url += '?' + '&'.join(k + '=' + urlquote(str(v), safe='') for k, v in _attrs.items()) + url += '?' + urlencode(_attrs) while True: with _gitlab_urlopen(url) as res: From c0c6c16a62627534778999ea1cfcd1b7ae70d276 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Tue, 3 Dec 2024 15:03:31 -0600 Subject: [PATCH 4/4] Add GitLab server escape hatch --- src/rosdistro/manifest_provider/gitlab.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/rosdistro/manifest_provider/gitlab.py b/src/rosdistro/manifest_provider/gitlab.py index ecef679e3..884504537 100644 --- a/src/rosdistro/manifest_provider/gitlab.py +++ b/src/rosdistro/manifest_provider/gitlab.py @@ -45,6 +45,7 @@ from rosdistro import logger GITLAB_PRIVATE_TOKEN = os.getenv('GITLAB_PRIVATE_TOKEN', None) +ROSDISTRO_GITLAB_SERVER = os.getenv('ROSDISTRO_GITLAB_SERVER', None) def _gitlab_urlopen(url): req = Request(url) @@ -91,7 +92,7 @@ def _gitlab_paged_api_query(server, path, resource, attrs): def gitlab_manifest_provider(_dist_name, repo, pkg_name): assert repo.version server, path = repo.get_url_parts() - if not server.startswith('gitlab.'): + if not server.endswith('gitlab.com') and server != ROSDISTRO_GITLAB_SERVER: logger.debug('Skip non-gitlab url "%s"' % repo.url) raise RuntimeError('can not handle non gitlab urls') @@ -110,7 +111,7 @@ def gitlab_manifest_provider(_dist_name, repo, pkg_name): def gitlab_source_manifest_provider(repo): assert repo.version server, path = repo.get_url_parts() - if not server.startswith('gitlab.'): + if not server.endswith('gitlab.com') and server != ROSDISTRO_GITLAB_SERVER: logger.debug('Skip non-gitlab url "%s"' % repo.url) raise RuntimeError('can not handle non gitlab urls')