From 102ba4b0795727771f8ffca24a55af6d8d605283 Mon Sep 17 00:00:00 2001 From: gregcorbett Date: Mon, 18 Sep 2023 14:29:23 +0000 Subject: [PATCH 01/32] Cap version of certifi used below 2020.4.5.2 for Python2.7 - certifi is used by the requests module - 2020.4.5.2 dropped Python 2 support, certifi/python-certifi@5efdd48 --- setup.py | 2 +- tox.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 47fcc39..8829a17 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ def get_ver(): REQUIREMENTS = [] if sys.version_info[0] == 2: - REQUIREMENTS = ['requests==2.20.0'], + REQUIREMENTS = ['requests==2.20.0', 'certifi<2020.4.5.2'], else: REQUIREMENTS = ['requests'], diff --git a/tox.ini b/tox.ini index 8ded7e0..71c31b2 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ deps = coverage httmock mock requests0: requests==2.20 + requests0: certifi<2020.4.5.2 requests260: requests==2.6.0 requests2125: requests==2.12.5 commands = coverage run --append -m pytest From e365369b4719cbfa4758877f632d58d132d08421 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 7 Mar 2024 22:46:20 +0100 Subject: [PATCH 02/32] Refine spec for Rocky 8 and 9 package build leaving the option for Centos 7 builds even for Python2 --- argo-ams-library.spec | 49 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/argo-ams-library.spec b/argo-ams-library.spec index 35e053b..d977676 100644 --- a/argo-ams-library.spec +++ b/argo-ams-library.spec @@ -1,5 +1,6 @@ %global underscore() %(echo %1 | sed 's/-/_/g') + %global sum A simple python library for interacting with the ARGO Messaging Service %global desc A simple python library for interacting with the ARGO Messaging Service @@ -21,6 +22,8 @@ BuildArch: noarch %{desc} + +%if 0%{?el7} %package -n python-%{name} Obsoletes: argo-ams-library Provides: argo-ams-library @@ -35,8 +38,8 @@ AutoReq: no %package -n python%{python3_pkgversion}-%{name} Summary: %{sum} -BuildRequires: python36-devel python36-setuptools -Requires: python36-requests +BuildRequires: python3-devel python3-setuptools +Requires: python3-requests AutoReq: no %description -n python%{python3_pkgversion}-%{name} %{desc} @@ -72,7 +75,49 @@ rm -rf %{buildroot} %{python_sitelib}/* + +%else + + + +%package -n python%{python3_pkgversion}-%{name} +Summary: %{sum} +BuildRequires: python3-devel python3-setuptools +Requires: python3-requests +AutoReq: no +%description -n python%{python3_pkgversion}-%{name} +%{desc} +%{?python_provide:%python_provide python3-%{name}} + + +%prep +%setup -q + + +%build +%{py3_build} + + +%install +rm -rf %{buildroot} +%{py3_install "--record=INSTALLED_FILES_PY3" } + + +%files -n python%{python3_pkgversion}-%{name} -f INSTALLED_FILES_PY3 +%doc examples/ README.md +%defattr(-,root,root,-) +%{python3_sitelib}/* +%doc examples/ README.md + + + +%endif + + + %changelog +* Thu Mar 7 2024 Daniel Vrcic - 0.6.2-1%{?dist} +- refine spec for Rocky 8 and Rocky9 python3 package build * Mon Feb 6 2023 agelostsal - 0.6.1-1%{?dist} - AM-314 Add projects:createUser functionality to ams library * Thu Nov 3 2022 Daniel Vrcic , agelostsal - 0.6.0-1%{?dist} From 74136d0f4a58b30024876b934f1709e9d9392b7e Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 7 Mar 2024 23:18:42 +0100 Subject: [PATCH 03/32] Separate section for el8 builds and add specific py39 RPM build for it --- Makefile | 2 +- argo-ams-library.spec | 27 ++++++++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 990e202..6cd5520 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ rpm: dist dist: rm -rf dist - python setup.py sdist + python3 setup.py sdist mv -f dist/${PKGNAME}-${PKGVERSION}.tar.gz . rm -rf dist diff --git a/argo-ams-library.spec b/argo-ams-library.spec index d977676..c159d86 100644 --- a/argo-ams-library.spec +++ b/argo-ams-library.spec @@ -1,5 +1,6 @@ %global underscore() %(echo %1 | sed 's/-/_/g') +%global python3_pkgversion_39 3.9 %global sum A simple python library for interacting with the ARGO Messaging Service %global desc A simple python library for interacting with the ARGO Messaging Service @@ -70,16 +71,14 @@ rm -rf %{buildroot} %else %files -n python2-%{name} -f INSTALLED_FILES_PY2 %endif -%doc examples/ README.md %defattr(-,root,root,-) %{python_sitelib}/* +%doc examples/ README.md +%endif -%else - - - +%if 0%{?el8} %package -n python%{python3_pkgversion}-%{name} Summary: %{sum} BuildRequires: python3-devel python3-setuptools @@ -89,6 +88,15 @@ AutoReq: no %{desc} %{?python_provide:%python_provide python3-%{name}} +%package -n python%{python3_pkgversion_39}-%{name} +Summary: %{sum} +BuildRequires: python39-devel python39-setuptools +Requires: python39-requests +AutoReq: no +%description -n python%{python3_pkgversion_39}-%{name} +%{desc} +%{?python_provide:%python_provide python%{python3_pkgversion_39}-%{name}} + %prep %setup -q @@ -96,11 +104,13 @@ AutoReq: no %build %{py3_build} +python3.9 setup.py build %install rm -rf %{buildroot} %{py3_install "--record=INSTALLED_FILES_PY3" } +python3.9 setup.py install --root=%{buildroot} --record=INSTALLED_FILES_PY3_39 %files -n python%{python3_pkgversion}-%{name} -f INSTALLED_FILES_PY3 @@ -109,8 +119,11 @@ rm -rf %{buildroot} %{python3_sitelib}/* %doc examples/ README.md - - +%files -n python%{python3_pkgversion_39}-%{name} -f INSTALLED_FILES_PY3_39 +%doc examples/ README.md +%defattr(-,root,root,-) +/usr/lib/python3.9/site-packages/* +%doc examples/ README.md %endif From 17b35ff49c930ac994af98d97e708948a97039b5 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Mar 2024 09:29:39 +0100 Subject: [PATCH 04/32] Add Rocky 9 Python 3.11 build --- argo-ams-library.spec | 60 +++++++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/argo-ams-library.spec b/argo-ams-library.spec index c159d86..c0f09a9 100644 --- a/argo-ams-library.spec +++ b/argo-ams-library.spec @@ -1,6 +1,7 @@ %global underscore() %(echo %1 | sed 's/-/_/g') %global python3_pkgversion_39 3.9 +%global python3_pkgversion_311 3.11 %global sum A simple python library for interacting with the ARGO Messaging Service %global desc A simple python library for interacting with the ARGO Messaging Service @@ -8,7 +9,7 @@ Name: argo-ams-library Summary: %{sum} -Version: 0.6.1 +Version: 0.6.2 Release: 1%{?dist} Group: Development/Libraries @@ -23,7 +24,6 @@ BuildArch: noarch %{desc} - %if 0%{?el7} %package -n python-%{name} Obsoletes: argo-ams-library @@ -36,7 +36,6 @@ AutoReq: no %{desc} %{?python_provide:%python_provide python-%{name}} - %package -n python%{python3_pkgversion}-%{name} Summary: %{sum} BuildRequires: python3-devel python3-setuptools @@ -46,22 +45,18 @@ AutoReq: no %{desc} %{?python_provide:%python_provide python3-%{name}} - %prep %setup -q - %build %{py_build} %{py3_build} - %install rm -rf %{buildroot} %{py_install "--record=INSTALLED_FILES_PY2" } %{py3_install "--record=INSTALLED_FILES_PY3" } - %files -n python%{python3_pkgversion}-%{name} -f INSTALLED_FILES_PY3 %doc examples/ README.md %defattr(-,root,root,-) @@ -77,7 +72,6 @@ rm -rf %{buildroot} %endif - %if 0%{?el8} %package -n python%{python3_pkgversion}-%{name} Summary: %{sum} @@ -97,22 +91,18 @@ AutoReq: no %{desc} %{?python_provide:%python_provide python%{python3_pkgversion_39}-%{name}} - %prep %setup -q - %build %{py3_build} python3.9 setup.py build - %install rm -rf %{buildroot} %{py3_install "--record=INSTALLED_FILES_PY3" } python3.9 setup.py install --root=%{buildroot} --record=INSTALLED_FILES_PY3_39 - %files -n python%{python3_pkgversion}-%{name} -f INSTALLED_FILES_PY3 %doc examples/ README.md %defattr(-,root,root,-) @@ -127,10 +117,54 @@ python3.9 setup.py install --root=%{buildroot} --record=INSTALLED_FILES_PY3_39 %endif +%if 0%{?el9} +%package -n python%{python3_pkgversion}-%{name} +Summary: %{sum} +BuildRequires: python3-devel python3-setuptools +Requires: python3-requests +AutoReq: no +%description -n python%{python3_pkgversion}-%{name} +%{desc} +%{?python_provide:%python_provide python3-%{name}} + +%package -n python%{python3_pkgversion_311}-%{name} +Summary: %{sum} +BuildRequires: python3.11-devel python3.11-setuptools +Requires: python3.11-requests +AutoReq: no +%description -n python%{python3_pkgversion_311}-%{name} +%{desc} +%{?python_provide:%python_provide python%{python3_pkgversion_311}-%{name}} + +%prep +%setup -q + +%build +%{py3_build} +python3.11 setup.py build + +%install +rm -rf %{buildroot} +%{py3_install "--record=INSTALLED_FILES_PY3" } +python3.11 setup.py install --root=%{buildroot} --record=INSTALLED_FILES_PY3_311 + +%files -n python%{python3_pkgversion}-%{name} -f INSTALLED_FILES_PY3 +%doc examples/ README.md +%defattr(-,root,root,-) +%{python3_sitelib}/* +%doc examples/ README.md + +%files -n python%{python3_pkgversion_311}-%{name} -f INSTALLED_FILES_PY3_311 +%doc examples/ README.md +%defattr(-,root,root,-) +/usr/lib/python3.11/site-packages/* +%doc examples/ README.md +%endif + %changelog * Thu Mar 7 2024 Daniel Vrcic - 0.6.2-1%{?dist} -- refine spec for Rocky 8 and Rocky9 python3 package build +- refine spec for Rocky 8 and Rocky 9 python3 package build * Mon Feb 6 2023 agelostsal - 0.6.1-1%{?dist} - AM-314 Add projects:createUser functionality to ams library * Thu Nov 3 2022 Daniel Vrcic , agelostsal - 0.6.0-1%{?dist} From e3d35a6b8a3f0d2f8c91387cd609c94373398b64 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Mar 2024 23:36:29 +0100 Subject: [PATCH 05/32] Correct pyenv version reference --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3a985e0..7ab5ec0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,8 +26,9 @@ pipeline { cd ${WORKSPACE}/$PROJECT_DIR rm -f .python-version &>/dev/null source $HOME/pyenv.sh + pyenv versions PY310V=$(pyenv versions | grep ams-py310) - pyenv local 3.7.15 3.8.15 3.9.15 ${PY310V// /} + pyenv local 3.7.17 3.8.18 3.9.18 ${PY310V// /} tox coverage xml --omit=*usr* --omit=*.tox* ''' From 0a96343e613336eb577392eb690a6933e889b1bf Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 08:29:28 +0100 Subject: [PATCH 06/32] Add Rocky 8 and Rocky 9 pipeline stages related to unit testing and RPM pkg building --- Jenkinsfile | 180 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 129 insertions(+), 51 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7ab5ec0..bd3af4f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,57 +11,135 @@ pipeline { } stages { - stage ('Test'){ - parallel { - stage ('Test Centos 7') { - agent { - docker { - image 'argo.registry:5000/epel-7-ams' - args '-u jenkins:jenkins' - } - } - steps { - echo 'Building Rpm...' - sh ''' - cd ${WORKSPACE}/$PROJECT_DIR - rm -f .python-version &>/dev/null - source $HOME/pyenv.sh - pyenv versions - PY310V=$(pyenv versions | grep ams-py310) - pyenv local 3.7.17 3.8.18 3.9.18 ${PY310V// /} - tox - coverage xml --omit=*usr* --omit=*.tox* - ''' - cobertura coberturaReportFile: '**/coverage.xml' - } - } - } - } - stage ('Build'){ - parallel { - stage ('Build Centos 7') { - agent { - docker { - image 'argo.registry:5000/epel-7-ams' - args '-u jenkins:jenkins' - } - } - steps { - echo 'Building Rpm...' - withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ - keyFileVariable: 'REPOKEY')]) { - sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d centos7 -p ${PROJECT_DIR} -s ${REPOKEY}" - } - archiveArtifacts artifacts: '**/*.rpm', fingerprint: true - } - post { - always { - cleanWs() - } - } - } - } - } + stage ('Test Centos 7') { + agent { + docker { + image 'argo.registry:5000/epel-7-ams' + args '-u jenkins:jenkins' + } + } + steps { + echo 'Executing unit tests...' + sh ''' + cd ${WORKSPACE}/$PROJECT_DIR + rm -f .python-version &>/dev/null + source $HOME/pyenv.sh + pyenv versions + PY310V=$(pyenv versions | grep ams-py310) + pyenv local 3.7.17 3.8.18 3.9.18 ${PY310V// /} + tox + coverage xml --omit=*usr* --omit=*.tox* + ''' + cobertura coberturaReportFile: '**/coverage.xml' + } + } + stage ('Build Centos 7') { + agent { + docker { + image 'argo.registry:5000/epel-7-ams' + args '-u jenkins:jenkins' + } + } + steps { + echo 'Building Rpm...' + withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ + keyFileVariable: 'REPOKEY')]) { + sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d centos7 -p ${PROJECT_DIR} -s ${REPOKEY}" + } + archiveArtifacts artifacts: '**/*.rpm', fingerprint: true + } + post { + always { + cleanWs() + } + } + } + stage ('Test Rocky 8') { + agent { + docker { + image 'argo.registry:5000/epel-8-ams' + args '-u jenkins:jenkins' + } + } + steps { + echo 'Executing unit tests...' + sh ''' + cd ${WORKSPACE}/$PROJECT_DIR + rm -f .python-version &>/dev/null + source $HOME/pyenv.sh + ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') + echo Found Python versions $ALLPYVERS + pyenv local $ALLPYVERS + tox + coverage xml --omit=*usr* --omit=*.tox* + ''' + cobertura coberturaReportFile: '**/coverage.xml' + } + } + stage ('Build Rocky 8') { + agent { + docker { + image 'argo.registry:5000/epel-8-ams' + args '-u jenkins:jenkins' + } + } + steps { + echo 'Building Rpm...' + withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ + keyFileVariable: 'REPOKEY')]) { + sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d rocky8 -p ${PROJECT_DIR} -s ${REPOKEY}" + } + archiveArtifacts artifacts: '**/*.rpm', fingerprint: true + } + post { + always { + cleanWs() + } + } + } + stage ('Test Rocky 9') { + agent { + docker { + image 'argo.registry:5000/epel-9-ams' + args '-u jenkins:jenkins' + } + } + steps { + echo 'Executing unit tests...' + sh ''' + cd ${WORKSPACE}/$PROJECT_DIR + rm -f .python-version &>/dev/null + source $HOME/pyenv.sh + ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') + echo Found Python versions $ALLPYVERS + pyenv local $ALLPYVERS + tox + coverage xml --omit=*usr* --omit=*.tox* + ''' + cobertura coberturaReportFile: '**/coverage.xml' + } + } + stage ('Build Rocky 9') { + agent { + docker { + image 'argo.registry:5000/epel-9-ams' + args '-u jenkins:jenkins' + } + } + steps { + echo 'Building Rpm...' + withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ + keyFileVariable: 'REPOKEY')]) { + sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d rocky9 -p ${PROJECT_DIR} -s ${REPOKEY}" + } + archiveArtifacts artifacts: '**/*.rpm', fingerprint: true + } + post { + always { + cleanWs() + } + } + } stage ('Upload to PyPI'){ when { branch 'master' From 06e5ec4333bc2f6006fff31a33aa33e06aac12a0 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 08:36:19 +0100 Subject: [PATCH 07/32] Do not hardcode Python versions on pyenv setup on Centos 7 target --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bd3af4f..d87db06 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,9 +24,10 @@ pipeline { cd ${WORKSPACE}/$PROJECT_DIR rm -f .python-version &>/dev/null source $HOME/pyenv.sh - pyenv versions + ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') PY310V=$(pyenv versions | grep ams-py310) - pyenv local 3.7.17 3.8.18 3.9.18 ${PY310V// /} + echo Found Python versions $ALLPYVERS $PY310V + pyenv local $ALLPYVERS ${PY310V// /} tox coverage xml --omit=*usr* --omit=*.tox* ''' From 4757b38cb524159c82dc3698d59a83a97924f5de Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 10:14:47 +0100 Subject: [PATCH 08/32] Let tox fire up tests in parallel manner --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d87db06..6e89bd0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,7 +28,7 @@ pipeline { PY310V=$(pyenv versions | grep ams-py310) echo Found Python versions $ALLPYVERS $PY310V pyenv local $ALLPYVERS ${PY310V// /} - tox + tox -p all coverage xml --omit=*usr* --omit=*.tox* ''' cobertura coberturaReportFile: '**/coverage.xml' @@ -71,7 +71,7 @@ pipeline { ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') echo Found Python versions $ALLPYVERS pyenv local $ALLPYVERS - tox + tox -p all coverage xml --omit=*usr* --omit=*.tox* ''' cobertura coberturaReportFile: '**/coverage.xml' @@ -114,7 +114,7 @@ pipeline { ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') echo Found Python versions $ALLPYVERS pyenv local $ALLPYVERS - tox + tox -p all coverage xml --omit=*usr* --omit=*.tox* ''' cobertura coberturaReportFile: '**/coverage.xml' From f36ee7beaa5b209b3b70d2a043c4daad60f888ce Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 10:17:22 +0100 Subject: [PATCH 09/32] Skip py27 environments for Rocky 8 and Rocky 9 --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6e89bd0..259d434 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -71,7 +71,7 @@ pipeline { ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') echo Found Python versions $ALLPYVERS pyenv local $ALLPYVERS - tox -p all + TOX_SKIP_ENV="py27.*" tox -p all coverage xml --omit=*usr* --omit=*.tox* ''' cobertura coberturaReportFile: '**/coverage.xml' @@ -114,7 +114,7 @@ pipeline { ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') echo Found Python versions $ALLPYVERS pyenv local $ALLPYVERS - tox -p all + TOX_SKIP_ENV="py27.*" tox -p all coverage xml --omit=*usr* --omit=*.tox* ''' cobertura coberturaReportFile: '**/coverage.xml' From 0c9e29ad39078b5cf2f66affcccf86b8802553df Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 10:20:44 +0100 Subject: [PATCH 10/32] Test with always latest requests for py36 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 71c31b2..e3d213a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27-requests0, py27-requests260, py36-requests0, py36-requests2125, py37-requests0, py38-requests0, py39-requests0, py310-requests0 +envlist = py27-requests0, py27-requests260, py36-requests0, py37-requests0, py38-requests0, py39-requests0, py310-requests0 [testenv] parallel_show_output = true From 959f98471e06344025ed43ea2484abc88f836c3d Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 11:22:43 +0100 Subject: [PATCH 11/32] Refine tox environments for match versions on Centos and Rocky distros --- tox.ini | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index e3d213a..08189a6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27-requests0, py27-requests260, py36-requests0, py37-requests0, py38-requests0, py39-requests0, py310-requests0 +envlist = py27-requests260, py36-requests0, py36-requests2125, py36-requests2200, py37-requests0, py38-requests0, py39-requests0, py39-requests2251, py310-requests0 [testenv] parallel_show_output = true @@ -7,8 +7,9 @@ deps = coverage pytest httmock mock - requests0: requests==2.20 - requests0: certifi<2020.4.5.2 requests260: requests==2.6.0 + requests260: certifi>=2020,<2020.4.5.2 requests2125: requests==2.12.5 + requests2251: requests==2.25.1 + requests2200: requests==2.20.0 commands = coverage run --append -m pytest From d08f47d6f4183b8b971cc1bc6704ad6c7e966ce1 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 11:31:59 +0100 Subject: [PATCH 12/32] Skip not interested environments for example py37 and py38 on Centos 7 as there are no such versions of Python there --- Jenkinsfile | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 259d434..c8e395b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,6 +28,7 @@ pipeline { PY310V=$(pyenv versions | grep ams-py310) echo Found Python versions $ALLPYVERS $PY310V pyenv local $ALLPYVERS ${PY310V// /} + export TOX_SKIP_ENV="py3[7,8,9,10].*" tox -p all coverage xml --omit=*usr* --omit=*.tox* ''' @@ -63,7 +64,7 @@ pipeline { } } steps { - echo 'Executing unit tests...' + echo 'Executing unit tests @ Rocky 8...' sh ''' cd ${WORKSPACE}/$PROJECT_DIR rm -f .python-version &>/dev/null @@ -71,7 +72,8 @@ pipeline { ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') echo Found Python versions $ALLPYVERS pyenv local $ALLPYVERS - TOX_SKIP_ENV="py27.*" tox -p all + export TOX_SKIP_ENV="py27.*" + tox -p all coverage xml --omit=*usr* --omit=*.tox* ''' cobertura coberturaReportFile: '**/coverage.xml' @@ -85,7 +87,7 @@ pipeline { } } steps { - echo 'Building Rpm...' + echo 'Building Rocky 8 RPM...' withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ keyFileVariable: 'REPOKEY')]) { sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d rocky8 -p ${PROJECT_DIR} -s ${REPOKEY}" @@ -106,7 +108,7 @@ pipeline { } } steps { - echo 'Executing unit tests...' + echo 'Executing unit tests @ Rocky 9...' sh ''' cd ${WORKSPACE}/$PROJECT_DIR rm -f .python-version &>/dev/null @@ -114,7 +116,8 @@ pipeline { ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') echo Found Python versions $ALLPYVERS pyenv local $ALLPYVERS - TOX_SKIP_ENV="py27.*" tox -p all + export TOX_SKIP_ENV="py27.*" + tox -p all coverage xml --omit=*usr* --omit=*.tox* ''' cobertura coberturaReportFile: '**/coverage.xml' @@ -128,7 +131,7 @@ pipeline { } } steps { - echo 'Building Rpm...' + echo 'Building Rocky 9 RPM...' withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ keyFileVariable: 'REPOKEY')]) { sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d rocky9 -p ${PROJECT_DIR} -s ${REPOKEY}" @@ -175,9 +178,9 @@ pipeline { success { script{ if ( env.BRANCH_NAME == 'devel' ) { - build job: '/ARGO/argodoc/devel', propagate: false + build job: '/ARGO/argo-ams-library/devel', propagate: false } else if ( env.BRANCH_NAME == 'master' ) { - build job: '/ARGO/argodoc/master', propagate: false + build job: '/ARGO/argo-ams-library/master', propagate: false } if ( env.BRANCH_NAME == 'master' || env.BRANCH_NAME == 'devel' ) { slackSend( message: ":rocket: New version for <$BUILD_URL|$PROJECT_DIR>:$BRANCH_NAME Job: $JOB_NAME !") From 4965fd4b58399ae34f19d36c377d96295ef3f7c2 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 12:10:07 +0100 Subject: [PATCH 13/32] Remove coverage append and use parallel and combine later and also clean any potential leftovers if containers is for ever reason reused --- Jenkinsfile | 260 +++++++++++++++++++++++++++------------------------- tox.ini | 2 +- 2 files changed, 134 insertions(+), 128 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c8e395b..46959d1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,139 +11,145 @@ pipeline { } stages { - stage ('Test Centos 7') { - agent { - docker { - image 'argo.registry:5000/epel-7-ams' - args '-u jenkins:jenkins' - } - } - steps { - echo 'Executing unit tests...' - sh ''' - cd ${WORKSPACE}/$PROJECT_DIR - rm -f .python-version &>/dev/null - source $HOME/pyenv.sh - ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') - PY310V=$(pyenv versions | grep ams-py310) + stage ('Test Centos 7') { + agent { + docker { + image 'argo.registry:5000/epel-7-ams' + args '-u jenkins:jenkins' + } + } + steps { + echo 'Executing unit tests...' + sh ''' + cd ${WORKSPACE}/$PROJECT_DIR + rm -f .python-version &>/dev/null + rm -rf .coverage* .tox/ coverage.xml &> /dev/null + source $HOME/pyenv.sh + ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') + PY310V=$(pyenv versions | grep ams-py310) echo Found Python versions $ALLPYVERS $PY310V - pyenv local $ALLPYVERS ${PY310V// /} + pyenv local $ALLPYVERS ${PY310V// /} export TOX_SKIP_ENV="py3[7,8,9,10].*" - tox -p all - coverage xml --omit=*usr* --omit=*.tox* - ''' - cobertura coberturaReportFile: '**/coverage.xml' - } - } - stage ('Build Centos 7') { - agent { - docker { - image 'argo.registry:5000/epel-7-ams' - args '-u jenkins:jenkins' - } - } - steps { - echo 'Building Rpm...' - withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ - keyFileVariable: 'REPOKEY')]) { - sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d centos7 -p ${PROJECT_DIR} -s ${REPOKEY}" - } - archiveArtifacts artifacts: '**/*.rpm', fingerprint: true - } - post { - always { - cleanWs() - } - } - } - stage ('Test Rocky 8') { - agent { - docker { - image 'argo.registry:5000/epel-8-ams' - args '-u jenkins:jenkins' - } - } - steps { - echo 'Executing unit tests @ Rocky 8...' - sh ''' - cd ${WORKSPACE}/$PROJECT_DIR - rm -f .python-version &>/dev/null - source $HOME/pyenv.sh - ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') + tox -p all + coverage combine + coverage xml --omit=*usr* --omit=*.tox* + ''' + cobertura coberturaReportFile: '**/coverage.xml' + } + } + stage ('Build Centos 7') { + agent { + docker { + image 'argo.registry:5000/epel-7-ams' + args '-u jenkins:jenkins' + } + } + steps { + echo 'Building Rpm...' + withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ + keyFileVariable: 'REPOKEY')]) { + sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d centos7 -p ${PROJECT_DIR} -s ${REPOKEY}" + } + archiveArtifacts artifacts: '**/*.rpm', fingerprint: true + } + post { + always { + cleanWs() + } + } + } + stage ('Test Rocky 8') { + agent { + docker { + image 'argo.registry:5000/epel-8-ams' + args '-u jenkins:jenkins' + } + } + steps { + echo 'Executing unit tests @ Rocky 8...' + sh ''' + cd ${WORKSPACE}/$PROJECT_DIR + rm -f .python-version &>/dev/null + rm -rf .coverage* .tox/ coverage.xml &> /dev/null + source $HOME/pyenv.sh + ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') echo Found Python versions $ALLPYVERS - pyenv local $ALLPYVERS + pyenv local $ALLPYVERS export TOX_SKIP_ENV="py27.*" - tox -p all - coverage xml --omit=*usr* --omit=*.tox* - ''' - cobertura coberturaReportFile: '**/coverage.xml' - } - } - stage ('Build Rocky 8') { - agent { - docker { - image 'argo.registry:5000/epel-8-ams' - args '-u jenkins:jenkins' - } - } - steps { - echo 'Building Rocky 8 RPM...' - withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ - keyFileVariable: 'REPOKEY')]) { - sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d rocky8 -p ${PROJECT_DIR} -s ${REPOKEY}" - } - archiveArtifacts artifacts: '**/*.rpm', fingerprint: true - } - post { - always { - cleanWs() - } - } - } - stage ('Test Rocky 9') { - agent { - docker { - image 'argo.registry:5000/epel-9-ams' - args '-u jenkins:jenkins' - } - } - steps { - echo 'Executing unit tests @ Rocky 9...' - sh ''' - cd ${WORKSPACE}/$PROJECT_DIR - rm -f .python-version &>/dev/null - source $HOME/pyenv.sh - ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') + tox -p all + coverage combine + coverage xml --omit=*usr* --omit=*.tox* + ''' + cobertura coberturaReportFile: '**/coverage.xml' + } + } + stage ('Build Rocky 8') { + agent { + docker { + image 'argo.registry:5000/epel-8-ams' + args '-u jenkins:jenkins' + } + } + steps { + echo 'Building Rocky 8 RPM...' + withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ + keyFileVariable: 'REPOKEY')]) { + sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d rocky8 -p ${PROJECT_DIR} -s ${REPOKEY}" + } + archiveArtifacts artifacts: '**/*.rpm', fingerprint: true + } + post { + always { + cleanWs() + } + } + } + stage ('Test Rocky 9') { + agent { + docker { + image 'argo.registry:5000/epel-9-ams' + args '-u jenkins:jenkins' + } + } + steps { + echo 'Executing unit tests @ Rocky 9...' + sh ''' + cd ${WORKSPACE}/$PROJECT_DIR + rm -f .python-version &>/dev/null + rm -rf .coverage* .tox/ coverage.xml &> /dev/null + source $HOME/pyenv.sh + ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') echo Found Python versions $ALLPYVERS - pyenv local $ALLPYVERS - export TOX_SKIP_ENV="py27.*" + pyenv local $ALLPYVERS + export TOX_SKIP_ENV="py27.*" tox -p all - coverage xml --omit=*usr* --omit=*.tox* - ''' - cobertura coberturaReportFile: '**/coverage.xml' - } - } - stage ('Build Rocky 9') { - agent { - docker { - image 'argo.registry:5000/epel-9-ams' - args '-u jenkins:jenkins' - } - } - steps { - echo 'Building Rocky 9 RPM...' - withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ - keyFileVariable: 'REPOKEY')]) { - sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d rocky9 -p ${PROJECT_DIR} -s ${REPOKEY}" - } - archiveArtifacts artifacts: '**/*.rpm', fingerprint: true - } - post { - always { - cleanWs() - } - } - } + coverage combine + coverage xml --omit=*usr* --omit=*.tox* + ''' + cobertura coberturaReportFile: '**/coverage.xml' + } + } + stage ('Build Rocky 9') { + agent { + docker { + image 'argo.registry:5000/epel-9-ams' + args '-u jenkins:jenkins' + } + } + steps { + echo 'Building Rocky 9 RPM...' + withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ + keyFileVariable: 'REPOKEY')]) { + sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d rocky9 -p ${PROJECT_DIR} -s ${REPOKEY}" + } + archiveArtifacts artifacts: '**/*.rpm', fingerprint: true + } + post { + always { + cleanWs() + } + } + } stage ('Upload to PyPI'){ when { branch 'master' diff --git a/tox.ini b/tox.ini index 08189a6..78158e3 100644 --- a/tox.ini +++ b/tox.ini @@ -12,4 +12,4 @@ deps = coverage requests2125: requests==2.12.5 requests2251: requests==2.25.1 requests2200: requests==2.20.0 -commands = coverage run --append -m pytest +commands = coverage run --parallel -m pytest From a2abd6085016dfd3ee1a356d4ab5ca944437118b Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 12:25:13 +0100 Subject: [PATCH 14/32] Skip Python 3.6 on Rocky 9 also --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 46959d1..cff1909 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -75,7 +75,7 @@ pipeline { ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') echo Found Python versions $ALLPYVERS pyenv local $ALLPYVERS - export TOX_SKIP_ENV="py27.*" + export TOX_SKIP_ENV="py27.*|py36.*" tox -p all coverage combine coverage xml --omit=*usr* --omit=*.tox* From adda62344bf40e1afdd5640f9e1c7493e648d572 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 12:28:27 +0100 Subject: [PATCH 15/32] Skip defined in correct place --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cff1909..5869941 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -75,7 +75,7 @@ pipeline { ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') echo Found Python versions $ALLPYVERS pyenv local $ALLPYVERS - export TOX_SKIP_ENV="py27.*|py36.*" + export TOX_SKIP_ENV="py27.*" tox -p all coverage combine coverage xml --omit=*usr* --omit=*.tox* @@ -121,7 +121,7 @@ pipeline { ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') echo Found Python versions $ALLPYVERS pyenv local $ALLPYVERS - export TOX_SKIP_ENV="py27.*" + export TOX_SKIP_ENV="py27.*|py36.*" tox -p all coverage combine coverage xml --omit=*usr* --omit=*.tox* From 5b5e010955dd44e8f1b6f63feffeb4ce09a673bc Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 12:36:44 +0100 Subject: [PATCH 16/32] No coverage parallel and combine as it's not needed anyway --- Jenkinsfile | 3 --- tox.ini | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5869941..94837f6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -31,7 +31,6 @@ pipeline { pyenv local $ALLPYVERS ${PY310V// /} export TOX_SKIP_ENV="py3[7,8,9,10].*" tox -p all - coverage combine coverage xml --omit=*usr* --omit=*.tox* ''' cobertura coberturaReportFile: '**/coverage.xml' @@ -77,7 +76,6 @@ pipeline { pyenv local $ALLPYVERS export TOX_SKIP_ENV="py27.*" tox -p all - coverage combine coverage xml --omit=*usr* --omit=*.tox* ''' cobertura coberturaReportFile: '**/coverage.xml' @@ -123,7 +121,6 @@ pipeline { pyenv local $ALLPYVERS export TOX_SKIP_ENV="py27.*|py36.*" tox -p all - coverage combine coverage xml --omit=*usr* --omit=*.tox* ''' cobertura coberturaReportFile: '**/coverage.xml' diff --git a/tox.ini b/tox.ini index 78158e3..d8062cf 100644 --- a/tox.ini +++ b/tox.ini @@ -12,4 +12,4 @@ deps = coverage requests2125: requests==2.12.5 requests2251: requests==2.25.1 requests2200: requests==2.20.0 -commands = coverage run --parallel -m pytest +commands = coverage run -m pytest From 45bd3a169e2db2f4cb4aed320f65def93a9d592b Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 12:39:40 +0100 Subject: [PATCH 17/32] Do not run various env tests in parallel for Rocky 8 --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 94837f6..d3200d3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -75,7 +75,7 @@ pipeline { echo Found Python versions $ALLPYVERS pyenv local $ALLPYVERS export TOX_SKIP_ENV="py27.*" - tox -p all + tox coverage xml --omit=*usr* --omit=*.tox* ''' cobertura coberturaReportFile: '**/coverage.xml' From 16a85200a9c305624afa7893580167473c515e07 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 12:51:45 +0100 Subject: [PATCH 18/32] Max 2 parallel instances for Rocky 8 tests --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d3200d3..d8b0ebc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,7 +19,7 @@ pipeline { } } steps { - echo 'Executing unit tests...' + echo 'Executing unit tests @ Centos 7...' sh ''' cd ${WORKSPACE}/$PROJECT_DIR rm -f .python-version &>/dev/null @@ -44,7 +44,7 @@ pipeline { } } steps { - echo 'Building Rpm...' + echo 'Building Centos 7 RPM...' withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ keyFileVariable: 'REPOKEY')]) { sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d centos7 -p ${PROJECT_DIR} -s ${REPOKEY}" @@ -75,7 +75,7 @@ pipeline { echo Found Python versions $ALLPYVERS pyenv local $ALLPYVERS export TOX_SKIP_ENV="py27.*" - tox + tox -p 2 coverage xml --omit=*usr* --omit=*.tox* ''' cobertura coberturaReportFile: '**/coverage.xml' From 13c4d0c824a40483da581af73439cb9d08d255db Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 15:52:10 +0100 Subject: [PATCH 19/32] Initial try of executing tests and builds for various distros in parallel --- Jenkinsfile | 270 ++++++++++++++++++++++++++-------------------------- 1 file changed, 137 insertions(+), 133 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d8b0ebc..736963e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,139 +11,143 @@ pipeline { } stages { - stage ('Test Centos 7') { - agent { - docker { - image 'argo.registry:5000/epel-7-ams' - args '-u jenkins:jenkins' - } - } - steps { - echo 'Executing unit tests @ Centos 7...' - sh ''' - cd ${WORKSPACE}/$PROJECT_DIR - rm -f .python-version &>/dev/null - rm -rf .coverage* .tox/ coverage.xml &> /dev/null - source $HOME/pyenv.sh - ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') - PY310V=$(pyenv versions | grep ams-py310) - echo Found Python versions $ALLPYVERS $PY310V - pyenv local $ALLPYVERS ${PY310V// /} - export TOX_SKIP_ENV="py3[7,8,9,10].*" - tox -p all - coverage xml --omit=*usr* --omit=*.tox* - ''' - cobertura coberturaReportFile: '**/coverage.xml' - } - } - stage ('Build Centos 7') { - agent { - docker { - image 'argo.registry:5000/epel-7-ams' - args '-u jenkins:jenkins' - } - } - steps { - echo 'Building Centos 7 RPM...' - withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ - keyFileVariable: 'REPOKEY')]) { - sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d centos7 -p ${PROJECT_DIR} -s ${REPOKEY}" - } - archiveArtifacts artifacts: '**/*.rpm', fingerprint: true - } - post { - always { - cleanWs() - } - } - } - stage ('Test Rocky 8') { - agent { - docker { - image 'argo.registry:5000/epel-8-ams' - args '-u jenkins:jenkins' - } - } - steps { - echo 'Executing unit tests @ Rocky 8...' - sh ''' - cd ${WORKSPACE}/$PROJECT_DIR - rm -f .python-version &>/dev/null - rm -rf .coverage* .tox/ coverage.xml &> /dev/null - source $HOME/pyenv.sh - ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') - echo Found Python versions $ALLPYVERS - pyenv local $ALLPYVERS - export TOX_SKIP_ENV="py27.*" - tox -p 2 - coverage xml --omit=*usr* --omit=*.tox* - ''' - cobertura coberturaReportFile: '**/coverage.xml' - } - } - stage ('Build Rocky 8') { - agent { - docker { - image 'argo.registry:5000/epel-8-ams' - args '-u jenkins:jenkins' - } - } - steps { - echo 'Building Rocky 8 RPM...' - withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ - keyFileVariable: 'REPOKEY')]) { - sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d rocky8 -p ${PROJECT_DIR} -s ${REPOKEY}" - } - archiveArtifacts artifacts: '**/*.rpm', fingerprint: true - } - post { - always { - cleanWs() - } - } - } - stage ('Test Rocky 9') { - agent { - docker { - image 'argo.registry:5000/epel-9-ams' - args '-u jenkins:jenkins' - } - } - steps { - echo 'Executing unit tests @ Rocky 9...' - sh ''' - cd ${WORKSPACE}/$PROJECT_DIR - rm -f .python-version &>/dev/null - rm -rf .coverage* .tox/ coverage.xml &> /dev/null - source $HOME/pyenv.sh - ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') - echo Found Python versions $ALLPYVERS - pyenv local $ALLPYVERS - export TOX_SKIP_ENV="py27.*|py36.*" - tox -p all - coverage xml --omit=*usr* --omit=*.tox* - ''' - cobertura coberturaReportFile: '**/coverage.xml' - } - } - stage ('Build Rocky 9') { - agent { - docker { - image 'argo.registry:5000/epel-9-ams' - args '-u jenkins:jenkins' - } - } - steps { - echo 'Building Rocky 9 RPM...' - withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ - keyFileVariable: 'REPOKEY')]) { - sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d rocky9 -p ${PROJECT_DIR} -s ${REPOKEY}" - } - archiveArtifacts artifacts: '**/*.rpm', fingerprint: true - } - post { - always { - cleanWs() + stage ('Testing and building...') { + parallel { + stage('Centos 7') { + agent { + docker { + image 'argo.registry:5000/epel-7-ams' + args '-u jenkins:jenkins' + } + } + stages { + stage ('Test Centos 7') { + steps { + echo 'Executing unit tests @ Centos 7...' + sh ''' + cd ${WORKSPACE}/$PROJECT_DIR + rm -f .python-version &>/dev/null + rm -rf .coverage* .tox/ coverage.xml &> /dev/null + source $HOME/pyenv.sh + ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') + PY310V=$(pyenv versions | grep ams-py310) + echo Found Python versions $ALLPYVERS $PY310V + pyenv local $ALLPYVERS ${PY310V// /} + export TOX_SKIP_ENV="py3[7,8,9,10].*" + tox -p all + coverage xml --omit=*usr* --omit=*.tox* + ''' + cobertura coberturaReportFile: '**/coverage.xml' + } + } + stage ('Build Centos 7') { + steps { + echo 'Building Centos 7 RPM...' + withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ + keyFileVariable: 'REPOKEY')]) { + sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d centos7 -p ${PROJECT_DIR} -s ${REPOKEY}" + } + archiveArtifacts artifacts: '**/*.rpm', fingerprint: true + } + post { + always { + cleanWs() + } + } + } + } + } + stage('Rocky 8') { + agent { + docker { + image 'argo.registry:5000/epel-8-ams' + args '-u jenkins:jenkins' + } + } + stages { + stage ('Test Rocky 8') { + steps { + echo 'Executing unit tests @ Rocky 8...' + sh ''' + cd ${WORKSPACE}/$PROJECT_DIR + rm -f .python-version &>/dev/null + rm -rf .coverage* .tox/ coverage.xml &> /dev/null + source $HOME/pyenv.sh + ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') + echo Found Python versions $ALLPYVERS + pyenv local $ALLPYVERS + export TOX_SKIP_ENV="py27.*" + tox -p 2 + coverage xml --omit=*usr* --omit=*.tox* + ''' + cobertura coberturaReportFile: '**/coverage.xml' + } + } + stage ('Build Rocky 8') { + agent { + docker { + image 'argo.registry:5000/epel-8-ams' + args '-u jenkins:jenkins' + } + } + steps { + echo 'Building Rocky 8 RPM...' + withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ + keyFileVariable: 'REPOKEY')]) { + sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d rocky8 -p ${PROJECT_DIR} -s ${REPOKEY}" + } + archiveArtifacts artifacts: '**/*.rpm', fingerprint: true + } + post { + always { + cleanWs() + } + } + } + } + } + stage('Rocky 9') { + agent { + docker { + image 'argo.registry:5000/epel-9-ams' + args '-u jenkins:jenkins' + } + } + stages { + stage ('Test Rocky 9') { + steps { + echo 'Executing unit tests @ Rocky 9...' + sh ''' + cd ${WORKSPACE}/$PROJECT_DIR + rm -f .python-version &>/dev/null + rm -rf .coverage* .tox/ coverage.xml &> /dev/null + source $HOME/pyenv.sh + ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') + echo Found Python versions $ALLPYVERS + pyenv local $ALLPYVERS + export TOX_SKIP_ENV="py27.*|py36.*" + tox -p all + coverage xml --omit=*usr* --omit=*.tox* + ''' + cobertura coberturaReportFile: '**/coverage.xml' + } + } + stage ('Build Rocky 9') { + steps { + echo 'Building Rocky 9 RPM...' + withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ + keyFileVariable: 'REPOKEY')]) { + sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d rocky9 -p ${PROJECT_DIR} -s ${REPOKEY}" + } + archiveArtifacts artifacts: '**/*.rpm', fingerprint: true + } + post { + always { + cleanWs() + } + } + } + } } } } From f95670fcdcd5fb9a572bf2c107016dc3494df7b8 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 16:11:51 +0100 Subject: [PATCH 20/32] Try Rocky 8 tests max parallel one more time --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 736963e..087842b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -77,7 +77,7 @@ pipeline { echo Found Python versions $ALLPYVERS pyenv local $ALLPYVERS export TOX_SKIP_ENV="py27.*" - tox -p 2 + tox -p all coverage xml --omit=*usr* --omit=*.tox* ''' cobertura coberturaReportFile: '**/coverage.xml' From 281d6e3b043a58b1a8afc2174a83a600f80cc8e3 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 16:14:27 +0100 Subject: [PATCH 21/32] Try max 4 --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 087842b..7a1fab5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -77,7 +77,7 @@ pipeline { echo Found Python versions $ALLPYVERS pyenv local $ALLPYVERS export TOX_SKIP_ENV="py27.*" - tox -p all + tox -p 4 coverage xml --omit=*usr* --omit=*.tox* ''' cobertura coberturaReportFile: '**/coverage.xml' From 425a087f46fc37732744ba46992bbade8bd433e5 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sat, 9 Mar 2024 16:20:57 +0100 Subject: [PATCH 22/32] Remove redundant agent as its already declared at the top of stage --- Jenkinsfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7a1fab5..ad8a983 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -84,12 +84,6 @@ pipeline { } } stage ('Build Rocky 8') { - agent { - docker { - image 'argo.registry:5000/epel-8-ams' - args '-u jenkins:jenkins' - } - } steps { echo 'Building Rocky 8 RPM...' withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ From eb98f18c03c84c2001aa1676a9b7f892ac18ea1d Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sun, 10 Mar 2024 11:21:43 +0100 Subject: [PATCH 23/32] Add Python 3.11 tests with requests 2.28.1 available on Rocky 9 for which RPM package is build --- Jenkinsfile | 4 ++-- tox.ini | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ad8a983..8d8d091 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,7 +33,7 @@ pipeline { PY310V=$(pyenv versions | grep ams-py310) echo Found Python versions $ALLPYVERS $PY310V pyenv local $ALLPYVERS ${PY310V// /} - export TOX_SKIP_ENV="py3[7,8,9,10].*" + export TOX_SKIP_ENV="py3[7,8,9,10,11].*" tox -p all coverage xml --omit=*usr* --omit=*.tox* ''' @@ -76,7 +76,7 @@ pipeline { ALLPYVERS=$(pyenv versions | grep '^[ ]*[0-9]' | tr '\n' ' ') echo Found Python versions $ALLPYVERS pyenv local $ALLPYVERS - export TOX_SKIP_ENV="py27.*" + export TOX_SKIP_ENV="py27.*|py311.*" tox -p 4 coverage xml --omit=*usr* --omit=*.tox* ''' diff --git a/tox.ini b/tox.ini index d8062cf..3846ba6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27-requests260, py36-requests0, py36-requests2125, py36-requests2200, py37-requests0, py38-requests0, py39-requests0, py39-requests2251, py310-requests0 +envlist = py27-requests260, py36-requests0, py36-requests2125, py36-requests2200, py37-requests0, py38-requests0, py39-requests0, py39-requests2251, py310-requests0, py311-requests-2281 [testenv] parallel_show_output = true @@ -12,4 +12,5 @@ deps = coverage requests2125: requests==2.12.5 requests2251: requests==2.25.1 requests2200: requests==2.20.0 + requests2281: requests==2.28.1 commands = coverage run -m pytest From 26c940359b4211c13bd3c2dca66d32981c2bc0e1 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Sun, 10 Mar 2024 11:54:53 +0100 Subject: [PATCH 24/32] Add Python 3.11 supported classifier --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 8829a17..5240bd1 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ def get_ver(): "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Intended Audience :: Developers", "Topic :: Software Development :: Libraries :: Python Modules" ], From 63e40e49e79c27d6a0f6a024f8d46cd369ab5f91 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Mon, 11 Mar 2024 12:04:40 +0100 Subject: [PATCH 25/32] Update README with new supported Python versions and Linux distributions --- README.md | 46 ++++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index d2729a9..b0ceaa7 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Jenkins [![Build Status](https://jenkins.argo.grnet.gr/job/argo-ams-library_devel/badge/icon)](https://jenkins.argo.grnet.gr/job/argo-ams-library_devel) -A simple python library for interacting with the ARGO Messaging Service. +A simple python library for interacting with the ARGO Messaging Service. The Messaging Services is implemented as a Publish/Subscribe Service. Instead of focusing on a single Messaging API specification for handling the logic of publishing/subscribing to the broker network the API focuses on creating nodes of Publishers and Subscribers as a Service. @@ -10,22 +10,24 @@ In the Publish/Subscribe paradigm, Publishers are users/systems that can send me You may find more information about [the ARGO Messaging Service documentation](http://argoeu.github.io/messaging/v1/) -## Library installation +## Library installation -Library is tested and should work with Python versions 2.6, 2.7, 3.4 and 3.6 running on CentOS 6 and CentOS 7 releases. +Library is tested and should work with Python versions 2.7, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11 running on CentOS 7, Rocky 8 and Rocky 9 releases. RPM packages are prepared for both CentOS releases and you may find it and download it from ARGO Repository. PyPI packages are prepared as well. RPM production packages: -http://rpm-repo.argo.grnet.gr/ARGO/prod/centos6/ http://rpm-repo.argo.grnet.gr/ARGO/prod/centos7/ +http://rpm-repo.argo.grnet.gr/ARGO/prod/rocky8/ +http://rpm-repo.argo.grnet.gr/ARGO/prod/rocky9/ RPM devel packages: -http://rpm-repo.argo.grnet.gr/ARGO/devel/centos6/ http://rpm-repo.argo.grnet.gr/ARGO/devel/centos7/ - +http://rpm-repo.argo.grnet.gr/ARGO/devel/rocky8/ +http://rpm-repo.argo.grnet.gr/ARGO/devel/rocky9/ + PyPI package: https://pypi.org/project/argo-ams-library/ @@ -59,33 +61,33 @@ In the folder `examples`, you may find examples of using the library: ### Publish messages This example explains how to publish messages in a topic with the use of the library. Topics are resources that can hold messages. Publishers (users/systems) can create topics on demand and name them (Usually with names that make sense and express the class of messages delivered in the topic). A topic name must be scoped to a project. - + You may find more information about [Topics in the ARGO Messaging Service documentation](http://argoeu.github.io/messaging/v1/api_topics/) - + ``` -publish.py --host=[the FQDN of AMS Service] ---token=[the user token] ---project=[the name of your project registered in AMS Service] +publish.py --host=[the FQDN of AMS Service] +--token=[the user token] +--project=[the name of your project registered in AMS Service] --topic=[the topic to publish your messages] ``` - -### Consume messages in pull mode - + +### Consume messages in pull mode + This example explains how to consume messages from a predefined subscription with the use of the library. A subscription is a named resource representing the stream of messages from a single, specific topic, to be delivered to the subscribing application. A subscription name must be scoped to a project. In pull delivery, your subscriber application initiates requests to the Pub/Sub server to retrieve messages. When you create a subscription, the system establishes a sync point. That is, your subscriber is guaranteed to receive any message published after this point. Messages published before the sync point may not be delivered. - + You may find more information about [Subscriptions in the ARGO Messaging Service documentation](http://argoeu.github.io/messaging/v1/api_subs/) - + ``` -consume-pull.py --host=[the FQDN of AMS Service] ---token=[the user token] ---project=[the name of your project registered in AMS Service] ---topic=[the topic from where the messages are delivered ] ---subscription=[the subscription name to pull the messages] +consume-pull.py --host=[the FQDN of AMS Service] +--token=[the user token] +--project=[the name of your project registered in AMS Service] +--topic=[the topic from where the messages are delivered ] +--subscription=[the subscription name to pull the messages] --nummsgs=[the num of messages to consume] ``` -### Retry +### Retry Library has self-implemented HTTP request retry ability to seamlesssly interact with the ARGO Messaging service. Specifically, requests will be retried in case of: * timeouts from AMS (HTTP `408`) or load balancer (HTTP `408` and `504`) From 7bcbf6f3a696d58f40d77eadaab03fa2fbd20a3b Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Mon, 11 Mar 2024 12:23:05 +0100 Subject: [PATCH 26/32] Update CHANGELOG info with new RPMs available --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 800b5f3..149f4c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [0.6.2] - 2024-03-11 + +### Added + +* ARGO-4425 Rocky 8 and 9 RPMs for argo-ams-library + ## [0.6.1] - 2023-02-06 ### Added From ba58ae1954ecc2ac1d7561fb28fe1563c9c27019 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Mon, 11 Mar 2024 16:58:16 +0100 Subject: [PATCH 27/32] remove slack sends and explicit build instructions --- Jenkinsfile | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8d8d091..f10df66 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -176,24 +176,5 @@ pipeline { always { cleanWs() } - success { - script{ - if ( env.BRANCH_NAME == 'devel' ) { - build job: '/ARGO/argo-ams-library/devel', propagate: false - } else if ( env.BRANCH_NAME == 'master' ) { - build job: '/ARGO/argo-ams-library/master', propagate: false - } - if ( env.BRANCH_NAME == 'master' || env.BRANCH_NAME == 'devel' ) { - slackSend( message: ":rocket: New version for <$BUILD_URL|$PROJECT_DIR>:$BRANCH_NAME Job: $JOB_NAME !") - } - } - } - failure { - script{ - if ( env.BRANCH_NAME == 'master' || env.BRANCH_NAME == 'devel' ) { - slackSend( message: ":rain_cloud: Build Failed for <$BUILD_URL|$PROJECT_DIR>:$BRANCH_NAME Job: $JOB_NAME") - } - } - } } } From 139188c8a6b662e801dd5cedead71050cbf1c81b Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Mon, 11 Mar 2024 17:02:55 +0100 Subject: [PATCH 28/32] Have global post with workspace clean --- Jenkinsfile | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f10df66..31c8e09 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -49,11 +49,6 @@ pipeline { } archiveArtifacts artifacts: '**/*.rpm', fingerprint: true } - post { - always { - cleanWs() - } - } } } } @@ -92,11 +87,6 @@ pipeline { } archiveArtifacts artifacts: '**/*.rpm', fingerprint: true } - post { - always { - cleanWs() - } - } } } } @@ -135,11 +125,6 @@ pipeline { } archiveArtifacts artifacts: '**/*.rpm', fingerprint: true } - post { - always { - cleanWs() - } - } } } } @@ -165,11 +150,6 @@ pipeline { ''' } } - post { - always { - cleanWs() - } - } } } post { From f0a6ae9ca1a75d42b7fc58621a2b9777ba4a3a90 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Mon, 11 Mar 2024 17:10:45 +0100 Subject: [PATCH 29/32] Have global report generate on success pipeline --- Jenkinsfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 31c8e09..42b44aa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -153,7 +153,14 @@ pipeline { } } post { + success { + steps { + echo 'Generating report' + cobertura coberturaReportFile: '**/coverage.xml' + } + } always { + echo 'Cleaning workspace and exiting' cleanWs() } } From d0d1041d2071c965079e0fc7f08507f1d05f6869 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Mon, 11 Mar 2024 17:22:01 +0100 Subject: [PATCH 30/32] No need to wrap in steps --- Jenkinsfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 42b44aa..446e9cd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -154,10 +154,8 @@ pipeline { } post { success { - steps { - echo 'Generating report' - cobertura coberturaReportFile: '**/coverage.xml' - } + echo 'Generating report' + cobertura coberturaReportFile: '**/coverage.xml' } always { echo 'Cleaning workspace and exiting' From 2451786a4f1254725c54df355ff8c5494bb5c5e3 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Mon, 11 Mar 2024 17:27:31 +0100 Subject: [PATCH 31/32] Actually keep the report for each distro stage --- Jenkinsfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 446e9cd..b55d631 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -153,10 +153,6 @@ pipeline { } } post { - success { - echo 'Generating report' - cobertura coberturaReportFile: '**/coverage.xml' - } always { echo 'Cleaning workspace and exiting' cleanWs() From f68819ed71e32a1f0cebaef728f47843b6d20f51 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 6 Jun 2024 12:39:19 +0200 Subject: [PATCH 32/32] Minor refinements of README --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index b0ceaa7..55d8267 100644 --- a/README.md +++ b/README.md @@ -14,19 +14,19 @@ You may find more information about [the ARGO Messaging Service documentation](h Library is tested and should work with Python versions 2.7, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11 running on CentOS 7, Rocky 8 and Rocky 9 releases. -RPM packages are prepared for both CentOS releases and you may find it and download it from ARGO Repository. PyPI packages are prepared as well. +RPM packages are prepared for CentOS and Rocky releases and you may find it and download it from ARGO Repository. PyPI packages are prepared as well. RPM production packages: -http://rpm-repo.argo.grnet.gr/ARGO/prod/centos7/ -http://rpm-repo.argo.grnet.gr/ARGO/prod/rocky8/ -http://rpm-repo.argo.grnet.gr/ARGO/prod/rocky9/ +* http://rpm-repo.argo.grnet.gr/ARGO/prod/centos7/ +* http://rpm-repo.argo.grnet.gr/ARGO/prod/rocky8/ +* http://rpm-repo.argo.grnet.gr/ARGO/prod/rocky9/ RPM devel packages: -http://rpm-repo.argo.grnet.gr/ARGO/devel/centos7/ -http://rpm-repo.argo.grnet.gr/ARGO/devel/rocky8/ -http://rpm-repo.argo.grnet.gr/ARGO/devel/rocky9/ +* http://rpm-repo.argo.grnet.gr/ARGO/devel/centos7/ +* http://rpm-repo.argo.grnet.gr/ARGO/devel/rocky8/ +* http://rpm-repo.argo.grnet.gr/ARGO/devel/rocky9/ PyPI package: