Move -install-pkg- jobs to be generated using gz-collections.yaml (#1093

) * Generate -install-pkg jobs on gazebo_libs * Remove install-pkg jobs from ignition.dsl * Automated change: update logs [skip ci] --------- Signed-off-by: Jose Luis Rivero <[email protected]> Co-authored-by: j-rivero <[email protected]>
gazebo-tooling · Dec 14, 2023 · c6c57ea · c6c57ea
1 parent bf7fb65
commit c6c57ea
Show file tree

Hide file tree

Showing 4 changed files with 129 additions and 106 deletions.
diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl
@@ -12,7 +12,7 @@ GITHUB_SUPPORT_ALL_BRANCHES = []
 ENABLE_GITHUB_PR_INTEGRATION = true
 
 def WRITE_JOB_LOG = System.getenv('WRITE_JOB_LOG') ?: false
-logging_list = [:]
+logging_list = [:].withDefault {[]}
 logging_list['branch_ci'] = []
 logging_list['asan_ci'] = []
 
@@ -179,7 +179,6 @@ void add_brew_shell_build_step(gz_brew_ci_job, lib_name, ws_checkout_dir)
 
 void generate_brew_ci_job(gz_brew_ci_job, lib_name, branch, ci_config)
 {
-  def script_name_prefix = cleanup_library_name(lib_name)
   def ws_checkout_dir = lib_name
   OSRFBrewCompilation.create(gz_brew_ci_job,
                              is_testing_enabled(lib_name, ci_config),
@@ -218,6 +217,36 @@ void generate_win_ci_job(gz_win_ci_job, lib_name, branch, ci_config)
   add_win_devel_bat_call(gz_win_ci_job, lib_name, ws_checkout_dir)
 }
 
+String generate_linux_install(src_name, lib_name, platform, arch)
+{
+  def script_name_prefix = cleanup_library_name(src_name)
+  def job_name = "${script_name_prefix}-install-pkg-${platform}-${arch}"
+  def install_default_job = job(job_name)
+  OSRFLinuxInstall.create(install_default_job)
+  install_default_job.with
+  {
+    triggers {
+      cron(Globals.CRON_EVERY_THREE_DAYS)
+    }
+
+    def dev_package = "lib${src_name}-dev"
+
+    steps {
+     shell("""\
+           #!/bin/bash -xe
+
+           ${GLOBAL_SHELL_CMD}
+           export DISTRO=${platform}
+           export ARCH=${arch}
+           export INSTALL_JOB_PKG=${dev_package}
+           export GZDEV_PROJECT_NAME="${src_name}"
+           /bin/bash -x ./scripts/jenkins-scripts/docker/generic-install-test-job.bash
+           """.stripIndent())
+    }
+  }
+  return job_name
+}
+
 def ciconf_per_lib_index = [:].withDefault { [:] }
 def pkgconf_per_src_index = [:].withDefault { [:] }
 generate_ciconfigs_by_lib(gz_collections_yaml, ciconf_per_lib_index, pkgconf_per_src_index)
@@ -378,17 +407,27 @@ pkgconf_per_src_index.each { pkg_src, pkg_src_configs ->
     def pkg_config = gz_collections_yaml.packaging_configs.find{ it.name == config_name }
     // lib_names are the same in all the entries
     def canonical_lib_name = pkg_src_config.getValue()[0].lib_name
-
     if (pkg_config.exclude?.contains(canonical_lib_name))
       return
-
+    def pkg_system = pkg_config.system
+    // --------------------------------------------------------------
     def gz_source_job = job("${pkg_src}-source")
     OSRFSourceCreation.create(gz_source_job, [
       PACKAGE: pkg_src,
       SOURCE_REPO_URI: "https://github.com/gazebosim/${canonical_lib_name}.git"])
     OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job,
       'repository_uploader_packages',
       '_releasepy')
+    // --------------------------------------------------------------
+    pkg_system.arch.each { arch ->
+      def job_name = generate_linux_install(
+        pkg_src, canonical_lib_name, pkg_system.version, arch)
+      pkg_src_config.getValue().each { index_entry ->
+        logging_list['install_ci'].add(
+          [collection: index_entry.collection,
+           job_name: job_name])
+      }
+    }
   }
 }
 

diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml
@@ -440,8 +440,6 @@ collections:
     packaging:
       configs:
         - jammy
-      exclude:
-          - __upcoming__
 ci_configs:
   - name: focal
     system:
@@ -569,13 +567,12 @@ packaging_configs:
       version: focal
       arch:
         - amd64
-        - arm64
   - name: jammy
     system:
       so: linux
       distribution: ubuntu
       version: jammy
       arch:
         - amd64
-        - arm64
-        - armhf
+    exclude:
+        - __upcoming__
diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl
@@ -354,37 +354,6 @@ boolean is_a_colcon_package(String gz_software_name)
   return false
 }
 
-void generate_install_job(prefix, gz_sw, major_version, distro, arch)
-{
-  def install_default_job = job("${prefix}_${gz_sw}${major_version}-install-pkg-${distro}-${arch}")
-  OSRFLinuxInstall.create(install_default_job)
-  include_gpu_label_if_needed(install_default_job, gz_sw)
-
-  install_default_job.with
-  {
-    triggers {
-      cron(Globals.CRON_EVERY_THREE_DAYS)
-    }
-
-    def dev_package = "lib${prefix}-${gz_sw}${major_version}-dev"
-    def gzdev_project = "${prefix}-${gz_sw}${major_version}"
-
-    steps {
-     shell("""\
-           #!/bin/bash -xe
-
-           ${GLOBAL_SHELL_CMD}
-
-           export DISTRO=${distro}
-           export ARCH=${arch}
-           export INSTALL_JOB_PKG=${dev_package}
-           export GZDEV_PROJECT_NAME="${gzdev_project}"
-           /bin/bash -x ./scripts/jenkins-scripts/docker/generic-install-test-job.bash
-           """.stripIndent())
-    }
-  }
-}
-
 // Need to be before the ci-pr_any so the abi job name is defined
 gz_software.each { gz_sw ->
   supported_arches.each { arch ->
@@ -422,72 +391,6 @@ gz_software.each { gz_sw ->
   } // end of arch
 } // end of gz_software
 
-
-// INSTALL PACKAGE ALL PLATFORMS / DAILY
-gz_software.each { gz_sw ->
-  // Exclusion list
-  if (gz_sw in gz_no_pkg_yet)
-    return
-
-  supported_arches.each { arch ->
-    supported_install_pkg_branches(gz_sw).each { major_version, supported_distros ->
-      supported_distros.each { distro ->
-        extra_repos_str=""
-        if ((gz_sw in gz_prerelease_pkgs) &&
-           (major_version in gz_prerelease_pkgs[gz_sw]) &&
-           (distro in gz_prerelease_pkgs[gz_sw][major_version]))
-          extra_repos_str="prerelease"
-
-        // No 1-dev or 0-dev packages (except special cases see
-        // gz_debbuild variable), unversioned
-        major_version_in_pkgname = major_version
-        if ("${major_version}" == "0" || "${major_version}" == "1")
-          major_version_in_pkgname = ""
-
-        // 1. gz_ prefix packages. All but not gazebo (replaced by sim)
-        generate_install_job("gz", gz_sw.replace('gazebo', 'sim'), major_version, distro, arch)
-
-        // 2. ignition_ prefix packages. gz software does not have ignition packages
-        if (major_version in supported_ign_branches(gz_sw))
-          generate_install_job("ignition", gz_sw, major_version_in_pkgname, distro, arch)
-      }
-    }
-  }
-}
-
-void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch,
-                     extra_cmake = '', extra_test = '', extra_cmd = [])
-{
-  OSRFLinuxCompilation.create(gz_ci_job, enable_testing(software_name))
-  OSRFGitHub.create(gz_ci_job,
-                        "gazebosim/ign-${software_name}",
-                        "${branch}", "ign-${software_name}")
-
-  include_gpu_label_if_needed(gz_ci_job, gz_sw)
-  gz_ci_job.with
-  {
-    if (gz_sw == 'physics') {
-      label Globals.nontest_label("large-memory")
-      extra_cmd += "export MAKE_JOBS=1"
-    }
-    if (gz_sw == 'gazebo')
-      gz_sw = 'sim'
-
-    steps {
-      shell("""#!/bin/bash -xe
-
-            ${GLOBAL_SHELL_CMD}
-            ${extra_cmd.join('\n')}
-            export BUILDING_EXTRA_CMAKE_PARAMS="${extra_cmake}"
-            export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}"
-            export DISTRO=${distro}
-            export ARCH=${arch}
-            /bin/bash -xe ./scripts/jenkins-scripts/docker/gz_${gz_sw.replaceAll('-','_')}-compilation.bash
-            """.stripIndent())
-    }
-  }
-}
-
 // --------------------------------------------------------------
 // DEBBUILD: linux package builder
 all_debbuilders().each { debbuilder_name ->

diff --git a/jenkins-scripts/dsl/logs/generated_jobs.txt b/jenkins-scripts/dsl/logs/generated_jobs.txt
@@ -318,6 +318,90 @@ branch_ci ionic gz_utils-main-win
 branch_ci ionic sdformat-ci-main-homebrew-amd64
 branch_ci ionic sdformat-ci-main-jammy-amd64
 branch_ci ionic sdformat-main-win
+install_ci __upcoming__ gz_tools3-install-pkg-jammy-amd64
+install_ci citadel gz_citadel-install-pkg-focal-amd64
+install_ci citadel gz_cmake2-install-pkg-focal-amd64
+install_ci citadel gz_common3-install-pkg-focal-amd64
+install_ci citadel gz_fuel_tools4-install-pkg-focal-amd64
+install_ci citadel gz_gui3-install-pkg-focal-amd64
+install_ci citadel gz_launch2-install-pkg-focal-amd64
+install_ci citadel gz_math6-install-pkg-focal-amd64
+install_ci citadel gz_msgs5-install-pkg-focal-amd64
+install_ci citadel gz_physics2-install-pkg-focal-amd64
+install_ci citadel gz_plugin1-install-pkg-focal-amd64
+install_ci citadel gz_rendering3-install-pkg-focal-amd64
+install_ci citadel gz_sensors3-install-pkg-focal-amd64
+install_ci citadel gz_sim3-install-pkg-focal-amd64
+install_ci citadel gz_tools1-install-pkg-focal-amd64
+install_ci citadel gz_transport8-install-pkg-focal-amd64
+install_ci citadel sdformat9-install-pkg-focal-amd64
+install_ci fortress gz_cmake2-install-pkg-focal-amd64
+install_ci fortress gz_common4-install-pkg-focal-amd64
+install_ci fortress gz_fortress-install-pkg-focal-amd64
+install_ci fortress gz_fuel_tools7-install-pkg-focal-amd64
+install_ci fortress gz_gui6-install-pkg-focal-amd64
+install_ci fortress gz_launch5-install-pkg-focal-amd64
+install_ci fortress gz_math6-install-pkg-focal-amd64
+install_ci fortress gz_msgs8-install-pkg-focal-amd64
+install_ci fortress gz_physics5-install-pkg-focal-amd64
+install_ci fortress gz_plugin1-install-pkg-focal-amd64
+install_ci fortress gz_rendering6-install-pkg-focal-amd64
+install_ci fortress gz_sensors6-install-pkg-focal-amd64
+install_ci fortress gz_sim6-install-pkg-focal-amd64
+install_ci fortress gz_tools1-install-pkg-focal-amd64
+install_ci fortress gz_transport11-install-pkg-focal-amd64
+install_ci fortress gz_utils1-install-pkg-focal-amd64
+install_ci fortress sdformat12-install-pkg-focal-amd64
+install_ci garden gz_cmake3-install-pkg-focal-amd64
+install_ci garden gz_common5-install-pkg-focal-amd64
+install_ci garden gz_fuel_tools8-install-pkg-focal-amd64
+install_ci garden gz_garden-install-pkg-focal-amd64
+install_ci garden gz_gui7-install-pkg-focal-amd64
+install_ci garden gz_launch6-install-pkg-focal-amd64
+install_ci garden gz_math7-install-pkg-focal-amd64
+install_ci garden gz_msgs9-install-pkg-focal-amd64
+install_ci garden gz_physics6-install-pkg-focal-amd64
+install_ci garden gz_plugin2-install-pkg-focal-amd64
+install_ci garden gz_rendering7-install-pkg-focal-amd64
+install_ci garden gz_sensors7-install-pkg-focal-amd64
+install_ci garden gz_sim7-install-pkg-focal-amd64
+install_ci garden gz_tools2-install-pkg-focal-amd64
+install_ci garden gz_transport12-install-pkg-focal-amd64
+install_ci garden gz_utils2-install-pkg-focal-amd64
+install_ci garden sdformat13-install-pkg-focal-amd64
+install_ci harmonic gz_cmake3-install-pkg-jammy-amd64
+install_ci harmonic gz_common5-install-pkg-jammy-amd64
+install_ci harmonic gz_fuel_tools9-install-pkg-jammy-amd64
+install_ci harmonic gz_gui8-install-pkg-jammy-amd64
+install_ci harmonic gz_harmonic-install-pkg-jammy-amd64
+install_ci harmonic gz_launch7-install-pkg-jammy-amd64
+install_ci harmonic gz_math7-install-pkg-jammy-amd64
+install_ci harmonic gz_msgs10-install-pkg-jammy-amd64
+install_ci harmonic gz_physics7-install-pkg-jammy-amd64
+install_ci harmonic gz_plugin2-install-pkg-jammy-amd64
+install_ci harmonic gz_rendering8-install-pkg-jammy-amd64
+install_ci harmonic gz_sensors8-install-pkg-jammy-amd64
+install_ci harmonic gz_sim8-install-pkg-jammy-amd64
+install_ci harmonic gz_tools2-install-pkg-jammy-amd64
+install_ci harmonic gz_transport13-install-pkg-jammy-amd64
+install_ci harmonic gz_utils2-install-pkg-jammy-amd64
+install_ci harmonic sdformat14-install-pkg-jammy-amd64
+install_ci ionic gz_cmake4-install-pkg-jammy-amd64
+install_ci ionic gz_common6-install-pkg-jammy-amd64
+install_ci ionic gz_fuel_tools10-install-pkg-jammy-amd64
+install_ci ionic gz_gui9-install-pkg-jammy-amd64
+install_ci ionic gz_launch8-install-pkg-jammy-amd64
+install_ci ionic gz_math8-install-pkg-jammy-amd64
+install_ci ionic gz_msgs11-install-pkg-jammy-amd64
+install_ci ionic gz_physics8-install-pkg-jammy-amd64
+install_ci ionic gz_plugin3-install-pkg-jammy-amd64
+install_ci ionic gz_rendering9-install-pkg-jammy-amd64
+install_ci ionic gz_sensors9-install-pkg-jammy-amd64
+install_ci ionic gz_sim9-install-pkg-jammy-amd64
+install_ci ionic gz_tools2-install-pkg-jammy-amd64
+install_ci ionic gz_transport14-install-pkg-jammy-amd64
+install_ci ionic gz_utils3-install-pkg-jammy-amd64
+install_ci ionic sdformat15-install-pkg-jammy-amd64
 unofficial_wrappers_install_pkg_ci garden ros_gzgarden_bridge-install-pkg_humble-ci-jammy-amd64
 unofficial_wrappers_install_pkg_ci garden ros_gzgarden_bridge-install-pkg_iron-ci-jammy-amd64
 unofficial_wrappers_install_pkg_ci harmonic ros_gzharmonic_bridge-install-pkg_iron-ci-jammy-amd64