diff --git a/ros_buildfarm/argument.py b/ros_buildfarm/argument.py index 4d07bd23e..ef7a15b9b 100644 --- a/ros_buildfarm/argument.py +++ b/ros_buildfarm/argument.py @@ -448,6 +448,12 @@ def add_argument_require_gpu_support(parser): help='GPU acceleration is needed in the build container') +def add_argument_shared_ccache(parser): + parser.add_argument( + '--shared-ccache', action='store_true', + help='User\'s ccache directory is shared in the build container') + + def add_argument_skip_cleanup(parser): parser.add_argument( '--skip-cleanup', action='store_true', diff --git a/ros_buildfarm/devel_job.py b/ros_buildfarm/devel_job.py index 87a9d74c3..47e75be0b 100644 --- a/ros_buildfarm/devel_job.py +++ b/ros_buildfarm/devel_job.py @@ -252,7 +252,8 @@ def configure_devel_job( build_targets=None, dry_run=False, run_abichecker=None, - require_gpu_support=None): + require_gpu_support=None, + shared_ccache=None): """ Configure a single Jenkins devel job. @@ -271,6 +272,9 @@ def configure_devel_job( if build_targets is not None: build_file.targets = build_targets + if shared_ccache is not None: + build_file.shared_ccache = shared_ccache + if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: diff --git a/ros_buildfarm/templates/ci/ci_job.xml.em b/ros_buildfarm/templates/ci/ci_job.xml.em index 3931b613a..47d376b99 100644 --- a/ros_buildfarm/templates/ci/ci_job.xml.em +++ b/ros_buildfarm/templates/ci/ci_job.xml.em @@ -300,12 +300,12 @@ parameters = [ '', ] + ([ 'echo "# BEGIN SECTION: ccache stats (before)"', - 'mkdir -p $HOME/.ccache', + 'mkdir -p "${CCACHE_DIR:-$HOME/.ccache}"', 'docker run' + ' --rm ' + ' --cidfile=$WORKSPACE/docker_build_and_install/docker_ccache_before.cid' + ' -e CCACHE_DIR=/home/buildfarm/.ccache' + - ' -v $HOME/.ccache:/home/buildfarm/.ccache' + + ' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' + ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name) + ' "ccache -s"', 'echo "# END SECTION"', @@ -320,7 +320,7 @@ parameters = [ ' --rm ' + ' --cidfile=$WORKSPACE/docker_build_and_install/docker.cid' + ((' -e CCACHE_DIR=/home/buildfarm/.ccache' + - ' -v $HOME/.ccache:/home/buildfarm/.ccache') if shared_ccache else '') + + ' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache') if shared_ccache else '') + ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' + ''.join([ ' -v %s:/tmp/ws%s/install_isolated:ro' % (space, i if i > 1 else '') @@ -337,7 +337,7 @@ parameters = [ ' --rm ' + ' --cidfile=$WORKSPACE/docker_build_and_install/docker_ccache_after.cid' + ' -e CCACHE_DIR=/home/buildfarm/.ccache' + - ' -v $HOME/.ccache:/home/buildfarm/.ccache' + + ' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' + ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name) + ' "ccache -s"', 'echo "# END SECTION"', @@ -374,12 +374,12 @@ parameters = [ '', ] + ([ 'echo "# BEGIN SECTION: ccache stats (before)"', - 'mkdir -p $HOME/.ccache', + 'mkdir -p "${CCACHE_DIR:-$HOME/.ccache}"', 'docker run' + ' --rm ' + ' --cidfile=$WORKSPACE/docker_build_and_test/docker_ccache_before.cid' + ' -e CCACHE_DIR=/home/buildfarm/.ccache' + - ' -v $HOME/.ccache:/home/buildfarm/.ccache' + + ' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' + ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name) + ' "ccache -s"', 'echo "# END SECTION"', @@ -396,7 +396,7 @@ parameters = [ ' --rm ' + ' --cidfile=$WORKSPACE/docker_build_and_test/docker.cid' + ((' -e CCACHE_DIR=/home/buildfarm/.ccache' + - ' -v $HOME/.ccache:/home/buildfarm/.ccache') if shared_ccache else '') + + ' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache') if shared_ccache else '') + ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' + ''.join([ ' -v %s:/tmp/ws%s/install_isolated:ro' % (space, i if i > 1 else '') @@ -413,7 +413,7 @@ parameters = [ ' --rm ' + ' --cidfile=$WORKSPACE/docker_build_and_test/docker_ccache_after.cid' + ' -e CCACHE_DIR=/home/buildfarm/.ccache' + - ' -v $HOME/.ccache:/home/buildfarm/.ccache' + + ' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' + ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name) + ' "ccache -s"', 'echo "# END SECTION"', diff --git a/ros_buildfarm/templates/devel/devel_job.xml.em b/ros_buildfarm/templates/devel/devel_job.xml.em index d0f322b63..935abbf74 100644 --- a/ros_buildfarm/templates/devel/devel_job.xml.em +++ b/ros_buildfarm/templates/devel/devel_job.xml.em @@ -202,7 +202,7 @@ if pull_request: '', 'echo "# BEGIN SECTION: Run Dockerfile - build and install"', ] + ([ - 'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi', + 'if [ ! -d "${CCACHE_DIR:-$HOME/.ccache}" ]; then mkdir "${CCACHE_DIR:-$HOME/.ccache}"; fi', ] if shared_ccache else []) + [ ('if [ ! -c /dev/nvidia[0-9] ]; then echo "--require-gpu-support is enabled but can not detect nvidia support installed" && exit 1; fi' if require_gpu_support else ''), 'docker run' + @@ -212,7 +212,7 @@ if pull_request: ' -e=TRAVIS=$TRAVIS' + ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' + ' -v $WORKSPACE/ws:/tmp/ws' + - (' -v $HOME/.ccache:/home/buildfarm/.ccache' if shared_ccache else '') + + (' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' if shared_ccache else '') + ' devel_build_and_install.%s_%s' % (rosdistro_name, source_repo_spec.name.lower()), 'cd -', # restore pwd when used in scripts 'echo "# END SECTION"', @@ -236,7 +236,7 @@ if pull_request: 'echo "# BEGIN SECTION: Run Dockerfile - build and test"', '', ] + ([ - 'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi', + 'if [ ! -d "${CCACHE_DIR:-$HOME/.ccache}" ]; then mkdir "${CCACHE_DIR:-$HOME/.ccache}"; fi', ] if shared_ccache else []) + [ 'docker run' + (' --env=DISPLAY=:0.0 --env=QT_X11_NO_MITSHM=1 --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw --gpus all' if require_gpu_support else '') + @@ -245,7 +245,7 @@ if pull_request: ' -e=TRAVIS=$TRAVIS' + ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' + ' -v $WORKSPACE/ws:/tmp/ws' + - (' -v $HOME/.ccache:/home/buildfarm/.ccache' if shared_ccache else '') + + (' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' if shared_ccache else '') + ' devel_build_and_test.%s_%s' % (rosdistro_name, source_repo_spec.name.lower()), 'cd -', # restore pwd when used in scripts 'echo "# END SECTION"', diff --git a/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em b/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em index 00d962db0..f9aa61d30 100644 --- a/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em +++ b/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em @@ -128,7 +128,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config 'mkdir -p $WORKSPACE/binarydeb', 'mkdir -p $WORKSPACE/docker_build_binarydeb', ] + ([ - 'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi', + 'if [ ! -d "${CCACHE_DIR:-$HOME/.ccache}" ]; then mkdir "${CCACHE_DIR:-$HOME/.ccache}"; fi', ] if shared_ccache else []) + [ 'docker run' + ' --rm ' + @@ -138,7 +138,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' + ' -v $WORKSPACE/binarydeb:/tmp/binarydeb' + ' -v $WORKSPACE/docker_build_binarydeb:/tmp/docker_build_binarydeb' + - (' -v $HOME/.ccache:/home/buildfarm/.ccache' if shared_ccache else '') + + (' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' if shared_ccache else '') + ' binarydeb_task_generation.%s_%s_%s_%s_%s' % (rosdistro_name, os_name, os_code_name, arch, pkg_name), 'echo "# END SECTION"', ]), @@ -162,7 +162,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config '# -e=HOME= is required to set a reasonable HOME for the user (not /)', '# otherwise apt-src will fail', ] + ([ - 'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi', + 'if [ ! -d "${CCACHE_DIR:-$HOME/.ccache}" ]; then mkdir "${CCACHE_DIR:-$HOME/.ccache}"; fi', ] if shared_ccache else []) + [ 'docker run' + ' --rm ' + @@ -172,7 +172,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config ' --net=host' + ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' + ' -v $WORKSPACE/binarydeb:/tmp/binarydeb' + - (' -v $HOME/.ccache:/home/buildfarm/.ccache' if shared_ccache else '') + + (' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' if shared_ccache else '') + ' binarydeb_build.%s_%s_%s_%s_%s' % (rosdistro_name, os_name, os_code_name, arch, pkg_name), 'echo "# END SECTION"', ]), diff --git a/ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em b/ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em index 8653c8aca..44b27224d 100644 --- a/ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em +++ b/ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em @@ -127,7 +127,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config 'rm -fr $WORKSPACE/binarypkg', 'mkdir -p $WORKSPACE/binarypkg/source', ] + ([ - 'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi', + 'if [ ! -d "${CCACHE_DIR:-$HOME/.ccache}" ]; then mkdir "${CCACHE_DIR:-$HOME/.ccache}"; fi', ] if shared_ccache else []) + [ 'docker run' + ' --rm' + @@ -137,7 +137,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config ' --net=host' + ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' + ' -v $WORKSPACE/binarypkg:/tmp/binarypkg' + - (' -v $HOME/.ccache:/home/buildfarm/.ccache' if shared_ccache else '') + + (' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' if shared_ccache else '') + ' binaryrpm.%s_%s_%s_%s_%s' % (rosdistro_name, os_name, os_code_name, arch, pkg_name), 'echo "# END SECTION"', ]), diff --git a/scripts/prerelease/generate_prerelease_script.py b/scripts/prerelease/generate_prerelease_script.py index 9bd94560e..1b6f5734a 100755 --- a/scripts/prerelease/generate_prerelease_script.py +++ b/scripts/prerelease/generate_prerelease_script.py @@ -33,6 +33,7 @@ from ros_buildfarm.argument import add_argument_os_name from ros_buildfarm.argument import add_argument_output_dir from ros_buildfarm.argument import add_argument_rosdistro_name +from ros_buildfarm.argument import add_argument_shared_ccache from ros_buildfarm.config import get_index as get_config_index from ros_buildfarm.config import get_release_build_files from ros_buildfarm.config import get_source_build_files @@ -54,6 +55,7 @@ def main(argv=sys.argv[1:]): add_argument_os_code_name(parser) add_argument_arch(parser) add_argument_build_tool(parser) + add_argument_shared_ccache(parser) add_argument_custom_rosdep_update_options(parser) add_argument_output_dir(parser, required=True) @@ -225,7 +227,8 @@ def beforeInclude(self, *_, **kwargs): index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=False, views=False, source_repository=source_repository, - build_targets=release_targets_combined) + build_targets=release_targets_combined, + shared_ccache=args.shared_ccache) templates.template_hooks = None