Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add --shared-ccache argument to generate_prerelease_script.py #876

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Add --shared-ccache argument to generate_prerelease_script.py #876

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1904d5c
Add --shared-ccache argument to generate_prerelease_script.py
mathias-luedtke May 4, 2021
8379ec8
Use optional CCACHE_DIR environment variable
mathias-luedtke May 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions ros_buildfarm/argument.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,12 @@ def add_argument_require_gpu_support(parser):
help='GPU acceleration is needed in the build container')


def add_argument_shared_ccache(parser):
parser.add_argument(
'--shared-ccache', action='store_true',
help='User\'s ccache directory is shared in the build container')


def add_argument_skip_cleanup(parser):
parser.add_argument(
'--skip-cleanup', action='store_true',
Expand Down
6 changes: 5 additions & 1 deletion ros_buildfarm/devel_job.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,8 @@ def configure_devel_job(
build_targets=None,
dry_run=False,
run_abichecker=None,
require_gpu_support=None):
require_gpu_support=None,
shared_ccache=None):
"""
Configure a single Jenkins devel job.

Expand All @@ -271,6 +272,9 @@ def configure_devel_job(
if build_targets is not None:
build_file.targets = build_targets

if shared_ccache is not None:
build_file.shared_ccache = shared_ccache

if index is None:
index = get_index(config.rosdistro_index_url)
if dist_file is None:
Expand Down
16 changes: 8 additions & 8 deletions ros_buildfarm/templates/ci/ci_job.xml.em
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -300,12 +300,12 @@ parameters = [
'',
] + ([
'echo "# BEGIN SECTION: ccache stats (before)"',
'mkdir -p $HOME/.ccache',
'mkdir -p "${CCACHE_DIR:-$HOME/.ccache}"',
'docker run' +
' --rm ' +
' --cidfile=$WORKSPACE/docker_build_and_install/docker_ccache_before.cid' +
' -e CCACHE_DIR=/home/buildfarm/.ccache' +
' -v $HOME/.ccache:/home/buildfarm/.ccache' +
' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' +
' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name) +
' "ccache -s"',
'echo "# END SECTION"',
Expand All @@ -320,7 +320,7 @@ parameters = [
' --rm ' +
' --cidfile=$WORKSPACE/docker_build_and_install/docker.cid' +
((' -e CCACHE_DIR=/home/buildfarm/.ccache' +
' -v $HOME/.ccache:/home/buildfarm/.ccache') if shared_ccache else '') +
' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache') if shared_ccache else '') +
' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
''.join([
' -v %s:/tmp/ws%s/install_isolated:ro' % (space, i if i > 1 else '')
Expand All @@ -337,7 +337,7 @@ parameters = [
' --rm ' +
' --cidfile=$WORKSPACE/docker_build_and_install/docker_ccache_after.cid' +
' -e CCACHE_DIR=/home/buildfarm/.ccache' +
' -v $HOME/.ccache:/home/buildfarm/.ccache' +
' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' +
' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name) +
' "ccache -s"',
'echo "# END SECTION"',
Expand Down Expand Up @@ -374,12 +374,12 @@ parameters = [
'',
] + ([
'echo "# BEGIN SECTION: ccache stats (before)"',
'mkdir -p $HOME/.ccache',
'mkdir -p "${CCACHE_DIR:-$HOME/.ccache}"',
'docker run' +
' --rm ' +
' --cidfile=$WORKSPACE/docker_build_and_test/docker_ccache_before.cid' +
' -e CCACHE_DIR=/home/buildfarm/.ccache' +
' -v $HOME/.ccache:/home/buildfarm/.ccache' +
' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' +
' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name) +
' "ccache -s"',
'echo "# END SECTION"',
Expand All @@ -396,7 +396,7 @@ parameters = [
' --rm ' +
' --cidfile=$WORKSPACE/docker_build_and_test/docker.cid' +
((' -e CCACHE_DIR=/home/buildfarm/.ccache' +
' -v $HOME/.ccache:/home/buildfarm/.ccache') if shared_ccache else '') +
' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache') if shared_ccache else '') +
' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
''.join([
' -v %s:/tmp/ws%s/install_isolated:ro' % (space, i if i > 1 else '')
Expand All @@ -413,7 +413,7 @@ parameters = [
' --rm ' +
' --cidfile=$WORKSPACE/docker_build_and_test/docker_ccache_after.cid' +
' -e CCACHE_DIR=/home/buildfarm/.ccache' +
' -v $HOME/.ccache:/home/buildfarm/.ccache' +
' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' +
' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name) +
' "ccache -s"',
'echo "# END SECTION"',
Expand Down
8 changes: 4 additions & 4 deletions ros_buildfarm/templates/devel/devel_job.xml.em
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ if pull_request:
'',
'echo "# BEGIN SECTION: Run Dockerfile - build and install"',
] + ([
'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi',
'if [ ! -d "${CCACHE_DIR:-$HOME/.ccache}" ]; then mkdir "${CCACHE_DIR:-$HOME/.ccache}"; fi',
] if shared_ccache else []) + [
('if [ ! -c /dev/nvidia[0-9] ]; then echo "--require-gpu-support is enabled but can not detect nvidia support installed" && exit 1; fi' if require_gpu_support else ''),
'docker run' +
Expand All @@ -212,7 +212,7 @@ if pull_request:
' -e=TRAVIS=$TRAVIS' +
' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
' -v $WORKSPACE/ws:/tmp/ws' +
(' -v $HOME/.ccache:/home/buildfarm/.ccache' if shared_ccache else '') +
(' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' if shared_ccache else '') +
' devel_build_and_install.%s_%s' % (rosdistro_name, source_repo_spec.name.lower()),
'cd -', # restore pwd when used in scripts
'echo "# END SECTION"',
Expand All @@ -236,7 +236,7 @@ if pull_request:
'echo "# BEGIN SECTION: Run Dockerfile - build and test"',
'',
] + ([
'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi',
'if [ ! -d "${CCACHE_DIR:-$HOME/.ccache}" ]; then mkdir "${CCACHE_DIR:-$HOME/.ccache}"; fi',
] if shared_ccache else []) + [
'docker run' +
(' --env=DISPLAY=:0.0 --env=QT_X11_NO_MITSHM=1 --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw --gpus all' if require_gpu_support else '') +
Expand All @@ -245,7 +245,7 @@ if pull_request:
' -e=TRAVIS=$TRAVIS' +
' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
' -v $WORKSPACE/ws:/tmp/ws' +
(' -v $HOME/.ccache:/home/buildfarm/.ccache' if shared_ccache else '') +
(' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' if shared_ccache else '') +
' devel_build_and_test.%s_%s' % (rosdistro_name, source_repo_spec.name.lower()),
'cd -', # restore pwd when used in scripts
'echo "# END SECTION"',
Expand Down
8 changes: 4 additions & 4 deletions ros_buildfarm/templates/release/deb/binarypkg_job.xml.em
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
'mkdir -p $WORKSPACE/binarydeb',
'mkdir -p $WORKSPACE/docker_build_binarydeb',
] + ([
'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi',
'if [ ! -d "${CCACHE_DIR:-$HOME/.ccache}" ]; then mkdir "${CCACHE_DIR:-$HOME/.ccache}"; fi',
] if shared_ccache else []) + [
'docker run' +
' --rm ' +
Expand All @@ -138,7 +138,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
' -v $WORKSPACE/binarydeb:/tmp/binarydeb' +
' -v $WORKSPACE/docker_build_binarydeb:/tmp/docker_build_binarydeb' +
(' -v $HOME/.ccache:/home/buildfarm/.ccache' if shared_ccache else '') +
(' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' if shared_ccache else '') +
' binarydeb_task_generation.%s_%s_%s_%s_%s' % (rosdistro_name, os_name, os_code_name, arch, pkg_name),
'echo "# END SECTION"',
]),
Expand All @@ -162,7 +162,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
'# -e=HOME= is required to set a reasonable HOME for the user (not /)',
'# otherwise apt-src will fail',
] + ([
'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi',
'if [ ! -d "${CCACHE_DIR:-$HOME/.ccache}" ]; then mkdir "${CCACHE_DIR:-$HOME/.ccache}"; fi',
] if shared_ccache else []) + [
'docker run' +
' --rm ' +
Expand All @@ -172,7 +172,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
' --net=host' +
' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
' -v $WORKSPACE/binarydeb:/tmp/binarydeb' +
(' -v $HOME/.ccache:/home/buildfarm/.ccache' if shared_ccache else '') +
(' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' if shared_ccache else '') +
' binarydeb_build.%s_%s_%s_%s_%s' % (rosdistro_name, os_name, os_code_name, arch, pkg_name),
'echo "# END SECTION"',
]),
Expand Down
4 changes: 2 additions & 2 deletions ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
'rm -fr $WORKSPACE/binarypkg',
'mkdir -p $WORKSPACE/binarypkg/source',
] + ([
'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi',
'if [ ! -d "${CCACHE_DIR:-$HOME/.ccache}" ]; then mkdir "${CCACHE_DIR:-$HOME/.ccache}"; fi',
] if shared_ccache else []) + [
'docker run' +
' --rm' +
Expand All @@ -137,7 +137,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
' --net=host' +
' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
' -v $WORKSPACE/binarypkg:/tmp/binarypkg' +
(' -v $HOME/.ccache:/home/buildfarm/.ccache' if shared_ccache else '') +
(' -v "${CCACHE_DIR:-$HOME/.ccache}":/home/buildfarm/.ccache' if shared_ccache else '') +
' binaryrpm.%s_%s_%s_%s_%s' % (rosdistro_name, os_name, os_code_name, arch, pkg_name),
'echo "# END SECTION"',
]),
Expand Down
5 changes: 4 additions & 1 deletion scripts/prerelease/generate_prerelease_script.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from ros_buildfarm.argument import add_argument_os_name
from ros_buildfarm.argument import add_argument_output_dir
from ros_buildfarm.argument import add_argument_rosdistro_name
from ros_buildfarm.argument import add_argument_shared_ccache
from ros_buildfarm.config import get_index as get_config_index
from ros_buildfarm.config import get_release_build_files
from ros_buildfarm.config import get_source_build_files
Expand All @@ -54,6 +55,7 @@ def main(argv=sys.argv[1:]):
add_argument_os_code_name(parser)
add_argument_arch(parser)
add_argument_build_tool(parser)
add_argument_shared_ccache(parser)
add_argument_custom_rosdep_update_options(parser)
add_argument_output_dir(parser, required=True)

Expand Down Expand Up @@ -225,7 +227,8 @@ def beforeInclude(self, *_, **kwargs):
index=index, dist_file=dist_file, dist_cache=dist_cache,
jenkins=False, views=False,
source_repository=source_repository,
build_targets=release_targets_combined)
build_targets=release_targets_combined,
shared_ccache=args.shared_ccache)

templates.template_hooks = None

Expand Down