From a7b65baff5a48bae5164e1e5c35f62e73361d1e9 Mon Sep 17 00:00:00 2001 From: Shingo Kitagawa Date: Mon, 20 May 2019 20:53:18 +0900 Subject: [PATCH 1/7] add coverage test in pfnci --- .pfnci/docker/devel-minimal/Dockerfile | 4 +++- .pfnci/docker/devel/Dockerfile | 2 ++ .pfnci/tests.sh | 3 +++ .pfnci/tests_gpu.sh | 3 +++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.pfnci/docker/devel-minimal/Dockerfile b/.pfnci/docker/devel-minimal/Dockerfile index e2e2d354f9..40b5d985c3 100644 --- a/.pfnci/docker/devel-minimal/Dockerfile +++ b/.pfnci/docker/devel-minimal/Dockerfile @@ -18,7 +18,9 @@ RUN pip2 install --no-cache-dir \ cython \ mock \ pytest \ + pytest-cov \ && pip3 install --no-cache-dir \ cython \ mock \ - pytest + pytest \ + pytest-cov diff --git a/.pfnci/docker/devel/Dockerfile b/.pfnci/docker/devel/Dockerfile index 5cea5eb81a..8978044432 100644 --- a/.pfnci/docker/devel/Dockerfile +++ b/.pfnci/docker/devel/Dockerfile @@ -79,6 +79,7 @@ RUN pip2 install --no-cache-dir \ mpi4py \ 'git+https://github.com/cocodataset/coco.git#egg=pycocotools&subdirectory=PythonAPI' \ pytest \ + pytest-cov \ scipy \ && pip3 install --no-cache-dir \ cython \ @@ -87,4 +88,5 @@ RUN pip2 install --no-cache-dir \ mpi4py \ 'git+https://github.com/cocodataset/coco.git#egg=pycocotools&subdirectory=PythonAPI' \ pytest \ + pytest-cov \ scipy diff --git a/.pfnci/tests.sh b/.pfnci/tests.sh index c6c75c080f..3631fecec0 100644 --- a/.pfnci/tests.sh +++ b/.pfnci/tests.sh @@ -17,10 +17,13 @@ docker run --interactive --rm \ pip${PYTHON} install --user pytest-xdist cd chainercv/ python${PYTHON} -m pytest --color=no -n $(nproc) \ + --cov=chainercv/ --cov-report= \ -m 'not pfnci_skip and not gpu and not mpi' tests/ if which mpiexec; then mpiexec -n 2 --allow-run-as-root \ python${PYTHON} -m pytest --color=no \ + --cov=chainercv/ --cov-report= --cov-append \ -m 'not pfnci_skip and not gpu and mpi' tests/ fi +coverage report --fail-under=90 EOD diff --git a/.pfnci/tests_gpu.sh b/.pfnci/tests_gpu.sh index 6e54949b8c..197de10150 100644 --- a/.pfnci/tests_gpu.sh +++ b/.pfnci/tests_gpu.sh @@ -11,10 +11,13 @@ docker run --runtime=nvidia --interactive --rm \ . ./install.sh cd chainercv/ python${PYTHON} -m pytest --color=no \ + --cov=chainercv/ --cov-report= \ -m 'not pfnci_skip and gpu and not mpi' tests/ if which mpiexec; then mpiexec -n 2 --allow-run-as-root \ python${PYTHON} -m pytest --color=no \ + --cov=chainercv/ --cov-report= --cov-append \ -m 'not pfnci_skip and gpu and mpi' tests/ fi +coverage report --fail-under=90 EOD From 48293eee16835057ce5fdde88364349e34bebf38 Mon Sep 17 00:00:00 2001 From: Toru Ogawa Date: Wed, 22 May 2019 02:17:59 +0900 Subject: [PATCH 2/7] download configs --- .pfnci/tests.sh | 2 ++ .pfnci/tests_gpu.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.pfnci/tests.sh b/.pfnci/tests.sh index 3631fecec0..07a441148e 100644 --- a/.pfnci/tests.sh +++ b/.pfnci/tests.sh @@ -8,6 +8,8 @@ gsutil -q cp gs://chainercv-pfn-public-ci/datasets-tiny.zip . unzip -q datasets-tiny.zip rm datasets-tiny.zip +gsutil -q cp gs://chainercv-pfn-public-ci/.coveralls.yml . + docker run --interactive --rm \ --volume $(pwd):/root/ --workdir /root/ \ --env MPLBACKEND=agg \ diff --git a/.pfnci/tests_gpu.sh b/.pfnci/tests_gpu.sh index 197de10150..3ebb61d48e 100644 --- a/.pfnci/tests_gpu.sh +++ b/.pfnci/tests_gpu.sh @@ -3,6 +3,8 @@ set -eux . $(dirname $0)/common.sh +gsutil -q cp gs://chainercv-pfn-public-ci/.coveralls.yml . + docker run --runtime=nvidia --interactive --rm \ --volume $(pwd):/root/ --workdir /root/ \ --env MPLBACKEND=agg \ From d753583e8571712ac11834e2616564041c67c457 Mon Sep 17 00:00:00 2001 From: Toru Ogawa Date: Wed, 22 May 2019 02:18:52 +0900 Subject: [PATCH 3/7] report --- .pfnci/tests.sh | 2 +- .pfnci/tests_gpu.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pfnci/tests.sh b/.pfnci/tests.sh index 07a441148e..b9231cd503 100644 --- a/.pfnci/tests.sh +++ b/.pfnci/tests.sh @@ -27,5 +27,5 @@ if which mpiexec; then --cov=chainercv/ --cov-report= --cov-append \ -m 'not pfnci_skip and not gpu and mpi' tests/ fi -coverage report --fail-under=90 +coveralls EOD diff --git a/.pfnci/tests_gpu.sh b/.pfnci/tests_gpu.sh index 3ebb61d48e..acc9ad70f1 100644 --- a/.pfnci/tests_gpu.sh +++ b/.pfnci/tests_gpu.sh @@ -21,5 +21,5 @@ if which mpiexec; then --cov=chainercv/ --cov-report= --cov-append \ -m 'not pfnci_skip and gpu and mpi' tests/ fi -coverage report --fail-under=90 +coveralls EOD From 7e9481ac37bdb85a5031ce422e156f4a971e744d Mon Sep 17 00:00:00 2001 From: Toru Ogawa Date: Wed, 22 May 2019 02:31:06 +0900 Subject: [PATCH 4/7] install coveralls --- .pfnci/docker/devel-minimal/Dockerfile | 2 ++ .pfnci/docker/devel/Dockerfile | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.pfnci/docker/devel-minimal/Dockerfile b/.pfnci/docker/devel-minimal/Dockerfile index 40b5d985c3..525688792f 100644 --- a/.pfnci/docker/devel-minimal/Dockerfile +++ b/.pfnci/docker/devel-minimal/Dockerfile @@ -15,11 +15,13 @@ RUN curl -LO https://bootstrap.pypa.io/get-pip.py \ && rm get-pip.py RUN pip2 install --no-cache-dir \ + coveralls \ cython \ mock \ pytest \ pytest-cov \ && pip3 install --no-cache-dir \ + coveralls \ cython \ mock \ pytest \ diff --git a/.pfnci/docker/devel/Dockerfile b/.pfnci/docker/devel/Dockerfile index 8978044432..d66cb4bf12 100644 --- a/.pfnci/docker/devel/Dockerfile +++ b/.pfnci/docker/devel/Dockerfile @@ -73,6 +73,7 @@ RUN cd $(mktemp -d) \ RUN echo /usr/local/lib/ > /etc/ld.so.conf.d/local.conf && ldconfig RUN pip2 install --no-cache-dir \ + coveralls \ cython \ matplotlib==2.1 \ mock \ @@ -82,6 +83,7 @@ RUN pip2 install --no-cache-dir \ pytest-cov \ scipy \ && pip3 install --no-cache-dir \ + coveralls \ cython \ matplotlib \ mock \ From 6ec630b4551bbe9c5115f1d15c9cbfc836d6cab1 Mon Sep 17 00:00:00 2001 From: Toru Ogawa Date: Wed, 22 May 2019 03:58:46 +0900 Subject: [PATCH 5/7] coveralls -> coveralls[yaml] --- .pfnci/docker/devel-minimal/Dockerfile | 4 ++-- .pfnci/docker/devel/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pfnci/docker/devel-minimal/Dockerfile b/.pfnci/docker/devel-minimal/Dockerfile index 525688792f..a7a1db982e 100644 --- a/.pfnci/docker/devel-minimal/Dockerfile +++ b/.pfnci/docker/devel-minimal/Dockerfile @@ -15,13 +15,13 @@ RUN curl -LO https://bootstrap.pypa.io/get-pip.py \ && rm get-pip.py RUN pip2 install --no-cache-dir \ - coveralls \ + coveralls[yaml] \ cython \ mock \ pytest \ pytest-cov \ && pip3 install --no-cache-dir \ - coveralls \ + coveralls[yaml] \ cython \ mock \ pytest \ diff --git a/.pfnci/docker/devel/Dockerfile b/.pfnci/docker/devel/Dockerfile index d66cb4bf12..e5be973d2d 100644 --- a/.pfnci/docker/devel/Dockerfile +++ b/.pfnci/docker/devel/Dockerfile @@ -73,7 +73,7 @@ RUN cd $(mktemp -d) \ RUN echo /usr/local/lib/ > /etc/ld.so.conf.d/local.conf && ldconfig RUN pip2 install --no-cache-dir \ - coveralls \ + coveralls[yaml] \ cython \ matplotlib==2.1 \ mock \ @@ -83,7 +83,7 @@ RUN pip2 install --no-cache-dir \ pytest-cov \ scipy \ && pip3 install --no-cache-dir \ - coveralls \ + coveralls[yaml] \ cython \ matplotlib \ mock \ From baffc8144cd328405e5f086f3594ca9823583c9c Mon Sep 17 00:00:00 2001 From: Toru Ogawa Date: Wed, 22 May 2019 04:20:15 +0900 Subject: [PATCH 6/7] fix path --- .pfnci/tests.sh | 2 +- .pfnci/tests_gpu.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pfnci/tests.sh b/.pfnci/tests.sh index b9231cd503..bc8365f385 100644 --- a/.pfnci/tests.sh +++ b/.pfnci/tests.sh @@ -8,7 +8,7 @@ gsutil -q cp gs://chainercv-pfn-public-ci/datasets-tiny.zip . unzip -q datasets-tiny.zip rm datasets-tiny.zip -gsutil -q cp gs://chainercv-pfn-public-ci/.coveralls.yml . +gsutil -q cp gs://chainercv-pfn-public-ci/.coveralls.yml chainercv/ docker run --interactive --rm \ --volume $(pwd):/root/ --workdir /root/ \ diff --git a/.pfnci/tests_gpu.sh b/.pfnci/tests_gpu.sh index acc9ad70f1..b65f4a7128 100644 --- a/.pfnci/tests_gpu.sh +++ b/.pfnci/tests_gpu.sh @@ -3,7 +3,7 @@ set -eux . $(dirname $0)/common.sh -gsutil -q cp gs://chainercv-pfn-public-ci/.coveralls.yml . +gsutil -q cp gs://chainercv-pfn-public-ci/.coveralls.yml chainercv/ docker run --runtime=nvidia --interactive --rm \ --volume $(pwd):/root/ --workdir /root/ \ From b13ca4a50c3112ff9e32c93b181ddf81437efd82 Mon Sep 17 00:00:00 2001 From: Toru Ogawa Date: Wed, 22 May 2019 05:09:27 +0900 Subject: [PATCH 7/7] enable git quickfix --- .pfnci/config.pbtxt | 24 ++++++++++++++++++++++++ .pfnci/gen_config.py | 6 +++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/.pfnci/config.pbtxt b/.pfnci/config.pbtxt index fdfdf41600..522726dff4 100644 --- a/.pfnci/config.pbtxt +++ b/.pfnci/config.pbtxt @@ -38,6 +38,7 @@ configs { key: "OPTIONAL_MODULES" value: "1" } + quickfix_checkout_dot_git: true } } configs { @@ -62,6 +63,7 @@ configs { key: "OPTIONAL_MODULES" value: "1" } + quickfix_checkout_dot_git: true } } configs { @@ -88,6 +90,7 @@ configs { key: "OPTIONAL_MODULES" value: "0" } + quickfix_checkout_dot_git: true } } configs { @@ -112,6 +115,7 @@ configs { key: "OPTIONAL_MODULES" value: "0" } + quickfix_checkout_dot_git: true } } configs { @@ -165,6 +169,7 @@ configs { key: "OPTIONAL_MODULES" value: "1" } + quickfix_checkout_dot_git: true } } configs { @@ -189,6 +194,7 @@ configs { key: "OPTIONAL_MODULES" value: "1" } + quickfix_checkout_dot_git: true } } configs { @@ -215,6 +221,7 @@ configs { key: "OPTIONAL_MODULES" value: "0" } + quickfix_checkout_dot_git: true } } configs { @@ -239,6 +246,7 @@ configs { key: "OPTIONAL_MODULES" value: "0" } + quickfix_checkout_dot_git: true } } configs { @@ -292,6 +300,7 @@ configs { key: "OPTIONAL_MODULES" value: "1" } + quickfix_checkout_dot_git: true } } configs { @@ -316,6 +325,7 @@ configs { key: "OPTIONAL_MODULES" value: "1" } + quickfix_checkout_dot_git: true } } configs { @@ -342,6 +352,7 @@ configs { key: "OPTIONAL_MODULES" value: "0" } + quickfix_checkout_dot_git: true } } configs { @@ -366,6 +377,7 @@ configs { key: "OPTIONAL_MODULES" value: "0" } + quickfix_checkout_dot_git: true } } configs { @@ -419,6 +431,7 @@ configs { key: "OPTIONAL_MODULES" value: "1" } + quickfix_checkout_dot_git: true } } configs { @@ -443,6 +456,7 @@ configs { key: "OPTIONAL_MODULES" value: "1" } + quickfix_checkout_dot_git: true } } configs { @@ -469,6 +483,7 @@ configs { key: "OPTIONAL_MODULES" value: "0" } + quickfix_checkout_dot_git: true } } configs { @@ -493,6 +508,7 @@ configs { key: "OPTIONAL_MODULES" value: "0" } + quickfix_checkout_dot_git: true } } configs { @@ -546,6 +562,7 @@ configs { key: "OPTIONAL_MODULES" value: "1" } + quickfix_checkout_dot_git: true } } configs { @@ -570,6 +587,7 @@ configs { key: "OPTIONAL_MODULES" value: "1" } + quickfix_checkout_dot_git: true } } configs { @@ -596,6 +614,7 @@ configs { key: "OPTIONAL_MODULES" value: "0" } + quickfix_checkout_dot_git: true } } configs { @@ -620,6 +639,7 @@ configs { key: "OPTIONAL_MODULES" value: "0" } + quickfix_checkout_dot_git: true } } configs { @@ -673,6 +693,7 @@ configs { key: "OPTIONAL_MODULES" value: "1" } + quickfix_checkout_dot_git: true } } configs { @@ -697,6 +718,7 @@ configs { key: "OPTIONAL_MODULES" value: "1" } + quickfix_checkout_dot_git: true } } configs { @@ -723,6 +745,7 @@ configs { key: "OPTIONAL_MODULES" value: "0" } + quickfix_checkout_dot_git: true } } configs { @@ -747,6 +770,7 @@ configs { key: "OPTIONAL_MODULES" value: "0" } + quickfix_checkout_dot_git: true } } configs { diff --git a/.pfnci/gen_config.py b/.pfnci/gen_config.py index b8bb5e6c8c..5a15ae83a7 100644 --- a/.pfnci/gen_config.py +++ b/.pfnci/gen_config.py @@ -33,10 +33,12 @@ def test_config(python, chainer, optional, target): value['requirement']['memory'] = 36 value['time_limit'] = {'seconds': 3600} value['command'] = 'sh .pfnci/tests.sh' + value['quickfix_checkout_dot_git'] = True elif target == 'gpu': key += '.gpu' value['requirement']['gpu'] = 1 value['command'] = 'sh .pfnci/tests_gpu.sh' + value['quickfix_checkout_dot_git'] = True elif target == 'examples': key += '.examples' value['requirement']['gpu'] = 2 @@ -79,7 +81,9 @@ def main(): def dump_pbtxt(key, value, level=0): indent = ' ' * level - if isinstance(value, int): + if isinstance(value, bool): + print('{}{}: {}'.format(indent, key, 'true' if value else 'false')) + elif isinstance(value, int): print('{}{}: {}'.format(indent, key, value)) elif isinstance(value, str): print('{}{}: "{}"'.format(indent, key, value))