From e3a227601de8c4179eae852e77c6673cc334b075 Mon Sep 17 00:00:00 2001 From: Kris Wilson Date: Fri, 4 Aug 2017 17:53:59 -0700 Subject: [PATCH 01/67] [pantsd] Add faulthandler support for stacktrace dumps. (#4784) Fixes #4626 Problem Early reports from alpha testers of pantsd revealed that some pantsd-runner processes were getting hung in the background on their machine with no apparent cause or repro. In order to better diagnose hangs or other such issues in pantsd itself or either normal pants or pantsd based runs, we need a way to easily collect a python stacktrace. Solution Add faulthandler support to pants, which is a C-level stack trace dumper. This permits us to kill -31 to get stack trace per thread from any of these 3 runtime contexts even if python is deadlocked. This will also automatically dump stacktraces on c/rust-level segfaults among other features described here: https://faulthandler.readthedocs.io/ Result [omerta pants (kwlzn/faulthandler)]$ ps -ef |grep "pantsd \[" 501 94833 1 0 4:26PM ?? 0:04.91 pantsd [/Users/kwilson/dev/pants] [omerta pants (kwlzn/faulthandler)]$ tail -0 -F .pants.d/pantsd/pantsd.log & [1] 72279 [omerta pants (kwlzn/faulthandler)]$ kill -31 94833 Thread 0x0000700010b98000 (most recent call first): File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 340 in wait File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/Queue.py", line 168 in get File "/Users/kwilson/dev/pants/build-support/pants_dev_deps.venv/lib/python2.7/site-packages/concurrent/futures/thread.py", line 65 in _worker File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 754 in run File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 801 in __bootstrap_inner File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 774 in __bootstrap Thread 0x0000700010795000 (most recent call first): File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 340 in wait File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/Queue.py", line 168 in get File "/Users/kwilson/dev/pants/build-support/pants_dev_deps.venv/lib/python2.7/site-packages/concurrent/futures/thread.py", line 65 in _worker File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 754 in run File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 801 in __bootstrap_inner File "/Users/k[omerta pants (kwlzn/faulthandler)]$ wilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 774 in __bootstrap Thread 0x0000700010392000 (most recent call first): File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 340 in wait File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/Queue.py", line 168 in get File "/Users/kwilson/dev/pants/build-support/pants_dev_deps.venv/lib/python2.7/site-packages/concurrent/futures/thread.py", line 65 in _worker File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 754 in run File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 801 in __bootstrap_inner File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 774 in __bootstrap Thread 0x000070000ff8f000 (most recent call first): File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 340 in wait File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/Queue.py", line 168 in get File "/Users/kwilson/dev/pants/build-support/pants_dev_deps.venv/lib/python2.7/site-packages/concurrent/futures/thread.py", line 65 in _worker File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 754 in run File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 801 in __bootstrap_inner File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 774 in __bootstrap Thread 0x000070000fb8c000 (most recent call first): File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/SocketServer.py", line 150 in _eintr_retry File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/SocketServer.py", line 271 in handle_request File "/Users/kwilson/dev/pants/src/python/pants/pantsd/service/pailgun_service.py", line 101 in run File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 754 in run File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 801 in __bootstrap_inner File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 774 in __bootstrap Thread 0x000070000f789000 (most recent call first): File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 359 in wait File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/Queue.py", line 177 in get File "/Users/kwilson/dev/pants/src/python/pants/pantsd/service/scheduler_service.py", line 73 in _process_event_queue File "/Users/kwilson/dev/pants/src/python/pants/pantsd/service/scheduler_service.py", line 103 in run File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 754 in run File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 801 in __bootstrap_inner File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 774 in __bootstrap Thread 0x000070000f386000 (most recent call first): File "/Users/kwilson/dev/pants/build-support/pants_dev_deps.venv/lib/python2.7/site-packages/pywatchman/__init__.py", line 168 in readBytes File "/Users/kwilson/dev/pants/build-support/pants_dev_deps.venv/lib/python2.7/site-packages/pywatchman/__init__.py", line 272 in receive File "/Users/kwilson/dev/pants/src/python/pants/pantsd/watchman_client.py", line 49 in stream_query File "/Users/kwilson/dev/pants/src/python/pants/pantsd/watchman.py", line 167 in subscribed File "/Users/kwilson/dev/pants/src/python/pants/pantsd/service/fs_event_service.py", line 115 in run File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 754 in run File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 801 in __bootstrap_inner File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 774 in __bootstrap Thread 0x0000700002674000 (most recent call first): File "/Users/kwilson/dev/pants/src/python/pants/reporting/report.py", line 34 in run File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 801 in __bootstrap_inner File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 774 in __bootstrap Current thread 0x00007fff98cfe3c0 (most recent call first): File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 359 in wait File "/Users/kwilson/Python/CPython-2.7.13/lib/python2.7/threading.py", line 951 in join File "/Users/kwilson/dev/pants/src/python/pants/pantsd/pants_daemon.py", line 169 in _run_services File "/Users/kwilson/dev/pants/src/python/pants/pantsd/pants_daemon.py", line 195 in _run File "/Users/kwilson/dev/pants/src/python/pants/pantsd/pants_daemon.py", line 210 in post_fork_child File "/Users/kwilson/dev/pants/src/python/pants/pantsd/process_manager.py", line 433 in daemonize File "/Users/kwilson/dev/pants/src/python/pants/init/pants_daemon_launcher.py", line 176 in _launch_pantsd File "/Users/kwilson/dev/pants/src/python/pants/init/pants_daemon_launcher.py", line 186 in maybe_launch File "/Users/kwilson/dev/pants/src/python/pants/bin/goal_runner.py", line 158 in _maybe_launch_pantsd File "/Users/kwilson/dev/pants/src/python/pants/bin/goal_runner.py", line 196 in setup File "/Users/kwilson/dev/pants/src/python/pants/bin/local_pants_runner.py", line 77 in _run File "/Users/kwilson/dev/pants/src/python/pants/bin/local_pants_runner.py", line 37 in run File "/Users/kwilson/dev/pants/src/python/pants/bin/pants_runner.py", line 46 in _run File "/Users/kwilson/dev/pants/src/python/pants/bin/pants_runner.py", line 57 in run File "/Users/kwilson/dev/pants/src/python/pants/bin/pants_exe.py", line 26 in main File "/Users/kwilson/dev/pants/src/python/pants/bin/pants_loader.py", line 58 in load_and_execute File "/Users/kwilson/dev/pants/src/python/pants/bin/pants_loader.py", line 65 in run File "/Users/kwilson/dev/pants/src/python/pants/bin/pants_loader.py", line 69 in main File "/Users/kwilson/dev/pants/src/python/pants/bin/pants_loader.py", line 73 in [omerta pants (kwlzn/faulthandler)]$ fg tail -0 -F .pants.d/pantsd/pantsd.log ^C --- 3rdparty/python/requirements.txt | 1 + src/docs/tshoot.md | 11 ++++++++ src/python/pants/base/BUILD | 9 +++++++ src/python/pants/{bin => base}/exiter.py | 11 +++++++- src/python/pants/bin/BUILD | 1 + src/python/pants/bin/daemon_pants_runner.py | 11 +++++--- src/python/pants/bin/pants_exe.py | 2 +- src/python/pants/java/nailgun_io.py | 3 +++ src/python/pants/logging/setup.py | 16 +++++++++--- src/python/pants/pantsd/BUILD | 1 + src/python/pants/pantsd/pants_daemon.py | 24 ++++++++++++------ tests/python/pants_test/base/BUILD | 10 ++++++++ .../pants_test/{bin => base}/test_exiter.py | 2 +- tests/python/pants_test/bin/BUILD | 10 -------- tests/python/pants_test/logging/test_setup.py | 2 +- .../pants_test/pantsd/test_pants_daemon.py | 10 ++++---- .../pantsd/test_pantsd_integration.py | 25 +++++++++++++++++++ 17 files changed, 116 insertions(+), 33 deletions(-) rename src/python/pants/{bin => base}/exiter.py (92%) rename tests/python/pants_test/{bin => base}/test_exiter.py (97%) diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 7f78c0e3b48..cfb86200264 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -4,6 +4,7 @@ cffi==1.7.0 coverage>=4.3.4,<4.4 docutils>=0.12,<0.13 fasteners==0.14.1 +faulthandler==2.6 futures==3.0.5 isort==4.2.5 Markdown==2.1.1 diff --git a/src/docs/tshoot.md b/src/docs/tshoot.md index 6ff0d0d7ef4..016fa431c15 100644 --- a/src/docs/tshoot.md +++ b/src/docs/tshoot.md @@ -135,6 +135,17 @@ by expanding the `stderr` nodes. 2015/03/12 03:55:57:451 PDT [DEBUG] header - -<< "X-Timer: S1426157757.333469,VS0,VE36[\r][\n]" ``` +Getting a stack trace from a running pants process +-------------------------------------------------- + +Pants implements `faulthandler` support in all execution contexts. To get a stack trace from a running pants process, send a `SIGUSR2` signal like so: + +``` +$ kill -31 +``` + +For traditional pants runs, this will dump a stack trace (for all threads) to stderr. For pantsd-based runs and for pantsd itself, the traces will end up in the `.pantsd/pantsd/pantsd.log` file. + Questions, Issues, Bug Reports ------------------------------ diff --git a/src/python/pants/base/BUILD b/src/python/pants/base/BUILD index 324704c20e0..e4156f1788b 100644 --- a/src/python/pants/base/BUILD +++ b/src/python/pants/base/BUILD @@ -187,3 +187,12 @@ python_library( '3rdparty/python:six', ], ) + +python_library( + name='exiter', + sources=['exiter.py'], + dependencies=[ + '3rdparty/python:faulthandler', + 'src/python/pants/util:dirutil' + ] +) diff --git a/src/python/pants/bin/exiter.py b/src/python/pants/base/exiter.py similarity index 92% rename from src/python/pants/bin/exiter.py rename to src/python/pants/base/exiter.py index 813a41bab29..f20fad39dbf 100644 --- a/src/python/pants/bin/exiter.py +++ b/src/python/pants/base/exiter.py @@ -8,9 +8,12 @@ import datetime import logging import os +import signal import sys import traceback +import faulthandler + from pants.util.dirutil import safe_open @@ -110,6 +113,12 @@ def _log_exception(self, msg): # we don't want to hide the original error. logger.error('Problem logging original exception: {}'.format(e)) - def set_except_hook(self): + def _setup_faulthandler(self, trace_stream): + faulthandler.enable(trace_stream) + # This permits a non-fatal `kill -31 ` for stacktrace retrieval. + faulthandler.register(signal.SIGUSR2, trace_stream, chain=True) + + def set_except_hook(self, trace_stream=None): """Sets the global exception hook.""" + self._setup_faulthandler(trace_stream or sys.stderr) sys.excepthook = self.handle_unhandled_exception diff --git a/src/python/pants/bin/BUILD b/src/python/pants/bin/BUILD index bea9e0c3caa..d720b9e0d78 100644 --- a/src/python/pants/bin/BUILD +++ b/src/python/pants/bin/BUILD @@ -9,6 +9,7 @@ python_library( 'src/python/pants/base:build_environment', 'src/python/pants/base:build_file', 'src/python/pants/base:cmd_line_spec_parser', + 'src/python/pants/base:exiter', 'src/python/pants/base:project_tree', 'src/python/pants/base:specs', 'src/python/pants/base:workunit', diff --git a/src/python/pants/bin/daemon_pants_runner.py b/src/python/pants/bin/daemon_pants_runner.py index 0966d0d604c..f28f772290d 100644 --- a/src/python/pants/bin/daemon_pants_runner.py +++ b/src/python/pants/bin/daemon_pants_runner.py @@ -9,11 +9,12 @@ import os import signal import socket +import sys from contextlib import contextmanager from setproctitle import setproctitle as set_process_title -from pants.bin.exiter import Exiter +from pants.base.exiter import Exiter from pants.bin.local_pants_runner import LocalPantsRunner from pants.init.util import clean_global_runtime_state from pants.java.nailgun_io import NailgunStreamWriter @@ -138,8 +139,12 @@ def pre_fork(self): def post_fork_child(self): """Post-fork child process callback executed via ProcessManager.daemonize().""" # Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception - # hook with socket-specific behavior. - self._exiter.set_except_hook() + # hook with socket-specific behavior. Note that this intentionally points the faulthandler + # trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to + # the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner + # process that the stacktrace output lands deterministically in a known place vs to a stray + # terminal window. + self._exiter.set_except_hook(sys.stderr) # Set context in the process title. set_process_title('pantsd-runner [{}]'.format(' '.join(self._args))) diff --git a/src/python/pants/bin/pants_exe.py b/src/python/pants/bin/pants_exe.py index 4ae22f6da36..f9f646e8e4f 100644 --- a/src/python/pants/bin/pants_exe.py +++ b/src/python/pants/bin/pants_exe.py @@ -5,7 +5,7 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) -from pants.bin.exiter import Exiter +from pants.base.exiter import Exiter from pants.bin.pants_runner import PantsRunner diff --git a/src/python/pants/java/nailgun_io.py b/src/python/pants/java/nailgun_io.py index 372f223f1bc..427a12ed3d3 100644 --- a/src/python/pants/java/nailgun_io.py +++ b/src/python/pants/java/nailgun_io.py @@ -110,3 +110,6 @@ def flush(self): def isatty(self): return self._isatty + + def fileno(self): + return self._socket.fileno() diff --git a/src/python/pants/logging/setup.py b/src/python/pants/logging/setup.py index ff31285261e..f37302e8a66 100644 --- a/src/python/pants/logging/setup.py +++ b/src/python/pants/logging/setup.py @@ -8,12 +8,17 @@ import logging import os import time +from collections import namedtuple from logging import Formatter, StreamHandler from logging.handlers import RotatingFileHandler from pants.util.dirutil import safe_mkdir +class LoggingSetupResult(namedtuple('LoggingSetupResult', ['log_filename', 'log_stream'])): + """A structured result for logging setup.""" + + # TODO: Once pantsd had a separate launcher entry point, and so no longer needs to call this # function, move this into the pants.init package, and remove the pants.logging package. def setup_logging(level, console_stream=None, log_dir=None, scope=None, log_name=None): @@ -42,12 +47,14 @@ def setup_logging(level, console_stream=None, log_dir=None, scope=None, log_name # logging control and we don't need to be the middleman plumbing an option for each python # standard logging knob. + log_filename = None + log_stream = None + logger = logging.getLogger(scope) for handler in logger.handlers: logger.removeHandler(handler) if console_stream: - log_file = None console_handler = StreamHandler(stream=console_stream) console_handler.setFormatter(Formatter(fmt='%(levelname)s] %(message)s')) console_handler.setLevel(level) @@ -55,8 +62,9 @@ def setup_logging(level, console_stream=None, log_dir=None, scope=None, log_name if log_dir: safe_mkdir(log_dir) - log_file = os.path.join(log_dir, log_name or 'pants.log') - file_handler = RotatingFileHandler(log_file, maxBytes=10 * 1024 * 1024, backupCount=4) + log_filename = os.path.join(log_dir, log_name or 'pants.log') + file_handler = RotatingFileHandler(log_filename, maxBytes=10 * 1024 * 1024, backupCount=4) + log_stream = file_handler.stream class GlogFormatter(Formatter): LEVEL_MAP = { @@ -87,4 +95,4 @@ def format(self, record): # This routes warnings through our loggers instead of straight to raw stderr. logging.captureWarnings(True) - return log_file + return LoggingSetupResult(log_filename, log_stream) diff --git a/src/python/pants/pantsd/BUILD b/src/python/pants/pantsd/BUILD index 747bb6e3ea2..6caa712d83b 100644 --- a/src/python/pants/pantsd/BUILD +++ b/src/python/pants/pantsd/BUILD @@ -47,6 +47,7 @@ python_library( sources = ['pants_daemon.py'], dependencies = [ '3rdparty/python:setproctitle', + 'src/python/pants/base:exiter', 'src/python/pants/goal:run_tracker', 'src/python/pants/logging', ':process_manager', diff --git a/src/python/pants/pantsd/pants_daemon.py b/src/python/pants/pantsd/pants_daemon.py index 76470a0c326..50e908a4a62 100644 --- a/src/python/pants/pantsd/pants_daemon.py +++ b/src/python/pants/pantsd/pants_daemon.py @@ -12,21 +12,25 @@ from setproctitle import setproctitle as set_process_title +from pants.base.exiter import Exiter from pants.goal.run_tracker import RunTracker from pants.logging.setup import setup_logging from pants.pantsd.process_manager import ProcessManager -class _StreamLogger(object): +class _LoggerStream(object): """A sys.{stdout,stderr} replacement that pipes output to a logger.""" - def __init__(self, logger, log_level): + def __init__(self, logger, log_level, logger_stream): """ :param logging.Logger logger: The logger instance to emit writes to. :param int log_level: The log level to use for the given logger. + :param file logger_stream: The underlying file object the logger is writing to, for + determining the fileno to support faulthandler logging. """ self._logger = logger self._log_level = log_level + self._stream = logger_stream def write(self, msg): for line in msg.rstrip().splitlines(): @@ -38,6 +42,9 @@ def flush(self): def isatty(self): return False + def fileno(self): + return self._stream.fileno() + class PantsDaemon(ProcessManager): """A daemon that manages PantsService instances.""" @@ -58,7 +65,6 @@ def __init__(self, build_root, work_dir, log_level, native, log_dir=None, servic :param tuple services: A tuple of PantsService instances to launch/manage. (Optional) :param callable reset_func: Called after the daemon is forked to reset any state inherited from the parent process. (Optional) - """ super(PantsDaemon, self).__init__(name='pantsd', metadata_base_dir=metadata_base_dir) self._logger = logging.getLogger(__name__) @@ -72,6 +78,7 @@ def __init__(self, build_root, work_dir, log_level, native, log_dir=None, servic self._socket_map = {} # N.B. This Event is used as nothing more than a convenient atomic flag - nothing waits on it. self._kill_switch = threading.Event() + self._exiter = Exiter() @property def is_killed(self): @@ -116,17 +123,19 @@ def _setup_logging(self, log_level): logging.shutdown() # Reinitialize logging for the daemon context. - setup_logging(log_level, console_stream=None, log_dir=self._log_dir, log_name=self.LOG_NAME) + result = setup_logging(log_level, log_dir=self._log_dir, log_name=self.LOG_NAME) # Close out pre-fork file descriptors. self._close_fds() # Redirect stdio to the root logger. - sys.stdout = _StreamLogger(logging.getLogger(), logging.INFO) - sys.stderr = _StreamLogger(logging.getLogger(), logging.WARN) + sys.stdout = _LoggerStream(logging.getLogger(), logging.INFO, result.log_stream) + sys.stderr = _LoggerStream(logging.getLogger(), logging.WARN, result.log_stream) self._logger.debug('logging initialized') + return result.log_stream + def _setup_services(self, services): for service in services: self._logger.info('setting up service {}'.format(service)) @@ -167,7 +176,8 @@ def _write_named_sockets(self, socket_map): def _run(self): """Synchronously run pantsd.""" # Switch log output to the daemon's log stream from here forward. - self._setup_logging(self._log_level) + log_stream = self._setup_logging(self._log_level) + self._exiter.set_except_hook(log_stream) self._logger.info('pantsd starting, log level is {}'.format(self._log_level)) # Purge as much state as possible from the pants run that launched us. diff --git a/tests/python/pants_test/base/BUILD b/tests/python/pants_test/base/BUILD index 40fd681070e..cf0f0a0106b 100644 --- a/tests/python/pants_test/base/BUILD +++ b/tests/python/pants_test/base/BUILD @@ -218,3 +218,13 @@ python_tests( 'src/python/pants/base:validation', ] ) + +python_tests( + name = 'exiter', + sources = ['test_exiter.py'], + dependencies = [ + 'src/python/pants/bin', + 'src/python/pants/util:contextutil', + 'tests/python/pants_test/option:testing', + ], +) diff --git a/tests/python/pants_test/bin/test_exiter.py b/tests/python/pants_test/base/test_exiter.py similarity index 97% rename from tests/python/pants_test/bin/test_exiter.py rename to tests/python/pants_test/base/test_exiter.py index 91e6fe9e2a9..e332e11afa0 100644 --- a/tests/python/pants_test/bin/test_exiter.py +++ b/tests/python/pants_test/base/test_exiter.py @@ -8,7 +8,7 @@ import os import unittest -from pants.bin.exiter import Exiter +from pants.base.exiter import Exiter from pants.util.contextutil import temporary_dir from pants_test.option.util.fakes import create_options diff --git a/tests/python/pants_test/bin/BUILD b/tests/python/pants_test/bin/BUILD index 0f843cdb7ff..058c0fbdda4 100644 --- a/tests/python/pants_test/bin/BUILD +++ b/tests/python/pants_test/bin/BUILD @@ -1,16 +1,6 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -python_tests( - name = 'exiter', - sources = ['test_exiter.py'], - dependencies = [ - 'src/python/pants/bin', - 'src/python/pants/util:contextutil', - 'tests/python/pants_test/option:testing', - ], -) - python_tests( name='loader_integration', sources=['test_loader_integration.py'], diff --git a/tests/python/pants_test/logging/test_setup.py b/tests/python/pants_test/logging/test_setup.py index b5980ab3818..d5cd41f8019 100644 --- a/tests/python/pants_test/logging/test_setup.py +++ b/tests/python/pants_test/logging/test_setup.py @@ -31,7 +31,7 @@ def logger(self, level, file_logging=False): with closing(six.StringIO()) as stream: with self.log_dir(file_logging) as log_dir: log_file = setup_logging(level, console_stream=stream, log_dir=log_dir, scope=logger.name) - yield logger, stream, log_file + yield logger, stream, log_file.log_filename def assertWarnInfoOutput(self, lines): """Check to see that only warn and info output appears in the stream. diff --git a/tests/python/pants_test/pantsd/test_pants_daemon.py b/tests/python/pants_test/pantsd/test_pants_daemon.py index 6e84525cc96..254c88a10ab 100644 --- a/tests/python/pants_test/pantsd/test_pants_daemon.py +++ b/tests/python/pants_test/pantsd/test_pants_daemon.py @@ -9,7 +9,7 @@ import mock -from pants.pantsd.pants_daemon import PantsDaemon, _StreamLogger +from pants.pantsd.pants_daemon import PantsDaemon, _LoggerStream from pants.pantsd.service.pants_service import PantsService from pants.util.contextutil import stdio_as from pants_test.base_test import BaseTest @@ -18,18 +18,18 @@ PATCH_OPTS = dict(autospec=True, spec_set=True) -class StreamLoggerTest(BaseTest): +class LoggerStreamTest(BaseTest): TEST_LOG_LEVEL = logging.INFO def test_write(self): mock_logger = mock.Mock() - _StreamLogger(mock_logger, self.TEST_LOG_LEVEL).write('testing 1 2 3') + _LoggerStream(mock_logger, self.TEST_LOG_LEVEL, None).write('testing 1 2 3') mock_logger.log.assert_called_once_with(self.TEST_LOG_LEVEL, 'testing 1 2 3') def test_write_multiline(self): mock_logger = mock.Mock() - _StreamLogger(mock_logger, self.TEST_LOG_LEVEL).write('testing\n1\n2\n3\n\n') + _LoggerStream(mock_logger, self.TEST_LOG_LEVEL, None).write('testing\n1\n2\n3\n\n') mock_logger.log.assert_has_calls([ mock.call(self.TEST_LOG_LEVEL, 'testing'), mock.call(self.TEST_LOG_LEVEL, '1'), @@ -38,7 +38,7 @@ def test_write_multiline(self): ]) def test_flush(self): - _StreamLogger(mock.Mock(), self.TEST_LOG_LEVEL).flush() + _LoggerStream(mock.Mock(), self.TEST_LOG_LEVEL, None).flush() class PantsDaemonTest(BaseTest): diff --git a/tests/python/pants_test/pantsd/test_pantsd_integration.py b/tests/python/pants_test/pantsd/test_pantsd_integration.py index ce55e720297..323d872e830 100644 --- a/tests/python/pants_test/pantsd/test_pantsd_integration.py +++ b/tests/python/pants_test/pantsd/test_pantsd_integration.py @@ -6,6 +6,7 @@ unicode_literals, with_statement) import os +import signal import time from contextlib import contextmanager @@ -159,3 +160,27 @@ def test_pantsd_broken_pipe(self): self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) checker.assert_stopped() + + def test_pantsd_stacktrace_dump(self): + config = {'watchman': {'socket_path': '/tmp/watchman.{}.sock'.format(os.getpid())}} + with self.pantsd_test_context(config) as (workdir, pantsd_config, checker): + print('log: {}/pantsd/pantsd.log'.format(workdir)) + # Explicitly kill any running pantsd instances for the current buildroot. + self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) + try: + # Start pantsd implicitly via a throwaway invocation. + self.assert_success(self.run_pants_with_workdir(['help'], workdir, pantsd_config)) + checker.await_pantsd(3) + + os.kill(checker.pid, signal.SIGUSR2) + checker.assert_running() + + # Wait for log flush. + time.sleep(2) + + self.assertIn('Current thread 0x', '\n'.join(read_pantsd_log(workdir))) + finally: + # Explicitly kill pantsd (from a pantsd-launched runner). + self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) + + checker.assert_stopped() From d8be9eb1ecaf5d41cc441c974258a9b9874af1e9 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Mon, 7 Aug 2017 13:42:47 -0700 Subject: [PATCH 02/67] Init the native engine from bootstrap options. (#4787) Previously the native engine was initialized by a Native.Factory subsystem. However we want to use the engine to parse options, meaning that the engine must be initialized using only bootstrap options. So the subsystem options were moved to the bootstrap options (their cmd-line and env names don't change, because we use the old subsystem scope as a prefix for the new global option names, however they will need to move to the global section in pants.ini). The code was moved out of pants/engine/subsystem into pants/engine directly. We could have renamed the subsystem directory to something else, but the relevant target both depends on and is depended on by targets in the parent dir, so it should probably just live there (circular deps at the directory level are icky). --- build-support/bin/native/bootstrap.sh | 10 +-- build-support/bin/native/utils.sh | 4 +- build-support/native-engine/bootstrap_cffi.py | 2 +- src/python/pants/bin/BUILD | 3 +- src/python/pants/bin/engine_initializer.py | 7 +- src/python/pants/bin/goal_runner.py | 7 +- .../pants/core_tasks/explain_options_task.py | 6 +- src/python/pants/engine/BUILD | 25 ++++++- src/python/pants/engine/README.md | 11 +++ .../pants/engine/{subsystem => }/native.py | 44 +++--------- src/python/pants/engine/native_engine_version | 1 + src/python/pants/engine/scheduler.py | 4 +- src/python/pants/engine/subsystem/BUILD | 21 ------ src/python/pants/engine/subsystem/README.md | 8 --- src/python/pants/engine/subsystem/__init__.py | 0 .../engine/subsystem/native_engine_version | 1 - src/python/pants/init/BUILD | 3 +- .../pants/init/pants_daemon_launcher.py | 71 +++++++------------ src/python/pants/option/BUILD | 1 + src/python/pants/option/global_options.py | 14 ++++ tests/python/pants_test/engine/BUILD | 26 ++++--- tests/python/pants_test/engine/examples/BUILD | 3 +- .../pants_test/engine/examples/visualizer.py | 24 ++++--- tests/python/pants_test/engine/test_rules.py | 6 +- tests/python/pants_test/engine/util.py | 11 +-- 25 files changed, 153 insertions(+), 160 deletions(-) rename src/python/pants/engine/{subsystem => }/native.py (94%) create mode 100644 src/python/pants/engine/native_engine_version delete mode 100644 src/python/pants/engine/subsystem/BUILD delete mode 100644 src/python/pants/engine/subsystem/README.md delete mode 100644 src/python/pants/engine/subsystem/__init__.py delete mode 100644 src/python/pants/engine/subsystem/native_engine_version diff --git a/build-support/bin/native/bootstrap.sh b/build-support/bin/native/bootstrap.sh index b09af0c2790..1d2ec0520bb 100644 --- a/build-support/bin/native/bootstrap.sh +++ b/build-support/bin/native/bootstrap.sh @@ -24,7 +24,7 @@ source ${REPO_ROOT}/build-support/bin/native/utils.sh readonly NATIVE_ROOT="${REPO_ROOT}/src/rust/engine" readonly NATIVE_ENGINE_MODULE="native_engine" readonly NATIVE_ENGINE_BINARY="${NATIVE_ENGINE_MODULE}.so" -readonly NATIVE_ENGINE_VERSION_RESOURCE="${REPO_ROOT}/src/python/pants/engine/subsystem/native_engine_version" +readonly NATIVE_ENGINE_VERSION_RESOURCE="${REPO_ROOT}/src/python/pants/engine/native_engine_version" readonly CFFI_BOOTSTRAPPER="${REPO_ROOT}/build-support/native-engine/bootstrap_cffi.py" # N.B. Set $MODE to "debug" to generate a binary with debugging symbols. @@ -45,7 +45,7 @@ function calculate_current_hash() { cd ${REPO_ROOT} git ls-files -c -o --exclude-standard \ "${NATIVE_ROOT}" \ - "${REPO_ROOT}/src/python/pants/engine/subsystem/native.py" \ + "${REPO_ROOT}/src/python/pants/engine/native.py" \ | git hash-object -t blob --stdin-paths | fingerprint_data ) } @@ -128,9 +128,9 @@ function bootstrap_native_code() { mkdir -p "$(dirname ${target_binary})" cp "${native_binary}" "${target_binary}" - # NB: The resource file emitted/over-written below is used by the `Native` subsystem to default - # the native engine library version used by pants. More info can be read here: - # src/python/pants/engine/subsystem/README.md + # NB: The resource file emitted/over-written below is used by the `Native` class to default + # the native engine library version used by pants. More info can be read at the end of this + # document: src/python/pants/engine/README.md echo ${native_engine_version} > ${NATIVE_ENGINE_VERSION_RESOURCE} fi } diff --git a/build-support/bin/native/utils.sh b/build-support/bin/native/utils.sh index 4c617382100..6225be48712 100644 --- a/build-support/bin/native/utils.sh +++ b/build-support/bin/native/utils.sh @@ -15,8 +15,8 @@ REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd ../../.. && pwd -P) function get_native_engine_version() { - ${REPO_ROOT}/pants options --scope=native-engine --name=version --output-format=json | \ - python -c 'import json, sys; print(json.load(sys.stdin)["native-engine.version"]["value"])' + ${REPO_ROOT}/pants options --scope=GLOBAL --name=native-engine-version --output-format=json | \ + python -c 'import json, sys; print(json.load(sys.stdin)["native_engine_version"]["value"])' } readonly RUST_OSX_MIN_VERSION=7 diff --git a/build-support/native-engine/bootstrap_cffi.py b/build-support/native-engine/bootstrap_cffi.py index 6c7b61bba76..f674cdb6dd2 100644 --- a/build-support/native-engine/bootstrap_cffi.py +++ b/build-support/native-engine/bootstrap_cffi.py @@ -7,7 +7,7 @@ import sys -from pants.engine.subsystem.native import bootstrap_c_source +from pants.engine.native import bootstrap_c_source if __name__ == '__main__': diff --git a/src/python/pants/bin/BUILD b/src/python/pants/bin/BUILD index d720b9e0d78..baa1f09527e 100644 --- a/src/python/pants/bin/BUILD +++ b/src/python/pants/bin/BUILD @@ -13,17 +13,18 @@ python_library( 'src/python/pants/base:project_tree', 'src/python/pants/base:specs', 'src/python/pants/base:workunit', + 'src/python/pants/binaries:binary_util', 'src/python/pants/build_graph', 'src/python/pants/core_tasks', 'src/python/pants/engine/legacy:address_mapper', 'src/python/pants/engine/legacy:change_calculator', 'src/python/pants/engine/legacy:graph', 'src/python/pants/engine/legacy:parser', - 'src/python/pants/engine/subsystem:native', 'src/python/pants/engine:build_files', 'src/python/pants/engine:fs', 'src/python/pants/engine:legacy_engine', 'src/python/pants/engine:mapper', + 'src/python/pants/engine:native', 'src/python/pants/engine:parser', 'src/python/pants/engine:scheduler', 'src/python/pants/engine:storage', diff --git a/src/python/pants/bin/engine_initializer.py b/src/python/pants/bin/engine_initializer.py index 06e084819db..1a76ebcc8ed 100644 --- a/src/python/pants/bin/engine_initializer.py +++ b/src/python/pants/bin/engine_initializer.py @@ -21,9 +21,9 @@ PythonTestsAdaptor, RemoteSourcesAdaptor, ScalaLibraryAdaptor, TargetAdaptor) from pants.engine.mapper import AddressMapper +from pants.engine.native import Native from pants.engine.parser import SymbolTable from pants.engine.scheduler import LocalScheduler -from pants.engine.subsystem.native import Native from pants.init.options_initializer import OptionsInitializer from pants.option.options_bootstrapper import OptionsBootstrapper from pants.util.memo import memoized_method @@ -131,7 +131,8 @@ def setup_legacy_graph(pants_ignore_patterns, :param list exclude_target_regexps: A list of regular expressions for excluding targets. :param list subproject_roots: Paths that correspond with embedded build roots under the current build root. - :param bool include_trace_on_error: If True, when an error occurs, the error message will include the graph trace. + :param bool include_trace_on_error: If True, when an error occurs, the error message will + include the graph trace. :returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls). """ @@ -150,7 +151,7 @@ def setup_legacy_graph(pants_ignore_patterns, subproject_roots=subproject_roots) # Load the native backend. - native = native or Native.Factory.global_instance().create() + native = native or Native.create() # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. diff --git a/src/python/pants/bin/goal_runner.py b/src/python/pants/bin/goal_runner.py index 51b6ac5db8d..9f6a6ced8eb 100644 --- a/src/python/pants/bin/goal_runner.py +++ b/src/python/pants/bin/goal_runner.py @@ -13,11 +13,12 @@ from pants.base.workunit import WorkUnit, WorkUnitLabel from pants.bin.engine_initializer import EngineInitializer from pants.bin.repro import Reproducer +from pants.binaries.binary_util import BinaryUtil from pants.build_graph.build_file_address_mapper import BuildFileAddressMapper from pants.build_graph.build_file_parser import BuildFileParser from pants.build_graph.mutable_build_graph import MutableBuildGraph +from pants.engine.native import Native from pants.engine.round_engine import RoundEngine -from pants.engine.subsystem.native import Native from pants.goal.context import Context from pants.goal.goal import Goal from pants.goal.run_tracker import RunTracker @@ -94,7 +95,7 @@ def _init_graph(self, use_engine, pants_ignore_patterns, build_ignore_patterns, if graph_helper or use_engine: # The daemon may provide a `graph_helper`. If that's present, use it for graph construction. if not graph_helper: - native = Native.Factory.global_instance().create() + native = Native.create(self._global_options) native.set_panic_handler() graph_helper = EngineInitializer.setup_legacy_graph( pants_ignore_patterns, @@ -231,7 +232,7 @@ def subsystems(cls): Reproducer, RunTracker, Changed.Factory, - Native.Factory, + BinaryUtil.Factory, PantsDaemonLauncher.Factory, } diff --git a/src/python/pants/core_tasks/explain_options_task.py b/src/python/pants/core_tasks/explain_options_task.py index d3d62171fa4..3e25f493b70 100644 --- a/src/python/pants/core_tasks/explain_options_task.py +++ b/src/python/pants/core_tasks/explain_options_task.py @@ -10,6 +10,7 @@ from colors import black, blue, cyan, green, magenta, red, white from packaging.version import Version +from pants.option.arg_splitter import GLOBAL_SCOPE from pants.option.ranked_value import RankedValue from pants.task.console_task import ConsoleTask from pants.version import PANTS_SEMVER @@ -26,7 +27,7 @@ class ExplainOptionsTask(ConsoleTask): @classmethod def register_options(cls, register): super(ExplainOptionsTask, cls).register_options(register) - register('--scope', help='Only show options in this scope.') + register('--scope', help='Only show options in this scope. Use GLOBAL for global scope.') register('--name', help='Only show options with this name.') register('--rank', choices=RankedValue.get_names(), help='Only show options with at least this importance.') @@ -41,7 +42,8 @@ def register_options(cls, register): def _scope_filter(self, scope): pattern = self.get_options().scope - return not pattern or scope.startswith(pattern) + return (not pattern or scope.startswith(pattern) or + (pattern == 'GLOBAL' and scope == GLOBAL_SCOPE)) def _option_filter(self, option): pattern = self.get_options().name diff --git a/src/python/pants/engine/BUILD b/src/python/pants/engine/BUILD index a6f3a64c265..6bafac0f386 100644 --- a/src/python/pants/engine/BUILD +++ b/src/python/pants/engine/BUILD @@ -6,6 +6,9 @@ page( source='README.md', ) +# TODO: Get rid of all these microtargets. The entire package should be one target, +# or be split into multiple packages. + python_library( name = 'legacy_engine', sources = ['legacy_engine.py', 'round_engine.py', 'round_manager.py'], @@ -162,12 +165,12 @@ python_library( ':addressable', ':fs', ':isolated_process', + ':native', ':nodes', ':rules', 'src/python/pants/base:exceptions', 'src/python/pants/base:specs', 'src/python/pants/build_graph', - 'src/python/pants/engine/subsystem:native', 'src/python/pants/util:objects', ] ) @@ -182,3 +185,23 @@ python_library( 'src/python/pants/util:meta', ] ) + +python_library( + name='native', + sources=['native.py'], + dependencies=[ + ':native_engine_version', + '3rdparty/python:cffi', + '3rdparty/python:setuptools', + 'src/python/pants/binaries:binary_util', + 'src/python/pants/engine:storage', + 'src/python/pants/util:dirutil', + 'src/python/pants/util:memo', + 'src/python/pants/util:objects' + ], +) + +resources( + name='native_engine_version', + sources=['native_engine_version'] +) diff --git a/src/python/pants/engine/README.md b/src/python/pants/engine/README.md index 07eec768598..34d40a368ae 100644 --- a/src/python/pants/engine/README.md +++ b/src/python/pants/engine/README.md @@ -148,3 +148,14 @@ $ ./pants --native-engine-visualize-to=viz list some/example/directory: $ ls viz run.0.dot ``` + +## Native Engine + +The native engine is integrated into the pants codebase via `native.py` in +this directory along with `build-support/bin/native/bootstrap.sh` which ensures a +pants native engine library is built and available for linking. The glue is the +sha1 hash of the native engine source code used as its version by the `Native` +class. This hash is maintained by `build-support/bin/native/bootstrap.sh` and +output to the `native_engine_version` file in this directory. Any modification +to this resource file's location will need adjustments in +`build-support/bin/native/bootstrap.sh` to ensure the linking continues to work. diff --git a/src/python/pants/engine/subsystem/native.py b/src/python/pants/engine/native.py similarity index 94% rename from src/python/pants/engine/subsystem/native.py rename to src/python/pants/engine/native.py index 0c2c827bb9b..2bdc47eb5cc 100644 --- a/src/python/pants/engine/subsystem/native.py +++ b/src/python/pants/engine/native.py @@ -14,13 +14,10 @@ import traceback import cffi -import pkg_resources import six from pants.binaries.binary_util import BinaryUtil from pants.engine.storage import Storage -from pants.option.custom_types import dir_option -from pants.subsystem.subsystem import Subsystem from pants.util.dirutil import safe_mkdir from pants.util.memo import memoized_property from pants.util.objects import datatype @@ -258,7 +255,7 @@ def bootstrap_c_source(output_dir, module_name=NATIVE_ENGINE_MODULE): # Write a shell script to be sourced at build time that contains inherited CFLAGS. print('generating {}'.format(env_script)) with open(env_script, 'wb') as f: - f.write('export CFLAGS="{}"\n'.format(get_build_cflags())) + f.write(b'export CFLAGS="{}"\n'.format(get_build_cflags())) def _initialize_externs(ffi): @@ -553,35 +550,16 @@ def from_key(self, cdata): class Native(object): - """Encapsulates fetching a platform specific version of the native portion of the engine. - """ - - class Factory(Subsystem): - options_scope = 'native-engine' - - @classmethod - def subsystem_dependencies(cls): - return (BinaryUtil.Factory,) - - @staticmethod - def _default_native_engine_version(): - return pkg_resources.resource_string(__name__, 'native_engine_version').strip() - - @classmethod - def register_options(cls, register): - register('--version', advanced=True, default=cls._default_native_engine_version(), - help='Native engine version.') - register('--supportdir', advanced=True, default='bin/native-engine', - help='Find native engine binaries under this dir. Used as part of the path to ' - 'lookup the binary with --binary-util-baseurls and --pants-bootstrapdir.') - register('--visualize-to', default=None, type=dir_option, - help='A directory to write execution and rule graphs to as `dot` files. The contents ' - 'of the directory will be overwritten if any filenames collide.') - - def create(self): - binary_util = BinaryUtil.Factory.create() - options = self.get_options() - return Native(binary_util, options.version, options.supportdir, options.visualize_to) + """Encapsulates fetching a platform specific version of the native portion of the engine.""" + + @staticmethod + def create(options): + """:param options: Any object that provides access to bootstrap option values.""" + binary_util = BinaryUtil.Factory.create() + return Native(binary_util, + options.native_engine_version, + options.native_engine_supportdir, + options.native_engine_visualize_to) def __init__(self, binary_util, version, supportdir, visualize_to_dir): """ diff --git a/src/python/pants/engine/native_engine_version b/src/python/pants/engine/native_engine_version new file mode 100644 index 00000000000..0b6fb4a89f6 --- /dev/null +++ b/src/python/pants/engine/native_engine_version @@ -0,0 +1 @@ +7ffa912ffa6f260c22a1788c36974b3c4d3b635e diff --git a/src/python/pants/engine/scheduler.py b/src/python/pants/engine/scheduler.py index 21129337518..65a5a5d3451 100644 --- a/src/python/pants/engine/scheduler.py +++ b/src/python/pants/engine/scheduler.py @@ -18,12 +18,12 @@ from pants.engine.addressable import SubclassesOf from pants.engine.fs import FileContent, FilesContent, Path, PathGlobs, Snapshot from pants.engine.isolated_process import _Snapshots, create_snapshot_rules +from pants.engine.native import Function, TypeConstraint, TypeId from pants.engine.nodes import Return, State, Throw from pants.engine.rules import RuleIndex, SingletonRule, TaskRule from pants.engine.selectors import (Select, SelectDependencies, SelectProjection, SelectTransitive, SelectVariant, constraint_for) from pants.engine.struct import HasProducts, Variants -from pants.engine.subsystem.native import Function, TypeConstraint, TypeId from pants.util.contextutil import temporary_file_path from pants.util.objects import datatype @@ -319,7 +319,7 @@ def __init__(self, :param rules: A set of Rules which is used to compute values in the product graph. :param project_tree: An instance of ProjectTree for the current build root. :param work_dir: The pants work dir. - :param native: An instance of engine.subsystem.native.Native. + :param native: An instance of engine.native.Native. :param include_trace_on_error: Include the trace through the graph upon encountering errors. :type include_trace_on_error: bool :param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph diff --git a/src/python/pants/engine/subsystem/BUILD b/src/python/pants/engine/subsystem/BUILD deleted file mode 100644 index f3b40a56aeb..00000000000 --- a/src/python/pants/engine/subsystem/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -python_library( - name='native', - sources=['native.py'], - dependencies=[ - ':native_engine_version', - '3rdparty/python:cffi', - '3rdparty/python:setuptools', - 'src/python/pants/binaries:binary_util', - 'src/python/pants/engine:fs', - 'src/python/pants/engine:storage', - 'src/python/pants/subsystem:subsystem', - ], -) - -resources( - name='native_engine_version', - sources=['native_engine_version'] -) diff --git a/src/python/pants/engine/subsystem/README.md b/src/python/pants/engine/subsystem/README.md deleted file mode 100644 index 0e1aacf53b6..00000000000 --- a/src/python/pants/engine/subsystem/README.md +++ /dev/null @@ -1,8 +0,0 @@ -NB: The native engine is integrated into the pants codebase via `native.py` in -this directory along with `build-support/bootstrap_native` which ensures a -pants native engine library is built and available for linking. The glue is the -sha1 hash of the native engine source code used as its version by the `Native` -subsystem. This hash is maintained by `build-support/bootstrap_native` and -output to the `native_engine_version` file in this directory. Any modification -to this resource files location will need adjustments in -`build-support/bootstrap_native` to ensure the linking continues to work. diff --git a/src/python/pants/engine/subsystem/__init__.py b/src/python/pants/engine/subsystem/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/python/pants/engine/subsystem/native_engine_version b/src/python/pants/engine/subsystem/native_engine_version deleted file mode 100644 index 3399380d61d..00000000000 --- a/src/python/pants/engine/subsystem/native_engine_version +++ /dev/null @@ -1 +0,0 @@ -bfa7d02b7a02d0a578d76a27569f67604ea870da diff --git a/src/python/pants/init/BUILD b/src/python/pants/init/BUILD index 4e07a619823..12f81db0aea 100644 --- a/src/python/pants/init/BUILD +++ b/src/python/pants/init/BUILD @@ -11,10 +11,11 @@ python_library( 'src/python/pants:version', 'src/python/pants/base:build_environment', 'src/python/pants/base:exceptions', + 'src/python/pants/binaries:binary_util', 'src/python/pants/build_graph', 'src/python/pants/core_tasks', + 'src/python/pants/engine:native', 'src/python/pants/engine/legacy:change_calculator', - 'src/python/pants/engine/subsystem:native', 'src/python/pants/goal', 'src/python/pants/goal:run_tracker', 'src/python/pants/logging', diff --git a/src/python/pants/init/pants_daemon_launcher.py b/src/python/pants/init/pants_daemon_launcher.py index c9263a6213b..e80fc2c44d3 100644 --- a/src/python/pants/init/pants_daemon_launcher.py +++ b/src/python/pants/init/pants_daemon_launcher.py @@ -9,7 +9,8 @@ import os from pants.base.build_environment import get_buildroot -from pants.engine.subsystem.native import Native +from pants.binaries.binary_util import BinaryUtil +from pants.engine.native import Native from pants.init.target_roots import TargetRoots from pants.init.util import clean_global_runtime_state from pants.pantsd.pants_daemon import PantsDaemon @@ -47,7 +48,8 @@ def register_options(cls, register): @classmethod def subsystem_dependencies(cls): return super(PantsDaemonLauncher.Factory, - cls).subsystem_dependencies() + (WatchmanLauncher.Factory, Subprocess.Factory, Native.Factory) + cls).subsystem_dependencies() + (WatchmanLauncher.Factory, Subprocess.Factory, + BinaryUtil.Factory) def create(self, engine_initializer=None): """ @@ -57,59 +59,36 @@ def create(self, engine_initializer=None): build_root = get_buildroot() options = self.global_instance().get_options() return PantsDaemonLauncher(build_root=build_root, - pants_workdir=options.pants_workdir, engine_initializer=engine_initializer, - log_dir=options.log_dir, - log_level=options.level.upper(), - pailgun_host=options.pailgun_host, - pailgun_port=options.pailgun_port, - fs_event_workers=options.fs_event_workers, - pants_ignore_patterns=options.pants_ignore, - build_ignore_patterns=options.build_ignore, - exclude_target_regexp=options.exclude_target_regexp, - subproject_roots=options.subproject_roots) + options=options) def __init__(self, build_root, - pants_workdir, engine_initializer, - log_dir, - log_level, - pailgun_host, - pailgun_port, - fs_event_workers, - pants_ignore_patterns, - build_ignore_patterns, - exclude_target_regexp, - subproject_roots): + options): """ :param str build_root: The path of the build root. - :param str pants_workdir: The path of the pants workdir. :param class engine_initializer: The class representing the EngineInitializer. - :param str log_dir: The path for pantsd logs. - :param str log_level: The log level for pantsd logs (derived from the pants log level). - :param str pailgun_host: The bind address for the Pailgun server. - :param int pailgun_port: The bind port for the Pailgun server. - :param int fs_event_workers: The number of workers to use for processing the fs event queue. - :param list pants_ignore_patterns: A list of path ignore patterns for filesystem operations. - :param list build_ignore_patterns: A list of path ignore patterns for BUILD file parsing. - :param list exclude_target_regexp: A list of target exclude regexps. - :param list subproject_roots: A list of subproject roots. - """ self._build_root = build_root - self._pants_workdir = pants_workdir self._engine_initializer = engine_initializer - self._log_dir = log_dir - self._log_level = log_level - self._pailgun_host = pailgun_host - self._pailgun_port = pailgun_port - self._fs_event_workers = fs_event_workers - self._pants_ignore_patterns = pants_ignore_patterns - self._build_ignore_patterns = build_ignore_patterns - self._exclude_target_regexp = exclude_target_regexp - self._subproject_roots = subproject_roots - self._native = Native.Factory.global_instance().create() + + # The options we register directly. + self._pailgun_host = options.pailgun_host + self._pailgun_port = options.pailgun_port + self._log_dir = options.log_dir + self._fs_event_workers = options.fs_event_workers + + # Values derived from global options (which our scoped options inherit). + self._pants_workdir = options.pants_workdir + self._log_level = options.level.upper() + self._pants_ignore_patterns = options.pants_ignore + self._build_ignore_patterns = options.build_ignore + self._exclude_target_regexp = options.exclude_target_regexp + self._subproject_roots = options.subproject_roots + # Native.create() reads global options, which, thanks to inheritance, it can + # read them via our scoped options. + self._native = Native.create(options) # TODO(kwlzn): Thread filesystem path ignores here to Watchman's subscription registration. lock_location = os.path.join(self._build_root, '.pantsd.startup') @@ -118,8 +97,8 @@ def __init__(self, @testable_memoized_property def pantsd(self): - return PantsDaemon(self._build_root, self._pants_workdir, self._log_level, self._native, self._log_dir, - reset_func=clean_global_runtime_state) + return PantsDaemon(self._build_root, self._pants_workdir, self._log_level, self._native, + self._log_dir, reset_func=clean_global_runtime_state) @testable_memoized_property def watchman_launcher(self): diff --git a/src/python/pants/option/BUILD b/src/python/pants/option/BUILD index c0c1ca22ccc..d82ea5a3cde 100644 --- a/src/python/pants/option/BUILD +++ b/src/python/pants/option/BUILD @@ -4,6 +4,7 @@ python_library( dependencies=[ '3rdparty/python:ansicolors', + '3rdparty/python:setuptools', '3rdparty/python:six', '3rdparty/python/twitter/commons:twitter.common.collections', 'src/python/pants/base:build_environment', diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py index 1a63ab74ee2..c6262f76d4f 100644 --- a/src/python/pants/option/global_options.py +++ b/src/python/pants/option/global_options.py @@ -9,9 +9,12 @@ import os import sys +import pkg_resources + from pants.base.build_environment import (get_buildroot, get_default_pants_config_file, get_pants_cachedir, get_pants_configdir, pants_version) from pants.option.arg_splitter import GLOBAL_SCOPE +from pants.option.custom_types import dir_option from pants.option.optionable import Optionable from pants.option.scope import ScopeInfo @@ -131,6 +134,17 @@ def register_bootstrap_options(cls, register): register('--enable-v2-engine', advanced=True, type=bool, default=True, help='Enables use of the v2 engine.') + # These facilitate configuring the native engine. + register('--native-engine-version', advanced=True, + default=pkg_resources.resource_string('pants.engine', 'native_engine_version').strip(), + help='Native engine version.') + register('--native-engine-supportdir', advanced=True, default='bin/native-engine', + help='Find native engine binaries under this dir. Used as part of the path to ' + 'lookup the binary with --binary-util-baseurls and --pants-bootstrapdir.') + register('--native-engine-visualize-to', advanced=True, default=None, type=dir_option, + help='A directory to write execution and rule graphs to as `dot` files. The contents ' + 'of the directory will be overwritten if any filenames collide.') + @classmethod def register_options(cls, register): """Register options not tied to any particular task or subsystem.""" diff --git a/tests/python/pants_test/engine/BUILD b/tests/python/pants_test/engine/BUILD index 50a4e5b5a3b..67b7dd1e1a8 100644 --- a/tests/python/pants_test/engine/BUILD +++ b/tests/python/pants_test/engine/BUILD @@ -1,6 +1,22 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +python_library( + name = 'util', + sources = ['util.py'], + dependencies = [ + 'src/python/pants/binaries:binary_util', + 'src/python/pants/engine:addressable', + 'src/python/pants/engine:native', + 'src/python/pants/engine:parser', + 'src/python/pants/engine:rules', + 'src/python/pants/engine:scheduler', + 'src/python/pants/engine:struct', + 'tests/python/pants_test/option/util', + 'tests/python/pants_test/subsystem:subsystem_utils', + ], +) + python_library( name = 'engine_test_base', sources = ['base_engine_test.py'], @@ -22,15 +38,6 @@ python_tests( ], ) -python_tests( - name = 'util', - sources = ['util.py'], - dependencies = [ - 'src/python/pants/engine/subsystem:native', - 'tests/python/pants_test/subsystem:subsystem_utils', - ], -) - python_tests( name='test_round_engine', sources=['test_round_engine.py'], @@ -171,7 +178,6 @@ python_tests( '3rdparty/python/twitter/commons:twitter.common.collections', ':util', ':scheduler_test_base', - 'src/python/pants/engine/subsystem:native', 'src/python/pants/engine:build_files', 'src/python/pants/engine:rules', 'src/python/pants/engine:selectors', diff --git a/tests/python/pants_test/engine/examples/BUILD b/tests/python/pants_test/engine/examples/BUILD index 7862694413a..ba766a169d0 100644 --- a/tests/python/pants_test/engine/examples/BUILD +++ b/tests/python/pants_test/engine/examples/BUILD @@ -58,11 +58,10 @@ python_library( 'src/python/pants/base:cmd_line_spec_parser', 'src/python/pants/binaries:binary_util', 'src/python/pants/build_graph', - 'src/python/pants/engine/subsystem:native', 'src/python/pants/engine:scheduler', 'src/python/pants/util:contextutil', 'src/python/pants/util:desktop', - 'tests/python/pants_test/subsystem:subsystem_utils', + 'tests/python/pants_test/engine:util', ] ) diff --git a/tests/python/pants_test/engine/examples/visualizer.py b/tests/python/pants_test/engine/examples/visualizer.py index 89ce86c4d36..c3532bcf3c1 100644 --- a/tests/python/pants_test/engine/examples/visualizer.py +++ b/tests/python/pants_test/engine/examples/visualizer.py @@ -12,13 +12,13 @@ from pants.base.cmd_line_spec_parser import CmdLineSpecParser from pants.engine.fs import PathGlobs -from pants.engine.subsystem.native import Native from pants.util import desktop from pants.util.contextutil import temporary_file_path from pants_test.engine.examples.planners import setup_json_scheduler -from pants_test.subsystem.subsystem_util import subsystem_instance +from pants_test.engine.util import init_native +# TODO: These aren't tests themselves, so they should be under examples/ or testprojects/? def visualize_execution_graph(scheduler): with temporary_file_path(cleanup=False, suffix='.dot') as dot_file: scheduler.visualize_graph_to_file(dot_file) @@ -31,13 +31,13 @@ def visualize_execution_graph(scheduler): def visualize_build_request(build_root, goals, subjects): - with subsystem_instance(Native.Factory) as native_factory: - scheduler = setup_json_scheduler(build_root, native_factory.create()) + native = init_native() + scheduler = setup_json_scheduler(build_root, native) - execution_request = scheduler.build_request(goals, subjects) - # NB: Calls `schedule` independently of `execute`, in order to render a graph before validating it. - scheduler.schedule(execution_request) - visualize_execution_graph(scheduler) + execution_request = scheduler.build_request(goals, subjects) + # NB: Calls `schedule` independently of `execute`, in order to render a graph before validating it. + scheduler.schedule(execution_request) + visualize_execution_graph(scheduler) def pop_build_root_and_goals(description, args): @@ -49,7 +49,7 @@ def usage(error_message): sys.exit(1) if len(args) < 2: - usage('Must supply at least the build root path and one goal.') + usage('Must supply at least the build root path and one goal: {}'.format(description)) build_root = args.pop(0) @@ -67,7 +67,8 @@ def is_goal(arg): return os.path.sep not in arg def main_addresses(): - build_root, goals, args = pop_build_root_and_goals('[build root path] [goal]+ [address spec]*', sys.argv[1:]) + build_root, goals, args = pop_build_root_and_goals( + '[build root path] [goal]+ [address spec]*', sys.argv[1:]) cmd_line_spec_parser = CmdLineSpecParser(build_root) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in args] @@ -75,7 +76,8 @@ def main_addresses(): def main_filespecs(): - build_root, goals, args = pop_build_root_and_goals('[build root path] [filespecs]*', sys.argv[1:]) + build_root, goals, args = pop_build_root_and_goals( + '[build root path] [filespecs]*', sys.argv[1:]) # Create PathGlobs for each arg relative to the buildroot. path_globs = PathGlobs.create('', include=args, exclude=[]) diff --git a/tests/python/pants_test/engine/test_rules.py b/tests/python/pants_test/engine/test_rules.py index 99b3853c9ba..bcabad61489 100644 --- a/tests/python/pants_test/engine/test_rules.py +++ b/tests/python/pants_test/engine/test_rules.py @@ -15,10 +15,10 @@ from pants.engine.rules import RootRule, RuleIndex, SingletonRule, TaskRule from pants.engine.scheduler import WrappedNativeScheduler from pants.engine.selectors import Select, SelectDependencies, SelectProjection, SelectTransitive -from pants.engine.subsystem.native import Native from pants_test.engine.examples.parsers import JsonParser from pants_test.engine.examples.planners import Goal -from pants_test.engine.util import TargetTable, assert_equal_with_printing, create_native_scheduler +from pants_test.engine.util import (TargetTable, assert_equal_with_printing, + create_native_scheduler, init_native) class AGoal(Goal): @@ -736,7 +736,7 @@ def test_successful_when_one_field_type_is_unfulfillable(self): subgraph) def create_scheduler(self, rule_index): - native = Native.Factory.global_instance().create() + native = init_native() scheduler = WrappedNativeScheduler( native=native, build_root='/tmp', diff --git a/tests/python/pants_test/engine/util.py b/tests/python/pants_test/engine/util.py index 56b1d4ba052..87e2c126449 100644 --- a/tests/python/pants_test/engine/util.py +++ b/tests/python/pants_test/engine/util.py @@ -7,19 +7,22 @@ import re +from pants.binaries.binary_util import BinaryUtil from pants.engine.addressable import SubclassesOf, addressable_list +from pants.engine.native import Native from pants.engine.parser import SymbolTable from pants.engine.rules import RuleIndex from pants.engine.scheduler import WrappedNativeScheduler from pants.engine.struct import HasProducts, Struct -from pants.engine.subsystem.native import Native +from pants_test.option.util.fakes import create_options_for_optionables from pants_test.subsystem.subsystem_util import init_subsystem def init_native(): - """Initialize and return the `Native` subsystem.""" - init_subsystem(Native.Factory) - return Native.Factory.global_instance().create() + """Initialize and return a `Native` instance.""" + init_subsystem(BinaryUtil.Factory) + opts = create_options_for_optionables([]) + return Native.create(opts.for_global_scope()) def create_native_scheduler(rules): From 17b905c5c79c7964e4e6c5ac26ec50567c4e7c3f Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 8 Aug 2017 12:01:54 -0600 Subject: [PATCH 03/67] Simplify `_validate_target_representation_args`. (#4791) Add some tests for this logic as well. --- src/python/pants/build_graph/target.py | 17 +++++++++-------- tests/python/pants_test/build_graph/BUILD | 1 + .../pants_test/build_graph/test_target.py | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/python/pants/build_graph/target.py b/src/python/pants/build_graph/target.py index 25c94e43107..39d5d828031 100644 --- a/src/python/pants/build_graph/target.py +++ b/src/python/pants/build_graph/target.py @@ -642,19 +642,18 @@ def traversable_dependency_specs(self): @staticmethod def _validate_target_representation_args(kwargs, payload): - assert kwargs is not None or payload is not None, 'must provide either kwargs or payload' - assert not (kwargs is not None and payload is not None), 'may not provide both kwargs and payload' - assert not (kwargs and not isinstance(kwargs, dict)), ( + assert (kwargs is None) ^ (payload is None), 'must provide either kwargs or payload' + assert (kwargs is None) or isinstance(kwargs, dict), ( 'expected a `dict` object for kwargs, instead found a {}'.format(type(kwargs)) ) - assert not (payload and not isinstance(payload, Payload)), ( + assert (payload is None) or isinstance(payload, Payload), ( 'expected a `Payload` object for payload, instead found a {}'.format(type(payload)) ) @classmethod def compute_injectable_specs(cls, kwargs=None, payload=None): - """Given either pre-Target.__init__() kwargs or a post-Target.__init__() payload, compute the specs - to inject as non-dependencies in the same vein as the prior `traversable_specs`. + """Given either pre-Target.__init__() kwargs or a post-Target.__init__() payload, compute the + specs to inject as non-dependencies in the same vein as the prior `traversable_specs`. :API: public @@ -663,7 +662,8 @@ def compute_injectable_specs(cls, kwargs=None, payload=None): :yields: Spec strings representing dependencies of this target. """ cls._validate_target_representation_args(kwargs, payload) - # N.B. This pattern turns this method into a non-yielding generator, which is helpful for subclassing. + # N.B. This pattern turns this method into a non-yielding generator, which is helpful for + # subclassing. return yield @@ -685,7 +685,8 @@ def compute_dependency_specs(cls, kwargs=None, payload=None): :yields: Spec strings representing dependencies of this target. """ cls._validate_target_representation_args(kwargs, payload) - # N.B. This pattern turns this method into a non-yielding generator, which is helpful for subclassing. + # N.B. This pattern turns this method into a non-yielding generator, which is helpful for + # subclassing. return yield diff --git a/tests/python/pants_test/build_graph/BUILD b/tests/python/pants_test/build_graph/BUILD index 15b78c3db7e..0e0fe342e7e 100644 --- a/tests/python/pants_test/build_graph/BUILD +++ b/tests/python/pants_test/build_graph/BUILD @@ -121,6 +121,7 @@ python_tests( dependencies = [ 'src/python/pants/base:exceptions', 'src/python/pants/base:fingerprint_strategy', + 'src/python/pants/base:payload', 'src/python/pants/build_graph', 'tests/python/pants_test:base_test', 'tests/python/pants_test/subsystem:subsystem_utils', diff --git a/tests/python/pants_test/build_graph/test_target.py b/tests/python/pants_test/build_graph/test_target.py index 7e56f690c95..551ad632d5f 100644 --- a/tests/python/pants_test/build_graph/test_target.py +++ b/tests/python/pants_test/build_graph/test_target.py @@ -10,6 +10,7 @@ from pants.base.exceptions import TargetDefinitionException from pants.base.fingerprint_strategy import DefaultFingerprintStrategy +from pants.base.payload import Payload from pants.build_graph.address import Address from pants.build_graph.target import Target from pants_test.base_test import BaseTest @@ -59,6 +60,24 @@ def test_empty_traversable_properties(self): self.assertSequenceEqual([], list(target.traversable_specs)) self.assertSequenceEqual([], list(target.compute_dependency_specs(payload=target.payload))) + def test_validate_target_representation_args_invalid_exactly_one(self): + with self.assertRaises(AssertionError): + Target._validate_target_representation_args(None, None) + + with self.assertRaises(AssertionError): + Target._validate_target_representation_args({}, Payload()) + + def test_validate_target_representation_args_invalid_type(self): + with self.assertRaises(AssertionError): + Target._validate_target_representation_args(kwargs=Payload(), payload=None) + + with self.assertRaises(AssertionError): + Target._validate_target_representation_args(kwargs=None, payload={}) + + def test_validate_target_representation_args_valid(self): + Target._validate_target_representation_args(kwargs={}, payload=None) + Target._validate_target_representation_args(kwargs=None, payload=Payload()) + def test_illegal_kwargs(self): init_subsystem(Target.Arguments) with self.assertRaises(Target.Arguments.UnknownArgumentError) as cm: From de9f76cdceebbd86f3ca47c967a58c57411f0655 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 8 Aug 2017 12:20:09 -0600 Subject: [PATCH 04/67] Cleanup cpp targets. (#4793) Noticed working #4780. --- .../python/pants/contrib/cpp/targets/cpp_binary.py | 11 +++-------- .../pants/contrib/cpp/targets/cpp_library.py | 6 ------ .../python/pants/contrib/cpp/targets/cpp_target.py | 14 ++++++-------- 3 files changed, 9 insertions(+), 22 deletions(-) diff --git a/contrib/cpp/src/python/pants/contrib/cpp/targets/cpp_binary.py b/contrib/cpp/src/python/pants/contrib/cpp/targets/cpp_binary.py index 78ec8659a3e..eab8b45ecb9 100644 --- a/contrib/cpp/src/python/pants/contrib/cpp/targets/cpp_binary.py +++ b/contrib/cpp/src/python/pants/contrib/cpp/targets/cpp_binary.py @@ -14,19 +14,14 @@ class CppBinary(CppTarget): """A C++ binary.""" - def __init__(self, - libraries=None, - *args, - **kwargs): + def __init__(self, libraries=None, payload=None, **kwargs): """ :param libraries: Libraries that this target depends on that are not pants targets. For example, 'm' or 'rt' that are expected to be installed on the local system. :type libraries: List of libraries to link against. """ - payload = Payload() - payload.add_fields({ - 'libraries': PrimitiveField(libraries) - }) + payload = payload or Payload() + payload.add_field('libraries', PrimitiveField(libraries)) super(CppBinary, self).__init__(payload=payload, **kwargs) @property diff --git a/contrib/cpp/src/python/pants/contrib/cpp/targets/cpp_library.py b/contrib/cpp/src/python/pants/contrib/cpp/targets/cpp_library.py index 72aa35b2917..4333da088c2 100644 --- a/contrib/cpp/src/python/pants/contrib/cpp/targets/cpp_library.py +++ b/contrib/cpp/src/python/pants/contrib/cpp/targets/cpp_library.py @@ -10,9 +10,3 @@ class CppLibrary(CppTarget): """A statically linked C++ library.""" - - # TODO: public headers - def __init__(self, - *args, - **kwargs): - super(CppLibrary, self).__init__(*args, **kwargs) diff --git a/contrib/cpp/src/python/pants/contrib/cpp/targets/cpp_target.py b/contrib/cpp/src/python/pants/contrib/cpp/targets/cpp_target.py index dafbedfbf18..347b57a826a 100644 --- a/contrib/cpp/src/python/pants/contrib/cpp/targets/cpp_target.py +++ b/contrib/cpp/src/python/pants/contrib/cpp/targets/cpp_target.py @@ -14,14 +14,12 @@ class CppTarget(Target): def __init__(self, address=None, payload=None, sources=None, **kwargs): """ - :param sources: Source code files to build. Paths are relative to the BUILD - file's directory. - :type sources: ``Fileset`` (from globs or rglobs) or list of strings + :param sources: Source code files to build. Paths are relative to the BUILD file's directory. + :type sources: :class:`pants.source.wrapped_globs.FilesetWithSpec` (from globs or rglobs) or + list of strings """ payload = payload or Payload() - payload.add_fields({ - 'sources': self.create_sources_field(sources=sources, - sources_rel_path=address.spec_path, - key_arg='sources'), - }) + payload.add_field('sources', self.create_sources_field(sources=sources, + sources_rel_path=address.spec_path, + key_arg='sources')) super(CppTarget, self).__init__(address=address, payload=payload, **kwargs) From 309e0e047a2d6a40798f20e061447760fb0f0d44 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 8 Aug 2017 12:35:49 -0600 Subject: [PATCH 05/67] Deprecate unused `go_thrift_library.import_path`. (#4794) --- .../go/src/python/pants/contrib/go/targets/BUILD | 1 + .../contrib/go/targets/go_thrift_library.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/contrib/go/src/python/pants/contrib/go/targets/BUILD b/contrib/go/src/python/pants/contrib/go/targets/BUILD index 34283ba4e29..516b9ade022 100644 --- a/contrib/go/src/python/pants/contrib/go/targets/BUILD +++ b/contrib/go/src/python/pants/contrib/go/targets/BUILD @@ -1,6 +1,7 @@ python_library( dependencies=[ 'src/python/pants/base:build_environment', + 'src/python/pants/base:deprecated', 'src/python/pants/base:exceptions', 'src/python/pants/base:parse_context', 'src/python/pants/base:payload', diff --git a/contrib/go/src/python/pants/contrib/go/targets/go_thrift_library.py b/contrib/go/src/python/pants/contrib/go/targets/go_thrift_library.py index c2a1cee5e5e..3ad6dc02d1d 100644 --- a/contrib/go/src/python/pants/contrib/go/targets/go_thrift_library.py +++ b/contrib/go/src/python/pants/contrib/go/targets/go_thrift_library.py @@ -5,6 +5,7 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +from pants.base.deprecated import deprecated_conditional from pants.base.payload import Payload from pants.build_graph.target import Target @@ -23,15 +24,18 @@ def __init__(self, **kwargs): """ :param sources: thrift source files - :type sources: ``Fileset`` or list of strings. Paths are relative to the - BUILD file's directory. - :param import_path: Go code will import this + :type sources: :class:`pants.source.wrapped_globs.FilesetWithSpec` or list of strings. Paths + are relative to the BUILD file's directory. + :param import_path: Deprecated: unused. """ + deprecated_conditional(lambda: import_path is not None, + removal_version='1.6.0.dev0', + entity_description='import_path', + hint_message='Remove this unused `{}` parameter'.format(self.alias())) payload = payload or Payload() - payload.add_fields({ - 'sources': self.create_sources_field(sources, address.spec_path, key_arg='sources'), - }) + payload.add_field('sources', + self.create_sources_field(sources, address.spec_path, key_arg='sources')) super(GoThriftLibrary, self).__init__(payload=payload, address=address, **kwargs) From 40428ed53afd6b6f10ff2da76911ed22de266b43 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 8 Aug 2017 15:51:27 -0600 Subject: [PATCH 06/67] Fix has_sources. (#4792) Previously the default implementation failed to handle an extension of None even though it accepted the argument first class. This is fixed with test coverage added and Resources is fixed to delegate to the fixed implementation after performing its special check. --- src/python/pants/build_graph/resources.py | 24 ++++++++++---- src/python/pants/build_graph/target.py | 22 ++++++++----- src/python/pants/source/payload_fields.py | 20 +++++------- tests/python/pants_test/build_graph/BUILD | 11 +++++++ .../pants_test/build_graph/test_resources.py | 32 +++++++++++++++++++ .../pants_test/build_graph/test_target.py | 29 +++++++++++++++++ 6 files changed, 111 insertions(+), 27 deletions(-) create mode 100644 tests/python/pants_test/build_graph/test_resources.py diff --git a/src/python/pants/build_graph/resources.py b/src/python/pants/build_graph/resources.py index 05445738e69..9499bf66832 100644 --- a/src/python/pants/build_graph/resources.py +++ b/src/python/pants/build_graph/resources.py @@ -28,9 +28,8 @@ def __init__(self, address=None, payload=None, sources=None, **kwargs): """ :API: public - :param sources: Files to "include". Paths are relative to the - BUILD file's directory. - :type sources: ``Fileset`` or list of strings + :param sources: Files to "include". Paths are relative to the BUILD file's directory. + :type sources: :class:`pants.source.wrapped_globs.FilesetWithSpec` or list of strings """ payload = payload or Payload() payload.add_fields({ @@ -40,10 +39,21 @@ def __init__(self, address=None, payload=None, sources=None, **kwargs): super(Resources, self).__init__(address=address, payload=payload, **kwargs) def has_sources(self, extension=None): - """``Resources`` never own sources of any particular native type, like for example - ``JavaLibrary``. + """`Resources` targets never logically "own" sources of any particular type (extension). + + `JavaLibrary` targets, in contrast, logically "own" `.java` files and so target consumers + (tasks) may collect targets to operate on by checking `has_sources('.java')` instead of + performing a target type test. + + A `Resources` target may have `.java` sources - for example, a java compiler test might use + loose `.java` source files in the test tree as compiler inputs - but it does not logically + "own" `.java` sources like `JavaLibrary` and so any query of `has_sources` with an `extension` + will return `False` to prevent standard compilation by a `javac` task in this example. :API: public + + :param string extension: Suffix of filenames to test for. + :return: `True` if this target owns at least one source file and `extension` is `None`. + :rtype: bool """ - # TODO(John Sirois): track down the reason for this hack and kill or explain better. - return extension is None + return extension is None and super(Resources, self).has_sources() diff --git a/src/python/pants/build_graph/target.py b/src/python/pants/build_graph/target.py index 39d5d828031..80ef594f8ce 100644 --- a/src/python/pants/build_graph/target.py +++ b/src/python/pants/build_graph/target.py @@ -535,29 +535,35 @@ def invalidate_dependee(dependee): dependee.mark_transitive_invalidation_hash_dirty() self._build_graph.walk_transitive_dependee_graph([self.address], work=invalidate_dependee) - @property + @memoized_property def _sources_field(self): sources_field = self.payload.get_field('sources') if sources_field is not None: return sources_field return SourcesField(sources=FilesetWithSpec.empty(self.address.spec_path)) - def has_sources(self, extension=''): - """ + def has_sources(self, extension=None): + """Return `True` if this target owns sources; optionally of the given `extension`. + :API: public - :param string extension: suffix of filenames to test for - :return: True if the target contains sources that match the optional extension suffix + :param string extension: Optional suffix of filenames to test for. + :return: `True` if the target contains sources that match the optional extension suffix. :rtype: bool """ - return self._sources_field.has_sources(extension) + source_paths = self._sources_field.source_paths + if not source_paths: + return False + if not extension: + return True + return any(source.endswith(extension) for source in source_paths) def sources_relative_to_buildroot(self): """ :API: public """ if self.has_sources(): - return self.payload.sources.relative_to_buildroot() + return self._sources_field.relative_to_buildroot() else: return [] @@ -581,7 +587,7 @@ def sources_relative_to_target_base(self): """ :API: public """ - return self.payload.sources.sources + return self._sources_field.sources @property def derived_from(self): diff --git a/src/python/pants/source/payload_fields.py b/src/python/pants/source/payload_fields.py index de81651a2f7..0109efd3cb7 100644 --- a/src/python/pants/source/payload_fields.py +++ b/src/python/pants/source/payload_fields.py @@ -17,6 +17,13 @@ class SourcesField(PayloadField): """A PayloadField encapsulating specified sources.""" + @staticmethod + def _validate_sources(sources): + if not isinstance(sources, FilesetWithSpec): + raise ValueError('Expected a FilesetWithSpec. `sources` should be ' + 'instantiated via `create_sources_field`.') + return sources + def __init__(self, sources, ref_address=None): """ :param sources: FilesetWithSpec representing the underlying sources. @@ -54,14 +61,9 @@ def source_paths(self): @property def address(self): - """Returns the address this sources field refers to (used by some derived classses)""" + """Returns the address this sources field refers to (used by some derived classes)""" return self._ref_address - def has_sources(self, extension=None): - if not self.source_paths: - return False - return any(source.endswith(extension) for source in self.source_paths) - def relative_to_buildroot(self): """All sources joined with their relative paths.""" return list(self.sources.paths_from_buildroot_iter()) @@ -71,9 +73,3 @@ def _compute_fingerprint(self): hasher.update(self.rel_path) hasher.update(self.sources.files_hash) return hasher.hexdigest() - - def _validate_sources(self, sources): - if not isinstance(sources, FilesetWithSpec): - raise ValueError('Expected a FilesetWithSpec. `sources` should be ' - 'instantiated via `create_sources_field`.') - return sources diff --git a/tests/python/pants_test/build_graph/BUILD b/tests/python/pants_test/build_graph/BUILD index 0e0fe342e7e..3834e022326 100644 --- a/tests/python/pants_test/build_graph/BUILD +++ b/tests/python/pants_test/build_graph/BUILD @@ -123,7 +123,18 @@ python_tests( 'src/python/pants/base:fingerprint_strategy', 'src/python/pants/base:payload', 'src/python/pants/build_graph', + 'src/python/pants/source', 'tests/python/pants_test:base_test', 'tests/python/pants_test/subsystem:subsystem_utils', ] ) + +python_tests( + name = 'resources', + sources = ['test_resources.py'], + dependencies = [ + 'src/python/pants/build_graph', + 'src/python/pants/source', + 'tests/python/pants_test:base_test', + ] +) \ No newline at end of file diff --git a/tests/python/pants_test/build_graph/test_resources.py b/tests/python/pants_test/build_graph/test_resources.py new file mode 100644 index 00000000000..9f035743ab4 --- /dev/null +++ b/tests/python/pants_test/build_graph/test_resources.py @@ -0,0 +1,32 @@ +# coding=utf-8 +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.build_graph.resources import Resources +from pants.source.wrapped_globs import Globs +from pants_test.base_test import BaseTest + + +class ResourcesTest(BaseTest): + @staticmethod + def sources(rel_path, *globs): + return Globs.create_fileset_with_spec(rel_path, *globs) + + def test_has_sources(self): + self.create_files('resources', ['a.txt', 'B.java']) + + no_resources = self.make_target('resources:none', + Resources, + sources=self.sources('resources', '*.rs')) + self.assertFalse(no_resources.has_sources()) + self.assertFalse(no_resources.has_sources('.java')) + + resources = self.make_target('resources:some', + Resources, + sources=self.sources('resources', '*.java')) + self.assertTrue(resources.has_sources()) + self.assertEqual(['resources/B.java'], resources.sources_relative_to_buildroot()) + self.assertFalse(resources.has_sources('.java')) diff --git a/tests/python/pants_test/build_graph/test_target.py b/tests/python/pants_test/build_graph/test_target.py index 551ad632d5f..8251426b840 100644 --- a/tests/python/pants_test/build_graph/test_target.py +++ b/tests/python/pants_test/build_graph/test_target.py @@ -13,6 +13,7 @@ from pants.base.payload import Payload from pants.build_graph.address import Address from pants.build_graph.target import Target +from pants.source.wrapped_globs import Globs from pants_test.base_test import BaseTest from pants_test.subsystem.subsystem_util import init_subsystem @@ -202,3 +203,31 @@ def direct(self, target): target_hash = target_c.invalidation_hash(fingerprint_strategy=fingerprint_strategy) hash_value = '{}.{}'.format(target_hash, dep_hash) self.assertEqual(hash_value, target_c.transitive_invalidation_hash(fingerprint_strategy=fingerprint_strategy)) + + def test_has_sources(self): + class SourcesTarget(Target): + def __init__(me, sources, address=None, **kwargs): + payload = Payload() + payload.add_field('sources', me.create_sources_field(sources, + sources_rel_path=address.spec_path, + key_arg='sources')) + super(SourcesTarget, me).__init__(address=address, payload=payload, **kwargs) + + def sources(rel_path, *args): + return Globs.create_fileset_with_spec(rel_path, *args) + + self.create_file('foo/bar/a.txt', 'a_contents') + + txt_sources = self.make_target('foo/bar:txt', + SourcesTarget, + sources=sources('foo/bar', '*.txt')) + self.assertTrue(txt_sources.has_sources()) + self.assertTrue(txt_sources.has_sources('.txt')) + self.assertFalse(txt_sources.has_sources('.rs')) + + no_sources = self.make_target('foo/bar:none', + SourcesTarget, + sources=sources('foo/bar', '*.rs')) + self.assertFalse(no_sources.has_sources()) + self.assertFalse(no_sources.has_sources('.txt')) + self.assertFalse(no_sources.has_sources('.rs')) From b1d496f25d821520048e7f8aaa6a763e3fb0ba68 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Wed, 9 Aug 2017 01:23:57 -0600 Subject: [PATCH 07/67] Work around bintray outage. (#4801) --- contrib/node/examples/src/node/preinstalled-project/BUILD | 5 ++++- pants.ini | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/contrib/node/examples/src/node/preinstalled-project/BUILD b/contrib/node/examples/src/node/preinstalled-project/BUILD index 983ed148b7e..41845d54948 100644 --- a/contrib/node/examples/src/node/preinstalled-project/BUILD +++ b/contrib/node/examples/src/node/preinstalled-project/BUILD @@ -3,8 +3,11 @@ node_preinstalled_module( sources=globs('package.json', 'src/*.js', 'test/*.js'), + # TODO(John Sirois): Update https://github.com/pantsbuild/node-preinstalled-modules to have a + # `sync-s3.sh` in place of the current `sync-bintray.sh`. + # See: https://github.com/pantsbuild/pants/issues/4800 dependencies_archive_url= - 'https://dl.bintray.com/pantsbuild/node-preinstalled-modules/node_modules.tar.gz' + 'https://s3.amazonaws.com/node-preinstalled-modules.pantsbuild.org/node_modules.tar.gz' ) node_test( diff --git a/pants.ini b/pants.ini index f9530e6ae8a..62383505f13 100644 --- a/pants.ini +++ b/pants.ini @@ -300,3 +300,8 @@ timeout_default: 60 materialize: True remote: True fail_floating: True + +[binaries] +# TODO(John Sirois): Consider making this the default. +# See: https://github.com/pantsbuild/pants/issues/4800 +baseurls: ['https://s3.amazonaws.com/binaries.pantsbuild.org'] From 1d5b3322f8f2f18640e640d7f015814df4274a89 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Wed, 9 Aug 2017 02:34:47 -0600 Subject: [PATCH 08/67] Upgrade default go to 1.8.3. (#4799) Release notes are here: https://golang.org/doc/go1.8 --- .../src/python/pants/contrib/go/subsystems/go_distribution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/go/src/python/pants/contrib/go/subsystems/go_distribution.py b/contrib/go/src/python/pants/contrib/go/subsystems/go_distribution.py index dcbd859fb9a..db17febd516 100644 --- a/contrib/go/src/python/pants/contrib/go/subsystems/go_distribution.py +++ b/contrib/go/src/python/pants/contrib/go/subsystems/go_distribution.py @@ -32,7 +32,7 @@ def register_options(cls, register): register('--supportdir', advanced=True, default='bin/go', help='Find the go distributions under this dir. Used as part of the path to lookup ' 'the distribution with --binary-util-baseurls and --pants-bootstrapdir') - register('--version', advanced=True, default='1.8', fingerprint=True, + register('--version', advanced=True, default='1.8.3', fingerprint=True, help='Go distribution version. Used as part of the path to lookup the distribution ' 'with --binary-util-baseurls and --pants-bootstrapdir') From 14350e54d730cedf15016cd80da79b9be50800bc Mon Sep 17 00:00:00 2001 From: John Sirois Date: Wed, 9 Aug 2017 03:01:24 -0600 Subject: [PATCH 09/67] Introduce a loose `Files` target. (#4798) This will be used initially to allow test targets to declare dependencies on loose files that should be included in test chroots. An example of this use is included which lists the correct `filedeps` for an existing integration test. Follow-up changes will add support for processing these targets to `PytestRun` and `JunitRun`. --- .../pants/backend/python/python_chroot.py | 2 + src/python/pants/build_graph/files.py | 52 +++++++++++++++++++ src/python/pants/build_graph/register.py | 2 + src/python/pants/build_graph/resources.py | 39 +------------- .../python/pants_test/backend/jvm/tasks/BUILD | 7 +-- .../jvm/tasks/ivy_utils_resources/BUILD | 1 + tests/python/pants_test/build_graph/BUILD | 4 +- .../pants_test/build_graph/test_files.py | 28 ++++++++++ .../pants_test/build_graph/test_resources.py | 32 ------------ 9 files changed, 93 insertions(+), 74 deletions(-) create mode 100644 src/python/pants/build_graph/files.py create mode 100644 tests/python/pants_test/backend/jvm/tasks/ivy_utils_resources/BUILD create mode 100644 tests/python/pants_test/build_graph/test_files.py delete mode 100644 tests/python/pants_test/build_graph/test_resources.py diff --git a/src/python/pants/backend/python/python_chroot.py b/src/python/pants/backend/python/python_chroot.py index a08e2c97085..52a91cbe2d0 100644 --- a/src/python/pants/backend/python/python_chroot.py +++ b/src/python/pants/backend/python/python_chroot.py @@ -28,6 +28,7 @@ from pants.backend.python.targets.python_tests import PythonTests from pants.backend.python.thrift_builder import PythonThriftBuilder from pants.base.build_environment import get_buildroot +from pants.build_graph.files import Files from pants.build_graph.prep_command import PrepCommand from pants.build_graph.resources import Resources from pants.build_graph.target import Target @@ -40,6 +41,7 @@ class PythonChroot(object): _VALID_DEPENDENCIES = { + Files: 'files', PrepCommand: 'prep', PythonLibrary: 'libraries', PythonRequirementLibrary: 'reqs', diff --git a/src/python/pants/build_graph/files.py b/src/python/pants/build_graph/files.py new file mode 100644 index 00000000000..76750d4760f --- /dev/null +++ b/src/python/pants/build_graph/files.py @@ -0,0 +1,52 @@ +# coding=utf-8 +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.base.payload import Payload +from pants.build_graph.target import Target + + +class Files(Target): + """A collection of loose files.""" + + @classmethod + def alias(cls): + return 'files' + + def __init__(self, address=None, payload=None, sources=None, **kwargs): + """ + :API: public + + :param sources: Files to "include". Paths are relative to the BUILD file's directory. + :type sources: :class:`pants.source.wrapped_globs.FilesetWithSpec` or list of strings + """ + payload = payload or Payload() + payload.add_fields({ + 'sources': self.create_sources_field(sources, + sources_rel_path=address.spec_path, + key_arg='sources'), + }) + super(Files, self).__init__(address=address, payload=payload, **kwargs) + + def has_sources(self, extension=None): + """`Files` targets never logically "own" sources of any particular type (extension). + + `JavaLibrary` targets, in contrast, logically "own" `.java` files and so target consumers + (tasks) may collect targets to operate on by checking `has_sources('.java')` instead of + performing a target type test. + + A `Files` target may have `.java` sources - for example, a java compiler test might use + loose `.java` source files in the test tree as compiler inputs - but it does not logically + "own" `.java` sources like `JavaLibrary` and so any query of `has_sources` with an `extension` + will return `False` to prevent standard compilation by a `javac` task in this example. + + :API: public + + :param string extension: Suffix of filenames to test for. + :return: `True` if this target owns at least one source file and `extension` is `None`. + :rtype: bool + """ + return extension is None and super(Files, self).has_sources() diff --git a/src/python/pants/build_graph/register.py b/src/python/pants/build_graph/register.py index 38fa9f1ae4d..327dbce24aa 100644 --- a/src/python/pants/build_graph/register.py +++ b/src/python/pants/build_graph/register.py @@ -10,6 +10,7 @@ from pants.base.build_environment import get_buildroot, pants_version from pants.build_graph.aliased_target import AliasTargetFactory from pants.build_graph.build_file_aliases import BuildFileAliases +from pants.build_graph.files import Files from pants.build_graph.intransitive_dependency import (IntransitiveDependencyFactory, ProvidedDependencyFactory) from pants.build_graph.prep_command import PrepCommand @@ -39,6 +40,7 @@ def build_file_aliases(): return BuildFileAliases( targets={ 'alias': AliasTargetFactory(), + 'files': Files, 'prep_command': PrepCommand, 'resources': Resources, 'remote_sources': RemoteSources, diff --git a/src/python/pants/build_graph/resources.py b/src/python/pants/build_graph/resources.py index 9499bf66832..f8a4f819330 100644 --- a/src/python/pants/build_graph/resources.py +++ b/src/python/pants/build_graph/resources.py @@ -5,11 +5,10 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) -from pants.base.payload import Payload -from pants.build_graph.target import Target +from pants.build_graph.files import Files -class Resources(Target): +class Resources(Files): """Resource files. Looking for loose files in your JVM application bundle? Those are `bundle <#bundle>`_\s. @@ -23,37 +22,3 @@ class Resources(Target): @classmethod def alias(cls): return 'resources' - - def __init__(self, address=None, payload=None, sources=None, **kwargs): - """ - :API: public - - :param sources: Files to "include". Paths are relative to the BUILD file's directory. - :type sources: :class:`pants.source.wrapped_globs.FilesetWithSpec` or list of strings - """ - payload = payload or Payload() - payload.add_fields({ - 'sources': self.create_sources_field(sources, - sources_rel_path=address.spec_path, key_arg='sources'), - }) - super(Resources, self).__init__(address=address, payload=payload, **kwargs) - - def has_sources(self, extension=None): - """`Resources` targets never logically "own" sources of any particular type (extension). - - `JavaLibrary` targets, in contrast, logically "own" `.java` files and so target consumers - (tasks) may collect targets to operate on by checking `has_sources('.java')` instead of - performing a target type test. - - A `Resources` target may have `.java` sources - for example, a java compiler test might use - loose `.java` source files in the test tree as compiler inputs - but it does not logically - "own" `.java` sources like `JavaLibrary` and so any query of `has_sources` with an `extension` - will return `False` to prevent standard compilation by a `javac` task in this example. - - :API: public - - :param string extension: Suffix of filenames to test for. - :return: `True` if this target owns at least one source file and `extension` is `None`. - :rtype: bool - """ - return extension is None and super(Resources, self).has_sources() diff --git a/tests/python/pants_test/backend/jvm/tasks/BUILD b/tests/python/pants_test/backend/jvm/tasks/BUILD index 2b81a57c3e6..c35244bc87b 100644 --- a/tests/python/pants_test/backend/jvm/tasks/BUILD +++ b/tests/python/pants_test/backend/jvm/tasks/BUILD @@ -295,14 +295,15 @@ python_tests( name = 'ivy_utils', sources = ['test_ivy_utils.py'], dependencies = [ - 'src/python/pants/backend/jvm/subsystems:jar_dependency_management', - 'src/python/pants/backend/jvm/targets:jvm', 'src/python/pants/backend/jvm:ivy_utils', - 'src/python/pants/java/jar', 'src/python/pants/backend/jvm:plugin', + 'src/python/pants/backend/jvm/subsystems:jar_dependency_management', + 'src/python/pants/backend/jvm/targets:jvm', 'src/python/pants/build_graph', 'src/python/pants/ivy', + 'src/python/pants/java/jar', 'src/python/pants/util:contextutil', + 'tests/python/pants_test/backend/jvm/tasks/ivy_utils_resources', 'tests/python/pants_test:base_test', 'tests/python/pants_test/subsystem:subsystem_utils', ] diff --git a/tests/python/pants_test/backend/jvm/tasks/ivy_utils_resources/BUILD b/tests/python/pants_test/backend/jvm/tasks/ivy_utils_resources/BUILD new file mode 100644 index 00000000000..83921cfbecd --- /dev/null +++ b/tests/python/pants_test/backend/jvm/tasks/ivy_utils_resources/BUILD @@ -0,0 +1 @@ +files(sources=globs('*.xml')) diff --git a/tests/python/pants_test/build_graph/BUILD b/tests/python/pants_test/build_graph/BUILD index 3834e022326..acf0f3d538d 100644 --- a/tests/python/pants_test/build_graph/BUILD +++ b/tests/python/pants_test/build_graph/BUILD @@ -130,8 +130,8 @@ python_tests( ) python_tests( - name = 'resources', - sources = ['test_resources.py'], + name = 'files', + sources = ['test_files.py'], dependencies = [ 'src/python/pants/build_graph', 'src/python/pants/source', diff --git a/tests/python/pants_test/build_graph/test_files.py b/tests/python/pants_test/build_graph/test_files.py new file mode 100644 index 00000000000..6c6d1459415 --- /dev/null +++ b/tests/python/pants_test/build_graph/test_files.py @@ -0,0 +1,28 @@ +# coding=utf-8 +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.build_graph.files import Files +from pants.source.wrapped_globs import Globs +from pants_test.base_test import BaseTest + + +class FilesTest(BaseTest): + @staticmethod + def sources(rel_path, *globs): + return Globs.create_fileset_with_spec(rel_path, *globs) + + def test_has_sources(self): + self.create_files('files', ['a.txt', 'B.java']) + + no_files = self.make_target('files:none', Files, sources=self.sources('files', '*.rs')) + self.assertFalse(no_files.has_sources()) + self.assertFalse(no_files.has_sources('.java')) + + files = self.make_target('files:some', Files, sources=self.sources('files', '*.java')) + self.assertTrue(files.has_sources()) + self.assertEqual(['files/B.java'], files.sources_relative_to_buildroot()) + self.assertFalse(files.has_sources('.java')) diff --git a/tests/python/pants_test/build_graph/test_resources.py b/tests/python/pants_test/build_graph/test_resources.py deleted file mode 100644 index 9f035743ab4..00000000000 --- a/tests/python/pants_test/build_graph/test_resources.py +++ /dev/null @@ -1,32 +0,0 @@ -# coding=utf-8 -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import (absolute_import, division, generators, nested_scopes, print_function, - unicode_literals, with_statement) - -from pants.build_graph.resources import Resources -from pants.source.wrapped_globs import Globs -from pants_test.base_test import BaseTest - - -class ResourcesTest(BaseTest): - @staticmethod - def sources(rel_path, *globs): - return Globs.create_fileset_with_spec(rel_path, *globs) - - def test_has_sources(self): - self.create_files('resources', ['a.txt', 'B.java']) - - no_resources = self.make_target('resources:none', - Resources, - sources=self.sources('resources', '*.rs')) - self.assertFalse(no_resources.has_sources()) - self.assertFalse(no_resources.has_sources('.java')) - - resources = self.make_target('resources:some', - Resources, - sources=self.sources('resources', '*.java')) - self.assertTrue(resources.has_sources()) - self.assertEqual(['resources/B.java'], resources.sources_relative_to_buildroot()) - self.assertFalse(resources.has_sources('.java')) From 10855a80949784662937c4fbaabe5b3f682b0721 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Wed, 9 Aug 2017 11:46:03 -0600 Subject: [PATCH 10/67] Add support for publishing native-engine to s3. (#4804) --- .gitignore | 3 +- .travis.yml | 53 +++++++++++++------ ...y-manifest.sh => prepare-binary-deploy.sh} | 6 +++ 3 files changed, 44 insertions(+), 18 deletions(-) rename build-support/bin/native/{generate-bintray-manifest.sh => prepare-binary-deploy.sh} (93%) diff --git a/.gitignore b/.gitignore index f64dbc477ac..93b623b5016 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ GRTAGS GSYMS GTAGS -# Generated by build-support/bin/generate-bintray-manifest.sh for +# Generated by build-support/bin/prepare-binary-deploy.sh for # use by Travis-CI bintray deploys. /native-engine.bintray.json +/build-support/bin/native/s3-upload/ diff --git a/.travis.yml b/.travis.yml index 3c7a8799678..25e9c69f593 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,7 @@ matrix: env: - SHARD="OSX Bintray Builder" script: - - ./build-support/bin/native/generate-bintray-manifest.sh + - ./build-support/bin/native/prepare-binary-deploy.sh - os: linux dist: trusty @@ -90,7 +90,7 @@ matrix: "${DOCKER_IMAGE}" sh -c " export PATH=/opt/python/cp27-cp27mu/bin:/opt/rh/devtoolset-2/root/usr/bin:${PATH} ; - cd $TRAVIS_BUILD_DIR && ./build-support/bin/native/generate-bintray-manifest.sh + cd $TRAVIS_BUILD_DIR && ./build-support/bin/native/prepare-binary-deploy.sh " - os: linux @@ -419,21 +419,40 @@ matrix: deploy: # See: https://docs.travis-ci.com/user/deployment/bintray/ - provider: bintray - # NB: This is generated in after_success in each shard above. - file: ./native-engine.bintray.json - user: ${BINTRAY_USER} - key: ${BINTRAY_KEY} - dry-run: false - on: - condition: -f ./native-engine.bintray.json - # NB: Deploys are always tagged as part of the deploy process encoded in - # `build-support/bin/release.sh`, so this ensures we release an appropriate native engine binary - # for all releases. Unfortunately, CI only runs after the release tag hits origin and so there - # will be a lag of roughly 30 minutes until a pypi release has its paired native engine version - # available on bintray for OSX and linux. This trade-off (vs releasing for all branch builds), - # helps us use bintray in a friendly way. - tags: true + - provider: bintray + # NB: This is generated in after_success in each shard above. + file: ./native-engine.bintray.json + user: ${BINTRAY_USER} + key: ${BINTRAY_KEY} + dry-run: false + on: + condition: -f ./native-engine.bintray.json + # NB: Deploys are always tagged as part of the deploy process encoded in + # `build-support/bin/release.sh`, so this ensures we release an appropriate native engine binary + # for all releases. Unfortunately, CI only runs after the release tag hits origin and so there + # will be a lag of roughly 30 minutes until a pypi release has its paired native engine version + # available on bintray for OSX and linux. This trade-off (vs releasing for all branch builds), + # helps us use bintray in a friendly way. + tags: true + repo: pantsbuild/pants + # See: https://docs.travis-ci.com/user/deployment/s3/ + - provider: s3 + access_key_id: AKIAIQHTQI5E42SQBSNA + secret_access_key: + secure: RQVzsNfZL8AgsXdjZ67j2tWs5Tjl/FKpmE1fyVgldMbua/xhW8dzdFrtOeWjTPX4/+sJZ4U7/tZectBtWejmrXUJiZQKJwJBnsyYxysENTWOV80BEYyoz2RPr8HSVbMZ1ZHtUafzO3OqV1x+Pvgpg8FUeUfsy3TGUk0JREO90Q0= + bucket: binaries.pantsbuild.org + local-dir: build-support/bin/native/s3-upload + acl: public_read + on: + condition: -f ./native-engine.bintray.json + # NB: Deploys are always tagged as part of the deploy process encoded in + # `build-support/bin/release.sh`, so this ensures we release an appropriate native engine binary + # for all releases. Unfortunately, CI only runs after the release tag hits origin and so there + # will be a lag of roughly 30 minutes until a pypi release has its paired native engine version + # available on bintray for OSX and linux. This trade-off (vs releasing for all branch builds), + # helps us use bintray in a friendly way. + tags: true + repo: pantsbuild/pants # We accept the default travis-ci email author+committer notification # for now which is enabled even with no `notifications` config. diff --git a/build-support/bin/native/generate-bintray-manifest.sh b/build-support/bin/native/prepare-binary-deploy.sh similarity index 93% rename from build-support/bin/native/generate-bintray-manifest.sh rename to build-support/bin/native/prepare-binary-deploy.sh index ef82d34de9e..3bdc4c63827 100755 --- a/build-support/bin/native/generate-bintray-manifest.sh +++ b/build-support/bin/native/prepare-binary-deploy.sh @@ -105,3 +105,9 @@ cat << __EOF__ >> ${REPO_ROOT}/native-engine.bintray.json ] } __EOF__ + +# Prepare a chroot for s3 deploy of the artifacts created above. +S3_UPLOAD_ROOT=${REPO_ROOT}/build-support/bin/native/s3-upload +rm -rf ${S3_UPLOAD_ROOT} +mkdir -p ${S3_UPLOAD_ROOT}/bin +ln -s ${CACHE_ROOT}/bin/native-engine ${S3_UPLOAD_ROOT}/bin/native-engine From e9030aa401df6ecc9dc6250da0718d9c124e795c Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Wed, 9 Aug 2017 15:14:41 -0700 Subject: [PATCH 11/67] Add per-target zinc compile stats (#4790) ### Problem Visibility into the size and compile time of individual JVM targets is currently not great. While log parsing is possible, it's not scalable for this usecase. ### Solution Now that we have an API for recording target information, begin recording zinc compilation info. ### Result The `target_data` entry in the posted JSON stats contains information like the following: ``` 'src/scala/org/pantsbuild/zinc/analysis:analysis': { u'compile.zinc': { u'compile': { u'classpath_len': 54, 'sources_len': 3, 'incremental': False, 'time': 6.252178907394409 } } }, 'src/scala/org/pantsbuild/zinc/extractor:extractor': { u'compile.zinc': { u'compile': { u'classpath_len': 61, 'sources_len': 3, 'incremental': False, 'time': 6.599011182785034 } } }, ``` ... in addition to any data recorded by junit. --- .../jvm/tasks/jvm_compile/jvm_compile.py | 46 +++++++++++++------ src/python/pants/goal/run_tracker.py | 17 ++++++- .../pants/task/testrunner_task_mixin.py | 5 +- 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py b/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py index bdf5e98a12b..8e726bbab61 100644 --- a/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py +++ b/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py @@ -40,6 +40,7 @@ from pants.build_graph.target_scopes import Scopes from pants.goal.products import MultipleRootedProducts from pants.reporting.reporting_utils import items_to_report_element +from pants.util.contextutil import Timer from pants.util.dirutil import (fast_relpath, read_file, safe_delete, safe_mkdir, safe_rmtree, safe_walk) from pants.util.fileutil import create_size_estimators @@ -881,7 +882,8 @@ def work_for_vts(vts, ctx): self._confs)) upstream_analysis = dict(self._upstream_analysis(compile_contexts, cp_entries)) - if not should_compile_incrementally(vts, ctx): + is_incremental = should_compile_incrementally(vts, ctx) + if not is_incremental: # Purge existing analysis file in non-incremental mode. safe_delete(ctx.analysis_file) # Work around https://github.com/pantsbuild/pants/issues/3670 @@ -890,20 +892,26 @@ def work_for_vts(vts, ctx): tgt, = vts.targets fatal_warnings = self._compute_language_property(tgt, lambda x: x.fatal_warnings) zinc_file_manager = self._compute_language_property(tgt, lambda x: x.zinc_file_manager) - self._compile_vts(vts, - ctx.target, - ctx.sources, - ctx.analysis_file, - upstream_analysis, - cp_entries, - ctx.classes_dir, - log_file, - ctx.zinc_args_file, - progress_message, - tgt.platform, - fatal_warnings, - zinc_file_manager, - counter) + with Timer() as timer: + self._compile_vts(vts, + ctx.target, + ctx.sources, + ctx.analysis_file, + upstream_analysis, + cp_entries, + ctx.classes_dir, + log_file, + ctx.zinc_args_file, + progress_message, + tgt.platform, + fatal_warnings, + zinc_file_manager, + counter) + self._record_target_stats(tgt, + len(cp_entries), + len(ctx.sources), + timer.elapsed, + is_incremental) self._analysis_tools.relativize(ctx.analysis_file, ctx.portable_analysis_file) # Write any additional resources for this target to the target workdir. @@ -938,6 +946,14 @@ def work_for_vts(vts, ctx): on_failure=ivts.force_invalidate)) return jobs + def _record_target_stats(self, target, classpath_len, sources_len, compiletime, is_incremental): + def record(k, v): + self.context.run_tracker.report_target_info(self.options_scope, target, ['compile', k], v) + record('time', compiletime) + record('classpath_len', classpath_len) + record('sources_len', sources_len) + record('incremental', is_incremental) + def _collect_invalid_compile_dependencies(self, compile_target, invalid_target_set): # Collects all invalid dependencies that are not dependencies of other invalid dependencies # within the closure of compile_target. diff --git a/src/python/pants/goal/run_tracker.py b/src/python/pants/goal/run_tracker.py index deff3f76a52..5218b9ecc4f 100644 --- a/src/python/pants/goal/run_tracker.py +++ b/src/python/pants/goal/run_tracker.py @@ -18,9 +18,11 @@ import requests from pants.base.build_environment import get_pants_cachedir +from pants.base.deprecated import deprecated_conditional from pants.base.run_info import RunInfo from pants.base.worker_pool import SubprocPool, WorkerPool from pants.base.workunit import WorkUnit +from pants.build_graph.target import Target from pants.goal.aggregated_timings import AggregatedTimings from pants.goal.artifact_cache_stats import ArtifactCacheStats from pants.reporting.report import Report @@ -495,13 +497,24 @@ def report_target_info(self, scope, target, keys, val): an error. :param string scope: The scope for which we are reporting the information. - :param string target: The target for which we want to store information. + :param Target target: The target for which we want to store information. :param list of string keys: The keys that will be recursively nested and pointing to the information being stored. :param primitive val: The value of the information being stored. :API: public """ - new_key_list = [target, scope] + if isinstance(target, Target): + target_spec = target.address.spec + else: + deprecated_conditional( + lambda: True, + '1.6.0.dev0', + 'The `target=` argument to `report_target_info`', + 'Should pass a Target instance rather than a string.' + ) + target_spec = target + + new_key_list = [target_spec, scope] new_key_list += keys self._merge_list_of_keys_into_dict(self._target_to_data, new_key_list, val, 0) diff --git a/src/python/pants/task/testrunner_task_mixin.py b/src/python/pants/task/testrunner_task_mixin.py index 2d2faf9437d..d824d9cf969 100644 --- a/src/python/pants/task/testrunner_task_mixin.py +++ b/src/python/pants/task/testrunner_task_mixin.py @@ -91,10 +91,9 @@ def _report_test_info(self, scope, target, keys, test_info): :param primitive test_info: The information being stored. """ if target and scope: - address = target.address.spec target_type = target.type_alias - self.context.run_tracker.report_target_info('GLOBAL', address, ['target_type'], target_type) - self.context.run_tracker.report_target_info(scope, address, keys, test_info) + self.context.run_tracker.report_target_info('GLOBAL', target, ['target_type'], target_type) + self.context.run_tracker.report_target_info(scope, target, keys, test_info) @staticmethod def parse_test_info(xml_path, error_handler, additional_testcase_attributes=None): From bd8f49adc8847c8e78d94294439478dd232586ec Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Wed, 9 Aug 2017 16:59:47 -0700 Subject: [PATCH 12/67] Switch default binary-baseurls to s3 (#4806) ### Problem bintray had already been performing spottily, and recently began rate limiting us. ### Solution Now that all artifacts have been migrated to s3 (and all repositories that used to push to bintray have been updated) switch the default baseurls to s3. ### Result @jsirois did a ton of other work to fix this, but as far as I can tell, this finalizes the switch. Fixes #4800. --- contrib/node/examples/src/node/preinstalled-project/BUILD | 3 --- src/python/pants/binaries/binary_util.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/contrib/node/examples/src/node/preinstalled-project/BUILD b/contrib/node/examples/src/node/preinstalled-project/BUILD index 41845d54948..5376c2cc783 100644 --- a/contrib/node/examples/src/node/preinstalled-project/BUILD +++ b/contrib/node/examples/src/node/preinstalled-project/BUILD @@ -3,9 +3,6 @@ node_preinstalled_module( sources=globs('package.json', 'src/*.js', 'test/*.js'), - # TODO(John Sirois): Update https://github.com/pantsbuild/node-preinstalled-modules to have a - # `sync-s3.sh` in place of the current `sync-bintray.sh`. - # See: https://github.com/pantsbuild/pants/issues/4800 dependencies_archive_url= 'https://s3.amazonaws.com/node-preinstalled-modules.pantsbuild.org/node_modules.tar.gz' ) diff --git a/src/python/pants/binaries/binary_util.py b/src/python/pants/binaries/binary_util.py index 2625453ca7a..2a23226b461 100644 --- a/src/python/pants/binaries/binary_util.py +++ b/src/python/pants/binaries/binary_util.py @@ -55,7 +55,7 @@ class Factory(Subsystem): @classmethod def register_options(cls, register): register('--baseurls', type=list, advanced=True, - default=['https://dl.bintray.com/pantsbuild/bin/build-support'], + default=['https://s3.amazonaws.com/binaries.pantsbuild.org'], help='List of urls from which binary tools are downloaded. Urls are searched in ' 'order until the requested path is found.') register('--fetch-timeout-secs', type=int, default=30, advanced=True, From caab6b101f558273ad6898c71b17195e9bf11d27 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Thu, 10 Aug 2017 16:13:31 -0700 Subject: [PATCH 13/67] Improve performance by not re-fingerprinting codegen'd sources. (#4789) ### Problem See #4598 ### Solution Passing an EagerFilsetWithSpec as the sources to the synthetic target avoids re-fingerprinting the output sources which improves performance. ### Result Internally, we have seen an approximate improvement of 2-3 seconds without the daemon (from 33 seconds to 30), and a 1-2 second improvement (from ~12 seconds to 10 or 11) with the daemon. --- .../contrib/scrooge/tasks/test_scrooge_gen.py | 8 +++- src/python/pants/task/simple_codegen_task.py | 30 +++++++++++++-- .../codegen/antlr/java/test_antlr_java_gen.py | 4 +- .../task/test_simple_codegen_task.py | 38 ++++++++++++++++++- 4 files changed, 74 insertions(+), 6 deletions(-) diff --git a/contrib/scrooge/tests/python/pants_test/contrib/scrooge/tasks/test_scrooge_gen.py b/contrib/scrooge/tests/python/pants_test/contrib/scrooge/tasks/test_scrooge_gen.py index 485f2738da5..e986d2bc68d 100644 --- a/contrib/scrooge/tests/python/pants_test/contrib/scrooge/tasks/test_scrooge_gen.py +++ b/contrib/scrooge/tests/python/pants_test/contrib/scrooge/tasks/test_scrooge_gen.py @@ -15,6 +15,7 @@ from pants.base.exceptions import TargetDefinitionException, TaskError from pants.build_graph.build_file_aliases import BuildFileAliases from pants.goal.context import Context +from pants.source.wrapped_globs import EagerFilesetWithSpec from pants_test.jvm.nailgun_task_test_base import NailgunTaskTestBase from twitter.common.collections import OrderedSet @@ -163,10 +164,15 @@ def _test_help(self, language, library_type, compiler_args, sources, rpc_style = self.assertEquals(call_kwargs['target_type'], library_type) self.assertEquals(call_kwargs['dependencies'], OrderedSet()) self.assertEquals(call_kwargs['provides'], None) - self.assertEquals(call_kwargs['sources'], []) self.assertEquals(call_kwargs['derived_from'], target) self.assertEquals(call_kwargs['strict_deps'], True) self.assertEquals(call_kwargs['fatal_warnings'], False) + sources = call_kwargs['sources'] + if isinstance(sources, EagerFilesetWithSpec): + self.assertEquals(sources.files, []) + else: + self.assertEquals(sources, []) + finally: Context.add_new_target = saved_add_new_target diff --git a/src/python/pants/task/simple_codegen_task.py b/src/python/pants/task/simple_codegen_task.py index a38bba320ba..4e2efb46ad3 100644 --- a/src/python/pants/task/simple_codegen_task.py +++ b/src/python/pants/task/simple_codegen_task.py @@ -17,6 +17,7 @@ from pants.base.workunit import WorkUnitLabel from pants.build_graph.address import Address from pants.build_graph.address_lookup_error import AddressLookupError +from pants.source.wrapped_globs import EagerFilesetWithSpec, FilesetRelPathWrapper from pants.task.task import Task from pants.util.dirutil import fast_relpath, safe_delete, safe_walk @@ -194,7 +195,7 @@ def execute(self): self._handle_duplicate_sources(vt.target, vt.results_dir) vt.update() # And inject a synthetic target to represent it. - self._inject_synthetic_target(vt.target, vt.results_dir) + self._inject_synthetic_target(vt.target, vt.results_dir, vt.cache_key) @property def _copy_target_attributes(self): @@ -210,11 +211,30 @@ def synthetic_target_dir(self, target, target_workdir): """ return target_workdir - def _inject_synthetic_target(self, target, target_workdir): + def _create_sources_with_fingerprint(self, target_workdir, fingerprint, files): + """Create an EagerFilesetWithSpec to pass to the sources argument for synthetic target injection. + + We are creating and passing an EagerFilesetWithSpec to the synthetic target injection in the + hopes that it will save the time of having to refingerprint the sources. + + :param target_workdir: The directory containing the generated code for the target. + :param fingerprint: the fingerprint of the VersionedTarget with which the EagerFilesetWithSpec + will be created. + :param files: a list of exact paths to generated sources. + """ + results_dir_relpath = os.path.relpath(target_workdir, get_buildroot()) + filespec = FilesetRelPathWrapper.to_filespec( + [os.path.join(results_dir_relpath, file) for file in files]) + return EagerFilesetWithSpec(results_dir_relpath, filespec=filespec, + files=files, files_hash='{}.{}'.format(fingerprint.id, fingerprint.hash)) + + def _inject_synthetic_target(self, target, target_workdir, fingerprint): """Create, inject, and return a synthetic target for the given target and workdir. :param target: The target to inject a synthetic target for. :param target_workdir: The work directory containing the generated code for the target. + :param fingerprint: The fingerprint to create the synthetic target + with to avoid re-fingerprinting """ copied_attributes = {} for attribute in self._copy_target_attributes: @@ -222,11 +242,15 @@ def _inject_synthetic_target(self, target, target_workdir): target_workdir = self.synthetic_target_dir(target, target_workdir) + sources = list(self.find_sources(target, target_workdir)) + if fingerprint: + sources = self._create_sources_with_fingerprint(target_workdir, fingerprint, sources) + synthetic_target = self.context.add_new_target( address=self._get_synthetic_address(target, target_workdir), target_type=self.synthetic_target_type(target), dependencies=self.synthetic_target_extra_dependencies(target, target_workdir), - sources=list(self.find_sources(target, target_workdir)), + sources=sources, derived_from=target, **copied_attributes ) diff --git a/tests/python/pants_test/backend/codegen/antlr/java/test_antlr_java_gen.py b/tests/python/pants_test/backend/codegen/antlr/java/test_antlr_java_gen.py index 7248686ce22..906287fb4ff 100644 --- a/tests/python/pants_test/backend/codegen/antlr/java/test_antlr_java_gen.py +++ b/tests/python/pants_test/backend/codegen/antlr/java/test_antlr_java_gen.py @@ -16,6 +16,7 @@ from pants.backend.codegen.antlr.java.java_antlr_library import JavaAntlrLibrary from pants.base.exceptions import TaskError from pants.build_graph.build_file_aliases import BuildFileAliases +from pants.invalidation.build_invalidator import CacheKey from pants.util.dirutil import safe_mkdtemp from pants_test.jvm.nailgun_task_test_base import NailgunTaskTestBase @@ -77,7 +78,8 @@ def execute_antlr_test(self, expected_package, target_workdir_fun=None): # Generate code, then create a synthetic target. task.execute_codegen(target, target_workdir) - syn_target = task._inject_synthetic_target(target, target_workdir) + fingerprint = CacheKey("test", target.invalidation_hash()) + syn_target = task._inject_synthetic_target(target, target_workdir, fingerprint) actual_sources = [s for s in Fileset.rglobs('*.java', root=target_workdir)] expected_sources = syn_target.sources_relative_to_source_root() diff --git a/tests/python/pants_test/task/test_simple_codegen_task.py b/tests/python/pants_test/task/test_simple_codegen_task.py index e0f8f933268..7ce6d32f839 100644 --- a/tests/python/pants_test/task/test_simple_codegen_task.py +++ b/tests/python/pants_test/task/test_simple_codegen_task.py @@ -12,6 +12,7 @@ from pants.build_graph.build_file_aliases import BuildFileAliases from pants.build_graph.register import build_file_aliases as register_core from pants.build_graph.target import Target +from pants.invalidation.build_invalidator import CacheKey from pants.task.simple_codegen_task import SimpleCodegenTask from pants.util.dirutil import safe_mkdtemp from pants_test.tasks.task_test_base import TaskTestBase, ensure_cached @@ -213,7 +214,8 @@ def execute(): target_workdir = target_workdirs[target] task.execute_codegen(target, target_workdir) task._handle_duplicate_sources(target, target_workdir) - syn_targets.append(task._inject_synthetic_target(target, target_workdir)) + fingerprint = CacheKey("test", target.invalidation_hash()) + syn_targets.append(task._inject_synthetic_target(target, target_workdir, fingerprint)) if should_fail: # If we're expected to fail, validate the resulting message. @@ -266,3 +268,37 @@ def test_copy_target_attributes(self): task = self._create_dummy_task(target_roots=targets, strategy='isolated') task.execute() self.assertEqual('copythis', task.codegen_targets()[0].copied) + + def test_invalidation_of_generated_sources(self): + self.create_file('src/thrift/com/foo/one.thrift', 'initial state') + + t1 = self.make_target(spec='src/thrift/com/foo:one', + target_type=DummyLibrary, + sources=['one.thrift']) + + task1 = self._create_dummy_task(target_roots=t1, strategy='isolated') + task1.execute() + + gen_targets = [self.build_graph.get_target(syn_addr) + for syn_addr in self.build_graph.synthetic_addresses] + syn_targets_for_t1 = [target for target in gen_targets if target.derived_from == t1] + + t1_hash = syn_targets_for_t1[0].invalidation_hash() + + self.reset_build_graph() + + self.create_file('src/thrift/com/foo/one.thrift', 'changed state') + + t2 = self.make_target(spec='src/thrift/com/foo:one', + target_type=DummyLibrary, + sources=['one.thrift']) + + task2 = self._create_dummy_task(target_roots=t2, strategy='isolated') + task2.execute() + + gen_targets = [self.build_graph.get_target(syn_addr) + for syn_addr in self.build_graph.synthetic_addresses] + syn_targets_for_t2 = [target for target in gen_targets if target.derived_from == t2] + + t2_hash = syn_targets_for_t2[0].invalidation_hash() + self.assertNotEqual(t1_hash, t2_hash) From 02704ef859ba86fb65de4e1ae2020266a1b6ac67 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 11 Aug 2017 12:03:46 -0600 Subject: [PATCH 14/67] Kill custom binaries.baseurls. (#4809) The same baseurls are now hardcoded as the default. --- pants.ini | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pants.ini b/pants.ini index 62383505f13..dd56f04ecc7 100644 --- a/pants.ini +++ b/pants.ini @@ -301,7 +301,3 @@ materialize: True remote: True fail_floating: True -[binaries] -# TODO(John Sirois): Consider making this the default. -# See: https://github.com/pantsbuild/pants/issues/4800 -baseurls: ['https://s3.amazonaws.com/binaries.pantsbuild.org'] From 9f415be3fe7514d86b6a682616cdb66535ae9351 Mon Sep 17 00:00:00 2001 From: Nick Howard Date: Fri, 11 Aug 2017 12:04:45 -0600 Subject: [PATCH 15/67] [python-repl] pass env through to repl (#4808) ### Problem The python repl zeros out the environment variables. See https://github.com/pantsbuild/pants/issues/4795 ### Solution Pass a copy of the current environment to the repl pex when it is run. ### Result The environment is accessible from the repl --- src/python/pants/backend/python/tasks2/python_repl.py | 5 ++++- .../pants_test/backend/python/tasks2/test_python_repl.py | 9 ++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/python/pants/backend/python/tasks2/python_repl.py b/src/python/pants/backend/python/tasks2/python_repl.py index 9dc0adc0736..20265a30517 100644 --- a/src/python/pants/backend/python/tasks2/python_repl.py +++ b/src/python/pants/backend/python/tasks2/python_repl.py @@ -5,6 +5,8 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +import os + from pex.pex_info import PexInfo from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary @@ -49,5 +51,6 @@ def setup_repl_session(self, targets): # NB: **pex_run_kwargs is used by tests only. def launch_repl(self, pex, **pex_run_kwargs): - po = pex.run(blocking=False, **pex_run_kwargs) + env = pex_run_kwargs.pop('env', os.environ).copy() + po = pex.run(blocking=False, env=env, **pex_run_kwargs) po.wait() diff --git a/tests/python/pants_test/backend/python/tasks2/test_python_repl.py b/tests/python/pants_test/backend/python/tasks2/test_python_repl.py index 0d8b1248ea4..0a9925cb851 100644 --- a/tests/python/pants_test/backend/python/tasks2/test_python_repl.py +++ b/tests/python/pants_test/backend/python/tasks2/test_python_repl.py @@ -19,7 +19,7 @@ from pants.build_graph.build_file_aliases import BuildFileAliases from pants.build_graph.target import Target from pants.task.repl_task_mixin import ReplTaskMixin -from pants.util.contextutil import temporary_dir +from pants.util.contextutil import environment_as, temporary_dir from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase @@ -177,6 +177,13 @@ def test_non_python_targets(self): expected=[''], targets=[self.non_python_target]) + def test_access_to_env(self): + with environment_as(SOME_ENV_VAR='twelve'): + self.do_test_repl(code=['import os', + 'print(os.environ.get("SOME_ENV_VAR"))'], + expected=['twelve'], + targets=[self.library]) + def test_ipython(self): # IPython supports shelling out with a leading !, so indirectly test its presence by reading # the head of this very file. From a965d45170875a1884f522a637b24df90054b842 Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Fri, 11 Aug 2017 14:23:21 -0700 Subject: [PATCH 16/67] Zinc 1.0.0-RC3 memory and output improvements (#4807) ### Problem While working through #4477, it became obvious that: 1. the performance achieved when the analysis cache is missed is pretty abysmal 2. a `log4j` JMX error was being logged during startup 3. the `forkJava` and `javaHome` settings are redundant: the only reason to pass `javaHome` would be to trigger forking Java 4. a large amount of unnecessary classloading was happening due to #4744 ### Solution 1. Set the default `zinc.analysis.cache.limit` value to `Int.MaxValue`, which will allow users who want to cap the size to set it lower, but otherwise not lead to performance cliffs when a target has more dependencies than the current limit. 2. Implicitly disable log4j JMX usage by setting the `log4j2.disable.jmx` property if it is not already set. 3. Remove the `forkJava` setting, to prepare to use the `javaHome` setting explicitly in #4729 4. Enable usage of `ClassLoaderCache` ### Result Fixes #4744, and removes a few more blockers for #4729. --- .../org/pantsbuild/zinc/analysis/AnalysisMap.scala | 9 ++++++++- .../pantsbuild/zinc/compiler/CompilerCacheKey.scala | 5 +---- .../org/pantsbuild/zinc/compiler/CompilerUtils.scala | 11 ++++++++--- src/scala/org/pantsbuild/zinc/compiler/Main.scala | 4 ++++ src/scala/org/pantsbuild/zinc/compiler/Settings.scala | 2 -- src/scala/org/pantsbuild/zinc/util/Util.scala | 8 ++++++++ 6 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/scala/org/pantsbuild/zinc/analysis/AnalysisMap.scala b/src/scala/org/pantsbuild/zinc/analysis/AnalysisMap.scala index 42b82b4e829..f35bffc9fff 100644 --- a/src/scala/org/pantsbuild/zinc/analysis/AnalysisMap.scala +++ b/src/scala/org/pantsbuild/zinc/analysis/AnalysisMap.scala @@ -134,7 +134,14 @@ class AnalysisMap private[AnalysisMap] ( } object AnalysisMap { - private val analysisCacheLimit = Util.intProperty("zinc.analysis.cache.limit", 100) + // Because the analysis cache uses Weak references, bounding its size is generally + // counterproductive. + private val analysisCacheLimit = + Util.intProperty( + "zinc.analysis.cache.limit", + Int.MaxValue + ) + /** * Static cache for compile analyses. Values must be Options because in get() we don't yet * know if, on a cache miss, the underlying file will yield a valid Analysis. diff --git a/src/scala/org/pantsbuild/zinc/compiler/CompilerCacheKey.scala b/src/scala/org/pantsbuild/zinc/compiler/CompilerCacheKey.scala index 593baf898d5..494a7d1ccf0 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/CompilerCacheKey.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/CompilerCacheKey.scala @@ -22,7 +22,6 @@ case class CompilerCacheKey( compilerBridgeSrc: File, compilerInterface: File, javaHome: Option[File], - forkJava: Boolean, cacheDir: File) object CompilerCacheKey { @@ -40,7 +39,6 @@ object CompilerCacheKey { compilerBridgeSrc, compilerInterface, settings.javaHome, - settings.forkJava, settings.zincCacheDir ) } @@ -55,7 +53,6 @@ object CompilerCacheKey { compilerBridgeSrc: File, compilerInterface: File, javaHomeDir: Option[File], - forkJava: Boolean, cacheDir: File ): CompilerCacheKey = { val normalise: File => File = { _.getAbsoluteFile } @@ -65,6 +62,6 @@ object CompilerCacheKey { val compilerBridgeJar = normalise(compilerBridgeSrc) val compilerInterfaceJar = normalise(compilerInterface) val javaHome = javaHomeDir map normalise - CompilerCacheKey(compilerJar, libraryJar, extraJars, compilerBridgeJar, compilerInterfaceJar, javaHome, forkJava, cacheDir) + CompilerCacheKey(compilerJar, libraryJar, extraJars, compilerBridgeJar, compilerInterfaceJar, javaHome, cacheDir) } } diff --git a/src/scala/org/pantsbuild/zinc/compiler/CompilerUtils.scala b/src/scala/org/pantsbuild/zinc/compiler/CompilerUtils.scala index 89f34b62c50..6df50cfcbac 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/CompilerUtils.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/CompilerUtils.scala @@ -15,6 +15,7 @@ import sbt.internal.inc.{ javac, ZincUtil } +import sbt.internal.inc.classpath.ClassLoaderCache import sbt.io.Path import sbt.io.syntax._ import sbt.util.Logger @@ -59,6 +60,12 @@ object CompilerUtils { CompilerCache.createCacheFor(maxCompilers) } + /** + * Cache of classloaders: see https://github.com/pantsbuild/pants/issues/4744 + */ + private val classLoaderCache: Option[ClassLoaderCache] = + Some(new ClassLoaderCache(new URLClassLoader(Array()))) + /** * Get or create a zinc compiler based on compiler setup. */ @@ -86,9 +93,7 @@ object CompilerUtils { ZincCompilerUtil.constantBridgeProvider(instance, interfaceJar), ClasspathOptionsUtil.auto, _ => (), - // TODO: Should likely use the classloader cache here: - // see https://github.com/pantsbuild/pants/issues/4744 - None + classLoaderCache ) /** diff --git a/src/scala/org/pantsbuild/zinc/compiler/Main.scala b/src/scala/org/pantsbuild/zinc/compiler/Main.scala index a58a36b7e71..702271b03bb 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Main.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Main.scala @@ -52,6 +52,10 @@ object Main { def printVersion(): Unit = println("%s (%s) %s" format (Command, Description, versionString)) def mkLogger(settings: Settings) = { + // If someone has not explicitly enabled log4j2 JMX, disable it. + if (!Util.isSetProperty("log4j2.disable.jmx")) { + Util.setProperty("log4j2.disable.jmx", "true") + } val cl = ConsoleLogger( out = ConsoleOut.systemOut, diff --git a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala index c18d2208f8a..4842133779b 100644 --- a/src/scala/org/pantsbuild/zinc/compiler/Settings.scala +++ b/src/scala/org/pantsbuild/zinc/compiler/Settings.scala @@ -41,7 +41,6 @@ case class Settings( scala: ScalaLocation = ScalaLocation(), scalacOptions: Seq[String] = Seq.empty, javaHome: Option[File] = None, - forkJava: Boolean = false, _zincCacheDir: Option[File] = None, javaOnly: Boolean = false, javacOptions: Seq[String] = Seq.empty, @@ -255,7 +254,6 @@ object Settings extends OptionSet[Settings] { header("Java options:"), file( "-java-home", "directory", "Select javac home directory (and fork)", (s: Settings, f: File) => s.copy(javaHome = Some(f))), - boolean( "-fork-java", "Run java compiler in separate process", (s: Settings) => s.copy(forkJava = true)), string( "-compile-order", "order", "Compile order for Scala and Java sources", (s: Settings, o: String) => s.copy(compileOrder = compileOrder(o))), boolean( "-java-only", "Don't add scala library to classpath", (s: Settings) => s.copy(javaOnly = true)), prefix( "-C", "", "Pass option to javac", (s: Settings, o: String) => s.copy(javacOptions = s.javacOptions :+ o)), diff --git a/src/scala/org/pantsbuild/zinc/util/Util.scala b/src/scala/org/pantsbuild/zinc/util/Util.scala index 9389490e26e..6af05ae26d6 100644 --- a/src/scala/org/pantsbuild/zinc/util/Util.scala +++ b/src/scala/org/pantsbuild/zinc/util/Util.scala @@ -118,6 +118,14 @@ object Util { // Properties // + def setProperty(name: String, value: String): Unit = { + System.setProperty(name, value) + } + + def isSetProperty(name: String): Boolean = { + System.getProperty(name) ne null + } + /** * Create int from system property. */ From 7ad822ab839440d225e73da4a2a2f65bd7a42a01 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Fri, 11 Aug 2017 16:06:52 -0700 Subject: [PATCH 17/67] Prepare the 1.4.0.dev8 release (#4810) --- src/python/pants/notes/master.rst | 75 +++++++++++++++++++++++++++++++ src/python/pants/version.py | 2 +- 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/src/python/pants/notes/master.rst b/src/python/pants/notes/master.rst index 712e1a363ca..2e1a54c26be 100644 --- a/src/python/pants/notes/master.rst +++ b/src/python/pants/notes/master.rst @@ -4,6 +4,81 @@ Master Pre-Releases This document describes ``dev`` releases which occur weekly from master, and which do not undergo the vetting associated with ``stable`` releases. +1.4.0.dev8 (8/11/2017) +---------------------- + +New Features +~~~~~~~~~~~~ + +* Add support for junit (successful) test caching. (#4771) + `PR #4771 `_ + +API Changes +~~~~~~~~~~~ + +* Kill custom binaries.baseurls. (#4809) + `PR #4809 `_ + +* Partition and pass JVM options to scalafmt (#4774) + `PR #4774 `_ + +Bugfixes +~~~~~~~~ + +* [python-repl] pass env through to repl (#4808) + `PR #4808 `_ + +* Switch default binary-baseurls to s3 (#4806) + `PR #4806 `_ + +* Work around bintray outage. (#4801) + `PR #4801 `_ + +* Fix has_sources. (#4792) + `PR #4792 `_ + +Refactoring, Improvements, and Tooling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Zinc 1.0.0-RC3 memory and output improvements (#4807) + `PR #4807 `_ + +* Improve performance by not re-fingerprinting codegen'd sources. (#4789) + `PR #4789 `_ + +* Add per-target zinc compile stats (#4790) + `PR #4790 `_ + +* Add support for publishing native-engine to s3. (#4804) + `PR #4804 `_ + +* Introduce a loose `Files` target. (#4798) + `PR #4798 `_ + +* Upgrade default go to 1.8.3. (#4799) + `PR #4799 `_ + +* Deprecate unused `go_thrift_library.import_path`. (#4794) + `PR #4794 `_ + +* Cleanup cpp targets. (#4793) + `PR #4793 `_ + +* Simplify `_validate_target_representation_args`. (#4791) + `PR #4791 `_ + +* Init the native engine from bootstrap options. (#4787) + `PR #4787 `_ + +* [pantsd] Add faulthandler support for stacktrace dumps. (#4784) + `PR #4784 `_ + +* Cleanup CI deprecation warnings. (#4781) + `PR #4781 `_ + +* Kill `-XX:-UseSplitVerifier`. (#4777) + `PR #4777 `_ + 1.4.0.dev7 (7/28/2017) ---------------------- diff --git a/src/python/pants/version.py b/src/python/pants/version.py index 914ce767877..504f0de332f 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -8,6 +8,6 @@ from packaging.version import Version -VERSION = '1.4.0.dev7' +VERSION = '1.4.0.dev8' PANTS_SEMVER = Version(VERSION) From c7c06583df9d6337de02f7ea325d6448ebcc1096 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Mon, 14 Aug 2017 11:52:08 -0700 Subject: [PATCH 18/67] Fix the S3 upload in the travis deploy. (#4813) - Without the `skip_cleanup: true` stanza travis does a `git stash --all` before deploying, which removes the `build-support/bin/native/s3-upload` dir. - Also fixed spelling of `local-dir` to `local_dir`. Travis apparently normalizes both to the latter, but we may as well match what appears in the travis documentation, to reduce confusion among future readers. - Also added some comments/TODOs, for things that weren't clear to me as a first-time reader of this code. --- .travis.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 25e9c69f593..72cfef2156f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -419,6 +419,8 @@ matrix: deploy: # See: https://docs.travis-ci.com/user/deployment/bintray/ + # TODO: Do we still need to deploy to bintray? If not we can remove this and also + # the code that generates ./native-engine.bintray.json. - provider: bintray # NB: This is generated in after_success in each shard above. file: ./native-engine.bintray.json @@ -441,16 +443,18 @@ deploy: secret_access_key: secure: RQVzsNfZL8AgsXdjZ67j2tWs5Tjl/FKpmE1fyVgldMbua/xhW8dzdFrtOeWjTPX4/+sJZ4U7/tZectBtWejmrXUJiZQKJwJBnsyYxysENTWOV80BEYyoz2RPr8HSVbMZ1ZHtUafzO3OqV1x+Pvgpg8FUeUfsy3TGUk0JREO90Q0= bucket: binaries.pantsbuild.org - local-dir: build-support/bin/native/s3-upload + local_dir: build-support/bin/native/s3-upload + skip_cleanup: true # Otherwise travis will stash build-support/bin/native/s3-upload and the deploy will fail. acl: public_read on: + # TODO: Do we need this condition? If so, document why, as it superficially appears to be irrelevant. condition: -f ./native-engine.bintray.json # NB: Deploys are always tagged as part of the deploy process encoded in # `build-support/bin/release.sh`, so this ensures we release an appropriate native engine binary # for all releases. Unfortunately, CI only runs after the release tag hits origin and so there # will be a lag of roughly 30 minutes until a pypi release has its paired native engine version - # available on bintray for OSX and linux. This trade-off (vs releasing for all branch builds), - # helps us use bintray in a friendly way. + # available on s3 for OSX and linux. This trade-off (vs releasing for all branch builds), + # helps us use s3 in a friendly way. tags: true repo: pantsbuild/pants From 9d21c9695e91b92a623cd27d42d3810133fb45a1 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 15 Aug 2017 12:27:13 -0700 Subject: [PATCH 19/67] Fix s3 deploy to use copies instead of a symlink. (#4814) In addition, cleanup unused cruft surrounding bintray deploys and re-enable native engine binary deploys for all green master commits. The latter should be sane since s3 storage is cheap, we'll only store changed native engine binaries in s3 (so noop on many master commits) and transfers (downloads) will only happen for released versions of pants as before this change. --- .gitignore | 4 +- .travis.yml | 65 +++------- .../bin/check_native_engine_version.sh | 12 ++ build-support/bin/native/bootstrap.sh | 34 ++++-- .../bin/native/prepare-binary-deploy.sh | 114 ++++++------------ build-support/bin/native/utils.sh | 13 +- build-support/bin/release.sh | 35 ++++-- pants | 13 ++ 8 files changed, 127 insertions(+), 163 deletions(-) diff --git a/.gitignore b/.gitignore index 93b623b5016..2390c7665c0 100644 --- a/.gitignore +++ b/.gitignore @@ -51,7 +51,5 @@ GRTAGS GSYMS GTAGS -# Generated by build-support/bin/prepare-binary-deploy.sh for -# use by Travis-CI bintray deploys. -/native-engine.bintray.json +# Generated by build-support/bin/prepare-binary-deploy.sh for use by Travis-CI binary deploys. /build-support/bin/native/s3-upload/ diff --git a/.travis.yml b/.travis.yml index 72cfef2156f..546d4e94cc8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,17 +3,6 @@ env: - PANTS_CONFIG_FILES="pants.travis-ci.ini" - ANDROID_SDK_INSTALL_LOCATION="${HOME}/opt/android-sdk-install" - ANDROID_HOME="$ANDROID_SDK_INSTALL_LOCATION/android-sdk-linux" - # Credentials for OSX syncing: GH_USER, GH_EMAIL, GH_TOKEN - # These are encrypted with a public key for our repo that only - # Travis-CI has the private key for. We are trusting Travis-CI - # here but no-one else. - # - # See: http://docs.travis-ci.com/user/encryption-keys/ - - secure: VvwbndU++a2/iNAjk9cd67ATiipDwqcKnxDR4/J2Ik3GH10wHEDUhJ1+MK4WLhedfaOakDOEmarZQS3GwtgvCHO3knpTJuJc8d/bCfZovYuSqdi//BEv4dS7hDt6tQeJfkbBjG0T4yNjPJ3W9R9KDWCy/vj2CUm90BGg2CmxUbg= - # User for bintray deploys: BINTRAY_USER - - secure: eXGuKvbp297wi/z74jFqGyxzRDCFic9HUb0z2UYDXDmobErILZdgT0KCvqeyAx7QX/JQSp5oQAQNisE8RLrain5lXAIa2ZuswTPsh6yXGmFgwdx/X+Am7CPO27b0P5OxiJAN3kfYglN7qY+opRM1jud4anzEaTJnm7jENFfDXwk= - # Key for bintray deploys: BINTRAY_KEY - - secure: qid/ot1XIWOpNaN+RhgdJq8IEajcpHI5EFvy2ywkYHJO2hKYawyX2M4gFd0Vq8+xmeGB4MUmpPW8D8gijLi5JB+0aZ3+5JHs5r9NWkK7HVMpVnok3CywknzXBgeo+UoEQv9ugYvRr1Sm9Dj9IezAhM0tw1uS95Ap+JLbnWFB830= before_cache: # Ensure permissions to do the below removals, which happen with or without caching enabled. @@ -58,7 +47,7 @@ matrix: - os: osx language: generic env: - - SHARD="OSX Bintray Builder" + - SHARD="OSX Native Engine Binary Builder" script: - ./build-support/bin/native/prepare-binary-deploy.sh @@ -73,7 +62,7 @@ matrix: language: python python: "2.7.13" env: - - SHARD="Linux Bintray Builder" + - SHARD="Linux Native Engine Binary Builder" # Use the standard python manylinux image for ideal binary compatibility. - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" before_install: @@ -418,45 +407,19 @@ matrix: - ./build-support/bin/ci.sh -x -fkmsrjlpnt -i 6/7 "${SHARD}" deploy: - # See: https://docs.travis-ci.com/user/deployment/bintray/ - # TODO: Do we still need to deploy to bintray? If not we can remove this and also - # the code that generates ./native-engine.bintray.json. - - provider: bintray - # NB: This is generated in after_success in each shard above. - file: ./native-engine.bintray.json - user: ${BINTRAY_USER} - key: ${BINTRAY_KEY} - dry-run: false - on: - condition: -f ./native-engine.bintray.json - # NB: Deploys are always tagged as part of the deploy process encoded in - # `build-support/bin/release.sh`, so this ensures we release an appropriate native engine binary - # for all releases. Unfortunately, CI only runs after the release tag hits origin and so there - # will be a lag of roughly 30 minutes until a pypi release has its paired native engine version - # available on bintray for OSX and linux. This trade-off (vs releasing for all branch builds), - # helps us use bintray in a friendly way. - tags: true - repo: pantsbuild/pants # See: https://docs.travis-ci.com/user/deployment/s3/ - - provider: s3 - access_key_id: AKIAIQHTQI5E42SQBSNA - secret_access_key: - secure: RQVzsNfZL8AgsXdjZ67j2tWs5Tjl/FKpmE1fyVgldMbua/xhW8dzdFrtOeWjTPX4/+sJZ4U7/tZectBtWejmrXUJiZQKJwJBnsyYxysENTWOV80BEYyoz2RPr8HSVbMZ1ZHtUafzO3OqV1x+Pvgpg8FUeUfsy3TGUk0JREO90Q0= - bucket: binaries.pantsbuild.org - local_dir: build-support/bin/native/s3-upload - skip_cleanup: true # Otherwise travis will stash build-support/bin/native/s3-upload and the deploy will fail. - acl: public_read - on: - # TODO: Do we need this condition? If so, document why, as it superficially appears to be irrelevant. - condition: -f ./native-engine.bintray.json - # NB: Deploys are always tagged as part of the deploy process encoded in - # `build-support/bin/release.sh`, so this ensures we release an appropriate native engine binary - # for all releases. Unfortunately, CI only runs after the release tag hits origin and so there - # will be a lag of roughly 30 minutes until a pypi release has its paired native engine version - # available on s3 for OSX and linux. This trade-off (vs releasing for all branch builds), - # helps us use s3 in a friendly way. - tags: true - repo: pantsbuild/pants + provider: s3 + access_key_id: AKIAIQHTQI5E42SQBSNA + secret_access_key: + secure: RQVzsNfZL8AgsXdjZ67j2tWs5Tjl/FKpmE1fyVgldMbua/xhW8dzdFrtOeWjTPX4/+sJZ4U7/tZectBtWejmrXUJiZQKJwJBnsyYxysENTWOV80BEYyoz2RPr8HSVbMZ1ZHtUafzO3OqV1x+Pvgpg8FUeUfsy3TGUk0JREO90Q0= + bucket: binaries.pantsbuild.org + local_dir: build-support/bin/native/s3-upload + # Otherwise travis will stash build-support/bin/native/s3-upload and the deploy will fail. + skip_cleanup: true + acl: public_read + on: + branch: master + repo: pantsbuild/pants # We accept the default travis-ci email author+committer notification # for now which is enabled even with no `notifications` config. diff --git a/build-support/bin/check_native_engine_version.sh b/build-support/bin/check_native_engine_version.sh index 4917a648bf4..c41e8e12d29 100755 --- a/build-support/bin/check_native_engine_version.sh +++ b/build-support/bin/check_native_engine_version.sh @@ -4,6 +4,18 @@ set -e REPO_ROOT="$(cd $(dirname "${BASH_SOURCE[0]}") && cd ../.. && pwd -P)" +# Defines: +# + CACHE_ROOT: The pants cache root dir. +# + NATIVE_ENGINE_CACHE_DIR: The native engine binary root cache directory. +# + NATIVE_ENGINE_CACHE_TARGET_DIR: The directory containing all versions of the native engine for +# the current OS. +# + NATIVE_ENGINE_BINARY: The basename of the native engine binary for the current OS. +# + NATIVE_ENGINE_VERSION_RESOURCE: The path of the resource file containing the native engine +# version hash. +# Exposes: +# + calculate_current_hash: Calculates the current native engine version hash and echoes it to +# stdout. +# + bootstrap_native_code: Builds target-specific native engine binaries. source "${REPO_ROOT}/build-support/bin/native/bootstrap.sh" readonly actual_native_engine_version="$(calculate_current_hash)" diff --git a/build-support/bin/native/bootstrap.sh b/build-support/bin/native/bootstrap.sh index 1d2ec0520bb..6d99c4a7f63 100644 --- a/build-support/bin/native/bootstrap.sh +++ b/build-support/bin/native/bootstrap.sh @@ -1,16 +1,24 @@ #!/usr/bin/env bash # Defines: -# + CACHE_TARGET_DIR: The directory containing all versions of the native engine for the current OS. +# + CACHE_ROOT: The pants cache root dir. +# + NATIVE_ENGINE_CACHE_DIR: The native engine binary root cache directory. +# + NATIVE_ENGINE_CACHE_TARGET_DIR: The directory containing all versions of the native engine for +# the current OS. +# + NATIVE_ENGINE_BINARY: The basename of the native engine binary for the current OS. +# + NATIVE_ENGINE_VERSION_RESOURCE: The path of the resource file containing the native engine +# version hash. # Exposes: -# + build_native_code: Builds target-specific native engine binaries. +# + calculate_current_hash: Calculates the current native engine version hash and echoes it to +# stdout. +# + bootstrap_native_code: Builds target-specific native engine binaries. REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd ../../.. && pwd -P) source ${REPO_ROOT}/build-support/common.sh # Defines: # + RUST_OSX_MIN_VERSION: The minimum minor version of OSX supported by Rust; eg 7 for OSX 10.7. -# + OSX_MAX_VERSION: The current latest OSX minor version; eg 12 for OSX Sierra 10.12 +# + OSX_MAX_VERSION: The current latest OSX minor version; eg 12 for OSX Sierra 10.12. # + LIB_EXTENSION: The extension of native libraries. # + KERNEL: The lower-cased name of the kernel as reported by uname. # + OS_NAME: The name of the OS as seen by pants. @@ -18,6 +26,7 @@ source ${REPO_ROOT}/build-support/common.sh # Exposes: # + get_native_engine_version: Echoes the current native engine version. # + get_rust_osx_versions: Produces the osx minor versions supported by Rust one per line. +# + get_rust_osx_ids: Produces the BinaryUtil osx os id paths supported by rust, one per line. # + get_rust_os_ids: Produces the BinaryUtil os id paths supported by rust, one per line. source ${REPO_ROOT}/build-support/bin/native/utils.sh @@ -35,7 +44,8 @@ case "$MODE" in esac readonly CACHE_ROOT=${XDG_CACHE_HOME:-$HOME/.cache}/pants -readonly CACHE_TARGET_DIR=${CACHE_ROOT}/bin/native-engine/${OS_ID} +readonly NATIVE_ENGINE_CACHE_DIR=${CACHE_ROOT}/bin/native-engine +readonly NATIVE_ENGINE_CACHE_TARGET_DIR=${NATIVE_ENGINE_CACHE_DIR}/${OS_ID} function calculate_current_hash() { # Cached and unstaged files, with ignored files excluded. @@ -50,12 +60,12 @@ function calculate_current_hash() { ) } -function ensure_cffi_sources() { +function _ensure_cffi_sources() { # N.B. Here we assume that higher level callers have already setup the pants' venv and $PANTS_SRCPATH. PYTHONPATH="${PANTS_SRCPATH}:${PYTHONPATH}" python "${CFFI_BOOTSTRAPPER}" "$@" } -function ensure_build_prerequisites() { +function _ensure_build_prerequisites() { # Control a pants-specific rust toolchain, optionally ensuring the given target toolchain is # installed. local readonly target=$1 @@ -83,11 +93,11 @@ function ensure_build_prerequisites() { fi } -function build_native_code() { +function _build_native_code() { # Builds the native code, optionally taking an explicit target triple arg, and echos the path of # the built binary. local readonly target=$1 - ensure_build_prerequisites ${target} + _ensure_build_prerequisites ${target} local readonly cargo="${CARGO_HOME}/bin/cargo" local readonly build_cmd="${cargo} build --manifest-path ${NATIVE_ROOT}/Cargo.toml ${MODE_FLAG}" @@ -105,14 +115,14 @@ function bootstrap_native_code() { # Bootstraps the native code and overwrites the native_engine_version to the resulting hash # version if needed. local native_engine_version="$(calculate_current_hash)" - local target_binary="${CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}" + local target_binary="${NATIVE_ENGINE_CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}" local cffi_output_dir="${NATIVE_ROOT}/src/cffi" local cffi_env_script="${cffi_output_dir}/${NATIVE_ENGINE_MODULE}.sh" if [ ! -f "${target_binary}" ] then - ensure_cffi_sources "${cffi_output_dir}" + _ensure_cffi_sources "${cffi_output_dir}" source "${cffi_env_script}" - local readonly native_binary="$(build_native_code)" + local readonly native_binary="$(_build_native_code)" # If bootstrapping the native engine fails, don't attempt to run pants # afterwards. @@ -123,7 +133,7 @@ function bootstrap_native_code() { # Pick up Cargo.lock changes if any caused by the `cargo build`. native_engine_version="$(calculate_current_hash)" - target_binary="${CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}" + target_binary="${NATIVE_ENGINE_CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}" mkdir -p "$(dirname ${target_binary})" cp "${native_binary}" "${target_binary}" diff --git a/build-support/bin/native/prepare-binary-deploy.sh b/build-support/bin/native/prepare-binary-deploy.sh index 3bdc4c63827..db9f86e7019 100755 --- a/build-support/bin/native/prepare-binary-deploy.sh +++ b/build-support/bin/native/prepare-binary-deploy.sh @@ -6,7 +6,7 @@ REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd ../../.. && pwd -P) # Indirectly defines: # + RUST_OSX_MIN_VERSION: The minimum minor version of OSX supported by Rust; eg 7 for OSX 10.7. -# + OSX_MAX_VERSION: The current latest OSX minor version; eg 12 for OSX Sierra 10.12 +# + OSX_MAX_VERSION: The current latest OSX minor version; eg 12 for OSX Sierra 10.12. # + LIB_EXTENSION: The extension of native libraries. # + KERNEL: The lower-cased name of the kernel as reported by uname. # + OS_NAME: The name of the OS as seen by pants. @@ -14,100 +14,54 @@ REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd ../../.. && pwd -P) # Indirectly exposes: # + get_native_engine_version: Echoes the current native engine version. # + get_rust_osx_versions: Produces the osx minor versions supported by Rust one per line. +# + get_rust_osx_ids: Produces the BinaryUtil osx os id paths supported by rust, one per line. # + get_rust_os_ids: Produces the BinaryUtil os id paths supported by rust, one per line. # Defines: -# + CACHE_TARGET_DIR: The directory containing all versions of the native engine for the current OS. +# + CACHE_ROOT: The pants cache root dir. +# + NATIVE_ENGINE_CACHE_DIR: The native engine binary root cache directory. +# + NATIVE_ENGINE_CACHE_TARGET_DIR: The directory containing all versions of the native engine for +# the current OS. +# + NATIVE_ENGINE_BINARY: The basename of the native engine binary for the current OS. +# + NATIVE_ENGINE_VERSION_RESOURCE: The path of the resource file containing the native engine +# version hash. # Exposes: -# + build_native_code: Builds target-specific native engine binaries. -source ${REPO_ROOT}/build-support/bin/native/bootstrap.sh +# + calculate_current_hash: Calculates the current native engine version hash and echoes it to +# stdout. +# + bootstrap_native_code: Builds target-specific native engine binaries. +source "${REPO_ROOT}/build-support/bin/native/bootstrap.sh" readonly native_engine_version=$(get_native_engine_version) +readonly cached_bin_path="${NATIVE_ENGINE_CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}" -cat << __EOF__ > ${REPO_ROOT}/native-engine.bintray.json -{ - "package": { - "subject": "pantsbuild", - "repo": "bin", - "name": "native-engine", - "desc": "The pants native engine library.", - "website_url": "http://www.pantsbuild.org", - "issue_tracker_url": "https://github.com/pantsbuild/pants/issues", - "vcs_url": "https://github.com/pantsbuild/pants.git", - "licenses": ["Apache-2.0"], - "public_download_numbers": true, - "public_stats": true, - "github_use_tag_release_notes": false, - "attributes": [], - "labels": [] - }, +readonly s3_upload_root="${REPO_ROOT}/build-support/bin/native/s3-upload" +readonly s3_native_engine_dir="${s3_upload_root}/bin/native-engine" - "version": { - "name": "${native_engine_version}", - "desc": "The native engine at sha1: ${native_engine_version}", - "released": "$(date +'%Y-%m-%d')", - "vcs_tag": "$(git rev-parse HEAD)", - "attributes": [], - "gpgSign": false - }, - - "publish": true, - - "files": [ -__EOF__ - -function emit_osx_files() { - local readonly cached_bin_path="${CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}" - ensure_file_exists "${cached_bin_path}" +function prepare_chroot() { + rm -rf "${s3_upload_root}" + mkdir -p "$(dirname ${s3_native_engine_dir})" + cp -vpr "${NATIVE_ENGINE_CACHE_DIR}" "${s3_native_engine_dir}" +} - # Rust targets OSX 10.7+ as noted here: https://doc.rust-lang.org/book/getting-started.html#tier-1 - for version in $(get_rust_osx_versions) +function prepare_osx_versions() { + for os_id in $(get_rust_osx_ids) do - local cached_link_path="${cached_bin_path}.10.${version}" - - if (( ${version} < ${OSX_MAX_VERSION} )) + if [ "${OS_ID}" != "${os_id}" ] then - local sep="," - else - local sep="" + local target="${s3_native_engine_dir}/${os_id}/${native_engine_version}/${NATIVE_ENGINE_BINARY}" + mkdir -p "$(dirname ${target})" + cp -vp "${cached_bin_path}" "${target}" fi - # It appears to be the case that upload de-dupes on includePattern keys; so we make a unique - # includePattern per uploadPattern via a symlink here per OSX version. - ln -fs "${cached_bin_path}" "${cached_link_path}" - cat << __EOF__ >> ${REPO_ROOT}/native-engine.bintray.json - { - "includePattern": "${cached_link_path}", - "uploadPattern": "build-support/bin/native-engine/mac/10.${version}/${native_engine_version}/${NATIVE_ENGINE_BINARY}" - }${sep} -__EOF__ done } -function emit_linux_files() { - local readonly cached_bin_path="${CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}" - ensure_file_exists "${cached_bin_path}" +# Sanity check the locally built native engine binary exists in the first place. +ensure_file_exists "${cached_bin_path}" - cat << __EOF__ >> ${REPO_ROOT}/native-engine.bintray.json - { - "includePattern": "${cached_bin_path}", - "uploadPattern": "build-support/bin/native-engine/linux/x86_64/${native_engine_version}/${NATIVE_ENGINE_BINARY}" - } -__EOF__ -} +# Prepare a chroot for s3 deploy of the binary(ies). +prepare_chroot +# Maybe add copies of the mac native engine binary for the other supported OSX versions. if [ "${OS_NAME}" == "mac" ] then - emit_osx_files -else - emit_linux_files -fi - -cat << __EOF__ >> ${REPO_ROOT}/native-engine.bintray.json - ] -} -__EOF__ - -# Prepare a chroot for s3 deploy of the artifacts created above. -S3_UPLOAD_ROOT=${REPO_ROOT}/build-support/bin/native/s3-upload -rm -rf ${S3_UPLOAD_ROOT} -mkdir -p ${S3_UPLOAD_ROOT}/bin -ln -s ${CACHE_ROOT}/bin/native-engine ${S3_UPLOAD_ROOT}/bin/native-engine + prepare_osx_versions +fi \ No newline at end of file diff --git a/build-support/bin/native/utils.sh b/build-support/bin/native/utils.sh index 6225be48712..68ebcb15bc2 100644 --- a/build-support/bin/native/utils.sh +++ b/build-support/bin/native/utils.sh @@ -10,6 +10,7 @@ # Exposes: # + get_native_engine_version: Echoes the current native engine version. # + get_rust_osx_versions: Produces the osx minor versions supported by Rust one per line. +# + get_rust_osx_ids: Produces the BinaryUtil osx os id paths supported by rust, one per line. # + get_rust_os_ids: Produces the BinaryUtil os id paths supported by rust, one per line. REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd ../../.. && pwd -P) @@ -19,6 +20,7 @@ function get_native_engine_version() { python -c 'import json, sys; print(json.load(sys.stdin)["native_engine_version"]["value"])' } +# Rust targets OSX 10.7+ as noted here: https://doc.rust-lang.org/book/getting-started.html#tier-1 readonly RUST_OSX_MIN_VERSION=7 # Bump this when there is a new OSX released: @@ -28,17 +30,18 @@ function get_rust_osx_versions() { seq ${RUST_OSX_MIN_VERSION} ${OSX_MAX_VERSION} } -function get_rust_os_ids() { - for os_id in linux/{i386,x86_64} - do - echo "${os_id}" - done +function get_rust_osx_ids() { for rev in $(get_rust_osx_versions) do echo "mac/10.${rev}" done } +function get_rust_os_ids() { + echo "linux/x86_64" + get_rust_osx_ids +} + # TODO(John Sirois): Eliminate this replication of BinaryUtil logic internal to pants code when # https://github.com/pantsbuild/pants/issues/4006 is complete. readonly KERNEL=$(uname -s | tr '[:upper:]' '[:lower:]') diff --git a/build-support/bin/release.sh b/build-support/bin/release.sh index 38252a3e0fc..2a902afe9e4 100755 --- a/build-support/bin/release.sh +++ b/build-support/bin/release.sh @@ -422,21 +422,34 @@ EOM fi } -# Defines: +# Indirectly defines: # + RUST_OSX_MIN_VERSION: The minimum minor version of OSX supported by Rust; eg 7 for OSX 10.7. -# + OSX_MAX_VERSION: The current latest OSX minor version; eg 12 for OSX Sierra 10.12 +# + OSX_MAX_VERSION: The current latest OSX minor version; eg 12 for OSX Sierra 10.12. # + LIB_EXTENSION: The extension of native libraries. # + KERNEL: The lower-cased name of the kernel as reported by uname. # + OS_NAME: The name of the OS as seen by pants. # + OS_ID: The ID of the current OS as seen by pants. -# Exposes: +# Indirectly exposes: # + get_native_engine_version: Echoes the current native engine version. # + get_rust_osx_versions: Produces the osx minor versions supported by Rust one per line. +# + get_rust_osx_ids: Produces the BinaryUtil osx os id paths supported by rust, one per line. # + get_rust_os_ids: Produces the BinaryUtil os id paths supported by rust, one per line. -source ${ROOT}/build-support/bin/native/utils.sh +# Defines: +# + CACHE_ROOT: The pants cache root dir. +# + NATIVE_ENGINE_CACHE_DIR: The native engine binary root cache directory. +# + NATIVE_ENGINE_CACHE_TARGET_DIR: The directory containing all versions of the native engine for +# the current OS. +# + NATIVE_ENGINE_BINARY: The basename of the native engine binary for the current OS. +# + NATIVE_ENGINE_VERSION_RESOURCE: The path of the resource file containing the native engine +# version hash. +# Exposes: +# + calculate_current_hash: Calculates the current native engine version hash and echoes it to +# stdout. +# + bootstrap_native_code: Builds target-specific native engine binaries. +source ${ROOT}/build-support/bin/native/bootstrap.sh -readonly BINTRAY_BASE_URL=https://dl.bintray.com/pantsbuild/bin -readonly NATIVE_ENGINE_BASE_URL=${BINTRAY_BASE_URL}/build-support/bin/native-engine +readonly BINARY_BASE_URL=https://s3.amazonaws.com/binaries.pantsbuild.org +readonly NATIVE_ENGINE_BASE_URL=${BINARY_BASE_URL}/bin/native-engine function check_native_engine() { local readonly native_engine_version=${NATIVE_ENGINE_VERSION:-$(get_native_engine_version)} @@ -446,7 +459,7 @@ function check_native_engine() { local result=0 for os_id in $(get_rust_os_ids) do - local url=${NATIVE_ENGINE_BASE_URL}/${os_id}/${native_engine_version}/native-engine + local url=${NATIVE_ENGINE_BASE_URL}/${os_id}/${native_engine_version}/${NATIVE_ENGINE_BINARY} echo -n " for ${os_id} -> ${url}... " curl --progress-bar --fail --head ${url} &> ${headers} && echo OK || { result=$(( ${result} + 1 )) && echo FAILURE && cat ${headers} && echo @@ -465,7 +478,7 @@ function usage() { echo "PyPi. Credentials are needed for this as described in the" echo "release docs: http://pantsbuild.org/release.html" echo - echo "Usage: $0 [-d] (-h|-n|-t|-l|-o)" + echo "Usage: $0 [-d] (-h|-n|-t|-l|-o|-e)" echo " -d Enables debug mode (verbose output, script pauses after venv creation)" echo " -h Prints out this help message." echo " -n Performs a release dry run." @@ -477,6 +490,7 @@ function usage() { echo " and can be installed in an ephemeral virtualenv." echo " -l Lists all pantsbuild packages that this script releases." echo " -o Lists all pantsbuild package owners." + echo " -e Check that native engine binaries are deployed for this release." echo echo "All options (except for '-d') are mutually exclusive." @@ -522,10 +536,7 @@ elif [[ "${test_release}" == "true" ]]; then else banner "Releasing packages to PyPi" && \ ( - # NB: Ideally we'd `check_native_engine`, but it won't get built until the tag created - # in `tag_release` is pushed to origin, triggering a TravisCI run. - # See: https://github.com/pantsbuild/pants/issues/4269 - check_origin && check_clean_branch && check_pgp && check_owners && \ + check_origin && check_clean_branch && check_pgp && check_native_engine && check_owners && \ dry_run_install && publish_packages && tag_release && publish_docs_if_master && \ banner "Successfully released packages to PyPi" ) || die "Failed to release packages to PyPi." diff --git a/pants b/pants index 84e75206503..9774344fad4 100755 --- a/pants +++ b/pants @@ -36,6 +36,19 @@ HERE=$(cd `dirname "${BASH_SOURCE[0]}"` && pwd) source ${HERE}/build-support/pants_venv + +# Defines: +# + CACHE_ROOT: The pants cache root dir. +# + NATIVE_ENGINE_CACHE_DIR: The native engine binary root cache directory. +# + NATIVE_ENGINE_CACHE_TARGET_DIR: The directory containing all versions of the native engine for +# the current OS. +# + NATIVE_ENGINE_BINARY: The basename of the native engine binary for the current OS. +# + NATIVE_ENGINE_VERSION_RESOURCE: The path of the resource file containing the native engine +# version hash. +# Exposes: +# + calculate_current_hash: Calculates the current native engine version hash and echoes it to +# stdout. +# + bootstrap_native_code: Builds target-specific native engine binaries. source ${HERE}/build-support/bin/native/bootstrap.sh PANTS_EXE="${HERE}/src/python/pants/bin/pants_loader.py" From 64287dccfde92f2e091a48b16520e78296018831 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 15 Aug 2017 14:35:06 -0700 Subject: [PATCH 20/67] Only attempt deploys on appropriate shards. (#4816) This averts deploy errors on shards that have not populated `build-support/bin/native/s3-upload`. Fixes #4815 --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 546d4e94cc8..57df8041ff5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,6 +48,7 @@ matrix: language: generic env: - SHARD="OSX Native Engine Binary Builder" + - NATIVE_ENGINE_DEPLOY=1 script: - ./build-support/bin/native/prepare-binary-deploy.sh @@ -63,6 +64,7 @@ matrix: python: "2.7.13" env: - SHARD="Linux Native Engine Binary Builder" + - NATIVE_ENGINE_DEPLOY=1 # Use the standard python manylinux image for ideal binary compatibility. - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" before_install: @@ -418,6 +420,7 @@ deploy: skip_cleanup: true acl: public_read on: + condition: $NATIVE_ENGINE_DEPLOY = 1 branch: master repo: pantsbuild/pants From 7017dc42eb114bbbedf66713fc994f7240f95795 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Wed, 16 Aug 2017 13:55:30 -0700 Subject: [PATCH 21/67] Fixup erroneous `exc` attribute access. (#4818) By inspection, a `ResolveError` has no `exc`, only a `Throw` does. This looks like it was a simple engine refactor ~typo/copypasta miss in edd039bf. Fixes #4812 --- src/python/pants/engine/legacy/graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/engine/legacy/graph.py b/src/python/pants/engine/legacy/graph.py index ab656c1a08d..d61d77c9c3a 100644 --- a/src/python/pants/engine/legacy/graph.py +++ b/src/python/pants/engine/legacy/graph.py @@ -240,7 +240,7 @@ def _inject(self, subjects): subjects) except ResolveError as e: # NB: ResolveError means that a target was not found, which is a common user facing error. - raise AddressLookupError(str(e.exc)) + raise AddressLookupError(str(e)) except Exception as e: raise AddressLookupError( 'Build graph construction failed: {} {}'.format(type(e).__name__, str(e)) From c2a19abd35f44283295014e1cf08077c6d549c29 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Thu, 17 Aug 2017 13:18:30 -0700 Subject: [PATCH 22/67] Turn on pytest successful test caching in CI. (#4819) Temporarily turn off code coverage. It's currently unused (coveralls integration is not working) and it generates too much log data for TravisCI unit test shards to complete successfully. --- build-support/bin/ci.sh | 1 - pants.travis-ci.ini | 9 +++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index 45b0274d38f..097f5e7d2fc 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -205,7 +205,6 @@ if [[ "${skip_python:-false}" == "false" ]]; then start_travis_section "CoreTests" "Running core python tests${shard_desc}" ( ./pants.pex --tag='-integration' ${PANTS_ARGS[@]} test.pytest \ - --coverage=pants \ --test-pytest-test-shard=${python_unit_shard} \ tests/python:: ) || die "Core python test failure" diff --git a/pants.travis-ci.ini b/pants.travis-ci.ini index 1fdc4377f13..17f31e84abb 100644 --- a/pants.travis-ci.ini +++ b/pants.travis-ci.ini @@ -4,12 +4,21 @@ # Turn off all nailgun use. use_nailgun: False +local_artifact_cache: %(pants_bootstrapdir)s/artifact_cache + [compile.zinc] # If we use the default of 1 worker per core, we see too many cores under travis # and get oomkilled from launching too many workers with too much total memory # overhead. worker_count: 4 +[cache.test.pytest] +# test.pytest supports successful test result caching, we use this to reduce CI times/costs. +read_from: ["%(local_artifact_cache)s"] +write_to: ["%(local_artifact_cache)s"] [test.pytest] +# This ensures test caching is maximally effective; ie: per-target. +fast: false + options: ['--duration=3'] \ No newline at end of file From 40ef7c0280da467bf53cd429d6a930b3ccd08240 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 18 Aug 2017 10:11:54 -0700 Subject: [PATCH 23/67] Ensure setup-py runs with all interpreter extras. (#4822) Previously setup-py was fixed to run with the proper `setuptools` extra from the interpreter, this change adds in the `wheel` extra as well, covering the common case of wanting to generate wheels using the `bdist_wheel` distutils setup command provided by `wheel`. Fixes #4820 --- .../pants/backend/python/tasks2/setup_py.py | 17 +++++++++++------ .../backend/python/tasks2/test_setup_py.py | 17 +++++++++++------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/python/pants/backend/python/tasks2/setup_py.py b/src/python/pants/backend/python/tasks2/setup_py.py index b761e17238b..4694c62c87f 100644 --- a/src/python/pants/backend/python/tasks2/setup_py.py +++ b/src/python/pants/backend/python/tasks2/setup_py.py @@ -47,21 +47,26 @@ class SetupPyRunner(InstallerBase): + _EXTRAS = ('setuptools', 'wheel') + def __init__(self, source_dir, setup_command, **kw): self.__setup_command = setup_command.split() super(SetupPyRunner, self).__init__(source_dir, **kw) def mixins(self): mixins = super(SetupPyRunner, self).mixins().copy() + extras = set(self._EXTRAS) for (key, version) in self._interpreter.extras: - if key == 'setuptools': - mixins['setuptools'] = 'setuptools=={}'.format(version) - break + if key in extras: + mixins[key] = '{}=={}'.format(key, version) + extras.remove(key) + if not extras: + break else: - # We know Pants sets up python interpreters with wheel and setuptools via the `PythonSetup` + # We know Pants sets up python interpreters with setuptools and wheel via the `PythonSetup` # subsystem; so this should never happen - raise AssertionError("Expected interpreter {} to have the extra 'setuptools'" - .format(self._interpreter)) + raise AssertionError("Expected interpreter {} to have the extras {}" + .format(self._interpreter, self._EXTRAS)) return mixins def _setup_command(self): diff --git a/tests/python/pants_test/backend/python/tasks2/test_setup_py.py b/tests/python/pants_test/backend/python/tasks2/test_setup_py.py index a732a16d135..83767d7d5ad 100644 --- a/tests/python/pants_test/backend/python/tasks2/test_setup_py.py +++ b/tests/python/pants_test/backend/python/tasks2/test_setup_py.py @@ -117,13 +117,18 @@ def run(self): with environment_as(PYTHONPATH=sdist_srcdir): with self.run_execute(foo): with open(os.path.join(sdist_srcdir, 'foo', 'commands', 'sys_path.txt')) as fp: - # The 1st element of the sys.path should be our custom SetupPyRunner Installer's - # setuptools mixin, which should match the setuptools version specified by the - # PythonSetup subsystem. - package = Package.from_href(fp.readline().strip()) - self.assertEqual('setuptools', package.name) + def assert_extra(name, expected_version): + package = Package.from_href(fp.readline().strip()) + self.assertEqual(name, package.name) + self.assertEqual(expected_version, package.raw_version) + + # The 1st two elements of the sys.path should be our custom SetupPyRunner Installer's + # setuptools and wheel mixins, which should match the setuptools and wheel versions + # specified by the PythonSetup subsystem. init_subsystem(PythonSetup) - self.assertEqual(PythonSetup.global_instance().setuptools_version, package.raw_version) + python_setup = PythonSetup.global_instance() + assert_extra('setuptools', python_setup.setuptools_version) + assert_extra('wheel', python_setup.wheel_version) class TestSetupPy(SetupPyTestBase): From b9923f3143818057b56b18f71964a3619ea56287 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 18 Aug 2017 16:14:16 -0700 Subject: [PATCH 24/67] Prepare the 1.4.0.dev9 release. (#4824) --- src/python/pants/notes/master.rst | 27 +++++++++++++++++++++++++++ src/python/pants/version.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/python/pants/notes/master.rst b/src/python/pants/notes/master.rst index 2e1a54c26be..28cc3790368 100644 --- a/src/python/pants/notes/master.rst +++ b/src/python/pants/notes/master.rst @@ -4,6 +4,33 @@ Master Pre-Releases This document describes ``dev`` releases which occur weekly from master, and which do not undergo the vetting associated with ``stable`` releases. +1.4.0.dev9 (8/18/2017) +---------------------- + +Bugfixes +~~~~~~~~ + +* Ensure setup-py runs with all interpreter extras. (#4822) + `PR #4822 `_ + +* Fixup erroneous `exc` attribute access. (#4818) + `PR #4818 `_ + +Refactoring, Improvements, and Tooling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Turn on pytest successful test caching in CI. (#4819) + `PR #4819 `_ + +* Only attempt deploys on appropriate shards. (#4816) + `PR #4816 `_ + +* Fix s3 deploy to use copies instead of a symlink. (#4814) + `PR #4814 `_ + +* Fix the S3 upload in the travis deploy. (#4813) + `PR #4813 `_ + 1.4.0.dev8 (8/11/2017) ---------------------- diff --git a/src/python/pants/version.py b/src/python/pants/version.py index 504f0de332f..cd5ba1ab0a3 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -8,6 +8,6 @@ from packaging.version import Version -VERSION = '1.4.0.dev8' +VERSION = '1.4.0.dev9' PANTS_SEMVER = Version(VERSION) From 8dd8eb0881d2dd05865e627d0f6203b1bbd92cd3 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Thu, 24 Aug 2017 14:31:29 -0400 Subject: [PATCH 25/67] Always return a bool from SetupPy.has_provides(). Fun Python fact: >>> type(True and 5) >>> type(False and 5) --- contrib/go/README.md | 2 +- src/python/pants/backend/python/tasks2/setup_py.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/go/README.md b/contrib/go/README.md index 67206182327..f01050c56a5 100644 --- a/contrib/go/README.md +++ b/contrib/go/README.md @@ -117,7 +117,7 @@ done ### Dependency metadata generation -Now that we have roper source root configuration and skeleton BUILD files, we can proceed to +Now that we have proper source root configuration and skeleton BUILD files, we can proceed to auto-generate the dependency information pants needs from BUILD files using the `buildgen` goal. Here we set options to emit the BUILD files to disk, include 3rdparty remote dependency BUILD files, and fail if any 3rdparty dependencies have un-pinned versions: diff --git a/src/python/pants/backend/python/tasks2/setup_py.py b/src/python/pants/backend/python/tasks2/setup_py.py index 4694c62c87f..1bfe4366236 100644 --- a/src/python/pants/backend/python/tasks2/setup_py.py +++ b/src/python/pants/backend/python/tasks2/setup_py.py @@ -312,7 +312,7 @@ def is_resources_target(target): @classmethod def has_provides(cls, target): - return cls.is_python_target(target) and target.provides + return cls.is_python_target(target) and target.provides is not None @classmethod def product_types(cls): From f7f56b0620b34331e46a4dc74dede3ab6b6d102c Mon Sep 17 00:00:00 2001 From: John Sirois Date: Thu, 24 Aug 2017 13:54:45 -0700 Subject: [PATCH 26/67] Add optional chrooting for junit tests. (#4823) Turn on chrooting for this repo and fixup tests to work with it or opt out as appropriate. --- pants.ini | 1 + .../pants/backend/jvm/targets/junit_tests.py | 3 +- .../pants/backend/jvm/tasks/junit_run.py | 93 ++++++++++---- tests/java/org/pantsbuild/tools/runner/BUILD | 2 + .../backend/jvm/tasks/test_junit_run.py | 118 +++++++++++++++++- .../jvm/tasks/test_junit_tests_integration.py | 2 + .../resources/org/pantsbuild/tools/ivy/BUILD | 4 +- 7 files changed, 194 insertions(+), 29 deletions(-) diff --git a/pants.ini b/pants.ini index dd56f04ecc7..edce85a1187 100644 --- a/pants.ini +++ b/pants.ini @@ -292,6 +292,7 @@ timeout_default: 60 [test.junit] +chroot: true timeouts: true timeout_default: 60 diff --git a/src/python/pants/backend/jvm/targets/junit_tests.py b/src/python/pants/backend/jvm/targets/junit_tests.py index acd295c8cc9..54d8110b0ee 100644 --- a/src/python/pants/backend/jvm/targets/junit_tests.py +++ b/src/python/pants/backend/jvm/targets/junit_tests.py @@ -55,7 +55,8 @@ def __init__(self, cwd=None, test_platform=None, payload=None, timeout=None, threads=None, **kwargs): """ :param str cwd: working directory (relative to the build root) for the tests under this - target. If unspecified (None), the working directory will be controlled by junit_run's --cwd. + target. If unspecified (None), the working directory will be controlled by junit_run's --cwd + and --chroot options. :param str test_platform: The name of the platform (defined under the jvm-platform subsystem) to use for running tests (that is, a key into the --jvm-platform-platforms dictionary). If unspecified, the platform will default to the same one used for compilation. diff --git a/src/python/pants/backend/jvm/tasks/junit_run.py b/src/python/pants/backend/jvm/tasks/junit_run.py index 82a80213959..e8162b1348f 100644 --- a/src/python/pants/backend/jvm/tasks/junit_run.py +++ b/src/python/pants/backend/jvm/tasks/junit_run.py @@ -5,7 +5,9 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +import functools import os +import shutil import sys from abc import abstractmethod from contextlib import contextmanager @@ -27,6 +29,7 @@ from pants.base.build_environment import get_buildroot from pants.base.exceptions import ErrorWhileTesting, TargetDefinitionException, TaskError from pants.base.workunit import WorkUnitLabel +from pants.build_graph.files import Files from pants.build_graph.target import Target from pants.build_graph.target_scopes import Scopes from pants.invalidation.cache_manager import VersionedTargetSet @@ -36,8 +39,8 @@ from pants.process.lock import OwnerPrintingInterProcessFileLock from pants.task.testrunner_task_mixin import TestRunnerTaskMixin from pants.util.argutil import ensure_arg, remove_arg -from pants.util.contextutil import environment_as -from pants.util.dirutil import safe_mkdir, safe_rmtree +from pants.util.contextutil import environment_as, temporary_dir +from pants.util.dirutil import safe_mkdir, safe_mkdir_for, safe_rmtree from pants.util.memo import memoized_method from pants.util.meta import AbstractClass from pants.util.strutil import pluralize @@ -160,7 +163,13 @@ def register_options(cls, register): 'All tests output also redirected to files in .pants.d/test/junit.') register('--cwd', advanced=True, fingerprint=True, help='Set the working directory. If no argument is passed, use the build root. ' - 'If cwd is set on a target, it will supersede this argument.') + 'If cwd is set on a target, it will supersede this option. It is an error to ' + 'use this option in combination with `--chroot`') + register('--chroot', advanced=True, fingerprint=True, type=bool, default=False, + help='Run tests in a chroot. Any loose files tests depend on via `{}` dependencies ' + 'will be copied to the chroot. If cwd is set on a target, it will supersede this' + 'option. It is an error to use this option in combination with `--cwd`' + .format(Files.alias())) register('--strict-jvm-version', type=bool, advanced=True, fingerprint=True, help='If true, will strictly require running junits with the same version of java as ' 'the platform -target level. Otherwise, the platform -target level will be ' @@ -205,6 +214,9 @@ def prepare(cls, options, round_manager): if cls.request_classes_by_source(options.test or ()): round_manager.require_data('classes_by_source') + class OptionError(TaskError): + """Indicates an invalid combination of options for this task.""" + def __init__(self, *args, **kwargs): super(JUnitRun, self).__init__(*args, **kwargs) @@ -212,7 +224,16 @@ def __init__(self, *args, **kwargs): self._tests_to_run = options.test self._batch_size = options.batch_size self._fail_fast = options.fail_fast - self._working_dir = options.cwd or get_buildroot() + + if options.cwd and options.chroot: + raise self.OptionError('Cannot set both `cwd` ({}) and ask for a `chroot` at the same time.' + .format(options.cwd)) + + if options.chroot: + self._working_dir = None + else: + self._working_dir = options.cwd or get_buildroot() + self._strict_jvm_version = options.strict_jvm_version self._failure_summary = options.failure_summary self._open = options.open @@ -342,6 +363,29 @@ def _collect_test_targets(self, targets): else: return test_registry + @staticmethod + def _copy_files(dest_dir, target): + if isinstance(target, Files): + for source in target.sources_relative_to_buildroot(): + src = os.path.join(get_buildroot(), source) + dest = os.path.join(dest_dir, source) + safe_mkdir_for(dest) + shutil.copy(src, dest) + + @contextmanager + def _chroot(self, targets, workdir): + if workdir is not None: + yield workdir + else: + root_dir = os.path.join(self.workdir, '_chroots') + safe_mkdir(root_dir) + with temporary_dir(root_dir=root_dir) as chroot: + self.context.build_graph.walk_transitive_dependency_graph( + addresses=[t.address for t in targets], + work=functools.partial(self._copy_files, chroot) + ) + yield chroot + def _run_tests(self, test_registry, output_dir, coverage): coverage.instrument() @@ -400,26 +444,27 @@ def parse_error_handler(parse_error): batch_test_specs = [test.render_test_spec() for test in batch] with argfile.safe_args(batch_test_specs, self.get_options()) as batch_tests: - self.context.log.debug('CWD = {}'.format(workdir)) - self.context.log.debug('platform = {}'.format(platform)) - with environment_as(**dict(target_env_vars)): - subprocess_result = self._spawn_and_wait( - executor=SubprocessExecutor(distribution), - distribution=distribution, - classpath=complete_classpath, - main=JUnit.RUNNER_MAIN, - jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options), - args=args + batch_tests, - workunit_factory=self.context.new_workunit, - workunit_name='run', - workunit_labels=[WorkUnitLabel.TEST], - cwd=workdir, - synthetic_jar_dir=output_dir, - create_synthetic_jar=self.synthetic_classpath, - ) - self.context.log.debug('JUnit subprocess exited with result ({})' - .format(subprocess_result)) - result += abs(subprocess_result) + with self._chroot(relevant_targets, workdir) as chroot: + self.context.log.debug('CWD = {}'.format(chroot)) + self.context.log.debug('platform = {}'.format(platform)) + with environment_as(**dict(target_env_vars)): + subprocess_result = self._spawn_and_wait( + executor=SubprocessExecutor(distribution), + distribution=distribution, + classpath=complete_classpath, + main=JUnit.RUNNER_MAIN, + jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options), + args=args + batch_tests, + workunit_factory=self.context.new_workunit, + workunit_name='run', + workunit_labels=[WorkUnitLabel.TEST], + cwd=chroot, + synthetic_jar_dir=output_dir, + create_synthetic_jar=self.synthetic_classpath, + ) + self.context.log.debug('JUnit subprocess exited with result ({})' + .format(subprocess_result)) + result += abs(subprocess_result) tests_info = self.parse_test_info(output_dir, parse_error_handler, ['classname']) for test_name, test_info in tests_info.items(): diff --git a/tests/java/org/pantsbuild/tools/runner/BUILD b/tests/java/org/pantsbuild/tools/runner/BUILD index 6197a05e0d7..2289ca7923a 100644 --- a/tests/java/org/pantsbuild/tools/runner/BUILD +++ b/tests/java/org/pantsbuild/tools/runner/BUILD @@ -9,4 +9,6 @@ junit_tests( '3rdparty:junit', 'src/java/org/pantsbuild/tools/runner:runner-library', ], + # This test needs to invoke ./pants at the buildroot. + cwd=get_buildroot() ) diff --git a/tests/python/pants_test/backend/jvm/tasks/test_junit_run.py b/tests/python/pants_test/backend/jvm/tasks/test_junit_run.py index f639c9db881..1dd752d680a 100644 --- a/tests/python/pants_test/backend/jvm/tasks/test_junit_run.py +++ b/tests/python/pants_test/backend/jvm/tasks/test_junit_run.py @@ -17,13 +17,14 @@ from pants.backend.python.targets.python_tests import PythonTests from pants.base.exceptions import TargetDefinitionException, TaskError from pants.build_graph.build_file_aliases import BuildFileAliases +from pants.build_graph.files import Files from pants.build_graph.resources import Resources from pants.ivy.bootstrapper import Bootstrapper from pants.ivy.ivy_subsystem import IvySubsystem from pants.java.distribution.distribution import DistributionLocator from pants.java.executor import SubprocessExecutor -from pants.util.contextutil import environment_as -from pants.util.dirutil import safe_file_dump +from pants.util.contextutil import environment_as, temporary_dir +from pants.util.dirutil import safe_file_dump, touch from pants.util.timeout import TimeoutReached from pants_test.jvm.jvm_tool_task_test_base import JvmToolTaskTestBase from pants_test.subsystem.subsystem_util import global_subsystem_instance, init_subsystem @@ -40,6 +41,7 @@ def task_type(cls): def alias_groups(self): return super(JUnitRunnerTest, self).alias_groups.merge(BuildFileAliases( targets={ + 'files': Files, 'junit_tests': JUnitTests, 'python_tests': PythonTests, }, @@ -433,3 +435,115 @@ def test_junt_run_with_too_many_args(self): self._execute_junit_runner(list_of_filename_content_tuples, target_name='tests/java/org/pantsbuild/foo:foo_test') + + @ensure_cached(JUnitRun, expected_num_artifacts=1) + def test_junit_run_chroot(self): + self.create_files('config/org/pantsbuild/foo', ['sentinel', 'another']) + files = self.make_target( + spec='config/org/pantsbuild/foo:sentinel', + target_type=Files, + sources=['sentinel'] + ) + self.make_target( + spec='tests/java/org/pantsbuild/foo:foo_test', + target_type=JUnitTests, + sources=['FooTest.java'], + dependencies=[files] + ) + content = dedent(""" + package org.pantsbuild.foo; + import java.io.File; + import org.junit.Test; + import static org.junit.Assert.assertFalse; + import static org.junit.Assert.assertTrue; + public class FooTest { + @Test + public void testFoo() { + assertTrue(new File("config/org/pantsbuild/foo/sentinel").exists()); + assertFalse(new File("config/org/pantsbuild/foo/another").exists()); + } + } + """) + self.set_options(chroot=True) + self._execute_junit_runner([('FooTest.java', content)], + target_name='tests/java/org/pantsbuild/foo:foo_test') + + @ensure_cached(JUnitRun, expected_num_artifacts=0) + def test_junit_run_chroot_cwd_mutex(self): + with temporary_dir() as chroot: + self.set_options(chroot=True, cwd=chroot) + with self.assertRaises(JUnitRun.OptionError): + self.execute(self.context()) + + @ensure_cached(JUnitRun, expected_num_artifacts=1) + def test_junit_run_target_cwd_trumps_chroot(self): + with temporary_dir() as target_cwd: + self.create_files('config/org/pantsbuild/foo', ['files_dep_sentinel']) + files = self.make_target( + spec='config/org/pantsbuild/foo:sentinel', + target_type=Files, + sources=['files_dep_sentinel'] + ) + self.make_target( + spec='tests/java/org/pantsbuild/foo:foo_test', + target_type=JUnitTests, + sources=['FooTest.java'], + dependencies=[files], + cwd=target_cwd + ) + content = dedent(""" + package org.pantsbuild.foo; + import java.io.File; + import org.junit.Test; + import static org.junit.Assert.assertFalse; + import static org.junit.Assert.assertTrue; + public class FooTest {{ + @Test + public void testFoo() {{ + assertTrue(new File("target_cwd_sentinel").exists()); + + // We declare a Files dependency on this file, but since we run in a CWD not in a + // chroot and not in the build root, we can't find it at the expected relative path. + assertFalse(new File("config/org/pantsbuild/foo/files_dep_sentinel").exists()); + + // As a sanity check, it is at the the expected absolute path though. + File buildRoot = new File("{}"); + assertTrue(new File(buildRoot, + "config/org/pantsbuild/foo/files_dep_sentinel").exists()); + }} + }} + """.format(self.build_root)) + touch(os.path.join(target_cwd, 'target_cwd_sentinel')) + self.set_options(chroot=True) + self._execute_junit_runner([('FooTest.java', content)], + target_name='tests/java/org/pantsbuild/foo:foo_test') + + @ensure_cached(JUnitRun, expected_num_artifacts=1) + def test_junit_run_target_cwd_trumps_cwd_option(self): + with temporary_dir() as target_cwd: + self.make_target( + spec='tests/java/org/pantsbuild/foo:foo_test', + target_type=JUnitTests, + sources=['FooTest.java'], + cwd=target_cwd + ) + content = dedent(""" + package org.pantsbuild.foo; + import java.io.File; + import org.junit.Test; + import static org.junit.Assert.assertFalse; + import static org.junit.Assert.assertTrue; + public class FooTest { + @Test + public void testFoo() { + assertTrue(new File("target_cwd_sentinel").exists()); + assertFalse(new File("option_cwd_sentinel").exists()); + } + } + """) + touch(os.path.join(target_cwd, 'target_cwd_sentinel')) + with temporary_dir() as option_cwd: + touch(os.path.join(option_cwd, 'option_cwd_sentinel')) + self.set_options(cwd=option_cwd) + self._execute_junit_runner([('FooTest.java', content)], + target_name='tests/java/org/pantsbuild/foo:foo_test') diff --git a/tests/python/pants_test/backend/jvm/tasks/test_junit_tests_integration.py b/tests/python/pants_test/backend/jvm/tasks/test_junit_tests_integration.py index 77ddb2fc5a9..bffe27914f2 100644 --- a/tests/python/pants_test/backend/jvm/tasks/test_junit_tests_integration.py +++ b/tests/python/pants_test/backend/jvm/tasks/test_junit_tests_integration.py @@ -90,6 +90,7 @@ def test_junit_test_requiring_cwd_passes_with_option_with_value_specified(self): '--python-setup-interpreter-constraints=CPython>=2.6,<3', '--python-setup-interpreter-constraints=CPython>=3.3', '--jvm-test-junit-options=-Dcwd.test.enabled=true', + '--no-test-junit-chroot', '--test-junit-cwd=testprojects/src/java/org/pantsbuild/testproject/cwdexample/subdir']) self.assert_success(pants_run) @@ -140,6 +141,7 @@ def test_junit_test_target_cwd_overrides_option(self): pants_run = self.run_pants([ 'test', 'testprojects/tests/java/org/pantsbuild/testproject/workdirs/onedir', + '--no-test-junit-chroot', '--test-junit-cwd=testprojects/tests/java/org/pantsbuild/testproject/dummies']) self.assert_success(pants_run) diff --git a/tests/resources/org/pantsbuild/tools/ivy/BUILD b/tests/resources/org/pantsbuild/tools/ivy/BUILD index a9fd7c1b8d9..8f16297a9b7 100644 --- a/tests/resources/org/pantsbuild/tools/ivy/BUILD +++ b/tests/resources/org/pantsbuild/tools/ivy/BUILD @@ -1,6 +1,6 @@ # Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -resources( - sources=globs('*.jar', '*.properties', '*.xml'), +files( + sources=rglobs('*.jar', '*.properties', '*.xml'), ) From 57b33b82d32f264e5cf709bf0170f19ae50a8d2a Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Fri, 25 Aug 2017 18:51:35 +0300 Subject: [PATCH 27/67] Stop dual-publishing the docsite. (#4828) Document how we now host it, while supporting HTTPS. --- build-support/bin/publish_docs.sh | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/build-support/bin/publish_docs.sh b/build-support/bin/publish_docs.sh index 4fe4cc23144..b0f9f3193cd 100755 --- a/build-support/bin/publish_docs.sh +++ b/build-support/bin/publish_docs.sh @@ -86,18 +86,21 @@ continue." ( ${REPO_ROOT}/src/docs/publish_via_git.sh \ https://github.com/pantsbuild/pantsbuild.github.io.git \ - ${publish_path} && \ - ${REPO_ROOT}/src/docs/publish_via_git.sh \ - https://github.com/benjyw/benjyw.github.io.git \ - ${publish_path} && \ + ${publish_path} do_open ${url}/index.html ) || die "Publish to ${url} failed." fi -# Note that we push the docs to two repos: pantsbuild.github.io and benjyw.github.io. -# The latter is where we redirect www.pantsbuild.org to via CNAME. We can't redirect -# to the former because then HTTPS won't work (the cert is for pantsbuild.github.io). -# This way we have working https URLs when needed, but also an http-only version at the -# more respectable domain of pantsbuild.org. -# This is a stopgap workaround until we can host the docsite ourselves, under our own -# SSL cert. +# Note that we use Cloudflare to enforce: +# +# - Flattening the apex domain pantsbuild.org to www.pantsbuild.org +# - Requiring HTTPS +# - Directing www.pantsbuild.org traffic via CNAME to pantsbuild.github.io. +# - Caching www.pantsbuild.org content. +# - Directing binaries.pantsbuild.org and node-preinstalled-modules.pantsbuild.org +# to the appropriate S3 buckets via CNAME. + +# Google domains is still our registrar, but we use it only to point to Cloudflare's nameservers. + +# See the DNS and Page Rules tabs in our Cloudflare acct, and also the GitHub Pages +# custom domain setup here: https://github.com/pantsbuild/pantsbuild.github.io/settings \ No newline at end of file From d99e30c5a48bee0d5ef375114367d87024e30ee6 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 25 Aug 2017 09:16:52 -0700 Subject: [PATCH 28/67] Point binary URLs off to CNAMES we own. (#4829) Cloudfare DNS allows us to get away with this now! --- build-support/bin/release.sh | 2 +- contrib/node/examples/src/node/preinstalled-project/BUILD | 2 +- src/python/pants/binaries/binary_util.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build-support/bin/release.sh b/build-support/bin/release.sh index 2a902afe9e4..268569aea4a 100755 --- a/build-support/bin/release.sh +++ b/build-support/bin/release.sh @@ -448,7 +448,7 @@ EOM # + bootstrap_native_code: Builds target-specific native engine binaries. source ${ROOT}/build-support/bin/native/bootstrap.sh -readonly BINARY_BASE_URL=https://s3.amazonaws.com/binaries.pantsbuild.org +readonly BINARY_BASE_URL=https://binaries.pantsbuild.org readonly NATIVE_ENGINE_BASE_URL=${BINARY_BASE_URL}/bin/native-engine function check_native_engine() { diff --git a/contrib/node/examples/src/node/preinstalled-project/BUILD b/contrib/node/examples/src/node/preinstalled-project/BUILD index 5376c2cc783..d41faac8507 100644 --- a/contrib/node/examples/src/node/preinstalled-project/BUILD +++ b/contrib/node/examples/src/node/preinstalled-project/BUILD @@ -4,7 +4,7 @@ node_preinstalled_module( sources=globs('package.json', 'src/*.js', 'test/*.js'), dependencies_archive_url= - 'https://s3.amazonaws.com/node-preinstalled-modules.pantsbuild.org/node_modules.tar.gz' + 'https://node-preinstalled-modules.pantsbuild.org/node_modules.tar.gz' ) node_test( diff --git a/src/python/pants/binaries/binary_util.py b/src/python/pants/binaries/binary_util.py index 2a23226b461..6da4a3f0c4e 100644 --- a/src/python/pants/binaries/binary_util.py +++ b/src/python/pants/binaries/binary_util.py @@ -55,7 +55,7 @@ class Factory(Subsystem): @classmethod def register_options(cls, register): register('--baseurls', type=list, advanced=True, - default=['https://s3.amazonaws.com/binaries.pantsbuild.org'], + default=['https://binaries.pantsbuild.org'], help='List of urls from which binary tools are downloaded. Urls are searched in ' 'order until the requested path is found.') register('--fetch-timeout-secs', type=int, default=30, advanced=True, From 94ae320b7acff5da5e1c28f0f6721f0f70362005 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 25 Aug 2017 11:25:11 -0600 Subject: [PATCH 29/67] Prepare the 1.4.0.dev10 release. (#4830) --- src/python/pants/notes/master.rst | 24 ++++++++++++++++++++++++ src/python/pants/version.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/python/pants/notes/master.rst b/src/python/pants/notes/master.rst index 28cc3790368..3becbc7fc32 100644 --- a/src/python/pants/notes/master.rst +++ b/src/python/pants/notes/master.rst @@ -4,6 +4,30 @@ Master Pre-Releases This document describes ``dev`` releases which occur weekly from master, and which do not undergo the vetting associated with ``stable`` releases. +1.4.0.dev10 (8/25/2017) +----------------------- + +New Features +~~~~~~~~~~~~ + +* Add optional chrooting for junit tests. (#4823) + `PR #4823 `_ + +Bugfixes +~~~~~~~~ + +* Always return a bool from SetupPy.has_provides(). + `PR #4826 `_ + +Refactoring, Improvements, and Tooling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Point binary URLs off to CNAMES we own. (#4829) + `PR #4829 `_ + +* Stop dual-publishing the docsite. (#4828) + `PR #4828 `_ + 1.4.0.dev9 (8/18/2017) ---------------------- diff --git a/src/python/pants/version.py b/src/python/pants/version.py index cd5ba1ab0a3..2f060d9ba4b 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -8,6 +8,6 @@ from packaging.version import Version -VERSION = '1.4.0.dev9' +VERSION = '1.4.0.dev10' PANTS_SEMVER = Version(VERSION) From 812c0a36e1690bb621991cf125103c95b11f919a Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 25 Aug 2017 20:05:34 -0600 Subject: [PATCH 30/67] Work around scalajs release test timeouts. --- contrib/release_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/release_packages.sh b/contrib/release_packages.sh index 98c19129910..65a073bc548 100644 --- a/contrib/release_packages.sh +++ b/contrib/release_packages.sh @@ -82,7 +82,7 @@ PKG_SCALAJS=( function pkg_scalajs_install_test() { execute_packaged_pants_with_internal_backends \ --plugins="['pantsbuild.pants.contrib.scalajs==$(local_version)']" \ - test contrib/scalajs:: + test.pytest --no-timeouts contrib/scalajs:: } PKG_PYTHON_CHECKS=( From b170aeb3fac52502ff248cf7ceac6172ad6fc75b Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 29 Aug 2017 12:59:53 -0600 Subject: [PATCH 31/67] Centralize options tracking in the Parser. (#4832) This also fixes a bug where details provenance of a merged value would be lost. --- src/python/pants/option/options.py | 5 ---- src/python/pants/option/parser.py | 25 ++++++++++++----- .../option/test_options_integration.py | 28 ++++++++++--------- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/src/python/pants/option/options.py b/src/python/pants/option/options.py index 6b0681b03e0..eed52704290 100644 --- a/src/python/pants/option/options.py +++ b/src/python/pants/option/options.py @@ -307,11 +307,6 @@ def for_scope(self, scope, inherit_from_enclosing_scope=True): # This makes the code a bit neater. values.update(self.for_scope(deprecated_scope)) - # Record the value derivation. - for option in values: - self._option_tracker.record_option(scope=scope, option=option, value=values[option], - rank=values.get_rank(option)) - # Cache the values. self._values_by_scope[scope] = values diff --git a/src/python/pants/option/parser.py b/src/python/pants/option/parser.py index 8bdd37cb51b..b771de1d10d 100644 --- a/src/python/pants/option/parser.py +++ b/src/python/pants/option/parser.py @@ -520,7 +520,17 @@ def expand(val_str): # instances of RankedValue (so none will be None, although they may wrap a None value). ranked_vals = list(reversed(list(RankedValue.prioritized_iter(*values_to_rank)))) - # Record info about the derivation of each of the values. + def record_option(value, rank, option_details=None): + deprecation_version = kwargs.get('removal_version') + self._option_tracker.record_option(scope=self._scope, + option=dest, + value=value, + rank=rank, + deprecation_version=deprecation_version, + details=option_details) + + # Record info about the derivation of each of the contributing values. + detail_history = [] for ranked_val in ranked_vals: if ranked_val.rank in (RankedValue.CONFIG, RankedValue.CONFIG_DEFAULT): details = config_details @@ -528,12 +538,9 @@ def expand(val_str): details = env_details else: details = None - self._option_tracker.record_option(scope=self._scope, - option=dest, - value=ranked_val.value, - rank=ranked_val.rank, - deprecation_version=kwargs.get('removal_version'), - details=details) + if details: + detail_history.append(details) + record_option(value=ranked_val.value, rank=ranked_val.rank, option_details=details) # Helper function to check various validity constraints on final option values. def check(val): @@ -569,6 +576,10 @@ def check(val): ret = ranked_vals[-1] check(ret.value) + # Record info about the derivation of the final value. + merged_details = ', '.join(detail_history) if detail_history else None + record_option(value=ret.value, rank=ret.rank, option_details=merged_details) + # All done! return ret diff --git a/tests/python/pants_test/option/test_options_integration.py b/tests/python/pants_test/option/test_options_integration.py index 845428419ef..15da2bc1653 100644 --- a/tests/python/pants_test/option/test_options_integration.py +++ b/tests/python/pants_test/option/test_options_integration.py @@ -111,7 +111,7 @@ def test_from_config(self): only_overridden: True show_history: True """)) - pants_run = self.run_pants(['--config-override={}'.format(config_path), 'options']) + pants_run = self.run_pants(['--pants-config-files={}'.format(config_path), 'options']) self.assert_success(pants_run) self.assertIn('options.only_overridden = True', pants_run.stdout_data) self.assertIn('(from CONFIG in {})'.format(config_path), pants_run.stdout_data) @@ -135,7 +135,7 @@ def test_options_deprecation_from_config(self): [options] colors: False """)) - pants_run = self.run_pants(['--config-override={}'.format(config_path), 'options']) + pants_run = self.run_pants(['--pants-config-files={}'.format(config_path), 'options']) self.assert_success(pants_run) @@ -161,7 +161,7 @@ def test_from_config_invalid_section(self): colors: False scope: options """)) - pants_run = self.run_pants(['--config-override={}'.format(config_path), + pants_run = self.run_pants(['--pants-config-files={}'.format(config_path), 'goals']) self.assert_failure(pants_run) self.assertIn('ERROR] Invalid scope [invalid_scope]', pants_run.stderr_data) @@ -180,7 +180,7 @@ def test_from_config_invalid_option(self): fail_fast: True invalid_option: True """)) - pants_run = self.run_pants(['--config-override={}'.format(config_path), + pants_run = self.run_pants(['--pants-config-files={}'.format(config_path), 'goals']) self.assert_failure(pants_run) self.assertIn("ERROR] Invalid option 'invalid_option' under [test.junit]", @@ -207,7 +207,7 @@ def test_from_config_invalid_global_option(self): [test.junit] fail_fast: True """)) - pants_run = self.run_pants(['--config-override={}'.format(config_path), + pants_run = self.run_pants(['--pants-config-files={}'.format(config_path), 'goals']) self.assert_failure(pants_run) self.assertIn("ERROR] Invalid option 'invalid_global' under [GLOBAL]", pants_run.stderr_data) @@ -232,16 +232,16 @@ def test_invalid_command_line_option_and_invalid_config(self): # Run with invalid config and invalid command line option. # Should error out with invalid command line option only. - pants_run = self.run_pants(['--config-override={}'.format(config_path), + pants_run = self.run_pants(['--pants-config-files={}'.format(config_path), '--test-junit-invalid=ALL', 'goals']) self.assert_failure(pants_run) - self.assertIn("Exception message: Unrecognized command line flags on scope 'test.junit': --invalid", - pants_run.stderr_data) + self.assertIn("Exception message: Unrecognized command line flags on scope 'test.junit': " + "--invalid", pants_run.stderr_data) # Run with invalid config only. # Should error out with `bad_option` and `invalid_scope` in config. - pants_run = self.run_pants(['--config-override={}'.format(config_path), + pants_run = self.run_pants(['--pants-config-files={}'.format(config_path), 'goals']) self.assert_failure(pants_run) self.assertIn("ERROR] Invalid option 'bad_option' under [test.junit]", pants_run.stderr_data) @@ -284,11 +284,12 @@ def test_pants_ignore_option(self): [GLOBAL] pants_ignore: +['some/random/dir'] """)) - pants_run = self.run_pants(['--config-override={}'.format(config_path), + pants_run = self.run_pants(['--pants-config-files={}'.format(config_path), '--no-colors', 'options']) self.assert_success(pants_run) - self.assertIn("pants_ignore = ['.*/', '/dist/', 'some/random/dir'] (from CONFIG)", + self.assertIn("pants_ignore = ['.*/', '/dist/', 'some/random/dir'] (from CONFIG in {})" + .format(config_path), pants_run.stdout_data) @ensure_engine @@ -301,9 +302,10 @@ def test_pants_ignore_option_non_default_dist_dir(self): pants_ignore: +['some/random/dir'] pants_distdir: some/other/dist/dir """)) - pants_run = self.run_pants(['--config-override={}'.format(config_path), + pants_run = self.run_pants(['--pants-config-files={}'.format(config_path), '--no-colors', 'options']) self.assert_success(pants_run) - self.assertIn("pants_ignore = ['.*/', '/some/other/dist/dir/', 'some/random/dir'] (from CONFIG)", + self.assertIn("pants_ignore = ['.*/', '/some/other/dist/dir/', 'some/random/dir'] " + "(from CONFIG in {})".format(config_path), pants_run.stdout_data) From 7ac68c6b8fe473a1a218d7e54f78cf6c344d8b59 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Fri, 1 Sep 2017 09:15:36 -0700 Subject: [PATCH 32/67] Bump petgraph to 0.4.5 (#4836) Rust recently updated their stable version which necessitates updating petgraph --- src/python/pants/engine/native_engine_version | 2 +- src/rust/engine/Cargo.lock | 6 +++--- src/rust/engine/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/python/pants/engine/native_engine_version b/src/python/pants/engine/native_engine_version index 0b6fb4a89f6..82a2c9095e6 100644 --- a/src/python/pants/engine/native_engine_version +++ b/src/python/pants/engine/native_engine_version @@ -1 +1 @@ -7ffa912ffa6f260c22a1788c36974b3c4d3b635e +27a5bba4e7fffd406e800ddce1c5800a9d497e22 diff --git a/src/rust/engine/Cargo.lock b/src/rust/engine/Cargo.lock index fa36b7115e1..78bcacdb1f1 100644 --- a/src/rust/engine/Cargo.lock +++ b/src/rust/engine/Cargo.lock @@ -11,7 +11,7 @@ dependencies = [ "ignore 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "ordermap 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "petgraph 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "petgraph 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "tar 0.4.12 (git+https://github.com/stuhood/tar-rs?rev=893a96e89c84f44f60f0e2398d27ae3267d196f8)", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -158,7 +158,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "petgraph" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "fixedbitset 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -302,7 +302,7 @@ dependencies = [ "checksum memchr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1dbccc0e46f1ea47b9f17e6d67c5a96bd27030519c519c9c91327e31275a47b4" "checksum num_cpus 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a18c392466409c50b87369414a2680c93e739aedeb498eb2bff7d7eb569744e2" "checksum ordermap 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "fdaf30b4edcc481e006518450305ada0149955758851ac87b78128bfbfdf310b" -"checksum petgraph 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "28814c774eeabfe38b20e1651f9e9a5ff6efeb3b2a66df25e056f579051ee4b3" +"checksum petgraph 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "14c6ae5ccb73b438781abc93d35615019b1ad6e24b44116377fb819cfd7587de" "checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d" "checksum regex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4278c17d0f6d62dfef0ab00028feb45bd7d2102843f80763474eeb1be8a10c01" "checksum regex-syntax 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9191b1f57603095f105d317e375d19b1c9c5c3185ea9633a99a6dcbed04457" diff --git a/src/rust/engine/Cargo.toml b/src/rust/engine/Cargo.toml index 4574575f559..b9ff40d7e9c 100644 --- a/src/rust/engine/Cargo.toml +++ b/src/rust/engine/Cargo.toml @@ -25,7 +25,7 @@ glob = "0.2.11" ignore = "0.1.8" lazy_static = "0.2.2" ordermap = "0.2.8" -petgraph = "0.4.4" +petgraph = "0.4.5" # TODO: See https://github.com/alexcrichton/tar-rs/pull/107. tar = { git = "https://github.com/stuhood/tar-rs", rev = "893a96e89c84f44f60f0e2398d27ae3267d196f8" } tempdir = "0.3.5" From 80eb957ae47ca16a0c58d635e94091e3c8173552 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 1 Sep 2017 13:10:07 -0700 Subject: [PATCH 33/67] Prepare the 1.4.0.dev11 release. (#4839) --- CONTRIBUTORS.md | 1 + src/python/pants/notes/master.rst | 15 +++++++++++++++ src/python/pants/version.py | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index d34c65925de..f49bf8b59de 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -37,6 +37,7 @@ Created by running `./build-support/bin/contributors.sh`. + Drew Rothstein + Dumitru Daniliuc + Ebube Chuba ++ Eitan Adler + Emily Caveness + Eric Ayers + Eric Danielson diff --git a/src/python/pants/notes/master.rst b/src/python/pants/notes/master.rst index 3becbc7fc32..a127a5df2a3 100644 --- a/src/python/pants/notes/master.rst +++ b/src/python/pants/notes/master.rst @@ -4,6 +4,21 @@ Master Pre-Releases This document describes ``dev`` releases which occur weekly from master, and which do not undergo the vetting associated with ``stable`` releases. +1.4.0.dev11 (9/1/2017) +---------------------- + +Bugfixes +~~~~~~~~ + +* Centralize options tracking in the Parser. (#4832) + `PR #4832 `_ + +Refactoring, Improvements, and Tooling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Bump petgraph to 0.4.5 (#4836) + `PR #4836 `_ + 1.4.0.dev10 (8/25/2017) ----------------------- diff --git a/src/python/pants/version.py b/src/python/pants/version.py index 2f060d9ba4b..383d2944019 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -8,6 +8,6 @@ from packaging.version import Version -VERSION = '1.4.0.dev10' +VERSION = '1.4.0.dev11' PANTS_SEMVER = Version(VERSION) From ecc30894b4d61fea1312ee2d4062109b0ed026b1 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Tue, 5 Sep 2017 10:43:10 -0700 Subject: [PATCH 34/67] Clean up stray pantsd-runner processes (#4835) Problem There are stray pantsd-runners that continue to live on even after a run has finished when an exception is thrown in pants. The error is detailed in issue 4827. Solution To ensure we are exiting out of the process when there is an exception, wrap the os._exit in a finally block. Result With this change, the pantsd-runner process closes and we don't have any stray processes running. --- src/python/pants/pantsd/process_manager.py | 12 ++++---- tests/python/pants_test/pantsd/BUILD | 3 +- .../pantsd/test_pantsd_integration.py | 29 +++++++++++++++++++ tests/python/pants_test/testutils/BUILD | 10 +++++++ .../pants_test/testutils/process_test_util.py | 18 ++++++++++++ 5 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 tests/python/pants_test/testutils/process_test_util.py diff --git a/src/python/pants/pantsd/process_manager.py b/src/python/pants/pantsd/process_manager.py index 08a40defbd3..a63b8448827 100644 --- a/src/python/pants/pantsd/process_manager.py +++ b/src/python/pants/pantsd/process_manager.py @@ -433,16 +433,16 @@ def daemonize(self, pre_fork_opts=None, post_fork_parent_opts=None, post_fork_ch self.post_fork_child(**post_fork_child_opts or {}) except Exception: logger.critical(traceback.format_exc()) - - os._exit(0) + finally: + os._exit(0) else: try: if write_pid: self.write_pid(second_pid) self.post_fork_parent(**post_fork_parent_opts or {}) except Exception: logger.critical(traceback.format_exc()) - - os._exit(0) + finally: + os._exit(0) else: # This prevents un-reaped, throw-away parent processes from lingering in the process table. os.waitpid(pid, 0) @@ -465,8 +465,8 @@ def daemon_spawn(self, pre_fork_opts=None, post_fork_parent_opts=None, post_fork self.post_fork_child(**post_fork_child_opts or {}) except Exception: logger.critical(traceback.format_exc()) - - os._exit(0) + finally: + os._exit(0) else: try: self.post_fork_parent(**post_fork_parent_opts or {}) diff --git a/tests/python/pants_test/pantsd/BUILD b/tests/python/pants_test/pantsd/BUILD index e414e7008ea..b08a59d0e5c 100644 --- a/tests/python/pants_test/pantsd/BUILD +++ b/tests/python/pants_test/pantsd/BUILD @@ -66,7 +66,8 @@ python_tests( sources = ['test_pantsd_integration.py'], dependencies = [ 'src/python/pants/pantsd:process_manager', - 'tests/python/pants_test:int-test' + 'tests/python/pants_test:int-test', + 'tests/python/pants_test/testutils:process_test_util' ], tags = {'integration'}, timeout = 120 diff --git a/tests/python/pants_test/pantsd/test_pantsd_integration.py b/tests/python/pants_test/pantsd/test_pantsd_integration.py index 323d872e830..b620a0a643c 100644 --- a/tests/python/pants_test/pantsd/test_pantsd_integration.py +++ b/tests/python/pants_test/pantsd/test_pantsd_integration.py @@ -12,6 +12,7 @@ from pants.pantsd.process_manager import ProcessManager from pants_test.pants_run_integration_test import PantsRunIntegrationTest +from pants_test.testutils.process_test_util import check_process_exists_by_command class PantsDaemonMonitor(ProcessManager): @@ -184,3 +185,31 @@ def test_pantsd_stacktrace_dump(self): self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) checker.assert_stopped() + + def test_pantsd_runner_dies_after_failed_run(self): + with self.pantsd_test_context() as (workdir, pantsd_config, checker): + self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) + try: + # Start pantsd implicitly via a throwaway invocation. + self.assert_success(self.run_pants_with_workdir(['help'], workdir, pantsd_config)) + checker.await_pantsd() + + # Run target that throws an exception in pants. + self.assert_failure( + self.run_pants_with_workdir( + ['bundle', 'testprojects/src/java/org/pantsbuild/testproject/bundle:missing-files'], + workdir, + pantsd_config) + ) + + # Check for no stray pantsd-runner prcesses. + self.assertFalse(check_process_exists_by_command('pantsd-runner')) + + # Assert pantsd is in a good functional state. + self.assert_success(self.run_pants_with_workdir(['help'], workdir, pantsd_config)) + + finally: + # Explicitly kill pantsd (from a pantsd-launched runner). + self.assert_success(self.run_pants_with_workdir(['kill-pantsd'], workdir, pantsd_config)) + + checker.assert_stopped() diff --git a/tests/python/pants_test/testutils/BUILD b/tests/python/pants_test/testutils/BUILD index 2d94e57ead4..af4869844a8 100644 --- a/tests/python/pants_test/testutils/BUILD +++ b/tests/python/pants_test/testutils/BUILD @@ -27,3 +27,13 @@ python_library( 'src/python/pants/util:contextutil', ], ) + +python_library( + name = 'process_test_util', + sources = [ + 'process_test_util.py', + ], + dependencies = [ + '3rdparty/python:psutil', + ], +) diff --git a/tests/python/pants_test/testutils/process_test_util.py b/tests/python/pants_test/testutils/process_test_util.py new file mode 100644 index 00000000000..b3cf6b81ba0 --- /dev/null +++ b/tests/python/pants_test/testutils/process_test_util.py @@ -0,0 +1,18 @@ +# coding=utf-8 +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import psutil + + +def check_process_exists_by_command(name): + for proc in psutil.process_iter(): + try: + if name in ''.join(proc.cmdline()): + return True + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + return False From 9fefd8aad907c62197c146ebd64a42b61babd544 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 5 Sep 2017 10:49:46 -0700 Subject: [PATCH 35/67] Upgrade pex to latest. (#4843) This brings in support for using subprocess32 when available which Pants will leverage to provide robust test timeouts for one. --- 3rdparty/python/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index cfb86200264..63052236780 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -12,7 +12,7 @@ mock==2.0.0 packaging==16.8 pathspec==0.5.0 pep8==1.6.2 -pex==1.2.8 +pex==1.2.11 psutil==4.3.0 pyflakes==1.1.0 Pygments==1.4 From e50ce611962437c766336d403ac3911d28f36f96 Mon Sep 17 00:00:00 2001 From: Kris Wilson Date: Wed, 6 Sep 2017 14:44:03 -0700 Subject: [PATCH 36/67] [pantsd] Improve locking. (#4847) Problem Currently, the daemon relies on a locked contextmanager threaded through via the scheduler service for guarding critical sections around pailgun forking. This does not make an affordance for e.g. locking around filesystem event processing or other scenarios. Solution Thread a daemon-global lock and helper method to PantsService and it's progeny for easier intra-daemon locking scenarios. Wrap some critical sections with it. Result Better locking with less race conditions. --- src/python/pants/bin/engine_initializer.py | 2 +- src/python/pants/engine/scheduler.py | 58 +++++++++---------- .../pants/init/pants_daemon_launcher.py | 47 ++++++++------- src/python/pants/pantsd/pants_daemon.py | 8 ++- .../pants/pantsd/service/fs_event_service.py | 3 +- .../pants/pantsd/service/pailgun_service.py | 9 +-- .../pants/pantsd/service/pants_service.py | 8 ++- .../pants/pantsd/service/scheduler_service.py | 52 +++++++++-------- .../init/test_pants_daemon_launcher.py | 2 +- .../pantsd/service/test_fs_event_service.py | 2 +- 10 files changed, 102 insertions(+), 89 deletions(-) diff --git a/src/python/pants/bin/engine_initializer.py b/src/python/pants/bin/engine_initializer.py index 1a76ebcc8ed..46c1c8d9a8b 100644 --- a/src/python/pants/bin/engine_initializer.py +++ b/src/python/pants/bin/engine_initializer.py @@ -94,7 +94,7 @@ def create_build_graph(self, target_roots, build_root=None): logger.debug('target_roots are: %r', target_roots) graph = LegacyBuildGraph.create(self.scheduler, self.symbol_table_cls) logger.debug('build_graph is: %s', graph) - with self.scheduler.locked(): + with self.scheduler.lock: # Ensure the entire generator is unrolled. for _ in graph.inject_specs_closure(target_roots.as_specs()): pass diff --git a/src/python/pants/engine/scheduler.py b/src/python/pants/engine/scheduler.py index 65a5a5d3451..0b99ddba0c4 100644 --- a/src/python/pants/engine/scheduler.py +++ b/src/python/pants/engine/scheduler.py @@ -10,7 +10,6 @@ import threading import time from collections import defaultdict -from contextlib import contextmanager from pants.base.exceptions import TaskError from pants.base.project_tree import Dir, File, Link @@ -86,30 +85,30 @@ def __init__(self, native, build_root, work_dir, ignore_patterns, rule_index): self._register_rules(rule_index) self._scheduler = native.new_scheduler( - self._tasks, - self._root_subject_types, - build_root, - work_dir, - ignore_patterns, - Snapshot, - _Snapshots, - FileContent, - FilesContent, - Path, - Dir, - File, - Link, - has_products_constraint, - constraint_for(Address), - constraint_for(Variants), - constraint_for(PathGlobs), - constraint_for(Snapshot), - constraint_for(_Snapshots), - constraint_for(FilesContent), - constraint_for(Dir), - constraint_for(File), - constraint_for(Link), - ) + self._tasks, + self._root_subject_types, + build_root, + work_dir, + ignore_patterns, + Snapshot, + _Snapshots, + FileContent, + FilesContent, + Path, + Dir, + File, + Link, + has_products_constraint, + constraint_for(Address), + constraint_for(Variants), + constraint_for(PathGlobs), + constraint_for(Snapshot), + constraint_for(_Snapshots), + constraint_for(FilesContent), + constraint_for(Dir), + constraint_for(File), + constraint_for(Link), + ) def _root_type_ids(self): return self._to_ids_buf(sorted(self._root_subject_types)) @@ -350,6 +349,10 @@ def __init__(self, self._scheduler.assert_ruleset_valid() + @property + def lock(self): + return self._product_graph_lock + def trace(self): """Yields a stringified 'stacktrace' starting from the scheduler's roots.""" with self._product_graph_lock: @@ -400,11 +403,6 @@ def execution_request(self, products, subjects): """ return ExecutionRequest(tuple((s, p) for s in subjects for p in products)) - @contextmanager - def locked(self): - with self._product_graph_lock: - yield - def root_entries(self, execution_request): """Returns the roots for the given ExecutionRequest as a list of tuples of: ((subject, product), State) diff --git a/src/python/pants/init/pants_daemon_launcher.py b/src/python/pants/init/pants_daemon_launcher.py index e80fc2c44d3..dc675386c83 100644 --- a/src/python/pants/init/pants_daemon_launcher.py +++ b/src/python/pants/init/pants_daemon_launcher.py @@ -97,8 +97,14 @@ def __init__(self, @testable_memoized_property def pantsd(self): - return PantsDaemon(self._build_root, self._pants_workdir, self._log_level, self._native, - self._log_dir, reset_func=clean_global_runtime_state) + return PantsDaemon( + self._build_root, + self._pants_workdir, + self._log_level, + self._native, + self._log_dir, + reset_func=clean_global_runtime_state + ) @testable_memoized_property def watchman_launcher(self): @@ -114,10 +120,6 @@ def _setup_services(self, watchman): # ultimately import the pantsd services in order to itself launch pantsd. from pants.bin.daemon_pants_runner import DaemonExiter, DaemonPantsRunner - services = [] - - fs_event_service = FSEventService(watchman, self._build_root, self._fs_event_workers) - legacy_graph_helper = self._engine_initializer.setup_legacy_graph( self._pants_ignore_patterns, self._pants_workdir, @@ -126,30 +128,35 @@ def _setup_services(self, watchman): exclude_target_regexps=self._exclude_target_regexp, subproject_roots=self._subproject_roots, ) - scheduler_service = SchedulerService(fs_event_service, legacy_graph_helper) - services.extend((fs_event_service, scheduler_service)) - pailgun_service = PailgunService(bind_addr=(self._pailgun_host, self._pailgun_port), - exiter_class=DaemonExiter, - runner_class=DaemonPantsRunner, - target_roots_class=TargetRoots, - scheduler_service=scheduler_service) - services.append(pailgun_service) - - # Construct a mapping of named ports used by the daemon's services. In the default case these - # will be randomly assigned by the underlying implementation so we can't reference via options. - port_map = dict(pailgun=pailgun_service.pailgun_port) + fs_event_service = FSEventService(watchman, self._build_root, self._fs_event_workers) + scheduler_service = SchedulerService(fs_event_service, legacy_graph_helper) + pailgun_service = PailgunService( + bind_addr=(self._pailgun_host, self._pailgun_port), + exiter_class=DaemonExiter, + runner_class=DaemonPantsRunner, + target_roots_class=TargetRoots, + scheduler_service=scheduler_service + ) - return tuple(services), port_map + return ( + # Use the schedulers reentrant lock as the daemon's global lock. + legacy_graph_helper.scheduler.lock, + # Services. + (fs_event_service, scheduler_service, pailgun_service), + # Port map. + dict(pailgun=pailgun_service.pailgun_port) + ) def _launch_pantsd(self): # Launch Watchman (if so configured). watchman = self.watchman_launcher.maybe_launch() # Initialize pantsd services. - services, port_map = self._setup_services(watchman) + lock, services, port_map = self._setup_services(watchman) # Setup and fork pantsd. + self.pantsd.set_lock(lock) self.pantsd.set_services(services) self.pantsd.set_socket_map(port_map) self.pantsd.daemonize() diff --git a/src/python/pants/pantsd/pants_daemon.py b/src/python/pants/pantsd/pants_daemon.py index 50e908a4a62..d3e5fb0a17a 100644 --- a/src/python/pants/pantsd/pants_daemon.py +++ b/src/python/pants/pantsd/pants_daemon.py @@ -79,11 +79,16 @@ def __init__(self, build_root, work_dir, log_level, native, log_dir=None, servic # N.B. This Event is used as nothing more than a convenient atomic flag - nothing waits on it. self._kill_switch = threading.Event() self._exiter = Exiter() + # Placeholder for a daemon-global lock for service<->service locking. + self._lock = None @property def is_killed(self): return self._kill_switch.is_set() + def set_lock(self, lock): + self._lock = lock + def set_services(self, services): self._services = services @@ -137,9 +142,10 @@ def _setup_logging(self, log_level): return result.log_stream def _setup_services(self, services): + assert self._lock is not None, 'PantsDaemon lock has not been set!' for service in services: self._logger.info('setting up service {}'.format(service)) - service.setup() + service.setup(self._lock) def _run_services(self, services): """Service runner main loop.""" diff --git a/src/python/pants/pantsd/service/fs_event_service.py b/src/python/pants/pantsd/service/fs_event_service.py index 83efe687cfd..5349e2aaf2b 100644 --- a/src/python/pants/pantsd/service/fs_event_service.py +++ b/src/python/pants/pantsd/service/fs_event_service.py @@ -40,7 +40,8 @@ def __init__(self, watchman, build_root, worker_count): self._executor = None self._handlers = {} - def setup(self, executor=None): + def setup(self, lock, executor=None): + super(FSEventService, self).setup(lock) self._executor = executor or ThreadPoolExecutor(max_workers=self._worker_count) def terminate(self): diff --git a/src/python/pants/pantsd/service/pailgun_service.py b/src/python/pants/pantsd/service/pailgun_service.py index c6e2a391fe8..d74e6a7ae7d 100644 --- a/src/python/pants/pantsd/service/pailgun_service.py +++ b/src/python/pants/pantsd/service/pailgun_service.py @@ -79,14 +79,7 @@ def runner_factory(sock, arguments, environment): @contextmanager def context_lock(): - """This lock is used to safeguard Pailgun request handling against a fork() with the - scheduler lock held by another thread (e.g. the FSEventService thread), which can - lead to a pailgun deadlock. - """ - if self._scheduler_service: - with self._scheduler_service.locked(): - yield - else: + with self.lock: yield return PailgunServer(self._bind_addr, runner_factory, context_lock) diff --git a/src/python/pants/pantsd/service/pants_service.py b/src/python/pants/pantsd/service/pants_service.py index 7ac98e5ebae..aea567964dc 100644 --- a/src/python/pants/pantsd/service/pants_service.py +++ b/src/python/pants/pantsd/service/pants_service.py @@ -34,8 +34,12 @@ def is_killed(self): def pre_fork(self): """Called pre-fork, before `run` to allow for service->service or other side-effecting setup.""" - def setup(self): - """Called before `run` to allow for service->service or other side-effecting setup.""" + def setup(self, lock): + """Called before `run` to allow for service->service or other side-effecting setup. + + :param threading.Lock lock: A global service<->service lock for guarding critical work. + """ + self.lock = lock @abstractmethod def run(self): diff --git a/src/python/pants/pantsd/service/scheduler_service.py b/src/python/pants/pantsd/service/scheduler_service.py index 503c65b725e..a5af2162296 100644 --- a/src/python/pants/pantsd/service/scheduler_service.py +++ b/src/python/pants/pantsd/service/scheduler_service.py @@ -7,6 +7,7 @@ import logging import Queue +import threading from pants.pantsd.service.pants_service import PantsService @@ -33,11 +34,7 @@ def __init__(self, fs_event_service, legacy_graph_helper): self._logger = logging.getLogger(__name__) self._event_queue = Queue.Queue(maxsize=64) - - @property - def locked(self): - """Surfaces the scheduler's `locked` method as part of the service's public API.""" - return self._scheduler.locked + self._ready = threading.Event() @property def change_calculator(self): @@ -48,8 +45,9 @@ def pre_fork(self): """Pre-fork controls.""" self._scheduler.pre_fork() - def setup(self): + def setup(self, lock): """Service setup.""" + super(SchedulerService, self).setup(lock) # Register filesystem event handlers on an FSEventService instance. self._fs_event_service.register_all_files_handler(self._enqueue_fs_event) @@ -61,10 +59,6 @@ def _enqueue_fs_event(self, event): def _handle_batch_event(self, files): self._logger.debug('handling change event for: %s', files) - if not self._scheduler: - self._logger.debug('no scheduler. ignoring event.') - return - self._scheduler.invalidate_files(files) def _process_event_queue(self): @@ -74,19 +68,25 @@ def _process_event_queue(self): except Queue.Empty: return - try: - subscription, is_initial_event, files = (event['subscription'], - event['is_fresh_instance'], - [f.decode('utf-8') for f in event['files']]) - except (KeyError, UnicodeDecodeError) as e: - self._logger.warn('%r raised by invalid watchman event: %s', e, event) - return + with self.lock: + try: + subscription, is_initial_event, files = (event['subscription'], + event['is_fresh_instance'], + [f.decode('utf-8') for f in event['files']]) + except (KeyError, UnicodeDecodeError) as e: + self._logger.warn('%r raised by invalid watchman event: %s', e, event) + return + + self._logger.debug('processing {} files for subscription {} (first_event={})' + .format(len(files), subscription, is_initial_event)) + + if is_initial_event: + # Once we've seen the initial watchman event, set the internal `Event`. + self._ready.set() + else: + # The first watchman event is a listing of all files - ignore it. + self._handle_batch_event(files) - self._logger.debug('processing {} files for subscription {} (first_event={})' - .format(len(files), subscription, is_initial_event)) - - if not is_initial_event: # Ignore the initial all files event from watchman. - self._handle_batch_event(files) self._event_queue.task_done() def warm_product_graph(self, spec_roots): @@ -94,8 +94,12 @@ def warm_product_graph(self, spec_roots): :returns: A `LegacyGraphHelper` instance for graph construction. """ - self._graph_helper.warm_product_graph(spec_roots) - return self._graph_helper + # Block warming until the initial watchman filesystem event is seen. + self._ready.wait() + + with self.lock: + self._graph_helper.warm_product_graph(spec_roots) + return self._graph_helper def run(self): """Main service entrypoint.""" diff --git a/tests/python/pants_test/init/test_pants_daemon_launcher.py b/tests/python/pants_test/init/test_pants_daemon_launcher.py index d1d9fa813f4..06bdb57602e 100644 --- a/tests/python/pants_test/init/test_pants_daemon_launcher.py +++ b/tests/python/pants_test/init/test_pants_daemon_launcher.py @@ -15,7 +15,7 @@ class PantsDaemonLauncherTest(BaseTest): - PDL_PATCH_OPTS = dict(autospec=True, spec_set=True, return_value=(None, None)) + PDL_PATCH_OPTS = dict(autospec=True, spec_set=True, return_value=(None, None, None)) def pants_daemon_launcher(self): factory = global_subsystem_instance(PantsDaemonLauncher.Factory) diff --git a/tests/python/pants_test/pantsd/service/test_fs_event_service.py b/tests/python/pants_test/pantsd/service/test_fs_event_service.py index eb6c8741726..5e2fafe10cd 100644 --- a/tests/python/pants_test/pantsd/service/test_fs_event_service.py +++ b/tests/python/pants_test/pantsd/service/test_fs_event_service.py @@ -37,7 +37,7 @@ def setUp(self): BaseTest.setUp(self) self.mock_watchman = mock.create_autospec(Watchman, spec_set=True) self.service = FSEventService(self.mock_watchman, self.BUILD_ROOT, self.WORKER_COUNT) - self.service.setup(TestExecutor()) + self.service.setup(None, executor=TestExecutor()) self.service.register_all_files_handler(lambda x: True, name='test') self.service.register_all_files_handler(lambda x: False, name='test2') From c34fa8669fa885fc6c6a188c13fa9226ecc9d366 Mon Sep 17 00:00:00 2001 From: Kris Wilson Date: Wed, 6 Sep 2017 22:19:54 -0700 Subject: [PATCH 37/67] Migrate BinaryUtil options to bootstrap options. (#4846) Problem In order to fully externalize pantsd lifecycle, we need to untangle pantsd's reliance on Subsystem dependencies and their options. This is an initial step in that direction. Solution Migrate BinaryUtil options to bootstrap options. Result Able to leverage BinaryUtil before subsystems are setup by directly calling BinaryUtil.__init__ with global options. --- src/python/pants/binaries/binary_util.py | 24 +++++++++-------------- src/python/pants/option/global_options.py | 12 ++++++++++++ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/python/pants/binaries/binary_util.py b/src/python/pants/binaries/binary_util.py index 6da4a3f0c4e..d5ec5fbc3c7 100644 --- a/src/python/pants/binaries/binary_util.py +++ b/src/python/pants/binaries/binary_util.py @@ -50,21 +50,11 @@ class Factory(Subsystem): """ :API: public """ + # N.B. `BinaryUtil` sources all of its options from bootstrap options, so that + # `BinaryUtil` instances can be created prior to `Subsystem` bootstrapping. So + # this options scope is unused, but required to remain a `Subsystem`. options_scope = 'binaries' - @classmethod - def register_options(cls, register): - register('--baseurls', type=list, advanced=True, - default=['https://binaries.pantsbuild.org'], - help='List of urls from which binary tools are downloaded. Urls are searched in ' - 'order until the requested path is found.') - register('--fetch-timeout-secs', type=int, default=30, advanced=True, - help='Timeout in seconds for url reads when fetching binary tools from the ' - 'repos specified by --baseurls') - register('--path-by-id', type=dict, advanced=True, - help='Maps output of uname for a machine to a binary search path. e.g. ' - '{ ("darwin", "15"): ["mac", "10.11"]), ("linux", "arm32"): ["linux", "arm32"] }') - @classmethod def create(cls): """ @@ -72,8 +62,12 @@ def create(cls): """ # NB: create is a class method to ~force binary fetch location to be global. options = cls.global_instance().get_options() - return BinaryUtil(options.baseurls, options.fetch_timeout_secs, options.pants_bootstrapdir, - options.path_by_id) + return BinaryUtil( + options.binaries_baseurls, + options.binaries_fetch_timeout_secs, + options.pants_bootstrapdir, + options.binaries_path_by_id + ) class MissingMachineInfo(TaskError): """Indicates that pants was unable to map this machine's OS to a binary path prefix.""" diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py index c6262f76d4f..4afb09232bc 100644 --- a/src/python/pants/option/global_options.py +++ b/src/python/pants/option/global_options.py @@ -145,6 +145,18 @@ def register_bootstrap_options(cls, register): help='A directory to write execution and rule graphs to as `dot` files. The contents ' 'of the directory will be overwritten if any filenames collide.') + # BinaryUtil options. + register('--binaries-baseurls', type=list, advanced=True, + default=['https://binaries.pantsbuild.org'], + help='List of URLs from which binary tools are downloaded. URLs are searched in ' + 'order until the requested path is found.') + register('--binaries-fetch-timeout-secs', type=int, default=30, advanced=True, + help='Timeout in seconds for URL reads when fetching binary tools from the ' + 'repos specified by --baseurls.') + register('--binaries-path-by-id', type=dict, advanced=True, + help=('Maps output of uname for a machine to a binary search path. e.g. ' + '{("darwin", "15"): ["mac", "10.11"]), ("linux", "arm32"): ["linux", "arm32"]}')) + @classmethod def register_options(cls, register): """Register options not tied to any particular task or subsystem.""" From 04a0c0aaf7888d2e47c3f332779ceb73ab327de2 Mon Sep 17 00:00:00 2001 From: Kris Wilson Date: Fri, 8 Sep 2017 11:12:39 -0700 Subject: [PATCH 38/67] Repair `BinaryNotFound` due to `sslv3 alert handshake failure`. (#4853) Problem The build recently started failing with SSL errors like: BinaryNotFound: Failed to fetch binary scripts/cloc/1.66/cloc from any source: (Failed to fetch binary from https://binaries.pantsbuild.org/scripts/cloc/1.66/cloc: Problem GETing data from https://binaries.pantsbuild.org/scripts/cloc/1.66/cloc: [Errno 1] _ssl.c:510: error:14077410:SSL routines:SSL23_GET_SERVER_HELLO:sslv3 alert handshake failure) when retrieving artifacts via BinaryUtil. Solution Bump the requests version range to permit the latest version (noting no apparent mention of API Changes between current minimal to pypi current based on the changelog here). Use the requests[security] extra to include a superior/modernized set of underlying SSL libraries (see here for more info). Bump the cffi version to include a fix for a cryptography build/cffi issue seen here. Add openssl and libssl-dev to Travis shards to support SSL related builds. Drop our reliance on the (now 2 versions) old Travis CI image based on observations in #4852, which presumably nets us a transitively better OpenSSL version via apt. Result Closes #4852 --- .travis.yml | 37 +++++++++++++++++++++----------- 3rdparty/python/requirements.txt | 4 ++-- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 57df8041ff5..8be920942e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required # Docker runs will write files as root, so avoid caching for this shard. cache: false @@ -86,7 +85,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -96,6 +94,8 @@ matrix: - lib32z1-dev - gcc-multilib - python-dev + - openssl + - libssl-dev language: python python: "2.7.13" before_install: @@ -113,7 +113,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -123,6 +122,8 @@ matrix: - lib32z1-dev - gcc-multilib - python-dev + - openssl + - libssl-dev language: python python: "2.7.13" before_install: @@ -140,7 +141,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -150,6 +150,8 @@ matrix: - lib32z1-dev - gcc-multilib - python-dev + - openssl + - libssl-dev language: python python: "2.7.13" before_install: @@ -167,7 +169,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -177,6 +178,8 @@ matrix: - lib32z1-dev - gcc-multilib - python-dev + - openssl + - libssl-dev language: python python: "2.7.13" before_install: @@ -194,7 +197,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -204,6 +206,8 @@ matrix: - lib32z1-dev - gcc-multilib - python-dev + - openssl + - libssl-dev language: python python: "2.7.13" before_install: @@ -221,7 +225,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -231,6 +234,8 @@ matrix: - lib32z1-dev - gcc-multilib - python-dev + - openssl + - libssl-dev language: python python: "2.7.13" before_install: @@ -248,7 +253,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -258,6 +262,8 @@ matrix: - lib32z1-dev - gcc-multilib - python-dev + - openssl + - libssl-dev language: python python: "2.7.13" before_install: @@ -275,7 +281,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -285,6 +290,8 @@ matrix: - lib32z1-dev - gcc-multilib - python-dev + - openssl + - libssl-dev language: python python: "2.7.13" before_install: @@ -302,7 +309,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -312,6 +318,8 @@ matrix: - lib32z1-dev - gcc-multilib - python-dev + - openssl + - libssl-dev language: python python: "2.7.13" before_install: @@ -329,7 +337,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -339,6 +346,8 @@ matrix: - lib32z1-dev - gcc-multilib - python-dev + - openssl + - libssl-dev language: python python: "2.7.13" before_install: @@ -356,7 +365,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -366,6 +374,8 @@ matrix: - lib32z1-dev - gcc-multilib - python-dev + - openssl + - libssl-dev language: python python: "2.7.13" before_install: @@ -383,7 +393,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -393,6 +402,8 @@ matrix: - lib32z1-dev - gcc-multilib - python-dev + - openssl + - libssl-dev language: python python: "2.7.13" before_install: diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 63052236780..578991f70c9 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -1,6 +1,6 @@ ansicolors==1.0.2 beautifulsoup4>=4.3.2,<4.4 -cffi==1.7.0 +cffi==1.10.0 coverage>=4.3.4,<4.4 docutils>=0.12,<0.13 fasteners==0.14.1 @@ -20,7 +20,7 @@ pystache==0.5.3 pytest-cov>=2.4,<2.5 pytest>=3.0.7,<4.0 pywatchman==1.3.0 -requests>=2.5.0,<2.6 +requests[security]>=2.5.0,<2.19 scandir==1.2 setproctitle==1.1.10 setuptools==30.0.0 From a0971382db5653f7abcc90e361e2555889783bce Mon Sep 17 00:00:00 2001 From: Mateo Rodriguez Date: Fri, 8 Sep 2017 21:11:15 -0400 Subject: [PATCH 39/67] Prepare the 1.4.0.dev12 release. (#4857) --- src/python/pants/notes/master.rst | 28 ++++++++++++++++++++++++++++ src/python/pants/version.py | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/python/pants/notes/master.rst b/src/python/pants/notes/master.rst index a127a5df2a3..64af9c7d336 100644 --- a/src/python/pants/notes/master.rst +++ b/src/python/pants/notes/master.rst @@ -4,6 +4,34 @@ Master Pre-Releases This document describes ``dev`` releases which occur weekly from master, and which do not undergo the vetting associated with ``stable`` releases. + +1.4.0.dev12 (9/8/2017) +---------------------- + +API Changes +~~~~~~~~~~~ + +* Migrate BinaryUtil options to bootstrap options. (#4846) + `PR #4846 `_ + +Bugfixes +~~~~~~~~ + +* Clean up stray pantsd-runner processes (#4835) + `PR #4835 `_ + +Refactoring, Improvements, and Tooling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Repair `BinaryNotFound` due to `sslv3 alert handshake failure`. (#4853) + `PR #4853 `_ + +* [pantsd] Improve locking. (#4847) + `PR #4847 `_ + +* Upgrade pex to latest. (#4843) + `PR #4843 `_ + 1.4.0.dev11 (9/1/2017) ---------------------- diff --git a/src/python/pants/version.py b/src/python/pants/version.py index 383d2944019..80c8bc3b146 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -8,6 +8,6 @@ from packaging.version import Version -VERSION = '1.4.0.dev11' +VERSION = '1.4.0.dev12' PANTS_SEMVER = Version(VERSION) From 1f6029ab50cc842f274029c76337be29d4bb0967 Mon Sep 17 00:00:00 2001 From: cheister Date: Mon, 11 Sep 2017 13:38:56 -0700 Subject: [PATCH 40/67] Use @files for javadoc so it runs with a longer command line and add doc exclude patterns option (#4842) --- .../org/pantsbuild/tools/jar/JarBuilder.java | 24 +++++++++++----- .../pants/backend/jvm/tasks/javadoc_gen.py | 18 ++++++++++-- .../pants/backend/jvm/tasks/jvmdoc_gen.py | 28 ++++++++++++++++--- 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/src/java/org/pantsbuild/tools/jar/JarBuilder.java b/src/java/org/pantsbuild/tools/jar/JarBuilder.java index beabe445bc2..066aa758d41 100644 --- a/src/java/org/pantsbuild/tools/jar/JarBuilder.java +++ b/src/java/org/pantsbuild/tools/jar/JarBuilder.java @@ -103,14 +103,14 @@ public static class DuplicateEntryException extends RuntimeException { } /** - * Returns the duplicate path. + * @return the duplicate path. */ public String getPath() { return entry.getJarPath(); } /** - * Returns the contents of the duplicate entry. + * @return the contents of duplicate entry */ public ByteSource getSource() { return entry.contents; @@ -182,7 +182,7 @@ public DuplicatePolicy(Predicate selector, DuplicateAction action) } /** - * Returns the action that should be applied when a duplicate entry falls under this policy's + * @return The action that should be applied when a duplicate entry falls under this policy's * jurisdiction. */ public DuplicateAction getAction() { @@ -212,6 +212,7 @@ public static class DuplicateHandler { * Creates a handler that always applies the given {@code action}. * * @param action The action to perform on all duplicate entries encountered. + * @return a handler */ public static DuplicateHandler always(DuplicateAction action) { Preconditions.checkNotNull(action); @@ -224,6 +225,8 @@ public static DuplicateHandler always(DuplicateAction action) { *

* Merged resources include META-INF/services/ files. *

+ * + * @return a handler */ public static DuplicateHandler skipDuplicatesConcatWellKnownMetadata() { DuplicatePolicy concatServices = @@ -238,6 +241,9 @@ public static DuplicateHandler skipDuplicatesConcatWellKnownMetadata() { /** * A convenience constructor equivalent to calling: * {@code DuplicateHandler(defaultAction, Arrays.asList(policies))} + * + * @param defaultAction The default action to apply when no policy matches. + * @param policies The policies to apply in preference order. */ public DuplicateHandler(DuplicateAction defaultAction, DuplicatePolicy... policies) { this(defaultAction, ImmutableList.copyOf(policies)); @@ -272,12 +278,15 @@ DuplicateAction actionFor(String jarPath) { public interface Source { /** - * Returns a name for this source. + * @return a name for this source. */ String name(); /** * Identifies a member of this source. + * + * @param name The name of the source + * @return identity */ String identify(String name); } @@ -437,17 +446,17 @@ public InputStream openStream() throws IOException { */ public interface Entry { /** - * Returns the source that contains the entry. + * @return the source that contains the entry. */ Source getSource(); /** - * Returns the name of the entry within its source. + * @return the name of the entry within its source. */ String getName(); /** - * Returns the path this entry will be added into the jar at. + * @return the path this entry will be added into the jar at. */ String getJarPath(); } @@ -668,6 +677,7 @@ public JarBuilder(File target) { * If the {@code target} does not exist a new jar will be created at its path. * * @param target The target jar file to write. + * @param listener A progress listener */ public JarBuilder(File target, Listener listener) { this.target = Preconditions.checkNotNull(target); diff --git a/src/python/pants/backend/jvm/tasks/javadoc_gen.py b/src/python/pants/backend/jvm/tasks/javadoc_gen.py index 23bed4d3b1a..c4c2f1a1f5c 100644 --- a/src/python/pants/backend/jvm/tasks/javadoc_gen.py +++ b/src/python/pants/backend/jvm/tasks/javadoc_gen.py @@ -5,6 +5,8 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +import os + from pants.backend.jvm.tasks.jvmdoc_gen import Jvmdoc, JvmdocGen from pants.java.distribution.distribution import DistributionLocator from pants.java.executor import SubprocessExecutor @@ -45,11 +47,12 @@ def create_javadoc_command(self, classpath, gendir, *targets): '-encoding', 'UTF-8', '-notimestamp', '-use', - '-classpath', ':'.join(classpath), + '-Xmaxerrs', '10000', # the default is 100 + '-Xmaxwarns', '10000', # the default is 100 '-d', gendir] # Always provide external linking for java API - offlinelinks = {'http://download.oracle.com/javase/6/docs/api/'} + offlinelinks = {'http://download.oracle.com/javase/8/docs/api/'} def link(target): for jar in target.jar_dependencies: @@ -63,7 +66,16 @@ def link(target): args.extend(self.args) - args.extend(sources) + javadoc_classpath_file = os.path.join(self.workdir, '{}.classpath'.format(os.path.basename(gendir))) + with open(javadoc_classpath_file, 'w') as f: + f.write('-classpath ') + f.write(':'.join(classpath)) + args.extend(['@{}'.format(javadoc_classpath_file)]) + + javadoc_sources_file = os.path.join(self.workdir, '{}.source.files'.format(os.path.basename(gendir))) + with open(javadoc_sources_file, 'w') as f: + f.write('\n'.join(sources)) + args.extend(['@{}'.format(javadoc_sources_file)]) java_executor = SubprocessExecutor(jdk) runner = java_executor.runner(jvm_options=self.jvm_options, diff --git a/src/python/pants/backend/jvm/tasks/jvmdoc_gen.py b/src/python/pants/backend/jvm/tasks/jvmdoc_gen.py index 90f85a08df3..dcb225f4911 100644 --- a/src/python/pants/backend/jvm/tasks/jvmdoc_gen.py +++ b/src/python/pants/backend/jvm/tasks/jvmdoc_gen.py @@ -9,12 +9,14 @@ import contextlib import multiprocessing import os +import re import subprocess from pants.backend.jvm.tasks.jvm_task import JvmTask from pants.base.exceptions import TaskError from pants.util import desktop from pants.util.dirutil import safe_mkdir, safe_walk +from pants.util.memo import memoized_property Jvmdoc = collections.namedtuple('Jvmdoc', ['tool_name', 'product_type']) @@ -54,6 +56,9 @@ def register_options(cls, register): fingerprint=True, help='Do not consider {0} errors to be build errors.'.format(tool_name)) + register('--exclude-patterns', type=list, default=[], fingerprint=True, + help='Patterns for targets to be excluded from doc generation.') + # TODO(John Sirois): This supports the JarPublish task and is an abstraction leak. # It allows folks doing a local-publish to skip an expensive and un-needed step. # Remove this flag and instead support conditional requirements being registered against @@ -78,6 +83,10 @@ def __init__(self, *args, **kwargs): self.ignore_failure = options.ignore_failure self.skip = options.skip + @memoized_property + def _exclude_patterns(self): + return [re.compile(x) for x in set(self.get_options().exclude_patterns or [])] + def generate_doc(self, language_predicate, create_jvmdoc_command): """ Generate an execute method given a language predicate and command to create documentation @@ -95,8 +104,19 @@ def generate_doc(self, language_predicate, create_jvmdoc_command): raise TaskError( 'Cannot provide {} target mappings for combined output'.format(self.jvmdoc().product_type)) - def docable(tgt): - return language_predicate(tgt) and (self._include_codegen or not tgt.is_synthetic) + def docable(target): + if not language_predicate(target): + self.context.log.debug('Skipping [{}] because it is does not pass the language predicate'.format(target.address.spec)) + return False + if not self._include_codegen and target.is_synthetic: + self.context.log.debug('Skipping [{}] because it is a synthetic target'.format(target.address.spec)) + return False + for pattern in self._exclude_patterns: + if pattern.search(target.address.spec): + self.context.log.debug( + "Skipping [{}] because it matches exclude pattern '{}'".format(target.address.spec, pattern.pattern)) + return False + return True targets = self.context.targets(predicate=docable) if not targets: @@ -185,9 +205,9 @@ def _generate_individual(self, targets, create_jvmdoc_command): def _handle_create_jvmdoc_result(self, targets, result, command): if result != 0: - targetlist = ", ".join(map(str, targets)) + targetlist = ", ".join(map(lambda target: target.address.spec, targets)) message = 'Failed to process {} for {} [{}]: {}'.format( - self.jvmdoc().tool_name, targetlist, result, command) + self.jvmdoc().tool_name, targetlist, result, " ".join(command)) if self.ignore_failure: self.context.log.warn(message) else: From c32ce824f65a6749cf3a5083d38d193be3fd1bec Mon Sep 17 00:00:00 2001 From: Mateo Rodriguez Date: Tue, 12 Sep 2017 05:06:41 -0400 Subject: [PATCH 41/67] Downgrade to vanilla requests. (#4858) requests[security] has a dependency on pyopenssl, which is currently triggering a depreaction due to an OpenSSL.rand call. pyopenssl has fixed this issue in code, but is waiting for a maintainer to return from vacation and cut a release. In the meantime, the release cannot be completed with the current dep due to the non-zero it raises. .7/site-packages/urllib3/contrib/pyopenssl.py:46: DeprecationWarning: OpenSSL.rand is deprecated - you should use os.urandom instead import OpenSSL.SSL **** Failed to install cryptography-2.0.3 (caused by: NonZeroExit("received exit code 1 during execution of This was a CI-fix around ssl handshakes, and since the CI is currently having its own issues, I figured we could unlock the release while we wait. If this commit lands, it should be short-term and be undone once the pyopenssl can be released. References: #4856 --- 3rdparty/python/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 578991f70c9..3371aaf298c 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -20,7 +20,7 @@ pystache==0.5.3 pytest-cov>=2.4,<2.5 pytest>=3.0.7,<4.0 pywatchman==1.3.0 -requests[security]>=2.5.0,<2.19 +requests==2.5.0,<2.19 scandir==1.2 setproctitle==1.1.10 setuptools==30.0.0 From 25a29ecae6364f818a1aa6610bb957bc01345aff Mon Sep 17 00:00:00 2001 From: Kris Wilson Date: Wed, 13 Sep 2017 13:04:42 -0700 Subject: [PATCH 42/67] Re-add requests[security] and pin pyOpenSSL==17.1.0 to avoid deprecation warning. (#4865) Problem Master is currently broken with SSLv3 issues on binary fetches - fixed once in #4853, then reverted in #4858. #4858 passed CI with a populated cache, but once cleared and re-run revealed the breakage was still there. Solution Re-apply requests[security] and pin pyOpenSSL to an older version that doesn't throw the deprecation warning. Result No deprecation warnings locally. --- 3rdparty/python/requirements.txt | 3 ++- contrib/go/src/python/pants/contrib/go/subsystems/BUILD | 1 + src/python/pants/cache/BUILD | 1 + src/python/pants/goal/BUILD | 1 + src/python/pants/net/BUILD | 1 + tests/python/pants_test/net/http/BUILD | 1 + 6 files changed, 7 insertions(+), 1 deletion(-) diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 3371aaf298c..4af4dfab331 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -16,11 +16,12 @@ pex==1.2.11 psutil==4.3.0 pyflakes==1.1.0 Pygments==1.4 +pyopenssl==17.1.0 pystache==0.5.3 pytest-cov>=2.4,<2.5 pytest>=3.0.7,<4.0 pywatchman==1.3.0 -requests==2.5.0,<2.19 +requests[security]==2.5.0,<2.19 scandir==1.2 setproctitle==1.1.10 setuptools==30.0.0 diff --git a/contrib/go/src/python/pants/contrib/go/subsystems/BUILD b/contrib/go/src/python/pants/contrib/go/subsystems/BUILD index 22347a79a60..dfd1b47f3a6 100644 --- a/contrib/go/src/python/pants/contrib/go/subsystems/BUILD +++ b/contrib/go/src/python/pants/contrib/go/subsystems/BUILD @@ -4,6 +4,7 @@ python_library( dependencies=[ '3rdparty/python:requests', + '3rdparty/python:pyopenssl', '3rdparty/python:six', 'contrib/go/src/python/pants/contrib/go/targets', 'src/python/pants/base:workunit', diff --git a/src/python/pants/cache/BUILD b/src/python/pants/cache/BUILD index ef33925985f..2de81f7962d 100644 --- a/src/python/pants/cache/BUILD +++ b/src/python/pants/cache/BUILD @@ -4,6 +4,7 @@ python_library( dependencies = [ '3rdparty/python:requests', + '3rdparty/python:pyopenssl', '3rdparty/python:six', 'src/python/pants/base:deprecated', 'src/python/pants/base:validation', diff --git a/src/python/pants/goal/BUILD b/src/python/pants/goal/BUILD index 4d9b05a3ba6..8004507ced7 100644 --- a/src/python/pants/goal/BUILD +++ b/src/python/pants/goal/BUILD @@ -77,6 +77,7 @@ python_library( ':aggregated_timings', ':artifact_cache_stats', '3rdparty/python:requests', + '3rdparty/python:pyopenssl', 'src/python/pants/base:build_environment', 'src/python/pants/base:run_info', 'src/python/pants/base:worker_pool', diff --git a/src/python/pants/net/BUILD b/src/python/pants/net/BUILD index d13ba5623d2..6a2db311dc6 100644 --- a/src/python/pants/net/BUILD +++ b/src/python/pants/net/BUILD @@ -5,6 +5,7 @@ python_library( sources = rglobs('*.py'), dependencies = [ '3rdparty/python:requests', + '3rdparty/python:pyopenssl', '3rdparty/python:six', 'src/python/pants/util:dirutil', 'src/python/pants/util:meta', diff --git a/tests/python/pants_test/net/http/BUILD b/tests/python/pants_test/net/http/BUILD index 81ec588b716..050e5516f16 100644 --- a/tests/python/pants_test/net/http/BUILD +++ b/tests/python/pants_test/net/http/BUILD @@ -5,6 +5,7 @@ python_tests( dependencies = [ '3rdparty/python:mock', '3rdparty/python:requests', + '3rdparty/python:pyopenssl', '3rdparty/python:six', 'src/python/pants/net', 'src/python/pants/util:contextutil', From dcb8c61f1d3adabf19d99efe77a9c60602597901 Mon Sep 17 00:00:00 2001 From: Kris Wilson Date: Wed, 13 Sep 2017 16:33:41 -0700 Subject: [PATCH 43/67] [pantsd] Bump watchman->v4.9.0, pywatchman->v1.4.1 (#4848) Problem The watchman version we're using is outdated. Solution Upgrade to the latest watchman version! Result Realize a handful of bugfixes, performance improvements and memory utilization fixes (e.g. a purported 40% memory reduction in 4.7.0) per the watchman release notes here. --- 3rdparty/python/requirements.txt | 2 +- src/python/pants/pantsd/subsystem/watchman_launcher.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 4af4dfab331..cde8245f28a 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -20,7 +20,7 @@ pyopenssl==17.1.0 pystache==0.5.3 pytest-cov>=2.4,<2.5 pytest>=3.0.7,<4.0 -pywatchman==1.3.0 +pywatchman==1.4.1 requests[security]==2.5.0,<2.19 scandir==1.2 setproctitle==1.1.10 diff --git a/src/python/pants/pantsd/subsystem/watchman_launcher.py b/src/python/pants/pantsd/subsystem/watchman_launcher.py index bd62aff55a8..bc15cae8a49 100644 --- a/src/python/pants/pantsd/subsystem/watchman_launcher.py +++ b/src/python/pants/pantsd/subsystem/watchman_launcher.py @@ -26,7 +26,7 @@ def subsystem_dependencies(cls): @classmethod def register_options(cls, register): - register('--version', advanced=True, default='4.5.0', + register('--version', advanced=True, default='4.9.0', help='Watchman version.') register('--supportdir', advanced=True, default='bin/watchman', help='Find watchman binaries under this dir. Used as part of the path to lookup ' From 0d93c1c6977a24d74a158c376e5bf8f4d032e25d Mon Sep 17 00:00:00 2001 From: Mateo Rodriguez Date: Wed, 13 Sep 2017 20:20:55 -0400 Subject: [PATCH 44/67] Updated the 1.4.0dev12 release notes (#4862) * Updated the 1.4.0dev12 release notes Some changes landed to unblock the release, so updating the notes. There was never any packages published under this version, the release cut from the coming tag will be the sole 1.4.0dev12 release. * Upgrade release to get pyopenssl workaround --- src/python/pants/notes/master.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/python/pants/notes/master.rst b/src/python/pants/notes/master.rst index 64af9c7d336..6931f68f407 100644 --- a/src/python/pants/notes/master.rst +++ b/src/python/pants/notes/master.rst @@ -5,23 +5,26 @@ This document describes ``dev`` releases which occur weekly from master, and whi not undergo the vetting associated with ``stable`` releases. -1.4.0.dev12 (9/8/2017) +1.4.0.dev12 (9/13/2017) ---------------------- API Changes ~~~~~~~~~~~ +* Use @files for javadoc so it runs with a longer command line and add doc exclude patterns option (#4842) + `PR #4842 `_ * Migrate BinaryUtil options to bootstrap options. (#4846) `PR #4846 `_ Bugfixes ~~~~~~~~ - * Clean up stray pantsd-runner processes (#4835) `PR #4835 `_ Refactoring, Improvements, and Tooling ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* Re-add requests[security] and pin pyOpenSSL==17.1.0 to avoid deprecation warning. (#4865) + `PR #4865 `_ * Repair `BinaryNotFound` due to `sslv3 alert handshake failure`. (#4853) `PR #4853 `_ From 1fcbbed29e7c41710356491d8df900795957e3ec Mon Sep 17 00:00:00 2001 From: Mateo Rodriguez Date: Thu, 14 Sep 2017 00:59:37 +0000 Subject: [PATCH 45/67] Fix markdown rendering typo. Submitting TBR to publish the docsite. --- src/python/pants/notes/master.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/notes/master.rst b/src/python/pants/notes/master.rst index 6931f68f407..17427e6050f 100644 --- a/src/python/pants/notes/master.rst +++ b/src/python/pants/notes/master.rst @@ -6,7 +6,7 @@ not undergo the vetting associated with ``stable`` releases. 1.4.0.dev12 (9/13/2017) ----------------------- +----------------------- API Changes ~~~~~~~~~~~ From 702d8c458caa76015579ef02f05a0b3d95d78e9d Mon Sep 17 00:00:00 2001 From: Kris Wilson Date: Wed, 13 Sep 2017 18:03:42 -0700 Subject: [PATCH 46/67] Re-pin to 2017Q2 TravisCI image. (#4869) Problem In #4853 I unpinned the TravisCI image because it didn't appear to effect the result of the build. After clearing the Travis cache, a test depending on python2.6 has been failing consistently. Solution Repin. Result Fixes #4866 --- .travis.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8be920942e9..a09492edd86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,6 +54,7 @@ matrix: - os: linux dist: trusty + group: deprecated-2017Q2 sudo: required # Docker runs will write files as root, so avoid caching for this shard. cache: false @@ -85,6 +86,7 @@ matrix: - os: linux dist: trusty + group: deprecated-2017Q2 sudo: required addons: apt: @@ -113,6 +115,7 @@ matrix: - os: linux dist: trusty + group: deprecated-2017Q2 sudo: required addons: apt: @@ -141,6 +144,7 @@ matrix: - os: linux dist: trusty + group: deprecated-2017Q2 sudo: required addons: apt: @@ -169,6 +173,7 @@ matrix: - os: linux dist: trusty + group: deprecated-2017Q2 sudo: required addons: apt: @@ -197,6 +202,7 @@ matrix: - os: linux dist: trusty + group: deprecated-2017Q2 sudo: required addons: apt: @@ -225,6 +231,7 @@ matrix: - os: linux dist: trusty + group: deprecated-2017Q2 sudo: required addons: apt: @@ -253,6 +260,7 @@ matrix: - os: linux dist: trusty + group: deprecated-2017Q2 sudo: required addons: apt: @@ -281,6 +289,7 @@ matrix: - os: linux dist: trusty + group: deprecated-2017Q2 sudo: required addons: apt: @@ -309,6 +318,7 @@ matrix: - os: linux dist: trusty + group: deprecated-2017Q2 sudo: required addons: apt: @@ -337,6 +347,7 @@ matrix: - os: linux dist: trusty + group: deprecated-2017Q2 sudo: required addons: apt: @@ -365,6 +376,7 @@ matrix: - os: linux dist: trusty + group: deprecated-2017Q2 sudo: required addons: apt: @@ -393,6 +405,7 @@ matrix: - os: linux dist: trusty + group: deprecated-2017Q2 sudo: required addons: apt: From 805e4aaec4f2cfdb741e6bf209d3e7c3b337d499 Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Thu, 14 Sep 2017 10:13:18 -0700 Subject: [PATCH 47/67] Move to SymbolTable/Parser instances (#4864) ### Problem Currently the `SymbolTable` and `Parser` used in v2 engine BUILD file parsing are static, and access the `BuildFileAliases` that they use by lazily triggering configuration parsing on use. This detracts from testability, and amounts to a "nasty escape hatch" that would have needed to be propagated in #4401. Part of the reason these were made static long ago was that pickling was used to compute object identities, and to support multiprocessing of all objects manipulated by the engine. Things have changed significantly since that time: 1. object identities need not be stable across processes, since they will never be used in cache keys (only process invokes will be cached) 2. multiprocessing for individual function invokes is no longer a goal, and so universal pickleability is no longer a goal. If we multiprocess, it will be at a much larger scope (ie, starting from pickleable inputs, redundantly compute BuildFileAliases, for example). ### Solution 1. Remove the requirement for pickleability of objects manipulated in the engine by switching back to memoization for the computation of object ids. 2. Switch to using instances of the SymbolTable and Parser, removing the requirement for static initialization 3. Drop the `storage.py` module based on the above understanding of content-addressability/caching. ### Result The nasty escape hatch is removed, and `setup_legacy_graph` accepts an explicit `BuildFileAliases` object to construct a `SymbolTable` from. Followup in #4401 will continue porting BaseTest to the v2 engine. --- src/python/pants/bin/BUILD | 1 - src/python/pants/bin/engine_initializer.py | 77 ++--- src/python/pants/bin/goal_runner.py | 1 + src/python/pants/engine/BUILD | 12 - src/python/pants/engine/build_files.py | 9 +- .../pants/engine/legacy/change_calculator.py | 6 +- src/python/pants/engine/legacy/graph.py | 14 +- src/python/pants/engine/legacy/parser.py | 68 +++-- src/python/pants/engine/mapper.py | 34 +-- src/python/pants/engine/native.py | 20 +- src/python/pants/engine/native_engine_version | 2 +- src/python/pants/engine/parser.py | 19 +- src/python/pants/engine/storage.py | 264 ------------------ tests/python/pants_test/engine/BUILD | 12 - tests/python/pants_test/engine/examples/BUILD | 1 - .../pants_test/engine/examples/parsers.py | 63 +++-- .../pants_test/engine/examples/planners.py | 9 +- .../pants_test/engine/legacy/test_graph.py | 44 +-- .../pants_test/engine/test_build_files.py | 27 +- tests/python/pants_test/engine/test_mapper.py | 14 +- .../python/pants_test/engine/test_parsers.py | 26 +- tests/python/pants_test/engine/test_rules.py | 6 +- .../python/pants_test/engine/test_storage.py | 87 ------ 23 files changed, 214 insertions(+), 602 deletions(-) delete mode 100644 src/python/pants/engine/storage.py delete mode 100644 tests/python/pants_test/engine/test_storage.py diff --git a/src/python/pants/bin/BUILD b/src/python/pants/bin/BUILD index baa1f09527e..3aba4107c20 100644 --- a/src/python/pants/bin/BUILD +++ b/src/python/pants/bin/BUILD @@ -27,7 +27,6 @@ python_library( 'src/python/pants/engine:native', 'src/python/pants/engine:parser', 'src/python/pants/engine:scheduler', - 'src/python/pants/engine:storage', 'src/python/pants/goal', 'src/python/pants/goal:context', 'src/python/pants/goal:run_tracker', diff --git a/src/python/pants/bin/engine_initializer.py b/src/python/pants/bin/engine_initializer.py index 46c1c8d9a8b..be0236f76f9 100644 --- a/src/python/pants/bin/engine_initializer.py +++ b/src/python/pants/bin/engine_initializer.py @@ -26,50 +26,49 @@ from pants.engine.scheduler import LocalScheduler from pants.init.options_initializer import OptionsInitializer from pants.option.options_bootstrapper import OptionsBootstrapper -from pants.util.memo import memoized_method logger = logging.getLogger(__name__) -# N.B. This should be top-level in the module for pickleability - don't nest it. class LegacySymbolTable(SymbolTable): """A v1 SymbolTable facade for use with the v2 engine.""" - @classmethod - @memoized_method - def aliases(cls): - """TODO: This is a nasty escape hatch to pass aliases to LegacyPythonCallbacksParser.""" - _, build_config = OptionsInitializer(OptionsBootstrapper()).setup(init_logging=False) - return build_config.registered_aliases() - - @classmethod - @memoized_method - def table(cls): - aliases = {alias: TargetAdaptor for alias in cls.aliases().target_types} + def __init__(self, build_file_aliases): + """ + :param build_file_aliases: BuildFileAliases to register. + :type build_file_aliases: :class:`pants.build_graph.build_file_aliases.BuildFileAliases` + """ + self._build_file_aliases = build_file_aliases + self._table = {alias: TargetAdaptor for alias in build_file_aliases.target_types} + # TODO: The alias replacement here is to avoid elevating "TargetAdaptors" into the public # API until after https://github.com/pantsbuild/pants/issues/3560 has been completed. # These should likely move onto Target subclasses as the engine gets deeper into beta # territory. for alias in ['java_library', 'java_agent', 'javac_plugin']: - aliases[alias] = JavaLibraryAdaptor + self._table[alias] = JavaLibraryAdaptor for alias in ['scala_library', 'scalac_plugin']: - aliases[alias] = ScalaLibraryAdaptor + self._table[alias] = ScalaLibraryAdaptor for alias in ['python_library', 'pants_plugin']: - aliases[alias] = PythonLibraryAdaptor + self._table[alias] = PythonLibraryAdaptor for alias in ['go_library', 'go_binary']: - aliases[alias] = GoTargetAdaptor + self._table[alias] = GoTargetAdaptor - aliases['junit_tests'] = JunitTestsAdaptor - aliases['jvm_app'] = JvmAppAdaptor - aliases['python_tests'] = PythonTestsAdaptor - aliases['python_binary'] = PythonTargetAdaptor - aliases['remote_sources'] = RemoteSourcesAdaptor + self._table['junit_tests'] = JunitTestsAdaptor + self._table['jvm_app'] = JvmAppAdaptor + self._table['python_tests'] = PythonTestsAdaptor + self._table['python_binary'] = PythonTargetAdaptor + self._table['remote_sources'] = RemoteSourcesAdaptor - return aliases + def aliases(self): + return self._build_file_aliases + def table(self): + return self._table -class LegacyGraphHelper(namedtuple('LegacyGraphHelper', ['scheduler', 'symbol_table_cls', + +class LegacyGraphHelper(namedtuple('LegacyGraphHelper', ['scheduler', 'symbol_table', 'change_calculator'])): """A container for the components necessary to construct a legacy BuildGraph facade.""" @@ -92,7 +91,7 @@ def create_build_graph(self, target_roots, build_root=None): :returns: A tuple of (BuildGraph, AddressMapper). """ logger.debug('target_roots are: %r', target_roots) - graph = LegacyBuildGraph.create(self.scheduler, self.symbol_table_cls) + graph = LegacyBuildGraph.create(self.scheduler, self.symbol_table) logger.debug('build_graph is: %s', graph) with self.scheduler.lock: # Ensure the entire generator is unrolled. @@ -112,7 +111,7 @@ def setup_legacy_graph(pants_ignore_patterns, workdir, build_root=None, native=None, - symbol_table_cls=None, + build_file_aliases=None, build_ignore_patterns=None, exclude_target_regexps=None, subproject_roots=None, @@ -124,8 +123,8 @@ def setup_legacy_graph(pants_ignore_patterns, :param str workdir: The pants workdir. :param str build_root: A path to be used as the build root. If None, then default is used. :param Native native: An instance of the native-engine subsystem. - :param SymbolTable symbol_table_cls: A SymbolTable class to use for build file parsing, or - None to use the default. + :param build_file_aliases: BuildFileAliases to register. + :type build_file_aliases: :class:`pants.build_graph.build_file_aliases.BuildFileAliases` :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD files, usually taken from the '--build-ignore' global option. :param list exclude_target_regexps: A list of regular expressions for excluding targets. @@ -133,19 +132,22 @@ def setup_legacy_graph(pants_ignore_patterns, under the current build root. :param bool include_trace_on_error: If True, when an error occurs, the error message will include the graph trace. - :returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls). + :returns: A tuple of (scheduler, engine, symbol_table, build_graph_cls). """ build_root = build_root or get_buildroot() scm = get_scm() - symbol_table_cls = symbol_table_cls or LegacySymbolTable + + if not build_file_aliases: + _, build_config = OptionsInitializer(OptionsBootstrapper()).setup(init_logging=False) + build_file_aliases = build_config.registered_aliases() + symbol_table = LegacySymbolTable(build_file_aliases) project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns) # Register "literal" subjects required for these tasks. - # TODO: Replace with `Subsystems`. - address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, - parser_cls=LegacyPythonCallbacksParser, + parser = LegacyPythonCallbacksParser(symbol_table, build_file_aliases) + address_mapper = AddressMapper(parser=parser, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_roots) @@ -156,13 +158,12 @@ def setup_legacy_graph(pants_ignore_patterns, # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. tasks = ( - create_legacy_graph_tasks(symbol_table_cls) + + create_legacy_graph_tasks(symbol_table) + create_fs_rules() + - create_graph_rules(address_mapper, symbol_table_cls) + create_graph_rules(address_mapper, symbol_table) ) - # TODO: Do not use the cache yet, as it incurs a high overhead. scheduler = LocalScheduler(workdir, dict(), tasks, project_tree, native, include_trace_on_error=include_trace_on_error) - change_calculator = EngineChangeCalculator(scheduler, symbol_table_cls, scm) if scm else None + change_calculator = EngineChangeCalculator(scheduler, symbol_table, scm) if scm else None - return LegacyGraphHelper(scheduler, symbol_table_cls, change_calculator) + return LegacyGraphHelper(scheduler, symbol_table, change_calculator) diff --git a/src/python/pants/bin/goal_runner.py b/src/python/pants/bin/goal_runner.py index 9f6a6ced8eb..09eb28f8922 100644 --- a/src/python/pants/bin/goal_runner.py +++ b/src/python/pants/bin/goal_runner.py @@ -101,6 +101,7 @@ def _init_graph(self, use_engine, pants_ignore_patterns, build_ignore_patterns, pants_ignore_patterns, workdir, native=native, + build_file_aliases=self._build_config.registered_aliases(), build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_build_roots, diff --git a/src/python/pants/engine/BUILD b/src/python/pants/engine/BUILD index 6bafac0f386..cecc9b634df 100644 --- a/src/python/pants/engine/BUILD +++ b/src/python/pants/engine/BUILD @@ -175,17 +175,6 @@ python_library( ] ) -python_library( - name='storage', - sources=['storage.py'], - dependencies=[ - ':nodes', - ':objects', - 'src/python/pants/util:dirutil', - 'src/python/pants/util:meta', - ] -) - python_library( name='native', sources=['native.py'], @@ -194,7 +183,6 @@ python_library( '3rdparty/python:cffi', '3rdparty/python:setuptools', 'src/python/pants/binaries:binary_util', - 'src/python/pants/engine:storage', 'src/python/pants/util:dirutil', 'src/python/pants/util:memo', 'src/python/pants/util:objects' diff --git a/src/python/pants/engine/build_files.py b/src/python/pants/engine/build_files.py index b3b146354ca..a1db435f58f 100644 --- a/src/python/pants/engine/build_files.py +++ b/src/python/pants/engine/build_files.py @@ -81,8 +81,7 @@ def parse_address_family(address_mapper, path, build_files): continue address_maps.append(AddressMap.parse(filecontent_product.path, filecontent_product.content, - address_mapper.symbol_table_cls, - address_mapper.parser_cls)) + address_mapper.parser)) return AddressFamily.create(path.path, address_maps) @@ -317,13 +316,13 @@ def _recursive_dirname(f): BuildFilesCollection = Collection.of(BuildFiles) -def create_graph_rules(address_mapper, symbol_table_cls): +def create_graph_rules(address_mapper, symbol_table): """Creates tasks used to parse Structs from BUILD files. :param address_mapper_key: The subject key for an AddressMapper instance. - :param symbol_table_cls: A SymbolTable class to provide symbols for Address lookups. + :param symbol_table: A SymbolTable instance to provide symbols for Address lookups. """ - symbol_table_constraint = symbol_table_cls.constraint() + symbol_table_constraint = symbol_table.constraint() return [ TaskRule(BuildFilesCollection, [SelectDependencies(BuildFiles, BuildDirs, field_types=(Dir,))], diff --git a/src/python/pants/engine/legacy/change_calculator.py b/src/python/pants/engine/legacy/change_calculator.py index a2078176b21..0110d67dc97 100644 --- a/src/python/pants/engine/legacy/change_calculator.py +++ b/src/python/pants/engine/legacy/change_calculator.py @@ -19,14 +19,14 @@ class EngineChangeCalculator(ChangeCalculator): """A ChangeCalculator variant that uses the v2 engine for source mapping.""" - def __init__(self, scheduler, symbol_table_cls, scm): + def __init__(self, scheduler, symbol_table, scm): """ :param Engine engine: The `Engine` instance to use for computing file to target mappings. :param Scm engine: The `Scm` instance to use for computing changes. """ super(EngineChangeCalculator, self).__init__(scm) self._scheduler = scheduler - self._symbol_table_cls = symbol_table_cls + self._symbol_table = symbol_table self._mapper = EngineSourceMapper(self._scheduler) def iter_changed_target_addresses(self, changed_request): @@ -46,7 +46,7 @@ def iter_changed_target_addresses(self, changed_request): return # For dependee finding, we need to parse all build files. - graph = LegacyBuildGraph.create(self._scheduler, self._symbol_table_cls) + graph = LegacyBuildGraph.create(self._scheduler, self._symbol_table) for _ in graph.inject_specs_closure([DescendantAddresses('')]): pass diff --git a/src/python/pants/engine/legacy/graph.py b/src/python/pants/engine/legacy/graph.py index d61d77c9c3a..0e922cdf837 100644 --- a/src/python/pants/engine/legacy/graph.py +++ b/src/python/pants/engine/legacy/graph.py @@ -50,15 +50,15 @@ class InvalidCommandLineSpecError(AddressLookupError): """Raised when command line spec is not a valid directory""" @classmethod - def create(cls, scheduler, symbol_table_cls): + def create(cls, scheduler, symbol_table): """Construct a graph given a Scheduler, Engine, and a SymbolTable class.""" - return cls(scheduler, cls._get_target_types(symbol_table_cls)) + return cls(scheduler, cls._get_target_types(symbol_table)) def __init__(self, scheduler, target_types): """Construct a graph given a Scheduler, Engine, and a SymbolTable class. :param scheduler: A Scheduler that is configured to be able to resolve HydratedTargets. - :param symbol_table_cls: A SymbolTable class used to instantiate Target objects. Must match + :param symbol_table: A SymbolTable instance used to instantiate Target objects. Must match the symbol table installed in the scheduler (TODO: see comment in `_instantiate_target`). """ self._scheduler = scheduler @@ -70,8 +70,8 @@ def clone_new(self): return LegacyBuildGraph(self._scheduler, self._target_types) @staticmethod - def _get_target_types(symbol_table_cls): - aliases = symbol_table_cls.aliases() + def _get_target_types(symbol_table): + aliases = symbol_table.aliases() target_types = dict(aliases.target_types) for alias, factory in aliases.target_macro_factories.items(): target_type, = factory.target_types @@ -356,9 +356,9 @@ def hydrate_bundles(bundles_field, snapshot_list): return HydratedField('bundles', bundles) -def create_legacy_graph_tasks(symbol_table_cls): +def create_legacy_graph_tasks(symbol_table): """Create tasks to recursively parse the legacy graph.""" - symbol_table_constraint = symbol_table_cls.constraint() + symbol_table_constraint = symbol_table.constraint() return [ transitive_hydrated_targets, TaskRule( diff --git a/src/python/pants/engine/legacy/parser.py b/src/python/pants/engine/legacy/parser.py index cdfc0296f92..9add40b9119 100644 --- a/src/python/pants/engine/legacy/parser.py +++ b/src/python/pants/engine/legacy/parser.py @@ -15,7 +15,7 @@ from pants.engine.mapper import UnaddressableObjectError from pants.engine.objects import Serializable from pants.engine.parser import Parser -from pants.util.memo import memoized_method, memoized_property +from pants.util.memo import memoized_property class LegacyPythonCallbacksParser(Parser): @@ -28,25 +28,31 @@ class LegacyPythonCallbacksParser(Parser): macros and target factories. """ - @classmethod - @memoized_method - def _get_symbols(cls, symbol_table_cls): - symbol_table = symbol_table_cls.table() - # TODO: Nasty escape hatch: see https://github.com/pantsbuild/pants/issues/3561 - aliases = symbol_table_cls.aliases() - + def __init__(self, symbol_table, aliases): + """ + :param symbol_table: A SymbolTable for this parser, which will be overlaid with the given + additional aliases. + :type symbol_table: :class:`pants.engine.parser.SymbolTable` + :param aliases: Additional BuildFileAliases to register. + :type aliases: :class:`pants.build_graph.build_file_aliases.BuildFileAliases` + """ + super(LegacyPythonCallbacksParser, self).__init__() + self._symbols, self._parse_context = self._generate_symbols(symbol_table, aliases) + + @staticmethod + def _generate_symbols(symbol_table, aliases): symbols = {} # Compute "per path" symbols. For performance, we use the same ParseContext, which we # mutate (in a critical section) to set the rel_path appropriately before it's actually used. - # This allows this memoized method to depend only on the symbol_table_cls, thus reusing - # the same symbols for all parses. Meanwhile we set the rel_path to None, so that we get - # a loud error if anything tries to use it before it's set. + # This allows this method to reuse the same symbols for all parses. Meanwhile we set the + # rel_path to None, so that we get a loud error if anything tries to use it before it's set. # TODO: See https://github.com/pantsbuild/pants/issues/3561 parse_context = ParseContext(rel_path=None, type_aliases=symbols) class Registrar(BuildFileTargetFactory): - def __init__(self, type_alias, object_type): + def __init__(self, parse_context, type_alias, object_type): + self._parse_context = parse_context self._type_alias = type_alias self._object_type = object_type self._serializable = Serializable.is_serializable_type(self._object_type) @@ -59,7 +65,7 @@ def __call__(self, *args, **kwargs): # Target names default to the name of the directory their BUILD file is in # (as long as it's not the root directory). if 'name' not in kwargs and issubclass(self._object_type, TargetAdaptor): - dirname = os.path.basename(parse_context.rel_path) + dirname = os.path.basename(self._parse_context.rel_path) if dirname: kwargs['name'] = dirname else: @@ -69,13 +75,13 @@ def __call__(self, *args, **kwargs): if name and self._serializable: kwargs.setdefault('type_alias', self._type_alias) obj = self._object_type(**kwargs) - parse_context._storage.add(obj) + self._parse_context._storage.add(obj) return obj else: return self._object_type(*args, **kwargs) - for alias, symbol in symbol_table.items(): - registrar = Registrar(alias, symbol) + for alias, symbol in symbol_table.table().items(): + registrar = Registrar(parse_context, alias, symbol) symbols[alias] = registrar symbols[symbol] = registrar @@ -89,29 +95,31 @@ def __call__(self, *args, **kwargs): underlying_symbol = symbols.get(alias, TargetAdaptor) symbols[alias] = target_macro_factory.target_macro(parse_context) for target_type in target_macro_factory.target_types: - symbols[target_type] = Registrar(alias, underlying_symbol) + symbols[target_type] = Registrar(parse_context, alias, underlying_symbol) # TODO: Replace builtins for paths with objects that will create wrapped PathGlobs objects. # The strategy for https://github.com/pantsbuild/pants/issues/3560 should account for # migrating these additional captured arguments to typed Sources. - def glob_wrapper(glob_type): - def glob_factory(*args, **kwargs): - return glob_type(*args, spec_path=parse_context.rel_path, **kwargs) - return glob_factory - symbols['globs'] = glob_wrapper(Globs) - symbols['rglobs'] = glob_wrapper(RGlobs) - symbols['zglobs'] = glob_wrapper(ZGlobs) + class GlobWrapper(object): + def __init__(self, parse_context, glob_type): + self._parse_context = parse_context + self._glob_type = glob_type + + def __call__(self, *args, **kwargs): + return self._glob_type(*args, spec_path=self._parse_context.rel_path, **kwargs) + + symbols['globs'] = GlobWrapper(parse_context, Globs) + symbols['rglobs'] = GlobWrapper(parse_context, RGlobs) + symbols['zglobs'] = GlobWrapper(parse_context, ZGlobs) symbols['bundle'] = BundleAdaptor return symbols, parse_context - @classmethod - def parse(cls, filepath, filecontent, symbol_table_cls): - symbols, parse_context = cls._get_symbols(symbol_table_cls) + def parse(self, filepath, filecontent): python = filecontent # Mutate the parse context for the new path, then exec, and copy the resulting objects. - parse_context._storage.clear(os.path.dirname(filepath)) - six.exec_(python, symbols) - return list(parse_context._storage.objects) + self._parse_context._storage.clear(os.path.dirname(filepath)) + six.exec_(python, self._symbols) + return list(self._parse_context._storage.objects) diff --git a/src/python/pants/engine/mapper.py b/src/python/pants/engine/mapper.py index 40945316688..affaedfead4 100644 --- a/src/python/pants/engine/mapper.py +++ b/src/python/pants/engine/mapper.py @@ -39,7 +39,7 @@ class AddressMap(datatype('AddressMap', ['path', 'objects_by_name'])): """ @classmethod - def parse(cls, filepath, filecontent, symbol_table_cls, parser_cls): + def parse(cls, filepath, filecontent, parser): """Parses a source for addressable Serializable objects. No matter the parser used, the parsed and mapped addressable objects are all 'thin'; ie: any @@ -48,13 +48,13 @@ def parse(cls, filepath, filecontent, symbol_table_cls, parser_cls): :param string filepath: The path to the byte source containing serialized objects. :param string filecontent: The content of byte source containing serialized objects to be parsed. - :param symbol_table_cls: The symbol table cls to expose a symbol table dict. - :type symbol_table_cls: A :class:`pants.engine.parser.SymbolTable`. - :param parser_cls: The parser cls to use. - :type parser_cls: A :class:`pants.engine.parser.Parser`. + :param symbol_table: The symbol table cls to expose a symbol table dict. + :type symbol_table: Instance of :class:`pants.engine.parser.SymbolTable`. + :param parser: The parser cls to use. + :type parser: A :class:`pants.engine.parser.Parser`. """ try: - objects = parser_cls.parse(filepath, filecontent, symbol_table_cls) + objects = parser.parse(filepath, filecontent) except Exception as e: raise MappingError('Failed to parse {}:\n{}'.format(filepath, e)) objects_by_name = {} @@ -166,8 +166,7 @@ class AddressMapper(object): """Configuration to parse build files matching a filename pattern.""" def __init__(self, - symbol_table_cls, - parser_cls, + parser, build_patterns=None, build_ignore_patterns=None, exclude_target_regexps=None, @@ -177,17 +176,14 @@ def __init__(self, Both the set of files that define a mappable BUILD files and the parser used to parse those files can be customized. See the `pants.engine.parsers` module for example parsers. - :param symbol_table_cls: The symbol table cls to expose a symbol table dict. - :type symbol_table_cls: A :class:`pants.engine.parser.SymbolTable`. - :param parser_cls: The BUILD file parser cls to use. - :type parser_cls: A :class:`pants.engine.parser.Parser`. + :param parser: The BUILD file parser to use. + :type parser: An instance of :class:`pants.engine.parser.Parser`. :param tuple build_patterns: A tuple of fnmatch-compatible patterns for identifying BUILD files used to resolve addresses. :param list build_ignore_patterns: A list of path ignore patterns used when searching for BUILD files. :param list exclude_target_regexps: A list of regular expressions for excluding targets. """ - self.symbol_table_cls = symbol_table_cls - self.parser_cls = parser_cls + self.parser = parser self.build_patterns = build_patterns or (b'BUILD', b'BUILD.*') self.build_ignore_patterns = PathSpec.from_lines(GitWildMatchPattern, build_ignore_patterns or []) self._exclude_target_regexps = exclude_target_regexps or [] @@ -199,20 +195,18 @@ def __eq__(self, other): return True if type(other) != type(self): return NotImplemented - return (other.symbol_table_cls == self.symbol_table_cls and - other.build_patterns == self.build_patterns and - other.parser_cls == self.parser_cls) + return (other.build_patterns == self.build_patterns and + other.parser == self.parser) def __ne__(self, other): return not (self == other) def __hash__(self): # Compiled regexes are not hashable. - return hash((self.symbol_table_cls, self.parser_cls)) + return hash(self.parser) def __repr__(self): - return 'AddressMapper(parser={}, symbol_table={}, build_patterns={})'.format( - self.parser_cls, self.symbol_table_cls, self.build_patterns) + return 'AddressMapper(parser={}, build_patterns={})'.format(self.parser, self.build_patterns) def __str__(self): return repr(self) diff --git a/src/python/pants/engine/native.py b/src/python/pants/engine/native.py index 2bdc47eb5cc..5302d8a2dc2 100644 --- a/src/python/pants/engine/native.py +++ b/src/python/pants/engine/native.py @@ -17,7 +17,6 @@ import six from pants.binaries.binary_util import BinaryUtil -from pants.engine.storage import Storage from pants.util.dirutil import safe_mkdir from pants.util.memo import memoized_property from pants.util.objects import datatype @@ -446,28 +445,25 @@ class ObjectIdMap(object): """ def __init__(self): - # Objects indexed by their keys, i.e, content digests - self._objects = Storage.create() # Memoized object Ids. - self._id_to_key = dict() - self._key_to_id = dict() + self._id_to_obj = dict() + self._obj_to_id = dict() self._next_id = 0 def put(self, obj): - key = self._objects.put(obj) new_id = self._next_id - oid = self._key_to_id.setdefault(key, new_id) + oid = self._obj_to_id.setdefault(obj, new_id) if oid is not new_id: # Object already existed. return oid # Object is new/unique. - self._id_to_key[oid] = key + self._id_to_obj[oid] = obj self._next_id += 1 return oid def get(self, oid): - return self._objects.get(self._id_to_key[oid]) + return self._id_to_obj[oid] class ExternContext(object): @@ -508,9 +504,9 @@ def utf8_buf_buf(self, strings): buf_buf = self._ffi.new('Buffer[]', bufs) return (buf_buf, len(bufs), self.to_value(buf_buf)) - def vals_buf(self, keys): - buf = self._ffi.new('Value[]', keys) - return (buf, len(keys), self.to_value(buf)) + def vals_buf(self, vals): + buf = self._ffi.new('Value[]', vals) + return (buf, len(vals), self.to_value(buf)) def type_ids_buf(self, types): buf = self._ffi.new('TypeId[]', types) diff --git a/src/python/pants/engine/native_engine_version b/src/python/pants/engine/native_engine_version index 82a2c9095e6..9452e0a4772 100644 --- a/src/python/pants/engine/native_engine_version +++ b/src/python/pants/engine/native_engine_version @@ -1 +1 @@ -27a5bba4e7fffd406e800ddce1c5800a9d497e22 +e846684b8e8a083b6629de31fc9d5e3aa163bb45 diff --git a/src/python/pants/engine/parser.py b/src/python/pants/engine/parser.py index 02cf0f09f66..d589c0f8629 100644 --- a/src/python/pants/engine/parser.py +++ b/src/python/pants/engine/parser.py @@ -16,34 +16,27 @@ class ParseError(Exception): class SymbolTable(AbstractClass): - """A one-classmethod interface exposing a symbol table dict. + """A one-classmethod interface exposing a symbol table dict.""" - SymbolTables exist as named classes because it allows them to be loaded by name as a python - module, rather than being pickled when they cross between processes. - """ - - @classmethod @abstractmethod - def table(cls): + def table(self): """Returns a dict of name to implementation class.""" - @classmethod - def constraint(cls): + def constraint(self): """Returns the typeconstraint for the symbol table""" # NB Sort types so that multiple calls get the same tuples. - symbol_table_types = sorted(set(cls.table().values())) + symbol_table_types = sorted(set(self.table().values())) return Exactly(*symbol_table_types, description='symbol table types') class Parser(AbstractClass): - @classmethod + @abstractmethod - def parse(cls, filepath, filecontent, symbol_table_cls): + def parse(self, filepath, filecontent): """ :param string filepath: The name of the file being parsed. The parser should not assume that the path is accessible, and should consume the filecontent. :param bytes filecontent: The raw byte content to parse. - :param dict symbol_table_cls: A symbol table to expose to the python file being parsed. :returns: A list of decoded addressable, Serializable objects. The callable will raise :class:`ParseError` if there were any problems encountered parsing the filecontent. :rtype: :class:`collections.Callable` diff --git a/src/python/pants/engine/storage.py b/src/python/pants/engine/storage.py deleted file mode 100644 index 8726188f035..00000000000 --- a/src/python/pants/engine/storage.py +++ /dev/null @@ -1,264 +0,0 @@ -# coding=utf-8 -# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import (absolute_import, division, generators, nested_scopes, print_function, - unicode_literals, with_statement) - -import cPickle as pickle -import cStringIO as StringIO -from binascii import hexlify -from collections import Counter -from contextlib import closing -from hashlib import sha1 -from struct import Struct as StdlibStruct - -from pants.engine.nodes import State -from pants.engine.objects import SerializationError - - -class Key(object): - """Holds the digest for the object, which uniquely identifies it. - - The `_hash` is a memoized 32 bit integer hashcode computed from the digest. - """ - - __slots__ = ['_digest', '_hash'] - - # The digest implementation used for Keys. - _DIGEST_IMPL = sha1 - _DIGEST_SIZE = _DIGEST_IMPL().digest_size - - # A struct.Struct definition for grabbing the first 4 bytes off of a digest of - # size DIGEST_SIZE, and discarding the rest. - _32_BIT_STRUCT = StdlibStruct(b' 0, 'No targets matched by {}'.format(addresses)) for address in addresses: - self.assertIn(TaggingSymbolTable.tag, graph.get_target(address).tags) + self.assertIn(tag, graph.get_target(address).tags) diff --git a/tests/python/pants_test/engine/test_build_files.py b/tests/python/pants_test/engine/test_build_files.py index ec5fb6c2a7c..aa6ac4d4a5b 100644 --- a/tests/python/pants_test/engine/test_build_files.py +++ b/tests/python/pants_test/engine/test_build_files.py @@ -79,8 +79,7 @@ def repos(self): class TestTable(SymbolTable): - @classmethod - def table(cls): + def table(self): return {'ApacheThriftConfig': ApacheThriftConfiguration, 'Struct': Struct, 'StructWithDeps': StructWithDeps, @@ -92,24 +91,22 @@ class GraphTestBase(unittest.TestCase, SchedulerTestBase): def setUp(self): super(GraphTestBase, self).setUp() - def create(self, build_patterns=None, parser_cls=None): - symbol_table_cls = TestTable + def create(self, build_patterns=None, parser=None): + address_mapper = AddressMapper(build_patterns=build_patterns, + parser=parser) + symbol_table = address_mapper.parser.symbol_table - address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, - build_patterns=build_patterns, - parser_cls=parser_cls) - - rules = create_fs_rules() + create_graph_rules(address_mapper, symbol_table_cls) + rules = create_fs_rules() + create_graph_rules(address_mapper, symbol_table) project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples')) scheduler = self.mk_scheduler(rules=rules, project_tree=project_tree) return scheduler def create_json(self): - return self.create(build_patterns=('*.BUILD.json',), parser_cls=JsonParser) + return self.create(build_patterns=('*.BUILD.json',), parser=JsonParser(TestTable())) def _populate(self, scheduler, address): """Perform an ExecutionRequest to parse the given Address into a Struct.""" - request = scheduler.execution_request([TestTable.constraint()], [address]) + request = scheduler.execution_request([TestTable().constraint()], [address]) root_entries = scheduler.execute(request).root_products self.assertEquals(1, len(root_entries)) return root_entries[0] @@ -169,12 +166,12 @@ def test_json(self): def test_python(self): scheduler = self.create(build_patterns=('*.BUILD.python',), - parser_cls=PythonAssignmentsParser) + parser=PythonAssignmentsParser(TestTable())) self.do_test_codegen_simple(scheduler) def test_python_classic(self): scheduler = self.create(build_patterns=('*.BUILD',), - parser_cls=PythonCallbacksParser) + parser=PythonCallbacksParser(TestTable())) self.do_test_codegen_simple(scheduler) def test_resolve_cache(self): @@ -328,10 +325,10 @@ def test_json_lazy(self): def test_python_lazy(self): scheduler = self.create(build_patterns=('*.BUILD.python',), - parser_cls=PythonAssignmentsParser) + parser=PythonAssignmentsParser(TestTable())) self.do_test_codegen_simple(scheduler) def test_python_classic_lazy(self): scheduler = self.create(build_patterns=('*.BUILD',), - parser_cls=PythonCallbacksParser) + parser=PythonCallbacksParser(TestTable())) self.do_test_codegen_simple(scheduler) diff --git a/tests/python/pants_test/engine/test_mapper.py b/tests/python/pants_test/engine/test_mapper.py index 29d67bc391a..e11c019000c 100644 --- a/tests/python/pants_test/engine/test_mapper.py +++ b/tests/python/pants_test/engine/test_mapper.py @@ -43,19 +43,18 @@ def __eq__(self, other): class ThingTable(SymbolTable): - @classmethod def table(cls): return {'thing': Thing} class AddressMapTest(unittest.TestCase): - _parser_cls = JsonParser - _symbol_table_cls = ThingTable + _symbol_table = ThingTable() + _parser = JsonParser(_symbol_table) @contextmanager def parse_address_map(self, json): path = '/dev/null' - address_map = AddressMap.parse(path, json, self._symbol_table_cls, self._parser_cls) + address_map = AddressMap.parse(path, json, self._parser) self.assertEqual(path, address_map.path) yield address_map @@ -143,11 +142,10 @@ def test_duplicate_names(self): class AddressMapperTest(unittest.TestCase, SchedulerTestBase): def setUp(self): # Set up a scheduler that supports address mapping. - symbol_table_cls = TargetTable - address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, - parser_cls=JsonParser, + symbol_table = TargetTable() + address_mapper = AddressMapper(parser=JsonParser(symbol_table), build_patterns=('*.BUILD.json',)) - rules = create_fs_rules() + create_graph_rules(address_mapper, symbol_table_cls) + rules = create_fs_rules() + create_graph_rules(address_mapper, symbol_table) # TODO handle updating the rule graph when passed unexpected root selectors. # Adding the following task allows us to get around the fact that SelectDependencies # requests are not currently supported. diff --git a/tests/python/pants_test/engine/test_parsers.py b/tests/python/pants_test/engine/test_parsers.py index 3d14c0dfc89..8f0e37355f8 100644 --- a/tests/python/pants_test/engine/test_parsers.py +++ b/tests/python/pants_test/engine/test_parsers.py @@ -47,18 +47,18 @@ def table(cls): return {'nancy': Bob} -def parse(parser, document, symbol_table_cls, **args): - return parser.parse('/dev/null', document, symbol_table_cls, **args) +def parse(parser, document, **args): + return parser.parse('/dev/null', document, **args) class JsonParserTest(unittest.TestCase): - def parse(self, document, symbol_table_cls=None, **kwargs): - symbol_table_cls = symbol_table_cls or EmptyTable - return parse(parsers.JsonParser, document, symbol_table_cls, **kwargs) + def parse(self, document, symbol_table=None, **kwargs): + symbol_table = symbol_table or EmptyTable() + return parse(parsers.JsonParser(symbol_table), document, **kwargs) - def round_trip(self, obj, symbol_table_cls=None): + def round_trip(self, obj, symbol_table=None): document = parsers.encode_json(obj, inline=True) - return self.parse(document, symbol_table_cls=symbol_table_cls) + return self.parse(document, symbol_table=symbol_table) def test_comments(self): document = dedent(""" @@ -93,10 +93,10 @@ def test_symbol_table(self): "hobbies": [1, 2, 3] } """) - results = self.parse(document, symbol_table_cls=TestTable) + results = self.parse(document, symbol_table=TestTable()) self.assertEqual(1, len(results)) self.assertEqual([Bob(hobbies=[1, 2, 3])], - self.round_trip(results[0], symbol_table_cls=TestTable)) + self.round_trip(results[0], symbol_table=TestTable())) self.assertEqual('bob', results[0]._asdict()['type_alias']) def test_nested_single(self): @@ -229,7 +229,7 @@ def test_error_presentation(self): """).strip() filepath = '/dev/null' with self.assertRaises(parser.ParseError) as exc: - parsers.JsonParser.parse(filepath, document, symbol_table_cls=EmptyTable) + parsers.JsonParser(EmptyTable()).parse(filepath, document) # Strip trailing whitespace from the message since our expected literal below will have # trailing ws stripped via editors and code reviews calling for it. @@ -327,7 +327,7 @@ def test_no_symbol_table(self): hobbies=[1, 2, 3] ) """) - results = parse(parsers.PythonAssignmentsParser, document, symbol_table_cls=EmptyTable) + results = parse(parsers.PythonAssignmentsParser(EmptyTable()), document) self.assertEqual([Bob(name='nancy', hobbies=[1, 2, 3])], results) # No symbol table was used so no `type_alias` plumbing can be expected. @@ -339,7 +339,7 @@ def test_symbol_table(self): hobbies=[1, 2, 3] ) """) - results = parse(parsers.PythonAssignmentsParser, document, symbol_table_cls=TestTable2) + results = parse(parsers.PythonAssignmentsParser(TestTable2()), document) self.assertEqual([Bob(name='bill', hobbies=[1, 2, 3])], results) self.assertEqual('nancy', results[0]._asdict()['type_alias']) @@ -352,6 +352,6 @@ def test(self): hobbies=[1, 2, 3] ) """) - results = parse(parsers.PythonCallbacksParser, document, symbol_table_cls=TestTable2) + results = parse(parsers.PythonCallbacksParser(TestTable2()), document) self.assertEqual([Bob(name='bill', hobbies=[1, 2, 3])], results) self.assertEqual('nancy', results[0]._asdict()['type_alias']) diff --git a/tests/python/pants_test/engine/test_rules.py b/tests/python/pants_test/engine/test_rules.py index bcabad61489..7dd3358da97 100644 --- a/tests/python/pants_test/engine/test_rules.py +++ b/tests/python/pants_test/engine/test_rules.py @@ -274,9 +274,9 @@ def test_smallest_full_test(self): }""").strip(), fullgraph) def test_full_graph_for_planner_example(self): - symbol_table_cls = TargetTable - address_mapper = AddressMapper(symbol_table_cls, JsonParser, '*.BUILD.json') - rules = create_graph_rules(address_mapper, symbol_table_cls) + create_fs_rules() + symbol_table = TargetTable() + address_mapper = AddressMapper(JsonParser(symbol_table), '*.BUILD.json') + rules = create_graph_rules(address_mapper, symbol_table) + create_fs_rules() rule_index = RuleIndex.create(rules) fullgraph_str = self.create_full_graph(rule_index) diff --git a/tests/python/pants_test/engine/test_storage.py b/tests/python/pants_test/engine/test_storage.py deleted file mode 100644 index ac227e9291b..00000000000 --- a/tests/python/pants_test/engine/test_storage.py +++ /dev/null @@ -1,87 +0,0 @@ -# coding=utf-8 -# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import (absolute_import, division, generators, nested_scopes, print_function, - unicode_literals, with_statement) - -import unittest - -from pants.base.project_tree import Dir, File -from pants.engine.nodes import Runnable -from pants.engine.storage import Cache, InvalidKeyError, Storage - - -def _runnable(an_arg): - return an_arg - - -class PickleableException(Exception): - def __eq__(self, other): - return type(self) == type(other) - - -class StorageTest(unittest.TestCase): - TEST_KEY = b'hello' - TEST_VALUE = b'world' - - TEST_PATH = File('/foo') - TEST_PATH2 = Dir('/bar') - - class SomeException(Exception): pass - - def setUp(self): - self.storage = Storage.create() - self.result = 'something' - self.request = Runnable(func=_runnable, args=('this is an arg',), cacheable=True) - - def test_storage(self): - key = self.storage.put(self.TEST_PATH) - self.assertEquals(self.TEST_PATH, self.storage.get(key)) - - with self.assertRaises(InvalidKeyError): - self.assertFalse(self.storage.get(self.TEST_KEY)) - - def test_storage_key_mappings(self): - key1 = self.storage.put(self.TEST_PATH) - key2 = self.storage.put(self.TEST_PATH2) - self.storage.add_mapping(key1, key2) - self.assertEquals(key2, self.storage.get_mapping(key1)) - - # key2 isn't mapped to any other key. - self.assertIsNone(self.storage.get_mapping(key2)) - - -class CacheTest(unittest.TestCase): - - def setUp(self): - """Setup cache as well as request and result.""" - self.storage = Storage.create() - self.cache = Cache.create(storage=self.storage) - self.request = Runnable(func=_runnable, args=('this is an arg',), cacheable=True) - self.result = 'something' - - def test_cache(self): - """Verify get and put.""" - self.assertIsNone(self.cache.get(self.request)[1]) - self._assert_hits_misses(hits=0, misses=1) - - request_key = self.storage.put_state(self.request) - self.cache.put(request_key, self.result) - - self.assertEquals(self.result, self.cache.get(self.request)[1]) - self._assert_hits_misses(hits=1, misses=1) - - def test_failure_to_update_mapping(self): - """Verify we can access cached result only if we save both result and the key mapping.""" - # This places result to the main storage without saving to key mapping. This - # simulates error might happen for saving key mapping after successfully saving the result. - self.cache._storage.put(self.result) - - self.assertIsNone(self.cache.get(self.request)[1]) - self._assert_hits_misses(hits=0, misses=1) - - def _assert_hits_misses(self, hits, misses): - self.assertEquals(hits, self.cache.get_stats().hits) - self.assertEquals(misses, self.cache.get_stats().misses) - self.assertEquals(hits+misses, self.cache.get_stats().total) From 12477cc2b263a76a32c659ffaeb4af93ad0d9bbd Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 14 Sep 2017 10:35:14 -0700 Subject: [PATCH 48/67] Error on task name reuse for a particular goal (#4863) We use name as a unique key in the engine, so if you register the same name more than once only one of the tasks will actually be called, and the others silently ignored. --- src/python/pants/goal/goal.py | 5 +++++ .../pants_test/core_tasks/test_list_goals.py | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/python/pants/goal/goal.py b/src/python/pants/goal/goal.py index 1ad514d7100..61801082cfb 100644 --- a/src/python/pants/goal/goal.py +++ b/src/python/pants/goal/goal.py @@ -141,6 +141,11 @@ def install(self, task_registrar, first=False, replace=False, before=None, after raise GoalError('Can only specify one of first, replace, before or after') task_name = task_registrar.name + if task_name in self._task_type_by_name: + raise GoalError( + 'Can only specify a task name once per goal, saw multiple values for {} in goal {}'.format( + task_name, + self.name)) Optionable.validate_scope_name_component(task_name) options_scope = Goal.scope(self.name, task_name) diff --git a/tests/python/pants_test/core_tasks/test_list_goals.py b/tests/python/pants_test/core_tasks/test_list_goals.py index 56dc94aa3c3..c90b41c7cc4 100644 --- a/tests/python/pants_test/core_tasks/test_list_goals.py +++ b/tests/python/pants_test/core_tasks/test_list_goals.py @@ -8,6 +8,7 @@ from unittest import expectedFailure from pants.core_tasks.list_goals import ListGoals +from pants.goal.error import GoalError from pants.goal.goal import Goal from pants.goal.task_registrar import TaskRegistrar from pants.task.task import Task @@ -81,6 +82,27 @@ def test_list_goals_all(self): options={'all': True} ) + def test_register_duplicate_task_name_is_error(self): + Goal.clear() + + class NoopTask(Task): + def execute(self): + pass + + class OtherNoopTask(Task): + def execute(self): + pass + + goal = Goal.register(self._LIST_GOALS_NAME, self._LIST_GOALS_DESC) + task_name = 'foo' + goal.install(TaskRegistrar(task_name, NoopTask)) + + with self.assertRaises(GoalError) as ctx: + goal.install(TaskRegistrar(task_name, OtherNoopTask)) + + self.assertIn('foo', ctx.exception.message) + self.assertIn(self._LIST_GOALS_NAME, ctx.exception.message) + # TODO(John Sirois): Re-enable when fixing up ListGoals `--graph` in # https://github.com/pantsbuild/pants/issues/918 @expectedFailure From b51c31f67956f0aad61f0564aaf6e206347edb83 Mon Sep 17 00:00:00 2001 From: Kris Wilson Date: Thu, 14 Sep 2017 15:16:56 -0700 Subject: [PATCH 49/67] Bump pyopenssl==17.3.0 (#4872) Problem In #4865 we transitively pinned PyOpenSSL to 17.1.0 to avoid a release-breaking deprecation warning present in 17.2.0 that was being triggered even tho the deprecated codepath wasn't being used. This was fixed in upstream master, but had not yet been released - until today. Solution Bump PyOpenSSL back to latest. Result Back on the latest/greatest version, sans spurious deprecation warnings. --- 3rdparty/python/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index cde8245f28a..9762b99bc7c 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -16,7 +16,7 @@ pex==1.2.11 psutil==4.3.0 pyflakes==1.1.0 Pygments==1.4 -pyopenssl==17.1.0 +pyopenssl==17.3.0 pystache==0.5.3 pytest-cov>=2.4,<2.5 pytest>=3.0.7,<4.0 From 2dbae122d3a11e9cc34e9ce754090192ccbfe08d Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Mon, 18 Sep 2017 13:32:54 -0700 Subject: [PATCH 50/67] Prepare the 1.4.0.dev13 release. (#4875) --- CONTRIBUTORS.md | 1 + src/python/pants/notes/master.rst | 24 ++++++++++++++++++++++++ src/python/pants/version.py | 2 +- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index f49bf8b59de..c3e535a6917 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -27,6 +27,7 @@ Created by running `./build-support/bin/contributors.sh`. + Dan Harrison + Daniel Anderson + Daniel Bentley ++ Daniel Wagner-Hall + Dave Brewster + David Taylor + David Turner diff --git a/src/python/pants/notes/master.rst b/src/python/pants/notes/master.rst index 17427e6050f..48c454cc957 100644 --- a/src/python/pants/notes/master.rst +++ b/src/python/pants/notes/master.rst @@ -5,6 +5,30 @@ This document describes ``dev`` releases which occur weekly from master, and whi not undergo the vetting associated with ``stable`` releases. +1.4.0.dev13 (9/18/2017) +----------------------- + +API Changes +~~~~~~~~~~~ + +* Bump pyopenssl==17.3.0 (#4872) + `PR #4872 `_ + +* Error on task name reuse for a particular goal (#4863) + `PR #4863 `_ + +Bugfixes +~~~~~~~~ + +* Re-pin to 2017Q2 TravisCI image. (#4869) + `PR #4869 `_ + +Refactoring, Improvements, and Tooling +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Move to SymbolTable/Parser instances (#4864) + `PR #4864 `_ + 1.4.0.dev12 (9/13/2017) ----------------------- diff --git a/src/python/pants/version.py b/src/python/pants/version.py index 80c8bc3b146..3df1a0257c9 100644 --- a/src/python/pants/version.py +++ b/src/python/pants/version.py @@ -8,6 +8,6 @@ from packaging.version import Version -VERSION = '1.4.0.dev12' +VERSION = '1.4.0.dev13' PANTS_SEMVER = Version(VERSION) From 7319b21030f020ffefabebda30e5f6fbf48cf248 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 19 Sep 2017 10:16:56 -0700 Subject: [PATCH 51/67] Customize native engine build through code (#4876) This reflects the required environment variables in the raw cffi_build script, so that things like IntelliJ can pick them up, rather than requiring that some shell file happens to have been sourced. --- build-support/bin/native/bootstrap.sh | 4 +-- src/python/pants/engine/native.py | 4 +-- src/python/pants/engine/native_engine_version | 2 +- src/rust/engine/src/cffi_build.rs | 32 ++++++++++++++++--- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/build-support/bin/native/bootstrap.sh b/build-support/bin/native/bootstrap.sh index 6d99c4a7f63..fee43e09f3a 100644 --- a/build-support/bin/native/bootstrap.sh +++ b/build-support/bin/native/bootstrap.sh @@ -117,11 +117,11 @@ function bootstrap_native_code() { local native_engine_version="$(calculate_current_hash)" local target_binary="${NATIVE_ENGINE_CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}" local cffi_output_dir="${NATIVE_ROOT}/src/cffi" - local cffi_env_script="${cffi_output_dir}/${NATIVE_ENGINE_MODULE}.sh" + local cffi_env_script="${cffi_output_dir}/${NATIVE_ENGINE_MODULE}.cflags" if [ ! -f "${target_binary}" ] then _ensure_cffi_sources "${cffi_output_dir}" - source "${cffi_env_script}" + export CFLAGS="$(cat "${cffi_env_script}")" local readonly native_binary="$(_build_native_code)" # If bootstrapping the native engine fails, don't attempt to run pants diff --git a/src/python/pants/engine/native.py b/src/python/pants/engine/native.py index 5302d8a2dc2..7f1bc5bfd4c 100644 --- a/src/python/pants/engine/native.py +++ b/src/python/pants/engine/native.py @@ -242,7 +242,7 @@ def bootstrap_c_source(output_dir, module_name=NATIVE_ENGINE_MODULE): output_prefix = os.path.join(output_dir, module_name) c_file = '{}.c'.format(output_prefix) - env_script = '{}.sh'.format(output_prefix) + env_script = '{}.cflags'.format(output_prefix) ffibuilder = cffi.FFI() ffibuilder.cdef(CFFI_TYPEDEFS) @@ -254,7 +254,7 @@ def bootstrap_c_source(output_dir, module_name=NATIVE_ENGINE_MODULE): # Write a shell script to be sourced at build time that contains inherited CFLAGS. print('generating {}'.format(env_script)) with open(env_script, 'wb') as f: - f.write(b'export CFLAGS="{}"\n'.format(get_build_cflags())) + f.write(get_build_cflags()) def _initialize_externs(ffi): diff --git a/src/python/pants/engine/native_engine_version b/src/python/pants/engine/native_engine_version index 9452e0a4772..8c616d60632 100644 --- a/src/python/pants/engine/native_engine_version +++ b/src/python/pants/engine/native_engine_version @@ -1 +1 @@ -e846684b8e8a083b6629de31fc9d5e3aa163bb45 +69fb4d2a155fa4ec8a5d96559b9936b410e9b8ea diff --git a/src/rust/engine/src/cffi_build.rs b/src/rust/engine/src/cffi_build.rs index 2c6d30c0b3c..0883e396a79 100644 --- a/src/rust/engine/src/cffi_build.rs +++ b/src/rust/engine/src/cffi_build.rs @@ -18,10 +18,32 @@ native engine binary, allowing us to address it both as an importable python mod */ +use std::fs; +use std::io::{Read, Result}; +use std::path::{Path, PathBuf}; + fn main() { - gcc::Config::new() - // N.B. The filename of this source code - at generation time - must line up 1:1 with the - // python import name, as python keys the initialization function name off of the import name. - .file("src/cffi/native_engine.c") - .compile("libnative_engine_ffi.a"); + let mut config = gcc::Config::new(); + + // N.B. The filename of this source code - at generation time - must line up 1:1 with the + // python import name, as python keys the initialization function name off of the import name. + let path = PathBuf::from("src/cffi/native_engine.c"); + + config.file(path.to_str().unwrap()); + for flag in make_flags(&path).unwrap() { + config.flag(flag.as_str()); + } + + config.compile("libnative_engine_ffi.a"); +} + +fn make_flags(c_file: &Path) -> Result> { + let mut path = PathBuf::from(c_file); + path.set_extension("cflags"); + + let mut contents = String::new(); + fs::File::open(path)?.read_to_string(&mut contents)?; + // It would be a shame if someone were to include a space in an actual quoted value. + // If they did that, I guess we'd need to implement shell tokenization or something. + return Ok(contents.trim().split(' ').map(str::to_owned).collect()); } From 681c6a4bdbb6e491c8ebad96cf27dd0507f1a572 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 19 Sep 2017 14:31:51 -0700 Subject: [PATCH 52/67] Leverage `subprocess32` subprocess backports. (#4851) Replace our bespoke test timeout infra with `subprocess(32).wait`. This change also kills redundant test timeout tests in mixee test tasks. The mixin test case properly covers testing its own behavior. --- .travis.yml | 6 +- 3rdparty/python/requirements.txt | 1 + .../python/pants/contrib/android/tasks/BUILD | 6 +- .../contrib/android/tasks/aapt_builder.py | 2 +- .../pants/contrib/android/tasks/aapt_gen.py | 2 +- .../pants/contrib/android/tasks/sign_apk.py | 2 +- .../pants/contrib/android/tasks/zipalign.py | 2 +- .../src/python/pants/contrib/cpp/tasks/BUILD | 3 +- .../pants/contrib/cpp/tasks/cpp_task.py | 3 +- .../python/pants/contrib/go/subsystems/BUILD | 1 + .../contrib/go/subsystems/go_distribution.py | 2 +- .../src/python/pants/contrib/go/tasks/BUILD | 3 +- .../contrib/go/tasks/go_fmt_task_base.py | 2 +- .../python/pants/contrib/go/tasks/go_go.py | 2 +- .../python/pants/contrib/go/tasks/go_task.py | 2 +- .../pants/contrib/go/tasks/go_thrift_gen.py | 2 +- .../pants_test/contrib/go/subsystems/BUILD | 1 + .../go/subsystems/test_go_distribution.py | 2 +- .../pants/contrib/node/subsystems/BUILD | 1 + .../node/subsystems/node_distribution.py | 2 +- .../pants_test/contrib/node/subsystems/BUILD | 1 + .../node/subsystems/test_node_distribution.py | 2 +- .../pants_test/contrib/node/tasks/BUILD | 17 -- .../contrib/node/tasks/test_node_test.py | 109 ----------- .../pants/backend/codegen/protobuf/java/BUILD | 3 +- .../codegen/protobuf/java/protobuf_gen.py | 2 +- .../pants/backend/codegen/ragel/java/BUILD | 1 + .../backend/codegen/ragel/java/ragel_gen.py | 2 +- .../pants/backend/codegen/thrift/lib/BUILD | 1 + .../thrift/lib/apache_thrift_gen_base.py | 2 +- .../pants/backend/graph_info/tasks/BUILD | 1 + .../pants/backend/graph_info/tasks/cloc.py | 2 +- src/python/pants/backend/jvm/tasks/BUILD | 6 +- .../pants/backend/jvm/tasks/jvmdoc_gen.py | 2 +- .../pants/backend/project_info/tasks/BUILD | 1 + .../project_info/tasks/idea_plugin_gen.py | 2 +- src/python/pants/backend/python/BUILD | 1 + src/python/pants/backend/python/tasks/BUILD | 1 - .../pants/backend/python/tasks/pytest_run.py | 3 +- .../backend/python/tasks/python_isort.py | 2 +- .../pants/backend/python/thrift_builder.py | 2 +- src/python/pants/console/BUILD | 4 +- src/python/pants/console/stty_utils.py | 3 +- src/python/pants/core_tasks/BUILD | 5 +- .../pants/core_tasks/run_prep_command.py | 2 +- src/python/pants/engine/BUILD | 1 + src/python/pants/engine/isolated_process.py | 2 +- src/python/pants/java/BUILD | 3 +- src/python/pants/java/distribution/BUILD | 3 +- .../pants/java/distribution/distribution.py | 2 +- src/python/pants/java/executor.py | 2 +- src/python/pants/java/util.py | 4 +- src/python/pants/pantsd/BUILD | 3 +- src/python/pants/pantsd/process_manager.py | 2 +- src/python/pants/process/BUILD | 1 + src/python/pants/process/xargs.py | 3 +- src/python/pants/scm/BUILD | 1 + src/python/pants/scm/git.py | 2 +- src/python/pants/task/BUILD | 2 +- .../pants/task/testrunner_task_mixin.py | 34 ++-- src/python/pants/util/BUILD | 10 +- src/python/pants/util/desktop.py | 3 +- src/python/pants/util/process_handler.py | 43 ++++- src/python/pants/util/timeout.py | 54 ------ tests/python/pants_test/BUILD | 2 +- .../backend/codegen/protobuf/java/BUILD | 1 + .../java/test_protobuf_integration.py | 2 +- .../backend/codegen/wire/java/BUILD | 1 + .../wire/java/test_wire_integration.py | 2 +- .../python/pants_test/backend/jvm/tasks/BUILD | 11 +- .../tasks/test_binary_create_integration.py | 2 +- .../jvm/tasks/test_bootstrap_jvm_tools.py | 2 +- .../backend/jvm/tasks/test_junit_run.py | 58 +----- .../jvm/tasks/test_junit_run_integration.py | 6 +- .../backend/project_info/tasks/BUILD | 3 +- .../tasks/test_export_integration.py | 2 +- tests/python/pants_test/backend/python/BUILD | 3 +- .../pants_test/backend/python/tasks/BUILD | 1 - .../backend/python/tasks/test_pytest_run.py | 26 --- .../backend/python/tasks/test_python_task.py | 2 +- .../pants_test/backend/python/tasks2/BUILD | 11 +- .../backend/python/tasks2/test_pytest_run.py | 27 --- .../tasks2/test_pytest_run_integration.py | 2 +- .../tasks2/test_resolve_requirements.py | 2 +- .../backend/python/test_python_chroot.py | 2 +- tests/python/pants_test/base/BUILD | 5 +- .../test_exclude_target_regexp_integration.py | 2 +- .../pants_test/base/test_pants_ignore_scm.py | 2 +- .../pants_test/base/test_scm_build_file.py | 3 +- tests/python/pants_test/engine/examples/BUILD | 1 + .../pants_test/engine/examples/visualizer.py | 2 +- tests/python/pants_test/engine/legacy/BUILD | 3 +- .../engine/legacy/test_changed_integration.py | 2 +- tests/python/pants_test/java/BUILD | 1 + .../python/pants_test/java/distribution/BUILD | 3 +- .../java/distribution/test_distribution.py | 2 +- tests/python/pants_test/java/test_executor.py | 2 +- .../pants_test/pants_run_integration_test.py | 2 +- tests/python/pants_test/pantsd/BUILD | 3 +- .../pants_test/pantsd/test_process_manager.py | 2 +- tests/python/pants_test/python/BUILD | 1 + .../test_interpreter_selection_integration.py | 2 +- tests/python/pants_test/scm/BUILD | 1 + tests/python/pants_test/scm/test_git.py | 2 +- tests/python/pants_test/task/BUILD | 4 +- .../task/test_testrunner_task_mixin.py | 180 +++++++++--------- tests/python/pants_test/tasks/BUILD | 6 +- .../python/pants_test/tasks/task_test_base.py | 2 +- .../tasks/test_clean_all_integration.py | 2 +- tests/python/pants_test/testutils/BUILD | 1 + tests/python/pants_test/testutils/git_util.py | 2 +- tests/python/pants_test/util/BUILD | 11 +- .../pants_test/util/test_contextutil.py | 2 +- .../pants_test/util/test_process_handler.py | 3 +- tests/python/pants_test/util/test_timeout.py | 75 -------- 115 files changed, 296 insertions(+), 600 deletions(-) delete mode 100644 contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_test.py delete mode 100644 src/python/pants/util/timeout.py delete mode 100644 tests/python/pants_test/util/test_timeout.py diff --git a/.travis.yml b/.travis.yml index a09492edd86..4ec443a74a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,8 +65,9 @@ matrix: env: - SHARD="Linux Native Engine Binary Builder" - NATIVE_ENGINE_DEPLOY=1 - # Use the standard python manylinux image for ideal binary compatibility. - - DOCKER_IMAGE="quay.io/pypa/manylinux1_x86_64" + # Use the standard python-2.7 docker Debian Wheezy image for binary compatibility with old + # linux distros. + - DOCKER_IMAGE="python:2.7.13-wheezy" before_install: # Remove any Ubuntu binary cruft before building. - git clean -fdx @@ -80,7 +81,6 @@ matrix: -e "HOME=${HOME}" "${DOCKER_IMAGE}" sh -c " - export PATH=/opt/python/cp27-cp27mu/bin:/opt/rh/devtoolset-2/root/usr/bin:${PATH} ; cd $TRAVIS_BUILD_DIR && ./build-support/bin/native/prepare-binary-deploy.sh " diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 9762b99bc7c..67c333fccac 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -25,6 +25,7 @@ requests[security]==2.5.0,<2.19 scandir==1.2 setproctitle==1.1.10 setuptools==30.0.0 +subprocess32==3.2.7 six>=1.9.0,<2 thrift>=0.9.1 wheel==0.29.0 diff --git a/contrib/android/src/python/pants/contrib/android/tasks/BUILD b/contrib/android/src/python/pants/contrib/android/tasks/BUILD index 0ad3491b39b..1cea8386fe3 100644 --- a/contrib/android/src/python/pants/contrib/android/tasks/BUILD +++ b/contrib/android/src/python/pants/contrib/android/tasks/BUILD @@ -33,6 +33,7 @@ python_library( 'contrib/android/src/python/pants/contrib/android/tasks:aapt_task', 'src/python/pants/base:exceptions', 'src/python/pants/base:workunit', + 'src/python/pants/util:process_handler', ], ) @@ -48,6 +49,7 @@ python_library( 'src/python/pants/base:workunit', 'src/python/pants/build_graph', 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', ], ) @@ -78,14 +80,15 @@ python_library( name = 'sign_apk', sources = ['sign_apk.py'], dependencies = [ + 'contrib/android/src/python/pants/contrib/android/targets:android', 'contrib/android/src/python/pants/contrib/android:android_config_util', 'contrib/android/src/python/pants/contrib/android:keystore_resolver', - 'contrib/android/src/python/pants/contrib/android/targets:android', 'src/python/pants/base:exceptions', 'src/python/pants/base:workunit', 'src/python/pants/java/distribution:distribution', 'src/python/pants/task', 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', ], ) @@ -114,5 +117,6 @@ python_library( 'src/python/pants/base:exceptions', 'src/python/pants/base:workunit', 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', ], ) diff --git a/contrib/android/src/python/pants/contrib/android/tasks/aapt_builder.py b/contrib/android/src/python/pants/contrib/android/tasks/aapt_builder.py index 4a489ea9624..16381451a47 100644 --- a/contrib/android/src/python/pants/contrib/android/tasks/aapt_builder.py +++ b/contrib/android/src/python/pants/contrib/android/tasks/aapt_builder.py @@ -7,10 +7,10 @@ import logging import os -import subprocess from pants.base.exceptions import TaskError from pants.base.workunit import WorkUnitLabel +from pants.util.process_handler import subprocess from pants.contrib.android.targets.android_resources import AndroidResources from pants.contrib.android.tasks.aapt_task import AaptTask diff --git a/contrib/android/src/python/pants/contrib/android/tasks/aapt_gen.py b/contrib/android/src/python/pants/contrib/android/tasks/aapt_gen.py index ff0ee9de049..1fd8b0dd33e 100644 --- a/contrib/android/src/python/pants/contrib/android/tasks/aapt_gen.py +++ b/contrib/android/src/python/pants/contrib/android/tasks/aapt_gen.py @@ -7,7 +7,6 @@ import logging import os -import subprocess from pants.backend.jvm.targets.jar_library import JarLibrary from pants.backend.jvm.targets.java_library import JavaLibrary @@ -17,6 +16,7 @@ from pants.build_graph.address import Address from pants.java.jar.jar_dependency import JarDependency from pants.util.dirutil import safe_mkdir +from pants.util.process_handler import subprocess from pants.contrib.android.targets.android_library import AndroidLibrary from pants.contrib.android.targets.android_resources import AndroidResources diff --git a/contrib/android/src/python/pants/contrib/android/tasks/sign_apk.py b/contrib/android/src/python/pants/contrib/android/tasks/sign_apk.py index 8529328d118..62e8a8435db 100644 --- a/contrib/android/src/python/pants/contrib/android/tasks/sign_apk.py +++ b/contrib/android/src/python/pants/contrib/android/tasks/sign_apk.py @@ -7,13 +7,13 @@ import logging import os -import subprocess from pants.base.exceptions import TaskError from pants.base.workunit import WorkUnitLabel from pants.java.distribution.distribution import DistributionLocator from pants.task.task import Task from pants.util.dirutil import safe_mkdir +from pants.util.process_handler import subprocess from pants.contrib.android.android_config_util import AndroidConfigUtil from pants.contrib.android.keystore.keystore_resolver import KeystoreResolver diff --git a/contrib/android/src/python/pants/contrib/android/tasks/zipalign.py b/contrib/android/src/python/pants/contrib/android/tasks/zipalign.py index ad2837c4882..3630bca705f 100644 --- a/contrib/android/src/python/pants/contrib/android/tasks/zipalign.py +++ b/contrib/android/src/python/pants/contrib/android/tasks/zipalign.py @@ -7,11 +7,11 @@ import logging import os -import subprocess from pants.base.exceptions import TaskError from pants.base.workunit import WorkUnitLabel from pants.util.dirutil import safe_mkdir +from pants.util.process_handler import subprocess from pants.contrib.android.targets.android_binary import AndroidBinary from pants.contrib.android.tasks.android_task import AndroidTask diff --git a/contrib/cpp/src/python/pants/contrib/cpp/tasks/BUILD b/contrib/cpp/src/python/pants/contrib/cpp/tasks/BUILD index 851dca4b718..6d16e8ac35a 100644 --- a/contrib/cpp/src/python/pants/contrib/cpp/tasks/BUILD +++ b/contrib/cpp/src/python/pants/contrib/cpp/tasks/BUILD @@ -11,11 +11,12 @@ python_library( 'cpp_task.py', ], dependencies=[ - 'contrib/cpp/src/python/pants/contrib/cpp/toolchain:toolchain', 'contrib/cpp/src/python/pants/contrib/cpp/targets:targets', + 'contrib/cpp/src/python/pants/contrib/cpp/toolchain:toolchain', 'src/python/pants/base:exceptions', 'src/python/pants/base:workunit', 'src/python/pants/task', 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', ], ) diff --git a/contrib/cpp/src/python/pants/contrib/cpp/tasks/cpp_task.py b/contrib/cpp/src/python/pants/contrib/cpp/tasks/cpp_task.py index e1c99e531cd..c073b8041ea 100644 --- a/contrib/cpp/src/python/pants/contrib/cpp/tasks/cpp_task.py +++ b/contrib/cpp/src/python/pants/contrib/cpp/tasks/cpp_task.py @@ -5,10 +5,9 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) -import subprocess - from pants.base.exceptions import TaskError from pants.task.task import Task +from pants.util.process_handler import subprocess from pants.contrib.cpp.targets.cpp_binary import CppBinary from pants.contrib.cpp.targets.cpp_library import CppLibrary diff --git a/contrib/go/src/python/pants/contrib/go/subsystems/BUILD b/contrib/go/src/python/pants/contrib/go/subsystems/BUILD index dfd1b47f3a6..9be79c79e27 100644 --- a/contrib/go/src/python/pants/contrib/go/subsystems/BUILD +++ b/contrib/go/src/python/pants/contrib/go/subsystems/BUILD @@ -16,5 +16,6 @@ python_library( 'src/python/pants/util:contextutil', 'src/python/pants/util:memo', 'src/python/pants/util:meta', + 'src/python/pants/util:process_handler', ], ) diff --git a/contrib/go/src/python/pants/contrib/go/subsystems/go_distribution.py b/contrib/go/src/python/pants/contrib/go/subsystems/go_distribution.py index db17febd516..11cdec2db97 100644 --- a/contrib/go/src/python/pants/contrib/go/subsystems/go_distribution.py +++ b/contrib/go/src/python/pants/contrib/go/subsystems/go_distribution.py @@ -6,7 +6,6 @@ unicode_literals, with_statement) import os -import subprocess from collections import OrderedDict, namedtuple from pants.base.workunit import WorkUnit, WorkUnitLabel @@ -15,6 +14,7 @@ from pants.subsystem.subsystem import Subsystem from pants.util.contextutil import temporary_dir from pants.util.memo import memoized_property +from pants.util.process_handler import subprocess class GoDistribution(object): diff --git a/contrib/go/src/python/pants/contrib/go/tasks/BUILD b/contrib/go/src/python/pants/contrib/go/tasks/BUILD index fffd6d7f089..a5459117425 100644 --- a/contrib/go/src/python/pants/contrib/go/tasks/BUILD +++ b/contrib/go/src/python/pants/contrib/go/tasks/BUILD @@ -14,12 +14,13 @@ python_library( 'src/python/pants/base:workunit', 'src/python/pants/binaries:thrift_util', 'src/python/pants/build_graph', + 'src/python/pants/option', 'src/python/pants/process', 'src/python/pants/source', 'src/python/pants/task', 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', 'src/python/pants/util:memo', - 'src/python/pants/option', + 'src/python/pants/util:process_handler', ] ) diff --git a/contrib/go/src/python/pants/contrib/go/tasks/go_fmt_task_base.py b/contrib/go/src/python/pants/contrib/go/tasks/go_fmt_task_base.py index e151538b51c..b67a1a078d5 100644 --- a/contrib/go/src/python/pants/contrib/go/tasks/go_fmt_task_base.py +++ b/contrib/go/src/python/pants/contrib/go/tasks/go_fmt_task_base.py @@ -6,10 +6,10 @@ unicode_literals, with_statement) import os -import subprocess from contextlib import contextmanager from pants.base.exceptions import TaskError +from pants.util.process_handler import subprocess from pants.contrib.go.targets.go_local_source import GoLocalSource from pants.contrib.go.tasks.go_workspace_task import GoWorkspaceTask diff --git a/contrib/go/src/python/pants/contrib/go/tasks/go_go.py b/contrib/go/src/python/pants/contrib/go/tasks/go_go.py index 7ea846344e7..95945317813 100644 --- a/contrib/go/src/python/pants/contrib/go/tasks/go_go.py +++ b/contrib/go/src/python/pants/contrib/go/tasks/go_go.py @@ -6,12 +6,12 @@ unicode_literals, with_statement) import os -import subprocess from abc import abstractmethod from colors import green, red, yellow from pants.base.exceptions import TaskError from pants.task.task import QuietTaskMixin +from pants.util.process_handler import subprocess from twitter.common.collections import OrderedSet from pants.contrib.go.tasks.go_workspace_task import GoWorkspaceTask diff --git a/contrib/go/src/python/pants/contrib/go/tasks/go_task.py b/contrib/go/src/python/pants/contrib/go/tasks/go_task.py index a0382d780d2..6def91c702e 100644 --- a/contrib/go/src/python/pants/contrib/go/tasks/go_task.py +++ b/contrib/go/src/python/pants/contrib/go/tasks/go_task.py @@ -7,12 +7,12 @@ import json import re -import subprocess from collections import namedtuple from pants.base.workunit import WorkUnit, WorkUnitLabel from pants.task.task import Task from pants.util.memo import memoized_method, memoized_property +from pants.util.process_handler import subprocess from twitter.common.collections.orderedset import OrderedSet from pants.contrib.go.subsystems.go_distribution import GoDistribution diff --git a/contrib/go/src/python/pants/contrib/go/tasks/go_thrift_gen.py b/contrib/go/src/python/pants/contrib/go/tasks/go_thrift_gen.py index f32ef900321..d940ca08aac 100644 --- a/contrib/go/src/python/pants/contrib/go/tasks/go_thrift_gen.py +++ b/contrib/go/src/python/pants/contrib/go/tasks/go_thrift_gen.py @@ -7,7 +7,6 @@ import os import re -import subprocess from pants.base.build_environment import get_buildroot from pants.base.exceptions import TaskError @@ -18,6 +17,7 @@ from pants.task.simple_codegen_task import SimpleCodegenTask from pants.util.dirutil import safe_mkdir from pants.util.memo import memoized_method, memoized_property +from pants.util.process_handler import subprocess from twitter.common.collections import OrderedSet from pants.contrib.go.targets.go_thrift_library import GoThriftGenLibrary, GoThriftLibrary diff --git a/contrib/go/tests/python/pants_test/contrib/go/subsystems/BUILD b/contrib/go/tests/python/pants_test/contrib/go/subsystems/BUILD index 08527940d90..59f2e51cab4 100644 --- a/contrib/go/tests/python/pants_test/contrib/go/subsystems/BUILD +++ b/contrib/go/tests/python/pants_test/contrib/go/subsystems/BUILD @@ -17,6 +17,7 @@ python_tests( dependencies=[ 'contrib/go/src/python/pants/contrib/go/subsystems', 'src/python/pants/util:contextutil', + 'src/python/pants/util:process_handler', 'tests/python/pants_test/subsystem:subsystem_utils', ] ) diff --git a/contrib/go/tests/python/pants_test/contrib/go/subsystems/test_go_distribution.py b/contrib/go/tests/python/pants_test/contrib/go/subsystems/test_go_distribution.py index 47c92d9202e..02e41d04a22 100644 --- a/contrib/go/tests/python/pants_test/contrib/go/subsystems/test_go_distribution.py +++ b/contrib/go/tests/python/pants_test/contrib/go/subsystems/test_go_distribution.py @@ -6,10 +6,10 @@ unicode_literals, with_statement) import os -import subprocess import unittest from pants.util.contextutil import environment_as +from pants.util.process_handler import subprocess from pants_test.subsystem.subsystem_util import global_subsystem_instance from pants.contrib.go.subsystems.go_distribution import GoDistribution diff --git a/contrib/node/src/python/pants/contrib/node/subsystems/BUILD b/contrib/node/src/python/pants/contrib/node/subsystems/BUILD index 096cbe45d22..6e54ced858a 100644 --- a/contrib/node/src/python/pants/contrib/node/subsystems/BUILD +++ b/contrib/node/src/python/pants/contrib/node/subsystems/BUILD @@ -11,5 +11,6 @@ python_library( 'src/python/pants/util:contextutil', 'src/python/pants/util:memo', 'src/python/pants/util:meta', + 'src/python/pants/util:process_handler', ], ) diff --git a/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py b/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py index cdda3aaf8c6..acaeb822dce 100644 --- a/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py +++ b/contrib/node/src/python/pants/contrib/node/subsystems/node_distribution.py @@ -7,7 +7,6 @@ import logging import os -import subprocess from collections import namedtuple from pants.base.exceptions import TaskError @@ -15,6 +14,7 @@ from pants.fs.archive import TGZ from pants.subsystem.subsystem import Subsystem from pants.util.memo import memoized_method +from pants.util.process_handler import subprocess logger = logging.getLogger(__name__) diff --git a/contrib/node/tests/python/pants_test/contrib/node/subsystems/BUILD b/contrib/node/tests/python/pants_test/contrib/node/subsystems/BUILD index a71082371dc..2956f48c015 100644 --- a/contrib/node/tests/python/pants_test/contrib/node/subsystems/BUILD +++ b/contrib/node/tests/python/pants_test/contrib/node/subsystems/BUILD @@ -6,6 +6,7 @@ python_tests( sources=['test_node_distribution.py'], dependencies=[ 'contrib/node/src/python/pants/contrib/node/subsystems:node_distribution', + 'src/python/pants/util:process_handler', 'tests/python/pants_test/subsystem:subsystem_utils', ] ) diff --git a/contrib/node/tests/python/pants_test/contrib/node/subsystems/test_node_distribution.py b/contrib/node/tests/python/pants_test/contrib/node/subsystems/test_node_distribution.py index aed5d03e693..c6428806e1b 100644 --- a/contrib/node/tests/python/pants_test/contrib/node/subsystems/test_node_distribution.py +++ b/contrib/node/tests/python/pants_test/contrib/node/subsystems/test_node_distribution.py @@ -7,9 +7,9 @@ import json import os -import subprocess import unittest +from pants.util.process_handler import subprocess from pants_test.subsystem.subsystem_util import global_subsystem_instance from pants.contrib.node.subsystems.node_distribution import NodeDistribution diff --git a/contrib/node/tests/python/pants_test/contrib/node/tasks/BUILD b/contrib/node/tests/python/pants_test/contrib/node/tasks/BUILD index 3ba2075f515..399726595ab 100644 --- a/contrib/node/tests/python/pants_test/contrib/node/tasks/BUILD +++ b/contrib/node/tests/python/pants_test/contrib/node/tasks/BUILD @@ -110,20 +110,3 @@ python_tests( 'tests/python/pants_test/tasks:task_test_base', ] ) - -python_tests( - name='node_test', - sources=['test_node_test.py'], - dependencies=[ - '3rdparty/python:mock', - 'contrib/node/src/python/pants/contrib/node/targets:node_module', - 'contrib/node/src/python/pants/contrib/node/targets:node_test', - 'contrib/node/src/python/pants/contrib/node/tasks:node_paths', - 'contrib/node/src/python/pants/contrib/node/tasks:node_resolve', - 'contrib/node/src/python/pants/contrib/node/tasks:node_test', - 'contrib/node/src/python/pants/contrib/node/subsystems/resolvers:npm_resolver', - 'src/python/pants/base:exceptions', - 'src/python/pants/util:timeout', - 'tests/python/pants_test/tasks:task_test_base', - ] -) diff --git a/contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_test.py b/contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_test.py deleted file mode 100644 index c99f73170e0..00000000000 --- a/contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_test.py +++ /dev/null @@ -1,109 +0,0 @@ -# coding=utf-8 -# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import (absolute_import, division, generators, nested_scopes, print_function, - unicode_literals, with_statement) - -import os -from textwrap import dedent - -from mock import patch -from pants.base.exceptions import ErrorWhileTesting -from pants.util.timeout import TimeoutReached -from pants_test.tasks.task_test_base import TaskTestBase - -from pants.contrib.node.subsystems.resolvers.npm_resolver import NpmResolver -from pants.contrib.node.targets.node_module import NodeModule -from pants.contrib.node.targets.node_test import NodeTest as NodeTestTarget -from pants.contrib.node.tasks.node_paths import NodePaths -from pants.contrib.node.tasks.node_resolve import NodeResolve -from pants.contrib.node.tasks.node_test import NodeTest as NodeTestTask - - -class NodeTestTest(TaskTestBase): - - @classmethod - def task_type(cls): - return NodeTestTask - - def setUp(self): - super(NodeTestTest, self).setUp() - NodeResolve.register_resolver_for_type(NodeModule, NpmResolver) - - def tearDown(self): - super(NodeTestTest, self).tearDown() - NodeResolve._clear_resolvers() - - def _create_node_module_target(self): - self.create_file('src/node/test_node_test/package.json', contents=dedent(""" - { - "name": "pantsbuild.pants.test.test_node_test", - "version": "0.0.0", - "scripts": { - "test": "echo 0" - } - } - """)) - return self.make_target(spec='src/node/test_node_test', - target_type=NodeModule, - sources=['package.json']) - - def _resolve_node_module_and_create_tests_task(self, node_module_target, test_targets, - passthru_args=None): - context = self.context(target_roots=test_targets, passthru_args=passthru_args) - - # Fake resolving so self.context.products.get_data(NodePaths) is populated for NodeTestTask. - node_module_target_root = os.path.join(self.build_root, node_module_target.address.spec_path) - node_paths = context.products.get_data(NodePaths, init_func=NodePaths) - node_paths.resolved(node_module_target, node_module_target_root) - - return self.create_task(context) - - def test_timeout(self): - target = self._create_node_module_target() - - test_target = self.make_target(spec='src/node/test_node_test:test', - target_type=NodeTestTarget, - dependencies=[target], - timeout=5) - - task = self._resolve_node_module_and_create_tests_task(target, [test_target]) - - with patch('pants.task.testrunner_task_mixin.Timeout') as mock_timeout: - mock_timeout().__exit__.side_effect = TimeoutReached(5) - - with self.assertRaises(ErrorWhileTesting): - task.execute() - - # Ensures that Timeout is instantiated with a 5 second timeout. - args, kwargs = mock_timeout.call_args - self.assertEqual(args, (5,)) - - def test_timeout_multiple_targets(self): - target = self._create_node_module_target() - - test_target1 = self.make_target(spec='src/node/test_node_test:test1', - target_type=NodeTestTarget, - dependencies=[target], - timeout=5) - test_target2 = self.make_target(spec='src/node/test_node_test:test2', - target_type=NodeTestTarget, - dependencies=[target], - timeout=5) - - task = self._resolve_node_module_and_create_tests_task(target, [test_target1, test_target2]) - - with patch('pants.task.testrunner_task_mixin.Timeout') as mock_timeout: - mock_timeout().__exit__.side_effect = TimeoutReached(5) - - with self.assertRaises(ErrorWhileTesting) as cm: - task.execute() - - # Verify that only the first target is in the failed_targets list, not all test targets. - self.assertEqual(len(cm.exception.failed_targets), 1) - self.assertEqual(cm.exception.failed_targets[0].address.spec, 'src/node/test_node_test:test1') - - # Ensures that Timeout is instantiated with a 5 second timeout. - args, kwargs = mock_timeout.call_args - self.assertEqual(args, (5,)) diff --git a/src/python/pants/backend/codegen/protobuf/java/BUILD b/src/python/pants/backend/codegen/protobuf/java/BUILD index 5ebceb782ca..1f7ee010e96 100644 --- a/src/python/pants/backend/codegen/protobuf/java/BUILD +++ b/src/python/pants/backend/codegen/protobuf/java/BUILD @@ -14,9 +14,10 @@ python_library( 'src/python/pants/base:workunit', 'src/python/pants/binaries:binary_util', 'src/python/pants/build_graph', - 'src/python/pants/goal:task_registrar', 'src/python/pants/fs', + 'src/python/pants/goal:task_registrar', 'src/python/pants/task', 'src/python/pants/util:memo', + 'src/python/pants/util:process_handler', ], ) diff --git a/src/python/pants/backend/codegen/protobuf/java/protobuf_gen.py b/src/python/pants/backend/codegen/protobuf/java/protobuf_gen.py index 37e3907201d..bea834c8a82 100644 --- a/src/python/pants/backend/codegen/protobuf/java/protobuf_gen.py +++ b/src/python/pants/backend/codegen/protobuf/java/protobuf_gen.py @@ -6,7 +6,6 @@ unicode_literals, with_statement) import os -import subprocess from collections import OrderedDict from hashlib import sha1 @@ -24,6 +23,7 @@ from pants.fs.archive import ZIP from pants.task.simple_codegen_task import SimpleCodegenTask from pants.util.memo import memoized_property +from pants.util.process_handler import subprocess class ProtobufGen(SimpleCodegenTask): diff --git a/src/python/pants/backend/codegen/ragel/java/BUILD b/src/python/pants/backend/codegen/ragel/java/BUILD index 315a6c43290..44260a239b2 100644 --- a/src/python/pants/backend/codegen/ragel/java/BUILD +++ b/src/python/pants/backend/codegen/ragel/java/BUILD @@ -12,5 +12,6 @@ python_library( 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', 'src/python/pants/util:memo', + 'src/python/pants/util:process_handler', ], ) diff --git a/src/python/pants/backend/codegen/ragel/java/ragel_gen.py b/src/python/pants/backend/codegen/ragel/java/ragel_gen.py index c201f1450a3..fb82438b2e0 100644 --- a/src/python/pants/backend/codegen/ragel/java/ragel_gen.py +++ b/src/python/pants/backend/codegen/ragel/java/ragel_gen.py @@ -7,7 +7,6 @@ import os import re -import subprocess from pants.backend.codegen.ragel.java.java_ragel_library import JavaRagelLibrary from pants.backend.jvm.targets.java_library import JavaLibrary @@ -17,6 +16,7 @@ from pants.task.simple_codegen_task import SimpleCodegenTask from pants.util.dirutil import safe_mkdir_for from pants.util.memo import memoized_property +from pants.util.process_handler import subprocess class RagelGen(SimpleCodegenTask): diff --git a/src/python/pants/backend/codegen/thrift/lib/BUILD b/src/python/pants/backend/codegen/thrift/lib/BUILD index 3ed39b606f7..d0f1bfab2fc 100644 --- a/src/python/pants/backend/codegen/thrift/lib/BUILD +++ b/src/python/pants/backend/codegen/thrift/lib/BUILD @@ -11,5 +11,6 @@ python_library( 'src/python/pants/option', 'src/python/pants/task', 'src/python/pants/util:memo', + 'src/python/pants/util:process_handler', ], ) diff --git a/src/python/pants/backend/codegen/thrift/lib/apache_thrift_gen_base.py b/src/python/pants/backend/codegen/thrift/lib/apache_thrift_gen_base.py index f4874b7138b..e18c633b76f 100644 --- a/src/python/pants/backend/codegen/thrift/lib/apache_thrift_gen_base.py +++ b/src/python/pants/backend/codegen/thrift/lib/apache_thrift_gen_base.py @@ -8,7 +8,6 @@ import os import re import shutil -import subprocess from twitter.common.collections import OrderedSet @@ -19,6 +18,7 @@ from pants.option.custom_types import target_option from pants.task.simple_codegen_task import SimpleCodegenTask from pants.util.memo import memoized_property +from pants.util.process_handler import subprocess class ApacheThriftGenBase(SimpleCodegenTask): diff --git a/src/python/pants/backend/graph_info/tasks/BUILD b/src/python/pants/backend/graph_info/tasks/BUILD index 2de5eb34200..9daa8e6aec9 100644 --- a/src/python/pants/backend/graph_info/tasks/BUILD +++ b/src/python/pants/backend/graph_info/tasks/BUILD @@ -15,6 +15,7 @@ python_library( 'src/python/pants/task', 'src/python/pants/util:contextutil', 'src/python/pants/util:filtering', + 'src/python/pants/util:process_handler', 'src/python/pants/util:strutil', ], ) diff --git a/src/python/pants/backend/graph_info/tasks/cloc.py b/src/python/pants/backend/graph_info/tasks/cloc.py index 6da5cc8231c..911e66360b8 100644 --- a/src/python/pants/backend/graph_info/tasks/cloc.py +++ b/src/python/pants/backend/graph_info/tasks/cloc.py @@ -6,7 +6,6 @@ unicode_literals, with_statement) import os -import subprocess from pants.base.build_environment import get_buildroot from pants.base.exceptions import TaskError @@ -14,6 +13,7 @@ from pants.binaries.binary_util import BinaryUtil from pants.task.console_task import ConsoleTask from pants.util.contextutil import temporary_dir +from pants.util.process_handler import subprocess class CountLinesOfCode(ConsoleTask): diff --git a/src/python/pants/backend/jvm/tasks/BUILD b/src/python/pants/backend/jvm/tasks/BUILD index 4efb5ab7650..5514fb41fb7 100644 --- a/src/python/pants/backend/jvm/tasks/BUILD +++ b/src/python/pants/backend/jvm/tasks/BUILD @@ -361,18 +361,18 @@ python_library( name = 'junit_run', sources = ['junit_run.py'], dependencies = [ - '3rdparty/python:six', '3rdparty/python/twitter/commons:twitter.common.collections', + '3rdparty/python:six', ':classpath_util', ':coverage', ':jvm_task', ':jvm_tool_task_mixin', ':reports', - 'src/python/pants/backend/jvm:argfile', 'src/python/pants/backend/jvm/subsystems:junit', 'src/python/pants/backend/jvm/subsystems:jvm_platform', 'src/python/pants/backend/jvm/targets:java', 'src/python/pants/backend/jvm/targets:jvm', + 'src/python/pants/backend/jvm:argfile', 'src/python/pants/base:build_environment', 'src/python/pants/base:exceptions', 'src/python/pants/base:workunit', @@ -388,6 +388,7 @@ python_library( 'src/python/pants/util:dirutil', 'src/python/pants/util:memo', 'src/python/pants/util:meta', + 'src/python/pants/util:process_handler', 'src/python/pants/util:strutil', ], ) @@ -517,6 +518,7 @@ python_library( 'src/python/pants/binaries:binary_util', 'src/python/pants/util:desktop', 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', ], ) diff --git a/src/python/pants/backend/jvm/tasks/jvmdoc_gen.py b/src/python/pants/backend/jvm/tasks/jvmdoc_gen.py index dcb225f4911..4e4580d96f4 100644 --- a/src/python/pants/backend/jvm/tasks/jvmdoc_gen.py +++ b/src/python/pants/backend/jvm/tasks/jvmdoc_gen.py @@ -10,13 +10,13 @@ import multiprocessing import os import re -import subprocess from pants.backend.jvm.tasks.jvm_task import JvmTask from pants.base.exceptions import TaskError from pants.util import desktop from pants.util.dirutil import safe_mkdir, safe_walk from pants.util.memo import memoized_property +from pants.util.process_handler import subprocess Jvmdoc = collections.namedtuple('Jvmdoc', ['tool_name', 'product_type']) diff --git a/src/python/pants/backend/project_info/tasks/BUILD b/src/python/pants/backend/project_info/tasks/BUILD index 0bad1337ecb..769359ac6e9 100644 --- a/src/python/pants/backend/project_info/tasks/BUILD +++ b/src/python/pants/backend/project_info/tasks/BUILD @@ -161,6 +161,7 @@ python_library( 'src/python/pants/util:contextutil', 'src/python/pants/util:desktop', 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', ], ) diff --git a/src/python/pants/backend/project_info/tasks/idea_plugin_gen.py b/src/python/pants/backend/project_info/tasks/idea_plugin_gen.py index b6eb92b42ca..202723c56a2 100644 --- a/src/python/pants/backend/project_info/tasks/idea_plugin_gen.py +++ b/src/python/pants/backend/project_info/tasks/idea_plugin_gen.py @@ -11,7 +11,6 @@ import pkgutil import re import shutil -import subprocess from pants.backend.jvm.targets.jvm_target import JvmTarget from pants.backend.python.targets.python_target import PythonTarget @@ -22,6 +21,7 @@ from pants.util import desktop from pants.util.contextutil import temporary_dir, temporary_file from pants.util.dirutil import safe_mkdir +from pants.util.process_handler import subprocess _TEMPLATE_BASEDIR = 'templates/idea' diff --git a/src/python/pants/backend/python/BUILD b/src/python/pants/backend/python/BUILD index fb3607466a6..a1e1a4146e0 100644 --- a/src/python/pants/backend/python/BUILD +++ b/src/python/pants/backend/python/BUILD @@ -148,5 +148,6 @@ python_library( 'src/python/pants/base:build_environment', 'src/python/pants/util:dirutil', 'src/python/pants/util:memo', + 'src/python/pants/util:process_handler', ] ) diff --git a/src/python/pants/backend/python/tasks/BUILD b/src/python/pants/backend/python/tasks/BUILD index c6cbcbecdc0..dadde1e8dac 100644 --- a/src/python/pants/backend/python/tasks/BUILD +++ b/src/python/pants/backend/python/tasks/BUILD @@ -39,6 +39,5 @@ python_library( 'src/python/pants/util:process_handler', 'src/python/pants/util:strutil', 'src/python/pants/util:xml_parser', - 'src/python/pants/util:timeout', ] ) diff --git a/src/python/pants/backend/python/tasks/pytest_run.py b/src/python/pants/backend/python/tasks/pytest_run.py index 75dc5a0f632..298b7e9e933 100644 --- a/src/python/pants/backend/python/tasks/pytest_run.py +++ b/src/python/pants/backend/python/tasks/pytest_run.py @@ -9,7 +9,6 @@ import os import re import shutil -import subprocess import time import traceback from contextlib import contextmanager @@ -31,7 +30,7 @@ from pants.util.contextutil import (environment_as, temporary_dir, temporary_file, temporary_file_path) from pants.util.dirutil import safe_mkdir, safe_open -from pants.util.process_handler import SubprocessProcessHandler +from pants.util.process_handler import SubprocessProcessHandler, subprocess from pants.util.strutil import safe_shlex_split diff --git a/src/python/pants/backend/python/tasks/python_isort.py b/src/python/pants/backend/python/tasks/python_isort.py index aafe1301a71..8ccf11812ac 100644 --- a/src/python/pants/backend/python/tasks/python_isort.py +++ b/src/python/pants/backend/python/tasks/python_isort.py @@ -7,7 +7,6 @@ import logging import os -import subprocess from pants.backend.python.targets.python_binary import PythonBinary from pants.backend.python.targets.python_library import PythonLibrary @@ -16,6 +15,7 @@ from pants.base.exceptions import TaskError from pants.binaries.binary_util import BinaryUtil from pants.task.task import Task +from pants.util.process_handler import subprocess class IsortPythonTask(Task): diff --git a/src/python/pants/backend/python/thrift_builder.py b/src/python/pants/backend/python/thrift_builder.py index 726916aa49e..5691822378c 100644 --- a/src/python/pants/backend/python/thrift_builder.py +++ b/src/python/pants/backend/python/thrift_builder.py @@ -7,7 +7,6 @@ import itertools import os -import subprocess import sys from twitter.common.collections import OrderedSet @@ -17,6 +16,7 @@ from pants.base.build_environment import get_buildroot from pants.util.dirutil import safe_walk from pants.util.memo import memoized_property +from pants.util.process_handler import subprocess class PythonThriftBuilder(CodeGenerator): diff --git a/src/python/pants/console/BUILD b/src/python/pants/console/BUILD index 3646d7443a7..464f606cb1e 100644 --- a/src/python/pants/console/BUILD +++ b/src/python/pants/console/BUILD @@ -4,5 +4,7 @@ python_library( name = 'stty_utils', sources = ['stty_utils.py'], - dependencies = [] + dependencies = [ + 'src/python/pants/util:process_handler', + ] ) diff --git a/src/python/pants/console/stty_utils.py b/src/python/pants/console/stty_utils.py index 317da283fcd..4d807ba539a 100644 --- a/src/python/pants/console/stty_utils.py +++ b/src/python/pants/console/stty_utils.py @@ -5,9 +5,10 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) -import subprocess from contextlib import contextmanager +from pants.util.process_handler import subprocess + @contextmanager def preserve_stty_settings(): diff --git a/src/python/pants/core_tasks/BUILD b/src/python/pants/core_tasks/BUILD index 795e36db263..d8d21a34127 100644 --- a/src/python/pants/core_tasks/BUILD +++ b/src/python/pants/core_tasks/BUILD @@ -3,10 +3,10 @@ python_library( dependencies=[ - ':templates', '3rdparty/python:ansicolors', - '3rdparty/python:setuptools', '3rdparty/python:packaging', + '3rdparty/python:setuptools', + ':templates', 'src/python/pants/base:build_environment', 'src/python/pants/base:deprecated', 'src/python/pants/base:exceptions', @@ -25,6 +25,7 @@ python_library( 'src/python/pants/util:contextutil', 'src/python/pants/util:desktop', 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', 'src/python/pants:version', ]) diff --git a/src/python/pants/core_tasks/run_prep_command.py b/src/python/pants/core_tasks/run_prep_command.py index 0c0b2beccae..ea31cda72f1 100644 --- a/src/python/pants/core_tasks/run_prep_command.py +++ b/src/python/pants/core_tasks/run_prep_command.py @@ -6,13 +6,13 @@ unicode_literals, with_statement) import os -import subprocess from collections import namedtuple from pants.base.exceptions import TaskError from pants.base.workunit import WorkUnit, WorkUnitLabel from pants.build_graph.prep_command import PrepCommand from pants.task.task import Task +from pants.util.process_handler import subprocess class RunPrepCommandBase(Task): diff --git a/src/python/pants/engine/BUILD b/src/python/pants/engine/BUILD index cecc9b634df..ad434ec8b76 100644 --- a/src/python/pants/engine/BUILD +++ b/src/python/pants/engine/BUILD @@ -96,6 +96,7 @@ python_library( ':struct', 'src/python/pants/build_graph', 'src/python/pants/util:objects', + 'src/python/pants/util:process_handler', ] ) diff --git a/src/python/pants/engine/isolated_process.py b/src/python/pants/engine/isolated_process.py index 777923d304d..8b0824642b0 100644 --- a/src/python/pants/engine/isolated_process.py +++ b/src/python/pants/engine/isolated_process.py @@ -8,7 +8,6 @@ import functools import logging import os -import subprocess from abc import abstractproperty from binascii import hexlify @@ -17,6 +16,7 @@ from pants.util.contextutil import open_tar, temporary_dir from pants.util.dirutil import safe_mkdir from pants.util.objects import datatype +from pants.util.process_handler import subprocess logger = logging.getLogger(__name__) diff --git a/src/python/pants/java/BUILD b/src/python/pants/java/BUILD index d9b5ea62a74..4b09436469d 100644 --- a/src/python/pants/java/BUILD +++ b/src/python/pants/java/BUILD @@ -5,12 +5,13 @@ python_library( name = 'executor', sources = ['executor.py'], dependencies = [ - '3rdparty/python:six', '3rdparty/python/twitter/commons:twitter.common.collections', + '3rdparty/python:six', 'src/python/pants/base:build_environment', 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', 'src/python/pants/util:meta', + 'src/python/pants/util:process_handler', ] ) diff --git a/src/python/pants/java/distribution/BUILD b/src/python/pants/java/distribution/BUILD index 304bdf89e37..b4898a25441 100644 --- a/src/python/pants/java/distribution/BUILD +++ b/src/python/pants/java/distribution/BUILD @@ -4,13 +4,14 @@ python_library( sources=[ 'distribution.py' ], dependencies=[ - ':resources', '3rdparty/python:six', + ':resources', 'src/python/pants/base:revision', 'src/python/pants/java:util', 'src/python/pants/subsystem', 'src/python/pants/util:contextutil', 'src/python/pants/util:osutil', + 'src/python/pants/util:process_handler', ], ) diff --git a/src/python/pants/java/distribution/distribution.py b/src/python/pants/java/distribution/distribution.py index 3f4396d8375..d2d86c935f0 100644 --- a/src/python/pants/java/distribution/distribution.py +++ b/src/python/pants/java/distribution/distribution.py @@ -10,7 +10,6 @@ import os import pkgutil import plistlib -import subprocess from abc import abstractproperty from collections import namedtuple from contextlib import contextmanager @@ -24,6 +23,7 @@ from pants.util.memo import memoized_method, memoized_property from pants.util.meta import AbstractClass from pants.util.osutil import OS_ALIASES, normalize_os_name +from pants.util.process_handler import subprocess logger = logging.getLogger(__name__) diff --git a/src/python/pants/java/executor.py b/src/python/pants/java/executor.py index 8691b1c3830..27f1f73d1f0 100644 --- a/src/python/pants/java/executor.py +++ b/src/python/pants/java/executor.py @@ -7,7 +7,6 @@ import logging import os -import subprocess from abc import abstractmethod, abstractproperty from contextlib import contextmanager @@ -18,6 +17,7 @@ from pants.util.contextutil import environment_as from pants.util.dirutil import relativize_paths from pants.util.meta import AbstractClass +from pants.util.process_handler import subprocess logger = logging.getLogger(__name__) diff --git a/src/python/pants/java/util.py b/src/python/pants/java/util.py index ae87b6ddeaa..f03ab2c4ec4 100644 --- a/src/python/pants/java/util.py +++ b/src/python/pants/java/util.py @@ -192,9 +192,9 @@ def execute_runner_async(runner, workunit_factory=None, workunit_name=None, work process = runner.spawn(stdout=workunit.output('stdout'), stderr=workunit.output('stderr')) class WorkUnitProcessHandler(ProcessHandler): - def wait(_): + def wait(_, timeout=None): try: - ret = process.wait() + ret = process.wait(timeout=timeout) workunit.set_outcome(WorkUnit.FAILURE if ret else WorkUnit.SUCCESS) workunit_generator.__exit__(None, None, None) return ret diff --git a/src/python/pants/pantsd/BUILD b/src/python/pants/pantsd/BUILD index 6caa712d83b..5499547678c 100644 --- a/src/python/pants/pantsd/BUILD +++ b/src/python/pants/pantsd/BUILD @@ -8,7 +8,8 @@ python_library( '3rdparty/python:psutil', 'src/python/pants/base:build_environment', 'src/python/pants/pantsd/subsystem:subprocess', - 'src/python/pants/util:dirutil' + 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', ] ) diff --git a/src/python/pants/pantsd/process_manager.py b/src/python/pants/pantsd/process_manager.py index a63b8448827..e49593db944 100644 --- a/src/python/pants/pantsd/process_manager.py +++ b/src/python/pants/pantsd/process_manager.py @@ -8,7 +8,6 @@ import logging import os import signal -import subprocess import time import traceback from contextlib import contextmanager @@ -18,6 +17,7 @@ from pants.base.build_environment import get_buildroot from pants.pantsd.subsystem.subprocess import Subprocess from pants.util.dirutil import read_file, rm_rf, safe_file_dump, safe_mkdir +from pants.util.process_handler import subprocess logger = logging.getLogger(__name__) diff --git a/src/python/pants/process/BUILD b/src/python/pants/process/BUILD index 696490cd5d9..ef6cc0d267b 100644 --- a/src/python/pants/process/BUILD +++ b/src/python/pants/process/BUILD @@ -6,5 +6,6 @@ python_library( '3rdparty/python:fasteners', '3rdparty/python:psutil', 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', ] ) diff --git a/src/python/pants/process/xargs.py b/src/python/pants/process/xargs.py index 8fb4ee98676..a8554ffc533 100644 --- a/src/python/pants/process/xargs.py +++ b/src/python/pants/process/xargs.py @@ -6,7 +6,8 @@ unicode_literals, with_statement) import errno -import subprocess + +from pants.util.process_handler import subprocess class Xargs(object): diff --git a/src/python/pants/scm/BUILD b/src/python/pants/scm/BUILD index 437710320a8..41d52eec19a 100644 --- a/src/python/pants/scm/BUILD +++ b/src/python/pants/scm/BUILD @@ -16,6 +16,7 @@ python_library( ':scm', 'src/python/pants/util:contextutil', 'src/python/pants/util:memo', + 'src/python/pants/util:process_handler', 'src/python/pants/util:strutil', ], ) diff --git a/src/python/pants/scm/git.py b/src/python/pants/scm/git.py index 1cb14b406d4..66ba9ca44e1 100644 --- a/src/python/pants/scm/git.py +++ b/src/python/pants/scm/git.py @@ -8,13 +8,13 @@ import logging import os import StringIO -import subprocess import traceback from contextlib import contextmanager from pants.scm.scm import Scm from pants.util.contextutil import pushd from pants.util.memo import memoized_method +from pants.util.process_handler import subprocess from pants.util.strutil import ensure_binary diff --git a/src/python/pants/task/BUILD b/src/python/pants/task/BUILD index 93b87c02ea5..e3ad3e94f0a 100644 --- a/src/python/pants/task/BUILD +++ b/src/python/pants/task/BUILD @@ -24,6 +24,6 @@ python_library( 'src/python/pants/util:dirutil', 'src/python/pants/util:memo', 'src/python/pants/util:meta', - 'src/python/pants/util:timeout', + 'src/python/pants/util:process_handler', ], ) diff --git a/src/python/pants/task/testrunner_task_mixin.py b/src/python/pants/task/testrunner_task_mixin.py index d824d9cf969..2667bf389a9 100644 --- a/src/python/pants/task/testrunner_task_mixin.py +++ b/src/python/pants/task/testrunner_task_mixin.py @@ -12,7 +12,7 @@ from threading import Timer from pants.base.exceptions import ErrorWhileTesting -from pants.util.timeout import Timeout, TimeoutReached +from pants.util.process_handler import subprocess class TestRunnerTaskMixin(object): @@ -186,37 +186,29 @@ def _spawn_and_wait(self, *args, **kwargs): process_handler = self._spawn(*args, **kwargs) - def _graceful_terminate(handler, wait_time): - """ - Returns a function which attempts to terminate the process gracefully. + def maybe_terminate(wait_time): + if process_handler.poll() < 0: + process_handler.terminate() - If terminate doesn't work after wait_time seconds, do a kill. - """ - - def terminator(): - handler.terminate() def kill_if_not_terminated(): - if handler.poll() is None: - # We can't use the context logger because it might not exist. + if process_handler.poll() < 0: + # We can't use the context logger because it might not exist when this delayed function + # is executed by the Timer below. import logging logger = logging.getLogger(__name__) logger.warn('Timed out test did not terminate gracefully after {} seconds, killing...' .format(wait_time)) - handler.kill() + process_handler.kill() timer = Timer(wait_time, kill_if_not_terminated) timer.start() - return terminator - try: - with Timeout(timeout, - threading_timer=Timer, - abort_handler=_graceful_terminate(process_handler, - self.get_options().timeout_terminate_wait)): - return process_handler.wait() - except TimeoutReached as e: + return process_handler.wait(timeout=timeout) + except subprocess.TimeoutExpired as e: raise ErrorWhileTesting(str(e), failed_targets=test_targets) + finally: + maybe_terminate(wait_time=self.get_options().timeout_terminate_wait) @abstractmethod def _spawn(self, *args, **kwargs): @@ -225,8 +217,6 @@ def _spawn(self, *args, **kwargs): :rtype: ProcessHandler """ - raise NotImplementedError - def _timeout_for_target(self, target): timeout = getattr(target, 'timeout', None) timeout_maximum = self.get_options().timeout_maximum diff --git a/src/python/pants/util/BUILD b/src/python/pants/util/BUILD index 23d09607584..29f30a16302 100644 --- a/src/python/pants/util/BUILD +++ b/src/python/pants/util/BUILD @@ -22,6 +22,7 @@ python_library( sources = ['desktop.py'], dependencies = [ ':osutil', + 'src/python/pants/util:process_handler', ], ) @@ -82,6 +83,10 @@ python_library( python_library( name = 'process_handler', sources = ['process_handler.py'], + dependencies=[ + '3rdparty/python:subprocess32', + ':meta', + ] ) python_library( @@ -118,11 +123,6 @@ python_library( ], ) -python_library( - name = 'timeout', - sources = ['timeout.py'], -) - python_library( name = 'xml_parser', sources = ['xml_parser.py'], diff --git a/src/python/pants/util/desktop.py b/src/python/pants/util/desktop.py index 35fe6d8300d..e033b555146 100644 --- a/src/python/pants/util/desktop.py +++ b/src/python/pants/util/desktop.py @@ -5,9 +5,8 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) -import subprocess - from pants.util.osutil import get_os_name +from pants.util.process_handler import subprocess class OpenError(Exception): diff --git a/src/python/pants/util/process_handler.py b/src/python/pants/util/process_handler.py index 4d460c9adb3..694abad3e37 100644 --- a/src/python/pants/util/process_handler.py +++ b/src/python/pants/util/process_handler.py @@ -5,40 +5,63 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +import os from abc import abstractmethod +import six -class ProcessHandler(object): - """A simple class to abstract process handling calls using the same interface as subprocess.Popen. +from pants.util.meta import AbstractClass + + +# Expose subprocess with python 3.2+ capabilities (namely timeouts) and bugfixes from this module. +# NB: As recommended here: https://github.com/google/python-subprocess32/blob/master/README.md +# which accounts for non-posix, ie: Windows. Although we don't support Windows yet, this sets the +# pattern up in anticipation. +if os.name == 'posix' and six.PY2: + import subprocess32 as subprocess3 +else: + import subprocess as subprocess3 +subprocess = subprocess3 + + +class ProcessHandler(AbstractClass): + """An abstraction of process handling calls using the same interface as subprocess(32).Popen. See SubprocessProcessHandler below for an example. """ @abstractmethod - def wait(self): - raise NotImplementedError + def wait(self, timeout=None): + """Wait for the underlying process to terminate. + + :param float timeout: The time to wait for the process to terminate in fractional seconds. Wait + forever by default. + :returns: The process exit code is it has terminated. + :rtype: int + :raises: :class:`subprocess.TimeoutExpired` + """ @abstractmethod def kill(self): - raise NotImplementedError + pass @abstractmethod def terminate(self): - raise NotImplementedError + pass @abstractmethod def poll(self): - raise NotImplementedError + pass class SubprocessProcessHandler(ProcessHandler): - """The simple passthrough class for a subprocess.Popen object.""" + """A `ProcessHandler` that delegates directly to a subprocess(32).Popen object.""" def __init__(self, process): self._process = process - def wait(self): - return self._process.wait() + def wait(self, timeout=None): + return self._process.wait(timeout=timeout) def kill(self): return self._process.kill() diff --git a/src/python/pants/util/timeout.py b/src/python/pants/util/timeout.py deleted file mode 100644 index 7dad069bbbe..00000000000 --- a/src/python/pants/util/timeout.py +++ /dev/null @@ -1,54 +0,0 @@ -# coding=utf-8 -# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import (absolute_import, division, generators, nested_scopes, print_function, - unicode_literals, with_statement) - -import threading - - -class TimeoutReached(Exception): - def __init__(self, seconds): - super(TimeoutReached, self).__init__("Timeout of {} seconds reached".format(seconds)) - - -class Timeout(object): - """Timeout generator. If seconds is None or 0, then the there is no timeout. - - try: - with Timeout(seconds): - - except TimeoutReached: - - """ - - def __init__(self, seconds, abort_handler=lambda: None, threading_timer=threading.Timer): - - # self._triggered is not protected by a mutex because boolean set/get is atomic in all the Python - # implementations we care about. - self._triggered = False - self._seconds = seconds - self._abort_handler = abort_handler - - if self._seconds: - self._timer = threading_timer(self._seconds, self._handle_timeout) - else: - self._timer = None - - def _handle_timeout(self): - self._triggered = True - self._abort_handler() - - def __enter__(self): - if self._timer is not None: - self._timer.start() - - def __exit__(self, type_, value, traceback): - """If triggered, raise TimeoutReached.""" - - if self._triggered: - raise TimeoutReached(self._seconds) - - elif self._timer is not None: - self._timer.cancel() diff --git a/tests/python/pants_test/BUILD b/tests/python/pants_test/BUILD index 45a51cdfc19..08b912ba4a4 100644 --- a/tests/python/pants_test/BUILD +++ b/tests/python/pants_test/BUILD @@ -3,7 +3,6 @@ python_library( name='test_infra', - sources=[], dependencies=[ 'tests/python/pants_test:base_test', 'tests/python/pants_test:int-test', @@ -55,6 +54,7 @@ python_library( 'src/python/pants/subsystem', 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', 'tests/python/pants_test/testutils:file_test_util', ] ) diff --git a/tests/python/pants_test/backend/codegen/protobuf/java/BUILD b/tests/python/pants_test/backend/codegen/protobuf/java/BUILD index ea70143c22c..47c04e30f35 100644 --- a/tests/python/pants_test/backend/codegen/protobuf/java/BUILD +++ b/tests/python/pants_test/backend/codegen/protobuf/java/BUILD @@ -21,6 +21,7 @@ python_tests( name = 'integration', sources = globs('*_integration.py'), dependencies = [ + 'src/python/pants/util:process_handler', 'tests/python/pants_test:int-test', ], tags = {'integration'}, diff --git a/tests/python/pants_test/backend/codegen/protobuf/java/test_protobuf_integration.py b/tests/python/pants_test/backend/codegen/protobuf/java/test_protobuf_integration.py index 015f282a35c..c7e8ca95c15 100644 --- a/tests/python/pants_test/backend/codegen/protobuf/java/test_protobuf_integration.py +++ b/tests/python/pants_test/backend/codegen/protobuf/java/test_protobuf_integration.py @@ -7,9 +7,9 @@ import os import re -import subprocess from pants.base.build_environment import get_buildroot +from pants.util.process_handler import subprocess from pants_test.pants_run_integration_test import PantsRunIntegrationTest diff --git a/tests/python/pants_test/backend/codegen/wire/java/BUILD b/tests/python/pants_test/backend/codegen/wire/java/BUILD index 073c6e23924..7c1575afd8f 100644 --- a/tests/python/pants_test/backend/codegen/wire/java/BUILD +++ b/tests/python/pants_test/backend/codegen/wire/java/BUILD @@ -25,6 +25,7 @@ python_tests( sources = globs('*_integration.py'), dependencies = [ 'src/python/pants/base:build_environment', + 'src/python/pants/util:process_handler', 'tests/python/pants_test:int-test', ], tags = {'integration'}, diff --git a/tests/python/pants_test/backend/codegen/wire/java/test_wire_integration.py b/tests/python/pants_test/backend/codegen/wire/java/test_wire_integration.py index 2ad6559dc06..36d0d6dc916 100644 --- a/tests/python/pants_test/backend/codegen/wire/java/test_wire_integration.py +++ b/tests/python/pants_test/backend/codegen/wire/java/test_wire_integration.py @@ -7,10 +7,10 @@ import os import re -import subprocess from pants.base.build_environment import get_buildroot from pants.util.contextutil import open_zip +from pants.util.process_handler import subprocess from pants_test.pants_run_integration_test import PantsRunIntegrationTest from pants_test.testutils.file_test_util import exact_files diff --git a/tests/python/pants_test/backend/jvm/tasks/BUILD b/tests/python/pants_test/backend/jvm/tasks/BUILD index c35244bc87b..6bfcd8d9f2f 100644 --- a/tests/python/pants_test/backend/jvm/tasks/BUILD +++ b/tests/python/pants_test/backend/jvm/tasks/BUILD @@ -15,10 +15,11 @@ python_tests( sources = ['test_binary_create.py'], dependencies = [ ':jvm_binary_task_test_base', - 'src/python/pants/java/jar', 'src/python/pants/backend/jvm/targets:jvm', 'src/python/pants/backend/jvm/tasks:binary_create', + 'src/python/pants/java/jar', 'src/python/pants/util:contextutil', + 'src/python/pants/util:process_handler', 'tests/python/pants_test/jvm:jvm_tool_task_test_base', ] ) @@ -27,15 +28,16 @@ python_tests( name = 'bootstrap_jvm_tools', sources = ['test_bootstrap_jvm_tools.py'], dependencies = [ - 'src/python/pants/java/jar', 'src/python/pants/backend/jvm/subsystems:shader', 'src/python/pants/backend/jvm/targets:jvm', 'src/python/pants/backend/jvm/tasks:bootstrap_jvm_tools', 'src/python/pants/backend/jvm/tasks:jvm_tool_task_mixin', 'src/python/pants/java/distribution', + 'src/python/pants/java/jar', 'src/python/pants/java:executor', 'src/python/pants/task', 'src/python/pants/util:contextutil', + 'src/python/pants/util:process_handler', 'tests/python/pants_test/jvm:jvm_tool_task_test_base', ] ) @@ -393,10 +395,10 @@ python_tests( 'src/python/pants/java:executor', 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', - 'src/python/pants/util:timeout', + 'src/python/pants/util:process_handler', 'tests/python/pants_test/jvm:jvm_tool_task_test_base', - 'tests/python/pants_test/tasks:task_test_base', 'tests/python/pants_test/subsystem:subsystem_utils', + 'tests/python/pants_test/tasks:task_test_base', ] ) @@ -405,6 +407,7 @@ python_tests( sources = ['test_junit_run_integration.py'], dependencies = [ ':missing_jvm_check', + 'src/python/pants/util:process_handler', 'tests/python/pants_test:int-test', ], tags = {'integration'}, diff --git a/tests/python/pants_test/backend/jvm/tasks/test_binary_create_integration.py b/tests/python/pants_test/backend/jvm/tasks/test_binary_create_integration.py index 5e47eea3fff..472c972156e 100644 --- a/tests/python/pants_test/backend/jvm/tasks/test_binary_create_integration.py +++ b/tests/python/pants_test/backend/jvm/tasks/test_binary_create_integration.py @@ -6,11 +6,11 @@ unicode_literals, with_statement) import os -import subprocess from pants.base.build_environment import get_buildroot from pants.util.contextutil import open_zip from pants.util.dirutil import safe_delete +from pants.util.process_handler import subprocess from pants_test.pants_run_integration_test import PantsRunIntegrationTest diff --git a/tests/python/pants_test/backend/jvm/tasks/test_bootstrap_jvm_tools.py b/tests/python/pants_test/backend/jvm/tasks/test_bootstrap_jvm_tools.py index c89a49774eb..72a6749a925 100644 --- a/tests/python/pants_test/backend/jvm/tasks/test_bootstrap_jvm_tools.py +++ b/tests/python/pants_test/backend/jvm/tasks/test_bootstrap_jvm_tools.py @@ -6,7 +6,6 @@ unicode_literals, with_statement) import os -import subprocess from contextlib import contextmanager from pants.backend.jvm.subsystems.shader import Shading @@ -18,6 +17,7 @@ from pants.java.jar.jar_dependency import JarDependency from pants.task.task import Task from pants.util.contextutil import open_zip +from pants.util.process_handler import subprocess from pants_test.jvm.jvm_tool_task_test_base import JvmToolTaskTestBase from pants_test.subsystem.subsystem_util import init_subsystem diff --git a/tests/python/pants_test/backend/jvm/tasks/test_junit_run.py b/tests/python/pants_test/backend/jvm/tasks/test_junit_run.py index 1dd752d680a..e0cebf2e0e3 100644 --- a/tests/python/pants_test/backend/jvm/tasks/test_junit_run.py +++ b/tests/python/pants_test/backend/jvm/tasks/test_junit_run.py @@ -6,11 +6,8 @@ unicode_literals, with_statement) import os -import subprocess from textwrap import dedent -from mock import patch - from pants.backend.jvm.subsystems.junit import JUnit from pants.backend.jvm.targets.junit_tests import JUnitTests from pants.backend.jvm.tasks.junit_run import JUnitRun @@ -25,7 +22,7 @@ from pants.java.executor import SubprocessExecutor from pants.util.contextutil import environment_as, temporary_dir from pants.util.dirutil import safe_file_dump, touch -from pants.util.timeout import TimeoutReached +from pants.util.process_handler import subprocess from pants_test.jvm.jvm_tool_task_test_base import JvmToolTaskTestBase from pants_test.subsystem.subsystem_util import global_subsystem_instance, init_subsystem from pants_test.tasks.task_test_base import ensure_cached @@ -101,59 +98,6 @@ def test_junit_runner_error(self): self.assertEqual([t.name for t in cm.exception.failed_targets], ['foo_test']) - @ensure_cached(JUnitRun, expected_num_artifacts=1) - def test_junit_runner_timeout_success(self): - """When we set a timeout and don't force failure, succeed.""" - - with patch('pants.task.testrunner_task_mixin.Timeout') as mock_timeout: - self.set_options(timeout_default=1) - self.set_options(timeouts=True) - self._execute_junit_runner( - [('FooTest.java', dedent(""" - import org.junit.Test; - import static org.junit.Assert.assertTrue; - public class FooTest { - @Test - public void testFoo() { - assertTrue(5 > 3); - } - } - """))] - ) - - # Ensures that Timeout is instantiated with a 1 second timeout. - args, kwargs = mock_timeout.call_args - self.assertEqual(args, (1,)) - - @ensure_cached(JUnitRun, expected_num_artifacts=0) - def test_junit_runner_timeout_fail(self): - """When we set a timeout and force a failure, fail.""" - - with patch('pants.task.testrunner_task_mixin.Timeout') as mock_timeout: - mock_timeout().__exit__.side_effect = TimeoutReached(1) - - self.set_options(timeout_default=1) - self.set_options(timeouts=True) - with self.assertRaises(TaskError) as cm: - self._execute_junit_runner( - [('FooTest.java', dedent(""" - import org.junit.Test; - import static org.junit.Assert.assertTrue; - public class FooTest { - @Test - public void testFoo() { - assertTrue(5 > 3); - } - } - """))] - ) - - self.assertEqual([t.name for t in cm.exception.failed_targets], ['foo_test']) - - # Ensures that Timeout is instantiated with a 1 second timeout. - args, kwargs = mock_timeout.call_args - self.assertEqual(args, (1,)) - def _execute_junit_runner(self, list_of_filename_content_tuples, create_some_resources=True, target_name=None): # Create the temporary base test directory diff --git a/tests/python/pants_test/backend/jvm/tasks/test_junit_run_integration.py b/tests/python/pants_test/backend/jvm/tasks/test_junit_run_integration.py index d50a05cdf92..d9c068abbda 100644 --- a/tests/python/pants_test/backend/jvm/tasks/test_junit_run_integration.py +++ b/tests/python/pants_test/backend/jvm/tasks/test_junit_run_integration.py @@ -68,7 +68,9 @@ def test_junit_run_timeout_succeeds(self): sleeping_target = 'testprojects/tests/java/org/pantsbuild/testproject/timeout:sleeping_target' pants_run = self.run_pants(['clean-all', 'test.junit', + '--timeouts', '--timeout-default=1', + '--timeout-terminate-wait=1', '--test=org.pantsbuild.testproject.timeout.ShortSleeperTest', sleeping_target]) self.assert_success(pants_run) @@ -78,7 +80,9 @@ def test_junit_run_timeout_fails(self): start = time.time() pants_run = self.run_pants(['clean-all', 'test.junit', + '--timeouts', '--timeout-default=1', + '--timeout-terminate-wait=1', '--test=org.pantsbuild.testproject.timeout.LongSleeperTest', sleeping_target]) end = time.time() @@ -88,7 +92,7 @@ def test_junit_run_timeout_fails(self): self.assertLess(end - start, 120) # Ensure that the timeout triggered. - self.assertIn("FAILURE: Timeout of 1 seconds reached", pants_run.stdout_data) + self.assertIn(" timed out after 1 seconds", pants_run.stdout_data) def test_junit_tests_using_cucumber(self): test_spec = 'testprojects/tests/java/org/pantsbuild/testproject/cucumber' diff --git a/tests/python/pants_test/backend/project_info/tasks/BUILD b/tests/python/pants_test/backend/project_info/tasks/BUILD index 37d332b567a..0eb1f3a4d49 100644 --- a/tests/python/pants_test/backend/project_info/tasks/BUILD +++ b/tests/python/pants_test/backend/project_info/tasks/BUILD @@ -88,13 +88,14 @@ python_tests( name = 'export_integration', sources = ['test_export_integration.py'], dependencies = [ - ':resolve_jars_test_mixin', '3rdparty/python/twitter/commons:twitter.common.collections', + ':resolve_jars_test_mixin', 'src/python/pants/base:build_environment', 'src/python/pants/build_graph', 'src/python/pants/ivy', 'src/python/pants/java/distribution', 'src/python/pants/util:contextutil', + 'src/python/pants/util:process_handler', 'tests/python/pants_test/subsystem:subsystem_utils', 'tests/python/pants_test:int-test', ], diff --git a/tests/python/pants_test/backend/project_info/tasks/test_export_integration.py b/tests/python/pants_test/backend/project_info/tasks/test_export_integration.py index 0afaf2e0503..9d1927ca76e 100644 --- a/tests/python/pants_test/backend/project_info/tasks/test_export_integration.py +++ b/tests/python/pants_test/backend/project_info/tasks/test_export_integration.py @@ -8,13 +8,13 @@ import json import os import re -import subprocess from twitter.common.collections import maybe_list from pants.base.build_environment import get_buildroot from pants.build_graph.intermediate_target_factory import hash_target from pants.ivy.ivy_subsystem import IvySubsystem +from pants.util.process_handler import subprocess from pants_test.backend.project_info.tasks.resolve_jars_test_mixin import ResolveJarsTestMixin from pants_test.pants_run_integration_test import PantsRunIntegrationTest, ensure_engine from pants_test.subsystem.subsystem_util import global_subsystem_instance diff --git a/tests/python/pants_test/backend/python/BUILD b/tests/python/pants_test/backend/python/BUILD index b5a9dbb6894..0adc39f4a65 100644 --- a/tests/python/pants_test/backend/python/BUILD +++ b/tests/python/pants_test/backend/python/BUILD @@ -20,17 +20,18 @@ python_tests( '3rdparty/python:pex', 'src/python/pants/backend/codegen/antlr/python', 'src/python/pants/backend/codegen/thrift/python', + 'src/python/pants/backend/python/subsystems', 'src/python/pants/backend/python/targets:python', 'src/python/pants/backend/python:interpreter_cache', 'src/python/pants/backend/python:python_chroot', 'src/python/pants/backend/python:python_requirement', - 'src/python/pants/backend/python/subsystems', 'src/python/pants/binaries:binary_util', 'src/python/pants/binaries:thrift_util', 'src/python/pants/ivy', 'src/python/pants/java/distribution', 'src/python/pants/python', 'src/python/pants/util:contextutil', + 'src/python/pants/util:process_handler', 'tests/python/pants_test/subsystem:subsystem_utils', 'tests/python/pants_test:base_test', ] diff --git a/tests/python/pants_test/backend/python/tasks/BUILD b/tests/python/pants_test/backend/python/tasks/BUILD index 9b6d8695f6d..4b2b12bd7fc 100644 --- a/tests/python/pants_test/backend/python/tasks/BUILD +++ b/tests/python/pants_test/backend/python/tasks/BUILD @@ -46,7 +46,6 @@ python_tests( ':python_task_test_base', 'src/python/pants/backend/python/tasks:python', 'src/python/pants/util:contextutil', - 'src/python/pants/util:timeout', ] ) diff --git a/tests/python/pants_test/backend/python/tasks/test_pytest_run.py b/tests/python/pants_test/backend/python/tasks/test_pytest_run.py index a307b416d43..d3623538da8 100644 --- a/tests/python/pants_test/backend/python/tasks/test_pytest_run.py +++ b/tests/python/pants_test/backend/python/tasks/test_pytest_run.py @@ -10,12 +10,9 @@ import xml.dom.minidom as DOM from textwrap import dedent -from mock import patch - from pants.backend.python.tasks.pytest_run import PytestRun from pants.base.exceptions import ErrorWhileTesting from pants.util.contextutil import pushd -from pants.util.timeout import TimeoutReached from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase @@ -289,29 +286,6 @@ def test_red_test_in_class(self): def test_mixed(self): self.run_failing_tests(targets=[self.green, self.red], failed_targets=[self.red]) - def test_one_timeout(self): - # When we have two targets, any of them doesn't have a timeout, and we have no default, - # then no timeout is set. - - with patch('pants.task.testrunner_task_mixin.Timeout') as mock_timeout: - self.run_tests(targets=[self.sleep_no_timeout, self.sleep_timeout]) - - # Ensures that Timeout is instantiated with no timeout. - args, kwargs = mock_timeout.call_args - self.assertEqual(args, (None,)) - - def test_timeout(self): - # Check that a failed timeout returns the right results. - - with patch('pants.task.testrunner_task_mixin.Timeout') as mock_timeout: - mock_timeout().__exit__.side_effect = TimeoutReached(1) - self.run_failing_tests(targets=[self.sleep_timeout], - failed_targets=[self.sleep_timeout]) - - # Ensures that Timeout is instantiated with a 1 second timeout. - args, kwargs = mock_timeout.call_args - self.assertEqual(args, (1,)) - def test_junit_xml_option(self): # We expect xml of the following form: # diff --git a/tests/python/pants_test/backend/python/tasks/test_python_task.py b/tests/python/pants_test/backend/python/tasks/test_python_task.py index 8a242ea4799..7ced6985178 100644 --- a/tests/python/pants_test/backend/python/tasks/test_python_task.py +++ b/tests/python/pants_test/backend/python/tasks/test_python_task.py @@ -5,12 +5,12 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) -import subprocess from contextlib import contextmanager from textwrap import dedent from pants.backend.python.tasks.python_task import PythonTask from pants.util.contextutil import temporary_file_path +from pants.util.process_handler import subprocess from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase diff --git a/tests/python/pants_test/backend/python/tasks2/BUILD b/tests/python/pants_test/backend/python/tasks2/BUILD index 426776bc1b1..6cabccf2683 100644 --- a/tests/python/pants_test/backend/python/tasks2/BUILD +++ b/tests/python/pants_test/backend/python/tasks2/BUILD @@ -10,12 +10,12 @@ python_tests( '3rdparty/python:coverage', '3rdparty/python:mock', '3rdparty/python:pex', - 'src/python/pants/backend/python:interpreter_cache', - 'src/python/pants/backend/python:python_artifact', - 'src/python/pants/backend/python:python_requirement', 'src/python/pants/backend/python/subsystems', 'src/python/pants/backend/python/targets', 'src/python/pants/backend/python/tasks2', + 'src/python/pants/backend/python:interpreter_cache', + 'src/python/pants/backend/python:python_artifact', + 'src/python/pants/backend/python:python_requirement', 'src/python/pants/base:build_root', 'src/python/pants/base:exceptions', 'src/python/pants/base:run_info', @@ -24,9 +24,9 @@ python_tests( 'src/python/pants/python', 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', - 'tests/python/pants_test/tasks:task_test_base', 'tests/python/pants_test/backend/python/tasks:python_task_test_base', - 'tests/python/pants_test/subsystem:subsystem_utils' + 'tests/python/pants_test/subsystem:subsystem_utils', + 'tests/python/pants_test/tasks:task_test_base', ] ) @@ -36,6 +36,7 @@ python_tests( sources=globs('*_integration.py'), dependencies=[ 'src/python/pants/util:contextutil', + 'src/python/pants/util:process_handler', 'tests/python/pants_test:int-test', ], tags = {'integration'}, diff --git a/tests/python/pants_test/backend/python/tasks2/test_pytest_run.py b/tests/python/pants_test/backend/python/tasks2/test_pytest_run.py index 45b17bb3c0b..2d5bb2126fa 100644 --- a/tests/python/pants_test/backend/python/tasks2/test_pytest_run.py +++ b/tests/python/pants_test/backend/python/tasks2/test_pytest_run.py @@ -9,7 +9,6 @@ from textwrap import dedent import coverage -from mock import patch from pants.backend.python.tasks2.gather_sources import GatherSources from pants.backend.python.tasks2.pytest_prep import PytestPrep @@ -19,7 +18,6 @@ from pants.base.exceptions import ErrorWhileTesting, TaskError from pants.util.contextutil import pushd from pants.util.dirutil import safe_mkdtemp, safe_rmtree -from pants.util.timeout import TimeoutReached from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase from pants_test.tasks.task_test_base import ensure_cached @@ -480,31 +478,6 @@ def test_red_test_in_class(self): def test_mixed(self): self.run_failing_tests(targets=[self.green, self.red], failed_targets=[self.red]) - @ensure_cached(PytestRun, expected_num_artifacts=1) - def test_none_timeout(self): - # When we have two targets, any of them doesn't have a timeout, and we have no default, - # then no timeout is set. - - with patch('pants.task.testrunner_task_mixin.Timeout') as mock_timeout: - self.run_tests(targets=[self.sleep_no_timeout, self.sleep_timeout]) - - # Ensures that Timeout is instantiated with no timeout. - args, kwargs = mock_timeout.call_args - self.assertEqual(args, (None,)) - - @ensure_cached(PytestRun, expected_num_artifacts=0) - def test_timeout(self): - # Check that a failed timeout returns the right results. - - with patch('pants.task.testrunner_task_mixin.Timeout') as mock_timeout: - mock_timeout().__exit__.side_effect = TimeoutReached(1) - self.run_failing_tests(targets=[self.sleep_timeout], - failed_targets=[self.sleep_timeout]) - - # Ensures that Timeout is instantiated with a 1 second timeout. - args, kwargs = mock_timeout.call_args - self.assertEqual(args, (1,)) - def coverage_data_file(self): return os.path.join(self.build_root, '.coverage') diff --git a/tests/python/pants_test/backend/python/tasks2/test_pytest_run_integration.py b/tests/python/pants_test/backend/python/tasks2/test_pytest_run_integration.py index b1a30a50d36..6bf8da2f97b 100644 --- a/tests/python/pants_test/backend/python/tasks2/test_pytest_run_integration.py +++ b/tests/python/pants_test/backend/python/tasks2/test_pytest_run_integration.py @@ -47,7 +47,7 @@ def test_pytest_run_timeout_fails(self): self.assertIn("No .coverage file was found! Skipping coverage reporting", pants_run.stderr_data) # Ensure that the timeout message triggered. - self.assertIn("FAILURE: Timeout of 1 seconds reached", pants_run.stdout_data) + self.assertIn(" timed out after 1 seconds", pants_run.stdout_data) @unittest.skip('https://github.com/pantsbuild/pants/issues/3255') def test_pytest_run_timeout_cant_terminate(self): diff --git a/tests/python/pants_test/backend/python/tasks2/test_resolve_requirements.py b/tests/python/pants_test/backend/python/tasks2/test_resolve_requirements.py index be62591a4dd..421bb0ba9ee 100644 --- a/tests/python/pants_test/backend/python/tasks2/test_resolve_requirements.py +++ b/tests/python/pants_test/backend/python/tasks2/test_resolve_requirements.py @@ -6,7 +6,6 @@ unicode_literals, with_statement) import os -import subprocess from pex.interpreter import PythonInterpreter @@ -18,6 +17,7 @@ from pants.base.build_environment import get_buildroot from pants.python.python_repos import PythonRepos from pants.util.contextutil import temporary_file +from pants.util.process_handler import subprocess from pants_test.tasks.task_test_base import TaskTestBase diff --git a/tests/python/pants_test/backend/python/test_python_chroot.py b/tests/python/pants_test/backend/python/test_python_chroot.py index c03adb79edc..efa90719799 100644 --- a/tests/python/pants_test/backend/python/test_python_chroot.py +++ b/tests/python/pants_test/backend/python/test_python_chroot.py @@ -6,7 +6,6 @@ unicode_literals, with_statement) import os -import subprocess from contextlib import contextmanager from textwrap import dedent @@ -29,6 +28,7 @@ from pants.java.distribution.distribution import DistributionLocator from pants.python.python_repos import PythonRepos from pants.util.contextutil import temporary_dir +from pants.util.process_handler import subprocess from pants_test.base_test import BaseTest from pants_test.subsystem.subsystem_util import global_subsystem_instance diff --git a/tests/python/pants_test/base/BUILD b/tests/python/pants_test/base/BUILD index cf0f0a0106b..45df8d255a0 100644 --- a/tests/python/pants_test/base/BUILD +++ b/tests/python/pants_test/base/BUILD @@ -36,6 +36,7 @@ python_tests( name = 'exclude_target_regexp_integration', sources = [ 'test_exclude_target_regexp_integration.py' ], dependencies = [ + 'src/python/pants/util:process_handler', 'tests/python/pants_test:int-test', ], tags = {'integration'}, @@ -129,6 +130,7 @@ python_tests( ':pants_ignore_test_base', 'src/python/pants/base:project_tree', 'src/python/pants/scm:git', + 'src/python/pants/util:process_handler', ] ) @@ -194,11 +196,12 @@ python_tests( name = 'scm_build_file', sources = ['test_scm_build_file.py'], dependencies = [ - ':build_file_test_base', '3rdparty/python/twitter/commons:twitter.common.collections', + ':build_file_test_base', 'src/python/pants/base:project_tree', 'src/python/pants/scm:git', 'src/python/pants/util:contextutil', + 'src/python/pants/util:process_handler', ] ) diff --git a/tests/python/pants_test/base/test_exclude_target_regexp_integration.py b/tests/python/pants_test/base/test_exclude_target_regexp_integration.py index 675a70a7e33..fcf349deb8f 100644 --- a/tests/python/pants_test/base/test_exclude_target_regexp_integration.py +++ b/tests/python/pants_test/base/test_exclude_target_regexp_integration.py @@ -6,10 +6,10 @@ unicode_literals, with_statement) import os -import subprocess from contextlib import contextmanager from pants.base.build_environment import get_buildroot +from pants.util.process_handler import subprocess from pants_test.pants_run_integration_test import PantsRunIntegrationTest, ensure_engine diff --git a/tests/python/pants_test/base/test_pants_ignore_scm.py b/tests/python/pants_test/base/test_pants_ignore_scm.py index 5c653a00e88..9245e8b964d 100644 --- a/tests/python/pants_test/base/test_pants_ignore_scm.py +++ b/tests/python/pants_test/base/test_pants_ignore_scm.py @@ -5,11 +5,11 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) -import subprocess import unittest from pants.base.scm_project_tree import ScmProjectTree from pants.scm.git import Git +from pants.util.process_handler import subprocess from pants_test.base.pants_ignore_test_base import PantsIgnoreTestBase diff --git a/tests/python/pants_test/base/test_scm_build_file.py b/tests/python/pants_test/base/test_scm_build_file.py index 9a59f7be0b1..745eea68c8e 100644 --- a/tests/python/pants_test/base/test_scm_build_file.py +++ b/tests/python/pants_test/base/test_scm_build_file.py @@ -5,13 +5,12 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) -import subprocess - from twitter.common.collections import OrderedSet from pants.base.scm_project_tree import ScmProjectTree from pants.scm.git import Git from pants.util.contextutil import pushd +from pants.util.process_handler import subprocess from pants_test.base.build_file_test_base import BuildFileTestBase diff --git a/tests/python/pants_test/engine/examples/BUILD b/tests/python/pants_test/engine/examples/BUILD index 9eba57960d2..e842344b293 100644 --- a/tests/python/pants_test/engine/examples/BUILD +++ b/tests/python/pants_test/engine/examples/BUILD @@ -60,6 +60,7 @@ python_library( 'src/python/pants/engine:scheduler', 'src/python/pants/util:contextutil', 'src/python/pants/util:desktop', + 'src/python/pants/util:process_handler', 'tests/python/pants_test/engine:util', ] ) diff --git a/tests/python/pants_test/engine/examples/visualizer.py b/tests/python/pants_test/engine/examples/visualizer.py index c3532bcf3c1..1ea50102b32 100644 --- a/tests/python/pants_test/engine/examples/visualizer.py +++ b/tests/python/pants_test/engine/examples/visualizer.py @@ -6,7 +6,6 @@ unicode_literals, with_statement) import os -import subprocess import sys from textwrap import dedent @@ -14,6 +13,7 @@ from pants.engine.fs import PathGlobs from pants.util import desktop from pants.util.contextutil import temporary_file_path +from pants.util.process_handler import subprocess from pants_test.engine.examples.planners import setup_json_scheduler from pants_test.engine.util import init_native diff --git a/tests/python/pants_test/engine/legacy/BUILD b/tests/python/pants_test/engine/legacy/BUILD index b8996008472..4e72b941f79 100644 --- a/tests/python/pants_test/engine/legacy/BUILD +++ b/tests/python/pants_test/engine/legacy/BUILD @@ -43,9 +43,10 @@ python_tests( 'src/python/pants/base:build_environment', 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', + 'tests/python/pants_test/testutils:git_util', 'tests/python/pants_test:base_test', 'tests/python/pants_test:int-test', - 'tests/python/pants_test/testutils:git_util', ], tags = {'integration'}, timeout = 600, diff --git a/tests/python/pants_test/engine/legacy/test_changed_integration.py b/tests/python/pants_test/engine/legacy/test_changed_integration.py index 3e15d851f5b..2e617913e59 100644 --- a/tests/python/pants_test/engine/legacy/test_changed_integration.py +++ b/tests/python/pants_test/engine/legacy/test_changed_integration.py @@ -7,7 +7,6 @@ import os import shutil -import subprocess import unittest from contextlib import contextmanager from textwrap import dedent @@ -15,6 +14,7 @@ from pants.base.build_environment import get_buildroot from pants.util.contextutil import environment_as, temporary_dir from pants.util.dirutil import safe_delete, safe_mkdir, safe_open, touch +from pants.util.process_handler import subprocess from pants_test.base_test import TestGenerator from pants_test.pants_run_integration_test import PantsRunIntegrationTest, ensure_engine from pants_test.testutils.git_util import initialize_repo diff --git a/tests/python/pants_test/java/BUILD b/tests/python/pants_test/java/BUILD index ea433860cda..c4a717ad033 100644 --- a/tests/python/pants_test/java/BUILD +++ b/tests/python/pants_test/java/BUILD @@ -9,6 +9,7 @@ python_tests( 'src/python/pants/java:executor', 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', ] ) diff --git a/tests/python/pants_test/java/distribution/BUILD b/tests/python/pants_test/java/distribution/BUILD index c63c76aaef5..ced419bcf9b 100644 --- a/tests/python/pants_test/java/distribution/BUILD +++ b/tests/python/pants_test/java/distribution/BUILD @@ -4,12 +4,13 @@ python_tests( sources = ['test_distribution.py'], dependencies = [ + '3rdparty/python/twitter/commons:twitter.common.collections', 'src/python/pants/base:revision', 'src/python/pants/java/distribution', 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', 'tests/python/pants_test/subsystem:subsystem_utils', - '3rdparty/python/twitter/commons:twitter.common.collections', ] ) diff --git a/tests/python/pants_test/java/distribution/test_distribution.py b/tests/python/pants_test/java/distribution/test_distribution.py index c58cc59d5ff..c9c25ae520f 100644 --- a/tests/python/pants_test/java/distribution/test_distribution.py +++ b/tests/python/pants_test/java/distribution/test_distribution.py @@ -6,7 +6,6 @@ unicode_literals, with_statement) import os -import subprocess import textwrap import unittest from contextlib import contextmanager @@ -19,6 +18,7 @@ _OSXEnvironment, _UnknownEnvironment) from pants.util.contextutil import environment_as, temporary_dir, temporary_file from pants.util.dirutil import chmod_plus_x, safe_open, touch +from pants.util.process_handler import subprocess from pants_test.subsystem.subsystem_util import global_subsystem_instance diff --git a/tests/python/pants_test/java/test_executor.py b/tests/python/pants_test/java/test_executor.py index f80411f292b..b9ef82026ca 100644 --- a/tests/python/pants_test/java/test_executor.py +++ b/tests/python/pants_test/java/test_executor.py @@ -6,7 +6,6 @@ unicode_literals, with_statement) import os -import subprocess import textwrap import unittest from contextlib import contextmanager @@ -15,6 +14,7 @@ from pants.java.executor import Executor, SubprocessExecutor from pants.util.contextutil import environment_as, temporary_dir from pants.util.dirutil import chmod_plus_x, safe_open +from pants.util.process_handler import subprocess class SubprocessExecutorTest(unittest.TestCase): diff --git a/tests/python/pants_test/pants_run_integration_test.py b/tests/python/pants_test/pants_run_integration_test.py index 7b4572f4136..82292089260 100644 --- a/tests/python/pants_test/pants_run_integration_test.py +++ b/tests/python/pants_test/pants_run_integration_test.py @@ -8,7 +8,6 @@ import ConfigParser import os import shutil -import subprocess import unittest from collections import namedtuple from contextlib import contextmanager @@ -23,6 +22,7 @@ from pants.subsystem.subsystem import Subsystem from pants.util.contextutil import environment_as, pushd, temporary_dir from pants.util.dirutil import safe_mkdir, safe_mkdir_for, safe_open +from pants.util.process_handler import subprocess from pants_test.testutils.file_test_util import check_symlinks, contains_exact_files diff --git a/tests/python/pants_test/pantsd/BUILD b/tests/python/pants_test/pantsd/BUILD index b08a59d0e5c..3a8cfa65bd2 100644 --- a/tests/python/pants_test/pantsd/BUILD +++ b/tests/python/pants_test/pantsd/BUILD @@ -15,7 +15,8 @@ python_tests( coverage = ['pants.pantsd.process_manager'], dependencies = [ ':test_deps', - 'src/python/pants/pantsd:process_manager' + 'src/python/pants/pantsd:process_manager', + 'src/python/pants/util:process_handler', ] ) diff --git a/tests/python/pants_test/pantsd/test_process_manager.py b/tests/python/pants_test/pantsd/test_process_manager.py index b41a62989b5..01c8770a975 100644 --- a/tests/python/pants_test/pantsd/test_process_manager.py +++ b/tests/python/pants_test/pantsd/test_process_manager.py @@ -7,7 +7,6 @@ import errno import os -import subprocess import sys from contextlib import contextmanager @@ -18,6 +17,7 @@ swallow_psutil_exceptions) from pants.util.contextutil import temporary_dir from pants.util.dirutil import safe_file_dump +from pants.util.process_handler import subprocess from pants_test.base_test import BaseTest diff --git a/tests/python/pants_test/python/BUILD b/tests/python/pants_test/python/BUILD index ffc140f8187..0a419017687 100644 --- a/tests/python/pants_test/python/BUILD +++ b/tests/python/pants_test/python/BUILD @@ -27,6 +27,7 @@ python_tests( sources = ['test_interpreter_selection_integration.py'], dependencies = [ 'src/python/pants/util:contextutil', + 'src/python/pants/util:process_handler', 'tests/python/pants_test:int-test', ], tags = {'integration'}, diff --git a/tests/python/pants_test/python/test_interpreter_selection_integration.py b/tests/python/pants_test/python/test_interpreter_selection_integration.py index 47d40bafa9a..a885aa6bcc6 100644 --- a/tests/python/pants_test/python/test_interpreter_selection_integration.py +++ b/tests/python/pants_test/python/test_interpreter_selection_integration.py @@ -6,9 +6,9 @@ unicode_literals, with_statement) import os -import subprocess from pants.util.contextutil import temporary_dir +from pants.util.process_handler import subprocess from pants_test.pants_run_integration_test import PantsRunIntegrationTest diff --git a/tests/python/pants_test/scm/BUILD b/tests/python/pants_test/scm/BUILD index 38401d0a728..feeb2727aa9 100644 --- a/tests/python/pants_test/scm/BUILD +++ b/tests/python/pants_test/scm/BUILD @@ -9,6 +9,7 @@ python_tests( 'src/python/pants/scm:git', 'src/python/pants/util:contextutil', 'src/python/pants/util:dirutil', + 'src/python/pants/util:process_handler', 'tests/python/pants_test/testutils:git_util', ] ) diff --git a/tests/python/pants_test/scm/test_git.py b/tests/python/pants_test/scm/test_git.py index 9d789dc348b..c80057072b0 100644 --- a/tests/python/pants_test/scm/test_git.py +++ b/tests/python/pants_test/scm/test_git.py @@ -6,7 +6,6 @@ unicode_literals, with_statement) import os -import subprocess import types import unittest from contextlib import contextmanager @@ -17,6 +16,7 @@ from pants.scm.scm import Scm from pants.util.contextutil import environment_as, pushd, temporary_dir from pants.util.dirutil import chmod_plus_x, safe_mkdir, safe_mkdtemp, safe_open, safe_rmtree, touch +from pants.util.process_handler import subprocess from pants_test.testutils.git_util import MIN_REQUIRED_GIT_VERSION, git_version diff --git a/tests/python/pants_test/task/BUILD b/tests/python/pants_test/task/BUILD index 8e6d0ef39a7..f73e3b5f19c 100644 --- a/tests/python/pants_test/task/BUILD +++ b/tests/python/pants_test/task/BUILD @@ -62,9 +62,9 @@ python_tests( name='testrunner_task_mixin', sources=['test_testrunner_task_mixin.py'], dependencies=[ + '3rdparty/python:mock', 'src/python/pants/task', - 'src/python/pants/util:timeout', + 'src/python/pants/util:process_handler', 'tests/python/pants_test/tasks:task_test_base', - '3rdparty/python:mock', ] ) diff --git a/tests/python/pants_test/task/test_testrunner_task_mixin.py b/tests/python/pants_test/task/test_testrunner_task_mixin.py index befa79720f7..517b7db4fa9 100644 --- a/tests/python/pants_test/task/test_testrunner_task_mixin.py +++ b/tests/python/pants_test/task/test_testrunner_task_mixin.py @@ -7,18 +7,18 @@ import collections import os +from contextlib import contextmanager from unittest import TestCase from xml.etree.ElementTree import ParseError -from mock import patch +from mock import Mock, patch from pants.base.exceptions import ErrorWhileTesting from pants.task.task import TaskBase from pants.task.testrunner_task_mixin import TestRunnerTaskMixin from pants.util.contextutil import temporary_dir from pants.util.dirutil import safe_open -from pants.util.process_handler import ProcessHandler -from pants.util.timeout import TimeoutReached +from pants.util.process_handler import ProcessHandler, subprocess from pants_test.tasks.task_test_base import TaskTestBase @@ -47,7 +47,7 @@ def _spawn(self, *args, **kwargs): self.call_list.append(['_spawn', args, kwargs]) class FakeProcessHandler(ProcessHandler): - def wait(_): + def wait(_, timeout=None): self.call_list.append(['process_handler.wait']) return 0 @@ -144,28 +144,28 @@ class TestRunnerTaskMixinSimpleTimeoutTest(TaskTestBase): @classmethod def task_type(cls): class TestRunnerTaskMixinTask(TestRunnerTaskMixin, TaskBase): - call_list = [] + waited_for = None def _execute(self, all_targets): - self.call_list.append(['_execute', all_targets]) self._spawn_and_wait() def _spawn(self, *args, **kwargs): - self.call_list.append(['_spawn', args, kwargs]) + timeouts = self.get_options().timeouts class FakeProcessHandler(ProcessHandler): - def wait(_): - self.call_list.append(['process_handler.wait']) + def wait(_, timeout=None): + self.waited_for = timeout + if timeouts and timeout: + raise subprocess.TimeoutExpired(cmd='', timeout=timeout) return 0 def kill(_): - self.call_list.append(['process_handler.kill']) + pass def terminate(_): - self.call_list.append(['process_handler.terminate']) + pass def poll(_): - self.call_list.append(['process_handler.poll']) return 0 return FakeProcessHandler() @@ -174,13 +174,10 @@ def _get_targets(self): return [targetB] def _test_target_filter(self): - def target_filter(target): - return True - - return target_filter + return lambda target: True def _validate_target(self, target): - self.call_list.append(['_validate_target', target]) + pass return TestRunnerTaskMixinTask @@ -188,38 +185,34 @@ def test_timeout(self): self.set_options(timeouts=True) task = self.create_task(self.context()) - with patch('pants.task.testrunner_task_mixin.Timeout') as mock_timeout: - mock_timeout().__exit__.side_effect = TimeoutReached(1) - - with self.assertRaises(ErrorWhileTesting): - task.execute() + with self.assertRaises(ErrorWhileTesting): + task.execute() - # Ensures that Timeout is instantiated with a 1 second timeout. - args, kwargs = mock_timeout.call_args - self.assertEqual(args, (1,)) + # Ensures that the wait is for 1 second. + self.assertEqual(task.waited_for, 1) def test_timeout_disabled(self): self.set_options(timeouts=False) task = self.create_task(self.context()) - with patch('pants.task.testrunner_task_mixin.Timeout') as mock_timeout: - task.execute() + task.execute() - # Ensures that Timeout is instantiated with no timeout. - args, kwargs = mock_timeout.call_args - self.assertEqual(args, (None,)) + # Ensures that the wait time is forever (no timeout). + self.assertIsNone(task.waited_for) class TestRunnerTaskMixinGracefulTimeoutTest(TaskTestBase): - def create_process_handler(self, return_none_first=True): + def create_process_handler(self, poll_returns): + poll_return_values = iter(poll_returns) + class FakeProcessHandler(ProcessHandler): call_list = [] poll_called = False - def wait(self): + def wait(self, timeout=None): self.call_list.append(['process_handler.wait']) - return 0 + raise subprocess.TimeoutExpired(cmd='', timeout=timeout) def kill(self): self.call_list.append(['process_handler.kill']) @@ -228,13 +221,8 @@ def terminate(self): self.call_list.append(['process_handler.terminate']) def poll(self): - print("poll called") self.call_list.append(['process_handler.poll']) - if not self.poll_called and return_none_first: - self.poll_called = True - return None - else: - return 0 + return next(poll_return_values) return FakeProcessHandler() @@ -269,49 +257,55 @@ def _validate_target(self, target): return TestRunnerTaskMixinTask - def test_graceful_terminate_if_poll_is_none(self): - self.process_handler = self.create_process_handler(return_none_first=True) + @contextmanager + def mock_timer(self): + with patch('threading._Timer') as timer_class: + timer = Mock() - self.set_options(timeouts=True) - task = self.create_task(self.context()) + def start(): + args, _ = timer_class.call_args + wait_time, action = args + self.assertEqual(10, wait_time) + action() - with patch('pants.task.testrunner_task_mixin.Timer') as mock_timer: - def set_handler(dummy, handler): - mock_timer_instance = mock_timer.return_value - mock_timer_instance.start.side_effect = handler - return mock_timer_instance + timer.start.side_effect = start + timer_class.return_value = timer - mock_timer.side_effect = set_handler + yield + def test_graceful_terminate_if_poll_is_none(self): + self.process_handler = self.create_process_handler(poll_returns=[None, -1]) + self.set_options(timeouts=True) + task = self.create_task(self.context()) + with self.mock_timer(): with self.assertRaises(ErrorWhileTesting): task.execute() - # Ensure that all the calls we want to kill the process gracefully are made. - self.assertEqual(self.process_handler.call_list, - [[u'process_handler.terminate'], [u'process_handler.poll'], [u'process_handler.kill'], [u'process_handler.wait']]) + # Ensure that all the calls we want to kill the process gracefully are made. + self.assertEqual(self.process_handler.call_list, + [[u'process_handler.wait'], + [u'process_handler.poll'], + [u'process_handler.terminate'], + [u'process_handler.poll'], + [u'process_handler.kill']]) def test_graceful_terminate_if_poll_is_zero(self): - self.process_handler = self.create_process_handler(return_none_first=False) + self.process_handler = self.create_process_handler(poll_returns=[-1, 0]) self.set_options(timeouts=True) task = self.create_task(self.context()) - with patch('pants.task.testrunner_task_mixin.Timer') as mock_timer: - def set_handler(dummy, handler): - mock_timer_instance = mock_timer.return_value - mock_timer_instance.start.side_effect = handler - return mock_timer_instance - - mock_timer.side_effect = set_handler - - + with self.mock_timer(): with self.assertRaises(ErrorWhileTesting): task.execute() - # Ensure that we only call terminate, and not kill. - self.assertEqual(self.process_handler.call_list, - [[u'process_handler.terminate'], [u'process_handler.poll'], [u'process_handler.wait']]) + # Ensure that we only call terminate, and not kill. + self.assertEqual(self.process_handler.call_list, + [[u'process_handler.wait'], + [u'process_handler.poll'], + [u'process_handler.terminate'], + [u'process_handler.poll']]) class TestRunnerTaskMixinMultipleTargets(TaskTestBase): @@ -319,21 +313,28 @@ class TestRunnerTaskMixinMultipleTargets(TaskTestBase): @classmethod def task_type(cls): class TestRunnerTaskMixinMultipleTargetsTask(TestRunnerTaskMixin, TaskBase): + wait_time = None + def _execute(self, all_targets): self._spawn_and_wait() def _spawn(self, *args, **kwargs): + timeouts = self.get_options().timeouts + class FakeProcessHandler(ProcessHandler): - def wait(self): + def wait(_, timeout=None): + self.wait_time = timeout + if timeouts and timeout: + raise subprocess.TimeoutExpired(cmd='', timeout=timeout) return 0 - def kill(self): + def kill(_): pass - def terminate(self): + def terminate(_): pass - def poll(self): + def poll(_): pass return FakeProcessHandler() @@ -344,32 +345,37 @@ def _test_target_filter(self): def _validate_target(self, target): pass - def _get_targets(self): - return [targetA, targetB] - def _get_test_targets_for_spawn(self): return self.current_targets return TestRunnerTaskMixinMultipleTargetsTask - def test_multiple_targets_single_target_timeout(self): - with patch('pants.task.testrunner_task_mixin.Timeout') as mock_timeout: - mock_timeout().__exit__.side_effect = TimeoutReached(1) + def test_multiple_targets(self): + self.set_options(timeouts=True) + task = self.create_task(self.context()) + + # The targetA has a `None` timeout which disables all timeouts for this batch. + task.current_targets = [targetA, targetB, targetC] + task.execute() + self.assertEqual(None, task.wait_time) - self.set_options(timeouts=True) - task = self.create_task(self.context()) + task.current_targets = [targetB] + with self.assertRaises(ErrorWhileTesting) as cm: + task.execute() + self.assertEqual(1, task.wait_time) + self.assertEqual([targetB], cm.exception.failed_targets) - task.current_targets = [targetA] - with self.assertRaises(ErrorWhileTesting) as cm: - task.execute() - self.assertEqual(len(cm.exception.failed_targets), 1) - self.assertEqual(cm.exception.failed_targets[0].address.spec, 'TargetA') + task.current_targets = [targetC] + with self.assertRaises(ErrorWhileTesting) as cm: + task.execute() + self.assertEqual(10, task.wait_time) + self.assertEqual([targetC], cm.exception.failed_targets) - task.current_targets = [targetB] - with self.assertRaises(ErrorWhileTesting) as cm: - task.execute() - self.assertEqual(len(cm.exception.failed_targets), 1) - self.assertEqual(cm.exception.failed_targets[0].address.spec, 'TargetB') + task.current_targets = [targetB, targetC] + with self.assertRaises(ErrorWhileTesting) as cm: + task.execute() + self.assertEqual(11, task.wait_time) # We should wait for the sum of the target timeouts. + self.assertEqual([targetB, targetC], cm.exception.failed_targets) class TestRunnerTaskMixinXmlParsing(TestRunnerTaskMixin, TestCase): diff --git a/tests/python/pants_test/tasks/BUILD b/tests/python/pants_test/tasks/BUILD index 824215356d3..1aa949e0116 100644 --- a/tests/python/pants_test/tasks/BUILD +++ b/tests/python/pants_test/tasks/BUILD @@ -7,8 +7,9 @@ python_library( dependencies = [ 'src/python/pants/goal:context', 'src/python/pants/ivy', - 'src/python/pants/util:contextutil', 'src/python/pants/task', + 'src/python/pants/util:contextutil', + 'src/python/pants/util:process_handler', 'tests/python/pants_test:base_test', ] ) @@ -57,8 +58,9 @@ python_tests( name = 'clean_all_integration', sources = ['test_clean_all_integration.py'], dependencies = [ - 'tests/python/pants_test:int-test', 'src/python/pants/util:contextutil', + 'src/python/pants/util:process_handler', + 'tests/python/pants_test:int-test', ], tags = {'integration'}, ) diff --git a/tests/python/pants_test/tasks/task_test_base.py b/tests/python/pants_test/tasks/task_test_base.py index ca9808958bf..17e12b30c95 100644 --- a/tests/python/pants_test/tasks/task_test_base.py +++ b/tests/python/pants_test/tasks/task_test_base.py @@ -6,7 +6,6 @@ unicode_literals, with_statement) import os -import subprocess from contextlib import closing from StringIO import StringIO @@ -14,6 +13,7 @@ from pants.ivy.bootstrapper import Bootstrapper from pants.task.console_task import ConsoleTask from pants.util.contextutil import temporary_dir +from pants.util.process_handler import subprocess from pants_test.base_test import BaseTest diff --git a/tests/python/pants_test/tasks/test_clean_all_integration.py b/tests/python/pants_test/tasks/test_clean_all_integration.py index c28157ab280..74d73d2a136 100644 --- a/tests/python/pants_test/tasks/test_clean_all_integration.py +++ b/tests/python/pants_test/tasks/test_clean_all_integration.py @@ -6,9 +6,9 @@ unicode_literals, with_statement) import os -import subprocess from pants.util.contextutil import temporary_dir +from pants.util.process_handler import subprocess from pants_test.pants_run_integration_test import PantsRunIntegrationTest diff --git a/tests/python/pants_test/testutils/BUILD b/tests/python/pants_test/testutils/BUILD index af4869844a8..b9851d04ca2 100644 --- a/tests/python/pants_test/testutils/BUILD +++ b/tests/python/pants_test/testutils/BUILD @@ -25,6 +25,7 @@ python_library( 'src/python/pants/base:revision', 'src/python/pants/scm:git', 'src/python/pants/util:contextutil', + 'src/python/pants/util:process_handler', ], ) diff --git a/tests/python/pants_test/testutils/git_util.py b/tests/python/pants_test/testutils/git_util.py index 7217a1d676c..e97288ad3c9 100644 --- a/tests/python/pants_test/testutils/git_util.py +++ b/tests/python/pants_test/testutils/git_util.py @@ -6,12 +6,12 @@ unicode_literals, with_statement) import re -import subprocess from contextlib import contextmanager from pants.base.revision import Revision from pants.scm.git import Git from pants.util.contextutil import environment_as, temporary_dir +from pants.util.process_handler import subprocess MIN_REQUIRED_GIT_VERSION = Revision.semver('1.7.10') diff --git a/tests/python/pants_test/util/BUILD b/tests/python/pants_test/util/BUILD index 44087a02951..a8672058cc8 100644 --- a/tests/python/pants_test/util/BUILD +++ b/tests/python/pants_test/util/BUILD @@ -17,6 +17,7 @@ python_tests( dependencies = [ '3rdparty/python:mock', 'src/python/pants/util:contextutil', + 'src/python/pants/util:process_handler', ] ) @@ -106,7 +107,7 @@ python_tests( name = 'process_handler', sources = ['test_process_handler.py'], dependencies = [ - 'src/python/pants/util:process_handler' + 'src/python/pants/util:process_handler', ] ) @@ -146,14 +147,6 @@ python_tests( ] ) -python_tests( - name = 'timeout', - sources = ['test_timeout.py'], - dependencies = [ - 'src/python/pants/util:timeout', - ] -) - python_library( name='xml_test_base', sources = ['xml_test_base.py'], diff --git a/tests/python/pants_test/util/test_contextutil.py b/tests/python/pants_test/util/test_contextutil.py index 260e46b6a28..18c94acfd2a 100644 --- a/tests/python/pants_test/util/test_contextutil.py +++ b/tests/python/pants_test/util/test_contextutil.py @@ -8,7 +8,6 @@ import os import pstats import shutil -import subprocess import sys import unittest import zipfile @@ -18,6 +17,7 @@ from pants.util.contextutil import (HardSystemExit, InvalidZipPath, Timer, environment_as, exception_logging, hard_exit_handler, maybe_profiled, open_zip, pushd, stdio_as, temporary_dir, temporary_file) +from pants.util.process_handler import subprocess PATCH_OPTS = dict(autospec=True, spec_set=True) diff --git a/tests/python/pants_test/util/test_process_handler.py b/tests/python/pants_test/util/test_process_handler.py index 4c574b47da3..667ba8a6138 100644 --- a/tests/python/pants_test/util/test_process_handler.py +++ b/tests/python/pants_test/util/test_process_handler.py @@ -5,10 +5,9 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) -import subprocess import unittest -from pants.util.process_handler import SubprocessProcessHandler +from pants.util.process_handler import SubprocessProcessHandler, subprocess class TestSubprocessProcessHandler(unittest.TestCase): diff --git a/tests/python/pants_test/util/test_timeout.py b/tests/python/pants_test/util/test_timeout.py deleted file mode 100644 index d3e23854760..00000000000 --- a/tests/python/pants_test/util/test_timeout.py +++ /dev/null @@ -1,75 +0,0 @@ -# coding=utf-8 -# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import (absolute_import, division, generators, nested_scopes, print_function, - unicode_literals, with_statement) - -import unittest - -from pants.util.timeout import Timeout, TimeoutReached - - -class FakeTimer(object): - def __init__(self, seconds, handler): - self._seconds = seconds - self._handler = handler - self._clock = 0 - self._started = False - - def cancel(self): - pass - - def start(self): - self._started = True - - def move_clock_forward(self, seconds): - if not self._started: - raise Exception("Timer not started but clock moved forward") - - self._clock += seconds - if self._clock > self._seconds: - self._handler() - - -class TestTimeout(unittest.TestCase): - def setUp(self): - super(TestTimeout, self).setUp() - self._fake_timer = None - self._aborted = False - - def _abort_handler(self): - self._aborted = True - - def _make_fake_timer(self, seconds, handler): - self._fake_timer = FakeTimer(seconds, handler) - return self._fake_timer - - def _move_clock_forward(self, seconds): - if self._fake_timer is not None: - self._fake_timer.move_clock_forward(seconds) - - def test_timeout_success(self): - with Timeout(2, threading_timer=self._make_fake_timer, abort_handler=self._abort_handler): - self._move_clock_forward(1) - - self.assertFalse(self._aborted) - - def test_timeout_failure(self): - with self.assertRaises(TimeoutReached): - with Timeout(2, threading_timer=self._make_fake_timer, abort_handler=self._abort_handler): - self._move_clock_forward(3) - - self.assertTrue(self._aborted) - - def test_timeout_none(self): - with Timeout(None, threading_timer=self._make_fake_timer, abort_handler=self._abort_handler): - self._move_clock_forward(3) - - self.assertFalse(self._aborted) - - def test_timeout_zero(self): - with Timeout(0, threading_timer=self._make_fake_timer, abort_handler=self._abort_handler): - self._move_clock_forward(3) - - self.assertFalse(self._aborted) From caa330cff03c7a5991c6a9fee0430ab14fba4614 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Wed, 20 Sep 2017 17:15:29 -0700 Subject: [PATCH 53/67] Refactor test partitioning. (#4879) This moves all the logic for partitioning batches of tests into a central helper method but otherwise leaves partitioning logic unchanged. --- .../pants/backend/jvm/tasks/junit_run.py | 153 +++++++++--------- 1 file changed, 77 insertions(+), 76 deletions(-) diff --git a/src/python/pants/backend/jvm/tasks/junit_run.py b/src/python/pants/backend/jvm/tasks/junit_run.py index e8162b1348f..77a97750c95 100644 --- a/src/python/pants/backend/jvm/tasks/junit_run.py +++ b/src/python/pants/backend/jvm/tasks/junit_run.py @@ -399,82 +399,74 @@ def parse_error_handler(parse_error): classpath_prepend = coverage.classpath_prepend classpath_append = coverage.classpath_append - tests_by_properties = test_registry.index( - lambda tgt: tgt.cwd if tgt.cwd is not None else self._working_dir, - lambda tgt: tgt.test_platform, - lambda tgt: tgt.payload.extra_jvm_options, - lambda tgt: tgt.payload.extra_env_vars, - lambda tgt: tgt.concurrency, - lambda tgt: tgt.threads) - - # the below will be None if not set, and we'll default back to runtime_classpath + # The 'instrument_classpath' product below below will be `None` if not set, and we'll default + # back to runtime_classpath classpath_product = self.context.products.get_data('instrument_classpath') result = 0 - for properties, tests in tests_by_properties.items(): + for properties, batch in self._partition(test_registry): (workdir, platform, target_jvm_options, target_env_vars, concurrency, threads) = properties - for batch in self._partition(tests): - # Batches of test classes will likely exist within the same targets: dedupe them. - relevant_targets = {test_registry.get_owning_target(t) for t in batch} - complete_classpath = OrderedSet() - complete_classpath.update(classpath_prepend) - complete_classpath.update(JUnit.global_instance().runner_classpath(self.context)) - complete_classpath.update(self.classpath(relevant_targets, - classpath_product=classpath_product)) - complete_classpath.update(classpath_append) - distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version) - - # Override cmdline args with values from junit_test() target that specify concurrency: - args = self._args(output_dir) + [u'-xmlreport'] - - if concurrency is not None: - args = remove_arg(args, '-default-parallel') - if concurrency == JUnitTests.CONCURRENCY_SERIAL: - args = ensure_arg(args, '-default-concurrency', param='SERIAL') - elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES: - args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES') - elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_METHODS: - args = ensure_arg(args, '-default-concurrency', param='PARALLEL_METHODS') - elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS: - args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES_AND_METHODS') - - if threads is not None: - args = remove_arg(args, '-parallel-threads', has_param=True) - args += ['-parallel-threads', str(threads)] - - batch_test_specs = [test.render_test_spec() for test in batch] - with argfile.safe_args(batch_test_specs, self.get_options()) as batch_tests: - with self._chroot(relevant_targets, workdir) as chroot: - self.context.log.debug('CWD = {}'.format(chroot)) - self.context.log.debug('platform = {}'.format(platform)) - with environment_as(**dict(target_env_vars)): - subprocess_result = self._spawn_and_wait( - executor=SubprocessExecutor(distribution), - distribution=distribution, - classpath=complete_classpath, - main=JUnit.RUNNER_MAIN, - jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options), - args=args + batch_tests, - workunit_factory=self.context.new_workunit, - workunit_name='run', - workunit_labels=[WorkUnitLabel.TEST], - cwd=chroot, - synthetic_jar_dir=output_dir, - create_synthetic_jar=self.synthetic_classpath, - ) - self.context.log.debug('JUnit subprocess exited with result ({})' - .format(subprocess_result)) - result += abs(subprocess_result) - - tests_info = self.parse_test_info(output_dir, parse_error_handler, ['classname']) - for test_name, test_info in tests_info.items(): - test_item = Test(test_info['classname'], test_name) - test_target = test_registry.get_owning_target(test_item) - self.report_all_info_for_single_test(self.options_scope, test_target, - test_name, test_info) - - if result != 0 and self._fail_fast: - break + # Batches of test classes will likely exist within the same targets: dedupe them. + relevant_targets = {test_registry.get_owning_target(t) for t in batch} + complete_classpath = OrderedSet() + complete_classpath.update(classpath_prepend) + complete_classpath.update(JUnit.global_instance().runner_classpath(self.context)) + complete_classpath.update(self.classpath(relevant_targets, + classpath_product=classpath_product)) + complete_classpath.update(classpath_append) + distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version) + + # Override cmdline args with values from junit_test() target that specify concurrency: + args = self._args(output_dir) + [u'-xmlreport'] + + if concurrency is not None: + args = remove_arg(args, '-default-parallel') + if concurrency == JUnitTests.CONCURRENCY_SERIAL: + args = ensure_arg(args, '-default-concurrency', param='SERIAL') + elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES: + args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES') + elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_METHODS: + args = ensure_arg(args, '-default-concurrency', param='PARALLEL_METHODS') + elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS: + args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES_AND_METHODS') + + if threads is not None: + args = remove_arg(args, '-parallel-threads', has_param=True) + args += ['-parallel-threads', str(threads)] + + batch_test_specs = [test.render_test_spec() for test in batch] + with argfile.safe_args(batch_test_specs, self.get_options()) as batch_tests: + with self._chroot(relevant_targets, workdir) as chroot: + self.context.log.debug('CWD = {}'.format(chroot)) + self.context.log.debug('platform = {}'.format(platform)) + with environment_as(**dict(target_env_vars)): + subprocess_result = self._spawn_and_wait( + executor=SubprocessExecutor(distribution), + distribution=distribution, + classpath=complete_classpath, + main=JUnit.RUNNER_MAIN, + jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options), + args=args + batch_tests, + workunit_factory=self.context.new_workunit, + workunit_name='run', + workunit_labels=[WorkUnitLabel.TEST], + cwd=chroot, + synthetic_jar_dir=output_dir, + create_synthetic_jar=self.synthetic_classpath, + ) + self.context.log.debug('JUnit subprocess exited with result ({})' + .format(subprocess_result)) + result += abs(subprocess_result) + + tests_info = self.parse_test_info(output_dir, parse_error_handler, ['classname']) + for test_name, test_info in tests_info.items(): + test_item = Test(test_info['classname'], test_name) + test_target = test_registry.get_owning_target(test_item) + self.report_all_info_for_single_test(self.options_scope, test_target, + test_name, test_info) + + if result != 0 and self._fail_fast: + break if result != 0: target_to_failed_test = parse_failed_targets(test_registry, output_dir, parse_error_handler) @@ -503,10 +495,19 @@ def render_owning_target(t): ) raise ErrorWhileTesting('\n'.join(error_message_lines), failed_targets=list(failed_targets)) - def _partition(self, tests): - stride = min(self._batch_size, len(tests)) - for i in range(0, len(tests), stride): - yield tests[i:i + stride] + def _partition(self, test_registry): + tests_by_properties = test_registry.index( + lambda tgt: tgt.cwd if tgt.cwd is not None else self._working_dir, + lambda tgt: tgt.test_platform, + lambda tgt: tgt.payload.extra_jvm_options, + lambda tgt: tgt.payload.extra_env_vars, + lambda tgt: tgt.concurrency, + lambda tgt: tgt.threads) + + for properties, tests in tests_by_properties.items(): + stride = min(self._batch_size, len(tests)) + for i in range(0, len(tests), stride): + yield properties, tests[i:i + stride] def _get_possible_tests_to_run(self): buildroot = get_buildroot() From 3dd3ae843d99d3de544fff0600b88ada09665dde Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 25 Sep 2017 12:14:57 -0600 Subject: [PATCH 54/67] Update the committer docs. (#4889) Also update the referenced COMMITTERS.md. --- COMMITTERS.md | 16 ++++++++-------- src/docs/committers.md | 3 +-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/COMMITTERS.md b/COMMITTERS.md index 8149cd19165..44990b83ebf 100644 --- a/COMMITTERS.md +++ b/COMMITTERS.md @@ -3,22 +3,14 @@ Active Committers * Benjy Weinberger * Chris Heisterkamp -* Eric Ayers -* Fedor Korotkov -* Garrett Malmquist * Ity Kaul * John Sirois * Kris Wilson * Larry Hosken * Mateo Rodriguez -* Matt Olsen * Nick Howard -* Patrick Lawson -* Peiyu Wang * Stu Hood -* Tejal Desai * Yi Cheng -* Yujie Chen Emeritus ======== @@ -26,4 +18,12 @@ Emeritus * Andy Reitz * David Taylor * David Turner +* Eric Ayers +* Fedor Korotkov +* Garrett Malmquist +* Matt Olsen +* Patrick Lawson +* Peiyu Wang +* Tejal Desai * Travis Crawford +* Yujie Chen diff --git a/src/docs/committers.md b/src/docs/committers.md index daf25a141ae..f0bf4e06445 100644 --- a/src/docs/committers.md +++ b/src/docs/committers.md @@ -21,12 +21,11 @@ After approval by a vote of the current committers, a new committer has access t - Commit access to the github repo. - Ability to Stop/restart builds on Travis-CI. - - An account on the code review site (RBCommons) that is part of the 'twitter’ team. - Access to the pants-committers@googlegroups.com group. A committer may also request access to publish changes to the org.pantsbuild groupId on the Maven Central Repository. -The list of current and past committers will be listed in a file named COMMITTERS.rb in the root of the pantsbuild/pants repo. +The list of current and past committers will be listed in a file named COMMITTERS.md in the root of the pantsbuild/pants repo. Committer Responsibilities -------------------------- From 09ea4cc894ddae2f072174081108f6122c7ca68e Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 25 Sep 2017 12:15:17 -0600 Subject: [PATCH 55/67] Only generate Android resource deps when needed. (#4888) Previously, `.aar` archives with empty `res/` dirs would trigger creation of a bugus synthetic `AndroidResources` dependency in the associated library. The existing tests are modified to exercise the case of empty `res/` dirs. --- .../python/pants/contrib/android/tasks/unpack_libraries.py | 2 +- .../pants_test/contrib/android/tasks/test_unpack_libraries.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/android/src/python/pants/contrib/android/tasks/unpack_libraries.py b/contrib/android/src/python/pants/contrib/android/tasks/unpack_libraries.py index f6c2556d6ff..7873bba2fd3 100644 --- a/contrib/android/src/python/pants/contrib/android/tasks/unpack_libraries.py +++ b/contrib/android/src/python/pants/contrib/android/tasks/unpack_libraries.py @@ -155,7 +155,7 @@ def create_android_library_target(self, binary, library, coordinate, unpacked_aa # Depending on the contents of the unpacked aar file, create the dependencies. deps = [] - if os.path.isdir(resource_dir): + if os.path.isdir(resource_dir) and os.listdir(resource_dir): new_resource_target = self.create_resource_target(library, coordinate, manifest, resource_dir) # # The new libraries resources must be compiled both by themselves and along with the dependent library. diff --git a/contrib/android/tests/python/pants_test/contrib/android/tasks/test_unpack_libraries.py b/contrib/android/tests/python/pants_test/contrib/android/tasks/test_unpack_libraries.py index 11edd7cb952..9bfadf4f136 100644 --- a/contrib/android/tests/python/pants_test/contrib/android/tasks/test_unpack_libraries.py +++ b/contrib/android/tests/python/pants_test/contrib/android/tasks/test_unpack_libraries.py @@ -39,8 +39,10 @@ def unpacked_aar_library(self, location, manifest=True, classes_jar=True, resour fp.close() if classes_jar: self.create_jarfile(location, filenames=filenames) + resource_dir = os.path.join(location, 'res') + safe_mkdir(resource_dir) if resources: - safe_mkdir(os.path.join(location, 'res')) + touch(os.path.join(resource_dir, 'resource_file')) return location def create_aarfile(self, location, name, filenames=None): From 65cc90b207c14b53da3546005fb7d4bd600072b2 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 25 Sep 2017 12:20:06 -0600 Subject: [PATCH 56/67] Support wheels when loading plugins. (#4887) Fixes #4877 --- 3rdparty/python/requirements.txt | 1 + src/python/pants/init/BUILD | 1 + src/python/pants/init/plugin_resolver.py | 32 ++++++++++++---- tests/python/pants_test/init/BUILD | 1 + .../pants_test/init/test_plugin_resolver.py | 37 +++++++++++-------- 5 files changed, 50 insertions(+), 22 deletions(-) diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 67c333fccac..0a8515bec7b 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -11,6 +11,7 @@ Markdown==2.1.1 mock==2.0.0 packaging==16.8 pathspec==0.5.0 +parameterized==0.6.1 pep8==1.6.2 pex==1.2.11 psutil==4.3.0 diff --git a/src/python/pants/init/BUILD b/src/python/pants/init/BUILD index 12f81db0aea..9b0bee315dd 100644 --- a/src/python/pants/init/BUILD +++ b/src/python/pants/init/BUILD @@ -7,6 +7,7 @@ python_library( ':plugins', '3rdparty/python:pex', '3rdparty/python:setuptools', + '3rdparty/python:wheel', '3rdparty/python/twitter/commons:twitter.common.collections', 'src/python/pants:version', 'src/python/pants/base:build_environment', diff --git a/src/python/pants/init/plugin_resolver.py b/src/python/pants/init/plugin_resolver.py index 62c04a57bd6..ef781cd6ec7 100644 --- a/src/python/pants/init/plugin_resolver.py +++ b/src/python/pants/init/plugin_resolver.py @@ -8,17 +8,18 @@ import hashlib import logging import os +import site from pex import resolver from pex.base import requirement_is_exact -from pex.package import EggPackage, SourcePackage from pkg_resources import working_set as global_working_set from pkg_resources import Requirement +from wheel.install import WheelFile from pants.option.global_options import GlobalOptionsRegistrar from pants.python.python_repos import PythonRepos from pants.subsystem.subsystem import Subsystem -from pants.util.dirutil import safe_open +from pants.util.dirutil import safe_mkdir, safe_open from pants.util.memo import memoized_property from pants.version import PANTS_SEMVER @@ -27,6 +28,26 @@ class PluginResolver(object): + @staticmethod + def _is_wheel(path): + return os.path.isfile(path) and path.endswith('.whl') + + @staticmethod + def _activate_wheel(wheel_path): + install_dir = '{}-install'.format(wheel_path) + safe_mkdir(install_dir, clean=True) + WheelFile(wheel_path).install(force=True, + overrides={ + 'purelib': install_dir, + 'headers': os.path.join(install_dir, 'headers'), + 'scripts': os.path.join(install_dir, 'bin'), + 'platlib': install_dir, + 'data': install_dir + }) + # Activate any .pth files installed above. + site.addsitedir(install_dir) + return install_dir + def __init__(self, options_bootstrapper): self._options_bootstrapper = options_bootstrapper @@ -44,6 +65,8 @@ def resolve(self, working_set=None): working_set = working_set or global_working_set if self._plugin_requirements: for plugin_location in self._resolve_plugin_locations(): + if self._is_wheel(plugin_location): + plugin_location = self._activate_wheel(plugin_location) working_set.add_entry(plugin_location) return working_set @@ -76,15 +99,10 @@ def _resolve_exact_plugin_locations(self): yield plugin_location.strip() def _resolve_plugins(self): - # When bootstrapping plugins without the full pants python backend machinery in-play, we are not - # guaranteed a properly initialized interpreter with wheel support so we enforce eggs only for - # bdists with this custom precedence. - precedence = (EggPackage, SourcePackage) logger.info('Resolving new plugins...:\n {}'.format('\n '.join(self._plugin_requirements))) return resolver.resolve(self._plugin_requirements, fetchers=self._python_repos.get_fetchers(), context=self._python_repos.get_network_context(), - precedence=precedence, cache=self.plugin_cache_dir, cache_ttl=10 * 365 * 24 * 60 * 60, # Effectively never expire. allow_prereleases=PANTS_SEMVER.is_prerelease) diff --git a/tests/python/pants_test/init/BUILD b/tests/python/pants_test/init/BUILD index 4fe79036579..2f845a123d4 100644 --- a/tests/python/pants_test/init/BUILD +++ b/tests/python/pants_test/init/BUILD @@ -5,6 +5,7 @@ python_tests( dependencies = [ '3rdparty/python:mock', + '3rdparty/python:parameterized', '3rdparty/python:pex', '3rdparty/python:setuptools', 'src/python/pants/base:exceptions', diff --git a/tests/python/pants_test/init/test_plugin_resolver.py b/tests/python/pants_test/init/test_plugin_resolver.py index 5f5c96c0fda..514e737971a 100644 --- a/tests/python/pants_test/init/test_plugin_resolver.py +++ b/tests/python/pants_test/init/test_plugin_resolver.py @@ -11,8 +11,9 @@ from contextlib import contextmanager from textwrap import dedent +from parameterized import parameterized from pex.crawler import Crawler -from pex.installer import Packager +from pex.installer import EggInstaller, Packager, WheelInstaller from pex.resolver import Unsatisfiable from pkg_resources import Requirement, WorkingSet @@ -24,10 +25,12 @@ req = Requirement.parse +INSTALLERS = [('sdist', Packager), ('egg', EggInstaller), ('whl', WheelInstaller)] + class PluginResolverTest(unittest.TestCase): @staticmethod - def create_plugin(distribution_repo_dir, plugin, version=None): + def create_plugin(distribution_repo_dir, plugin, version=None, packager_cls=None): with safe_open(os.path.join(distribution_repo_dir, plugin, 'setup.py'), 'w') as fp: fp.write(dedent(""" from setuptools import setup @@ -35,12 +38,13 @@ def create_plugin(distribution_repo_dir, plugin, version=None): setup(name="{plugin}", version="{version}") """).format(plugin=plugin, version=version or '0.0.0')) - packager = Packager(source_dir=os.path.join(distribution_repo_dir, plugin), - install_dir=distribution_repo_dir) + packager_cls = packager_cls or Packager + packager = packager_cls(source_dir=os.path.join(distribution_repo_dir, plugin), + install_dir=distribution_repo_dir) packager.run() @contextmanager - def plugin_resolution(self, chroot=None, plugins=None): + def plugin_resolution(self, chroot=None, plugins=None, packager_cls=None): @contextmanager def provide_chroot(existing): if existing: @@ -64,7 +68,7 @@ def provide_chroot(existing): plugin, version = plugin plugin_list.append('{}=={}'.format(plugin, version) if version else plugin) if create_artifacts: - self.create_plugin(repo_dir, plugin, version) + self.create_plugin(repo_dir, plugin, version, packager_cls=packager_cls) env['PANTS_PLUGINS'] = '[{}]'.format(','.join(map(repr, plugin_list))) configpath = os.path.join(root_dir, 'pants.ini') @@ -81,11 +85,10 @@ def test_no_plugins(self): with self.plugin_resolution() as (working_set, _, _, _): self.assertEqual([], working_set.entries) - def test_plugins(self): - with self.plugin_resolution(plugins=[('jake', '1.2.3'), 'jane']) as (working_set, - _, - _, - cache_dir): + @parameterized.expand(INSTALLERS) + def test_plugins(self, unused_test_name, packager_cls): + with self.plugin_resolution(plugins=[('jake', '1.2.3'), 'jane'], + packager_cls=packager_cls) as (working_set, _, _, cache_dir): self.assertEqual(2, len(working_set.entries)) dist = working_set.find(req('jake')) @@ -98,8 +101,10 @@ def test_plugins(self): self.assertEqual(os.path.realpath(cache_dir), os.path.realpath(os.path.dirname(dist.location))) - def test_exact_requirements(self): - with self.plugin_resolution(plugins=[('jake', '1.2.3'), ('jane', '3.4.5')]) as results: + @parameterized.expand(INSTALLERS) + def test_exact_requirements(self, unused_test_name, packager_cls): + with self.plugin_resolution(plugins=[('jake', '1.2.3'), ('jane', '3.4.5')], + packager_cls=packager_cls) as results: working_set, chroot, repo_dir, cache_dir = results self.assertEqual(2, len(working_set.entries)) @@ -114,8 +119,10 @@ def test_exact_requirements(self): self.assertEqual(working_set.entries, working_set2.entries) - def test_inexact_requirements(self): - with self.plugin_resolution(plugins=[('jake', '1.2.3'), 'jane']) as results: + @parameterized.expand(INSTALLERS) + def test_inexact_requirements(self, unused_test_name, packager_cls): + with self.plugin_resolution(plugins=[('jake', '1.2.3'), 'jane'], + packager_cls=packager_cls) as results: working_set, chroot, repo_dir, cache_dir = results self.assertEqual(2, len(working_set.entries)) From 98a03827364789446a5b7e290d889b50e04b24e8 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Mon, 25 Sep 2017 12:33:48 -0700 Subject: [PATCH 57/67] Exit with error on error bootstrapping cffi (#4891) --- build-support/native-engine/bootstrap_cffi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build-support/native-engine/bootstrap_cffi.py b/build-support/native-engine/bootstrap_cffi.py index f674cdb6dd2..416d754157c 100644 --- a/build-support/native-engine/bootstrap_cffi.py +++ b/build-support/native-engine/bootstrap_cffi.py @@ -15,5 +15,6 @@ output_dir = sys.argv[1] except Exception: print('usage: {} '.format(sys.argv[0])) + sys.exit(1) bootstrap_c_source(output_dir) From 44da70072ab094c58aee7fb0c66e51ee94a2c027 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 25 Sep 2017 15:15:40 -0600 Subject: [PATCH 58/67] Remove python 2.6 support completely. (#4871) Pants itself has required 2.7 to run for a while now and the latest pex upgrade drops support for 2.6 in our python backend execution infrastructure so it's now safe to drop 2.6 support completely. --- .travis.yml | 13 ------- examples/src/python/example/README.md | 39 ++++++++++++------- .../eclipse/pydevproject-3.7.mustache | 2 +- .../backend/python/subsystems/python_setup.py | 2 +- .../pants/backend/python/tasks/setup_py.py | 6 +-- .../pants/backend/python/tasks2/setup_py.py | 6 +-- src/python/pants/fs/archive.py | 9 +---- src/python/pants/option/global_options.py | 2 +- src/python/pants/util/contextutil.py | 2 +- .../src/python/interpreter_selection/BUILD | 10 ++--- .../jvm/tasks/test_junit_tests_integration.py | 8 ++-- .../jvm/tasks/test_jvm_run_integration.py | 4 +- .../tasks2/test_python_repl_integration.py | 2 +- .../tasks2/test_python_run_integration.py | 23 +++++------ .../pants_test/pants_run_integration_test.py | 2 +- .../test_interpreter_selection_integration.py | 11 +++--- 16 files changed, 67 insertions(+), 74 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4ec443a74a0..dc4b67a30af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required # Docker runs will write files as root, so avoid caching for this shard. cache: false @@ -86,7 +85,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -115,7 +113,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -144,7 +141,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -173,7 +169,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -202,7 +197,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -231,7 +225,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -260,7 +253,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -289,7 +281,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -318,7 +309,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -347,7 +337,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -376,7 +365,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: @@ -405,7 +393,6 @@ matrix: - os: linux dist: trusty - group: deprecated-2017Q2 sudo: required addons: apt: diff --git a/examples/src/python/example/README.md b/examples/src/python/example/README.md index ad4688eccf4..39abf4ce02c 100644 --- a/examples/src/python/example/README.md +++ b/examples/src/python/example/README.md @@ -135,14 +135,17 @@ Use `test` to run the tests. This uses `pytest`: 13:29:29 00:01 [pytest] 13:29:29 00:01 [run] ============== test session starts =============== - platform darwin -- Python 2.6.8 -- py-1.4.20 -- pytest-2.5.2 - plugins: cov, timeout + platform linux2 -- Python 2.7.12, pytest-3.0.7, py-1.4.32, pluggy-0.4.0 + rootdir: /home/jsirois, inifile: + plugins: cov-2.4.0, timeout-1.2.0 collected 2 items - examples/tests/python/example_test/hello/greet/test_greet.py .. + .pants.d/pyprep/sources/48bd113ee4f5fa26f55357fbd9bb6d31382241fa/example_test/hello/greet/test_greet.py .. - ============ 1 passed in 0.02 seconds ============ + generated xml file: /home/jsirois/dev/pantsbuild/jsirois-pants2/.pants.d/test/pytest/examples.tests.python.example_test.hello.greet.greet/junitxml/TEST-examples.tests.python.example_test.hello.greet.greet.xml + ============ 2 passed in 0.01 seconds ============ + examples.tests.python.example_test.hello.greet.greet ..... SUCCESS 13:30:18 00:50 [junit] 13:30:18 00:50 [specs] SUCCESS @@ -279,16 +282,19 @@ Pants runs Python tests with `pytest`. You can pass CLI options to `pytest` with you could run: :::bash - $ ./pants test.pytest --options='-k req' examples/tests/python/example_test/hello/greet + $ ./pants test.pytest --options='-k foo' examples/tests/python/example_test/hello/greet ... ============== test session starts =============== - platform darwin -- Python 2.6.8 -- py-1.4.20 -- pytest-2.5.2 - plugins: cov, timeout + platform linux2 -- Python 2.7.12, pytest-3.0.7, py-1.4.32, pluggy-0.4.0 + rootdir: /home/jsirois, inifile: + plugins: cov-2.4.0, timeout-1.2.0 collected 2 items - ========= 2 tests deselected by '-kfoo' ========== + generated xml file: /home/jsirois/dev/pantsbuild/jsirois-pants2/.pants.d/test/pytest/examples.tests.python.example_test.hello.greet.greet/junitxml/TEST-examples.tests.python.example_test.hello.greet.greet.xml + =============== 2 tests deselected =============== ========== 2 deselected in 0.01 seconds ========== + examples.tests.python.example_test.hello.greet.greet ..... SUCCESS 13:34:28 00:02 [junit] 13:34:28 00:02 [specs] SUCCESS @@ -305,16 +311,19 @@ parameters: 10:43:04 00:01 [prep_command] 10:43:04 00:01 [pytest] 10:43:04 00:01 [run] - ============== test session starts =============== - platform darwin -- Python 2.7.5 -- py-1.4.26 -- pytest-2.6.4 - plugins: cov, timeout - collected 2 items + ============== test session starts =============== + platform linux2 -- Python 2.7.12, pytest-3.0.7, py-1.4.32, pluggy-0.4.0 + rootdir: /home/jsirois, inifile: + plugins: cov-2.4.0, timeout-1.2.0 + collected 2 items - examples/tests/python/example_test/hello/greet/test_greet.py . + .pants.d/pyprep/sources/48bd113ee4f5fa26f55357fbd9bb6d31382241fa/example_test/hello/greet/test_greet.py . - ========= 1 tests deselected by '-kreq' ========== - ===== 1 passed, 1 deselected in 0.05 seconds ===== + generated xml file: /home/jsirois/dev/pantsbuild/jsirois-pants2/.pants.d/test/pytest/examples.tests.python.example_test.hello.greet.greet/junitxml/TEST-examples.tests.python.example_test.hello.greet.greet.xml + =============== 1 tests deselected =============== + ===== 1 passed, 1 deselected in 0.01 seconds ===== + examples.tests.python.example_test.hello.greet.greet ..... SUCCESS 10:43:05 00:02 [junit] 10:43:05 00:02 [specs] SUCCESS diff --git a/src/python/pants/backend/project_info/tasks/templates/eclipse/pydevproject-3.7.mustache b/src/python/pants/backend/project_info/tasks/templates/eclipse/pydevproject-3.7.mustache index 07bd942d938..a0e0b781f97 100644 --- a/src/python/pants/backend/project_info/tasks/templates/eclipse/pydevproject-3.7.mustache +++ b/src/python/pants/backend/project_info/tasks/templates/eclipse/pydevproject-3.7.mustache @@ -10,7 +10,7 @@ Licensed under the Apache License, Version 2.0 (see LICENSE). {{! TODO(John Sirois): support python project setups with interpreter of choice.}} Default - python 2.6 + python 2.7 {{#project.pythonpaths}} {{#includes?}} diff --git a/src/python/pants/backend/python/subsystems/python_setup.py b/src/python/pants/backend/python/subsystems/python_setup.py index 282c77c898c..e8fab0e6a94 100644 --- a/src/python/pants/backend/python/subsystems/python_setup.py +++ b/src/python/pants/backend/python/subsystems/python_setup.py @@ -34,7 +34,7 @@ def register_options(cls, register): register('--interpreter-constraints', advanced=True, default=[], type=list, metavar='', help="Constrain the selected Python interpreter. Specify with requirement syntax, " - "e.g. 'CPython>=2.6,<3' or 'PyPy'. Multiple constraints will be ORed together. " + "e.g. 'CPython>=2.7,<3' or 'PyPy'. Multiple constraints will be ORed together. " "These constraints are applied in addition to any compatibilities required by " "the relevant targets.") register('--setuptools-version', advanced=True, default='30.0.0', diff --git a/src/python/pants/backend/python/tasks/setup_py.py b/src/python/pants/backend/python/tasks/setup_py.py index b5376496e2e..c78491e43c2 100644 --- a/src/python/pants/backend/python/tasks/setup_py.py +++ b/src/python/pants/backend/python/tasks/setup_py.py @@ -551,9 +551,9 @@ def convert(input): # pprint.pformat embeds u's in the string itself during conversion. # For that reason we convert each unicode string independently. # - # hoth:~ travis$ python - # Python 2.6.8 (unknown, Aug 25 2013, 00:04:29) - # [GCC 4.2.1 Compatible Apple LLVM 5.0 (clang-500.0.68)] on darwin + # jsirois@gill ~ $ python2 + # Python 2.7.13 (default, Jul 21 2017, 03:24:34) + # [GCC 7.1.1 20170630] on linux2 # Type "help", "copyright", "credits" or "license" for more information. # >>> import pprint # >>> data = {u'entry_points': {u'console_scripts': [u'pants = pants.bin.pants_exe:main']}} diff --git a/src/python/pants/backend/python/tasks2/setup_py.py b/src/python/pants/backend/python/tasks2/setup_py.py index 1bfe4366236..036821fd79a 100644 --- a/src/python/pants/backend/python/tasks2/setup_py.py +++ b/src/python/pants/backend/python/tasks2/setup_py.py @@ -544,9 +544,9 @@ def convert(input): # pprint.pformat embeds u's in the string itself during conversion. # For that reason we convert each unicode string independently. # - # hoth:~ travis$ python - # Python 2.6.8 (unknown, Aug 25 2013, 00:04:29) - # [GCC 4.2.1 Compatible Apple LLVM 5.0 (clang-500.0.68)] on darwin + # jsirois@gill ~ $ python2 + # Python 2.7.13 (default, Jul 21 2017, 03:24:34) + # [GCC 7.1.1 20170630] on linux2 # Type "help", "copyright", "credits" or "license" for more information. # >>> import pprint # >>> data = {u'entry_points': {u'console_scripts': [u'pants = pants.bin.pants_exe:main']}} diff --git a/src/python/pants/fs/archive.py b/src/python/pants/fs/archive.py index fefa0860c47..e1e3bf1bba6 100644 --- a/src/python/pants/fs/archive.py +++ b/src/python/pants/fs/archive.py @@ -90,13 +90,8 @@ def extract(cls, path, outdir, filter_func=None): # While we're at it, we also perform this safety test. if name.startswith(b'/') or name.startswith(b'..'): raise ValueError('Zip file contains unsafe path: {}'.format(name)) - # Ignore directories. extract() will create parent dirs as needed. - # OS X's python 2.6.1 has a bug in zipfile that makes it unzip directories as regular files. - # This method should work on for python 2.6-3.x. - # TODO(Eric Ayers) Pants no longer builds with python 2.6. Can this be removed? - if not name.endswith(b'/'): - if (not filter_func or filter_func(name)): - archive_file.extract(name, outdir) + if (not filter_func or filter_func(name)): + archive_file.extract(name, outdir) def __init__(self, compression, extension): """ diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py index 4afb09232bc..148ff464bc1 100644 --- a/src/python/pants/option/global_options.py +++ b/src/python/pants/option/global_options.py @@ -184,7 +184,7 @@ def register_options(cls, register): removal_version='1.5.0.dev0', removal_hint='Use --interpreter-constraints in scope python-setup instead.', help="Constrain what Python interpreters to use. Uses Requirement format from " - "pkg_resources, e.g. 'CPython>=2.6,<3' or 'PyPy'. By default, no constraints " + "pkg_resources, e.g. 'CPython>=2.7,<3' or 'PyPy'. By default, no constraints " "are used. Multiple constraints may be added. They will be ORed together.") register('--exclude-target-regexp', advanced=True, type=list, default=[], metavar='', diff --git a/src/python/pants/util/contextutil.py b/src/python/pants/util/contextutil.py index f2050fb0c56..252ad616d55 100644 --- a/src/python/pants/util/contextutil.py +++ b/src/python/pants/util/contextutil.py @@ -31,7 +31,7 @@ def environment_as(**kwargs): """Update the environment to the supplied values, for example: with environment_as(PYTHONPATH='foo:bar:baz', - PYTHON='/usr/bin/python2.6'): + PYTHON='/usr/bin/python2.7'): subprocess.Popen(foo).wait() """ new_environment = kwargs diff --git a/testprojects/src/python/interpreter_selection/BUILD b/testprojects/src/python/interpreter_selection/BUILD index ff1d13fd640..d496c547be2 100644 --- a/testprojects/src/python/interpreter_selection/BUILD +++ b/testprojects/src/python/interpreter_selection/BUILD @@ -9,16 +9,16 @@ python_library( sources = ['echo_interpreter_version.py'], dependencies = [], # Play with this to test interpreter selection in the pex machinery. - compatibility = ['CPython>=2.6,<3'] + compatibility = ['CPython>=2.7,<3', 'CPython>=3.3'] ) python_binary( - name = 'echo_interpreter_version_2.6', + name = 'echo_interpreter_version_3', dependencies = [ ':echo_interpreter_version_lib', ], entry_point = 'interpreter_selection.echo_interpreter_version', - compatibility = ['CPython>=2.6,<2.7'] + compatibility = ['CPython>=3.3'] ) python_binary( @@ -46,14 +46,14 @@ python_binary( ':echo_interpreter_version_lib', ], entry_point = 'interpreter_selection.echo_interpreter_version', - compatibility = ['CPython<2.6'] + compatibility = ['CPython<2.7'] ) python_library( name = 'die_lib', sources = ['die.py'], dependencies = [], - compatibility = ['CPython>=2.6,<3'] + compatibility = ['CPython>=2.7,<3'] ) python_binary( diff --git a/tests/python/pants_test/backend/jvm/tasks/test_junit_tests_integration.py b/tests/python/pants_test/backend/jvm/tasks/test_junit_tests_integration.py index bffe27914f2..1558cb21522 100644 --- a/tests/python/pants_test/backend/jvm/tasks/test_junit_tests_integration.py +++ b/tests/python/pants_test/backend/jvm/tasks/test_junit_tests_integration.py @@ -27,7 +27,7 @@ def test_junit_test_custom_interpreter(self): 'test', 'examples/tests/java/org/pantsbuild/example/hello/greet', 'examples/tests/scala/org/pantsbuild/example/hello/welcome', - '--python-setup-interpreter-constraints=CPython>=2.6,<3', + '--python-setup-interpreter-constraints=CPython>=2.7,<3', '--python-setup-interpreter-constraints=CPython>=3.3'], workdir) self.assert_success(pants_run) @@ -78,7 +78,7 @@ def test_junit_test_requiring_cwd_fails_without_option_specified(self): pants_run = self.run_pants([ 'test', 'testprojects/tests/java/org/pantsbuild/testproject/cwdexample', - '--python-setup-interpreter-constraints=CPython>=2.6,<3', + '--python-setup-interpreter-constraints=CPython>=2.7,<3', '--python-setup-interpreter-constraints=CPython>=3.3', '--jvm-test-junit-options=-Dcwd.test.enabled=true']) self.assert_failure(pants_run) @@ -87,7 +87,7 @@ def test_junit_test_requiring_cwd_passes_with_option_with_value_specified(self): pants_run = self.run_pants([ 'test', 'testprojects/tests/java/org/pantsbuild/testproject/cwdexample', - '--python-setup-interpreter-constraints=CPython>=2.6,<3', + '--python-setup-interpreter-constraints=CPython>=2.7,<3', '--python-setup-interpreter-constraints=CPython>=3.3', '--jvm-test-junit-options=-Dcwd.test.enabled=true', '--no-test-junit-chroot', @@ -98,7 +98,7 @@ def test_junit_test_requiring_cwd_fails_with_option_with_no_value_specified(self pants_run = self.run_pants([ 'test', 'testprojects/tests/java/org/pantsbuild/testproject/cwdexample', - '--python-setup-interpreter-constraints=CPython>=2.6,<3', + '--python-setup-interpreter-constraints=CPython>=2.7,<3', '--python-setup-interpreter-constraints=CPython>=3.3', '--jvm-test-junit-options=-Dcwd.test.enabled=true']) self.assert_failure(pants_run) diff --git a/tests/python/pants_test/backend/jvm/tasks/test_jvm_run_integration.py b/tests/python/pants_test/backend/jvm/tasks/test_jvm_run_integration.py index 753f36fc33b..03645af95c5 100644 --- a/tests/python/pants_test/backend/jvm/tasks/test_jvm_run_integration.py +++ b/tests/python/pants_test/backend/jvm/tasks/test_jvm_run_integration.py @@ -19,7 +19,7 @@ def _exec_run(self, target, *args): # Avoid some known-to-choke-on interpreters. command = ['run', target, - '--python-setup-interpreter-constraints=CPython>=2.6,<3', + '--python-setup-interpreter-constraints=CPython>=2.7,<3', '--python-setup-interpreter-constraints=CPython>=3.3'] + list(args) pants_run = self.run_pants(command) self.assert_success(pants_run) @@ -44,7 +44,7 @@ def test_no_run_cwd(self): # Make sure the test fails if you don't specify a directory pants_run = self.run_pants(['run', 'testprojects/src/java/org/pantsbuild/testproject/cwdexample', - '--python-setup-interpreter-constraints=CPython>=2.6,<3', + '--python-setup-interpreter-constraints=CPython>=2.7,<3', '--python-setup-interpreter-constraints=CPython>=3.3']) self.assert_failure(pants_run) self.assertIn('Neither ExampleCwd.java nor readme.txt found.', pants_run.stderr_data) diff --git a/tests/python/pants_test/backend/python/tasks2/test_python_repl_integration.py b/tests/python/pants_test/backend/python/tasks2/test_python_repl_integration.py index 358c594d94e..4ef3db41087 100644 --- a/tests/python/pants_test/backend/python/tasks2/test_python_repl_integration.py +++ b/tests/python/pants_test/backend/python/tasks2/test_python_repl_integration.py @@ -14,7 +14,7 @@ def test_run_repl(self): # Run a repl on a library target. Avoid some known-to-choke-on interpreters. command = ['repl', 'testprojects/src/python/interpreter_selection:echo_interpreter_version_lib', - '--python-setup-interpreter-constraints=CPython>=2.6,<3', + '--python-setup-interpreter-constraints=CPython>=2.7,<3', '--python-setup-interpreter-constraints=CPython>=3.3', '--quiet'] program = 'from interpreter_selection.echo_interpreter_version import say_hello; say_hello()' diff --git a/tests/python/pants_test/backend/python/tasks2/test_python_run_integration.py b/tests/python/pants_test/backend/python/tasks2/test_python_run_integration.py index 00a9a5c3425..9d4fa3ad4ce 100644 --- a/tests/python/pants_test/backend/python/tasks2/test_python_run_integration.py +++ b/tests/python/pants_test/backend/python/tasks2/test_python_run_integration.py @@ -12,13 +12,13 @@ class PythonRunIntegrationTest(PantsRunIntegrationTest): testproject = 'testprojects/src/python/interpreter_selection' - def test_run_26(self): - self._maybe_run_version('2.6') + def test_run_3(self): + self._maybe_run_version('3') def test_run_27(self): self._maybe_run_version('2.7') - def test_run_27_and_then_26(self): + def test_run_27_and_then_3(self): with temporary_dir() as interpreters_cache: pants_ini_config = {'python-setup': {'interpreter_cache_dir': interpreters_cache}} pants_run_27 = self.run_pants( @@ -26,18 +26,18 @@ def test_run_27_and_then_26(self): config=pants_ini_config ) self.assert_success(pants_run_27) - pants_run_26 = self.run_pants( - command=['run', '{}:echo_interpreter_version_2.6'.format(self.testproject), - '--python-setup-interpreter-constraints=CPython>=2.6,<3', + pants_run_3 = self.run_pants( + command=['run', '{}:echo_interpreter_version_3'.format(self.testproject), + '--python-setup-interpreter-constraints=CPython>=2.7,<3', '--python-setup-interpreter-constraints=CPython>=3.3'], config=pants_ini_config ) - self.assert_success(pants_run_26) + self.assert_success(pants_run_3) def test_die(self): command = ['run', '{}:die'.format(self.testproject), - '--python-setup-interpreter-constraints=CPython>=2.6,<3', + '--python-setup-interpreter-constraints=CPython>=2.7,<3', '--python-setup-interpreter-constraints=CPython>=3.3', '--quiet'] pants_run = self.run_pants(command=command) @@ -59,9 +59,10 @@ def _maybe_run_version(self, version): if self.has_python_version(version): print('Found python {}. Testing running on it.'.format(version)) echo = self._run_echo_version(version) - v = echo.split('.') # E.g., 2.6.8. + v = echo.split('.') # E.g., 2.7.13. self.assertTrue(len(v) > 2, 'Not a valid version string: {}'.format(v)) - self.assertEquals(version, '{}.{}'.format(v[0], v[1])) + expected_components = version.split('.') + self.assertEquals(expected_components, v[:len(expected_components,)]) else: print('No python {} found. Skipping.'.format(version)) self.skipTest('No python {} on system'.format(version)) @@ -73,7 +74,7 @@ def _run_echo_version(self, version): # Avoid some known-to-choke-on interpreters. command = ['run', binary_target, - '--python-setup-interpreter-constraints=CPython>=2.6,<3', + '--python-setup-interpreter-constraints=CPython>=2.7,<3', '--python-setup-interpreter-constraints=CPython>=3.3', '--quiet'] pants_run = self.run_pants(command=command) diff --git a/tests/python/pants_test/pants_run_integration_test.py b/tests/python/pants_test/pants_run_integration_test.py index 82292089260..39b25e5cb40 100644 --- a/tests/python/pants_test/pants_run_integration_test.py +++ b/tests/python/pants_test/pants_run_integration_test.py @@ -97,7 +97,7 @@ def hermetic_env_whitelist(cls): def has_python_version(cls, version): """Returns true if the current system has the specified version of python. - :param version: A python version string, such as 2.6, 3. + :param version: A python version string, such as 2.7, 3. """ try: subprocess.call(['python%s' % version, '-V']) diff --git a/tests/python/pants_test/python/test_interpreter_selection_integration.py b/tests/python/pants_test/python/test_interpreter_selection_integration.py index a885aa6bcc6..738dee0e688 100644 --- a/tests/python/pants_test/python/test_interpreter_selection_integration.py +++ b/tests/python/pants_test/python/test_interpreter_selection_integration.py @@ -21,8 +21,8 @@ def test_conflict(self): self.assert_failure(pants_run, 'Unexpected successful build of {binary}.'.format(binary=binary_target)) - def test_select_26(self): - self._maybe_test_version('2.6') + def test_select_3(self): + self._maybe_test_version('3') def test_select_27(self): self._maybe_test_version('2.7') @@ -31,9 +31,10 @@ def _maybe_test_version(self, version): if self.has_python_version(version): print('Found python {}. Testing interpreter selection against it.'.format(version)) echo = self._echo_version(version) - v = echo.split('.') # E.g., 2.6.8 . + v = echo.split('.') # E.g., 2.7.13. self.assertTrue(len(v) > 2, 'Not a valid version string: {}'.format(v)) - self.assertEquals(version, '{}.{}'.format(v[0], v[1])) + expected_components = version.split('.') + self.assertEquals(expected_components, v[:len(expected_components)]) else: print('No python {} found. Skipping.'.format(version)) self.skipTest('No python {} on system'.format(version)) @@ -60,6 +61,6 @@ def _build_pex(self, binary_target, config=None): # Avoid some known-to-choke-on interpreters. command = ['binary', binary_target, - '--python-setup-interpreter-constraints=CPython>=2.6,<3', + '--python-setup-interpreter-constraints=CPython>=2.7,<3', '--python-setup-interpreter-constraints=CPython>=3.3'] return self.run_pants(command=command, config=config) From 55311533cbe5e5c58606a8dfebe5c52cda824c5a Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Mon, 25 Sep 2017 15:45:12 -0700 Subject: [PATCH 59/67] Reduce BUILD file parse pollution (#4892) ### Problem As described on #4880, the parser used in the v2 engine was re-using a symbol table across parses, which (in the presence of non-deterministic execution orders due to engine concurrency) could cause non-deterministic successful parses of otherwise invalid code. ### Solution Shallow clone the symbol table for each use, which should defend against 95% of errors in this area. The remaining 5% would require deep cloning (or recreating) the symbol table for each file parse, which doesn't seem worth the benefit. ### Result A failing test that imports in parsed files now succeeds, and non-determinism due to imports should be eliminated. Fixes #4880 --- src/python/pants/engine/legacy/parser.py | 6 ++++- src/python/pants/engine/parser.py | 5 ++++ tests/python/pants_test/engine/legacy/BUILD | 10 ++++++++ .../pants_test/engine/legacy/test_parser.py | 24 +++++++++++++++++++ .../pants_test/engine/scheduler_test_base.py | 7 ------ .../python/pants_test/engine/test_parsers.py | 12 +++------- 6 files changed, 47 insertions(+), 17 deletions(-) create mode 100644 tests/python/pants_test/engine/legacy/test_parser.py diff --git a/src/python/pants/engine/legacy/parser.py b/src/python/pants/engine/legacy/parser.py index 9add40b9119..3b632e5cb81 100644 --- a/src/python/pants/engine/legacy/parser.py +++ b/src/python/pants/engine/legacy/parser.py @@ -120,6 +120,10 @@ def parse(self, filepath, filecontent): python = filecontent # Mutate the parse context for the new path, then exec, and copy the resulting objects. + # We execute with a (shallow) clone of the symbols as a defense against accidental + # pollution of the namespace via imports or variable definitions. Defending against + # _intentional_ mutation would require a deep clone, which doesn't seem worth the cost at + # this juncture. self._parse_context._storage.clear(os.path.dirname(filepath)) - six.exec_(python, self._symbols) + six.exec_(python, dict(self._symbols)) return list(self._parse_context._storage.objects) diff --git a/src/python/pants/engine/parser.py b/src/python/pants/engine/parser.py index d589c0f8629..fc1533be460 100644 --- a/src/python/pants/engine/parser.py +++ b/src/python/pants/engine/parser.py @@ -29,6 +29,11 @@ def constraint(self): return Exactly(*symbol_table_types, description='symbol table types') +class EmptyTable(SymbolTable): + def table(self): + return {} + + class Parser(AbstractClass): @abstractmethod diff --git a/tests/python/pants_test/engine/legacy/BUILD b/tests/python/pants_test/engine/legacy/BUILD index 4e72b941f79..10f307143ed 100644 --- a/tests/python/pants_test/engine/legacy/BUILD +++ b/tests/python/pants_test/engine/legacy/BUILD @@ -133,6 +133,16 @@ python_tests( tags = {'integration'} ) +python_tests( + name = 'parser', + sources = ['test_parser.py'], + dependencies = [ + 'src/python/pants/build_graph', + 'src/python/pants/engine/legacy:parser', + 'src/python/pants/engine:parser', + ] +) + python_tests( name = 'structs', sources = ['test_structs.py'], diff --git a/tests/python/pants_test/engine/legacy/test_parser.py b/tests/python/pants_test/engine/legacy/test_parser.py new file mode 100644 index 00000000000..13d43f4f61c --- /dev/null +++ b/tests/python/pants_test/engine/legacy/test_parser.py @@ -0,0 +1,24 @@ +# coding=utf-8 +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import unittest + +from pants.build_graph.build_file_aliases import BuildFileAliases +from pants.engine.legacy.parser import LegacyPythonCallbacksParser +from pants.engine.parser import EmptyTable + + +class LegacyPythonCallbacksParserTest(unittest.TestCase): + + def test_no_import_sideeffects(self): + # A parser with no symbols registered. + parser = LegacyPythonCallbacksParser(EmptyTable(), BuildFileAliases()) + # Call to import a module should succeed. + parser.parse('/dev/null', '''import os; os.path.join('x', 'y')''') + # But the imported module should not be visible as a symbol in further parses. + with self.assertRaises(NameError): + parser.parse('/dev/null', '''os.path.join('x', 'y')''') diff --git a/tests/python/pants_test/engine/scheduler_test_base.py b/tests/python/pants_test/engine/scheduler_test_base.py index 66bfa9fee01..e62ef6b75d9 100644 --- a/tests/python/pants_test/engine/scheduler_test_base.py +++ b/tests/python/pants_test/engine/scheduler_test_base.py @@ -10,19 +10,12 @@ from pants.base.file_system_project_tree import FileSystemProjectTree from pants.engine.nodes import Return -from pants.engine.parser import SymbolTable from pants.engine.scheduler import LocalScheduler from pants.util.contextutil import temporary_file_path from pants.util.dirutil import safe_mkdtemp, safe_rmtree from pants_test.engine.util import init_native -class EmptyTable(SymbolTable): - @classmethod - def table(cls): - return {} - - class SchedulerTestBase(object): """A mixin for classes (tests, presumably) which need to create temporary schedulers. diff --git a/tests/python/pants_test/engine/test_parsers.py b/tests/python/pants_test/engine/test_parsers.py index 8f0e37355f8..3f72f1ebf36 100644 --- a/tests/python/pants_test/engine/test_parsers.py +++ b/tests/python/pants_test/engine/test_parsers.py @@ -29,12 +29,6 @@ def __eq__(self, other): return isinstance(other, Bob) and self._key() == other._key() -class EmptyTable(parser.SymbolTable): - @classmethod - def table(cls): - return {} - - class TestTable(parser.SymbolTable): @classmethod def table(cls): @@ -53,7 +47,7 @@ def parse(parser, document, **args): class JsonParserTest(unittest.TestCase): def parse(self, document, symbol_table=None, **kwargs): - symbol_table = symbol_table or EmptyTable() + symbol_table = symbol_table or parser.EmptyTable() return parse(parsers.JsonParser(symbol_table), document, **kwargs) def round_trip(self, obj, symbol_table=None): @@ -229,7 +223,7 @@ def test_error_presentation(self): """).strip() filepath = '/dev/null' with self.assertRaises(parser.ParseError) as exc: - parsers.JsonParser(EmptyTable()).parse(filepath, document) + parsers.JsonParser(parser.EmptyTable()).parse(filepath, document) # Strip trailing whitespace from the message since our expected literal below will have # trailing ws stripped via editors and code reviews calling for it. @@ -327,7 +321,7 @@ def test_no_symbol_table(self): hobbies=[1, 2, 3] ) """) - results = parse(parsers.PythonAssignmentsParser(EmptyTable()), document) + results = parse(parsers.PythonAssignmentsParser(parser.EmptyTable()), document) self.assertEqual([Bob(name='nancy', hobbies=[1, 2, 3])], results) # No symbol table was used so no `type_alias` plumbing can be expected. From c244eb23c8917a48d095d396da902a092315f1e3 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 25 Sep 2017 17:38:40 -0600 Subject: [PATCH 60/67] Add default routing for OSX High Sierra binaries. (#4894) These are now uploaded to https://binaries.pantsbuild.org. Fixes #4893 --- src/python/pants/binaries/binary_util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/pants/binaries/binary_util.py b/src/python/pants/binaries/binary_util.py index d5ec5fbc3c7..5242f7de13c 100644 --- a/src/python/pants/binaries/binary_util.py +++ b/src/python/pants/binaries/binary_util.py @@ -34,6 +34,7 @@ ('darwin', '14'): ('mac', '10.10'), ('darwin', '15'): ('mac', '10.11'), ('darwin', '16'): ('mac', '10.12'), + ('darwin', '17'): ('mac', '10.13'), } From 6606016ad50eea570efb549bb84c22ff33320957 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 25 Sep 2017 21:01:34 -0600 Subject: [PATCH 61/67] Prepare the 1.4.0.dev13 release. (#4895) NB: This is the second attempt at this particular release and rolls up changes from the 1.4.0.dev12 tag, which went unreleased due to technical difficulties. --- src/python/pants/notes/master.rst | 45 ++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/src/python/pants/notes/master.rst b/src/python/pants/notes/master.rst index 48c454cc957..9c1bd8b1052 100644 --- a/src/python/pants/notes/master.rst +++ b/src/python/pants/notes/master.rst @@ -5,12 +5,21 @@ This document describes ``dev`` releases which occur weekly from master, and whi not undergo the vetting associated with ``stable`` releases. -1.4.0.dev13 (9/18/2017) +1.4.0.dev13 (9/25/2017) ----------------------- +New Features +~~~~~~~~~~~~ + +* Support wheels when loading plugins. (#4887) + `PR #4887 `_ + API Changes ~~~~~~~~~~~ +* Remove python 2.6 support completely. (#4871) + `PR #4871 `_ + * Bump pyopenssl==17.3.0 (#4872) `PR #4872 `_ @@ -20,17 +29,47 @@ API Changes Bugfixes ~~~~~~~~ +* Add default routing for OSX High Sierra binaries. (#4894) + `PR #4894 `_ + +* Reduce BUILD file parse pollution (#4892) + `PR #4892 `_ + +* Exit with error on error bootstrapping cffi (#4891) + `PR #4891 `_ + +* Only generate Android resource deps when needed. (#4888) + `PR #4888 `_ + * Re-pin to 2017Q2 TravisCI image. (#4869) `PR #4869 `_ +Documentation Updates +~~~~~~~~~~~~~~~~~~~~~ + +* Update the committer docs. (#4889) + `PR #4889 `_ + Refactoring, Improvements, and Tooling ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* Refactor test partitioning. (#4879) + `PR #4879 `_ + +* Leverage `subprocess32` subprocess backports. (#4851) + `PR #4851 `_ + +* Customize native engine build through code (#4876) + `PR #4876 `_ + * Move to SymbolTable/Parser instances (#4864) `PR #4864 `_ -1.4.0.dev12 (9/13/2017) ------------------------ +1.4.0.dev12 (9/13/2017) [UNRELEASED] +------------------------------------ + +NB: 1.4.0.dev12 was never released to pypi due to technical difficulties; its changes were rolled +up into 1.4.0.dev13 and released with it. API Changes ~~~~~~~~~~~ From c3de4ab06f13918de8dc637be6753b9b5dfb0e3c Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 25 Sep 2017 22:00:13 -0600 Subject: [PATCH 62/67] Release native engine binaries for OSX 10.13. (#4898) Fixes #4893 --- build-support/bin/native/utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-support/bin/native/utils.sh b/build-support/bin/native/utils.sh index 68ebcb15bc2..805897699a8 100644 --- a/build-support/bin/native/utils.sh +++ b/build-support/bin/native/utils.sh @@ -24,7 +24,7 @@ function get_native_engine_version() { readonly RUST_OSX_MIN_VERSION=7 # Bump this when there is a new OSX released: -readonly OSX_MAX_VERSION=12 +readonly OSX_MAX_VERSION=13 function get_rust_osx_versions() { seq ${RUST_OSX_MIN_VERSION} ${OSX_MAX_VERSION} From e24d1bcdfcd2b3ab8f89632039e817a345c4cd75 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 25 Sep 2017 22:02:57 -0600 Subject: [PATCH 63/67] Fixup release notes for 1.4.0.dev13. Include the native-engine binary release fix for 10.13 (#4898). --- src/python/pants/notes/master.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/python/pants/notes/master.rst b/src/python/pants/notes/master.rst index 9c1bd8b1052..ca040b3eb53 100644 --- a/src/python/pants/notes/master.rst +++ b/src/python/pants/notes/master.rst @@ -29,6 +29,9 @@ API Changes Bugfixes ~~~~~~~~ +* Release native engine binaries for OSX 10.13. (#4898) + `PR #4898 `_ + * Add default routing for OSX High Sierra binaries. (#4894) `PR #4894 `_ From 5a2f91ab295092d86744312e24f73ea7933c5004 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 26 Sep 2017 17:41:23 +0100 Subject: [PATCH 64/67] Fix -Wstrict-prototypes warnings (#4902) --- src/python/pants/engine/native.py | 4 ++-- src/python/pants/engine/native_engine_version | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python/pants/engine/native.py b/src/python/pants/engine/native.py index 7f1bc5bfd4c..74977cc482d 100644 --- a/src/python/pants/engine/native.py +++ b/src/python/pants/engine/native.py @@ -146,7 +146,7 @@ extern_ptr_invoke_runnable, TypeId); -Tasks* tasks_create(); +Tasks* tasks_create(void); void tasks_task_begin(Tasks*, Function, TypeConstraint); void tasks_add_select(Tasks*, TypeConstraint); void tasks_add_select_variant(Tasks*, TypeConstraint, Buffer); @@ -202,7 +202,7 @@ void nodes_destroy(RawNodes*); -void set_panic_handler(); +void set_panic_handler(void); ''' CFFI_EXTERNS = ''' diff --git a/src/python/pants/engine/native_engine_version b/src/python/pants/engine/native_engine_version index 8c616d60632..a72065f27f3 100644 --- a/src/python/pants/engine/native_engine_version +++ b/src/python/pants/engine/native_engine_version @@ -1 +1 @@ -69fb4d2a155fa4ec8a5d96559b9936b410e9b8ea +3b7f1c8c665357aee1b4ecf45afa5ae3da8d2359 From b6a1ac01fab22202e3a1ec3c35d29c8ad3d767cc Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Tue, 26 Sep 2017 13:17:40 -0400 Subject: [PATCH 65/67] managed_jar_dependencies: allow target()'s with jar_library dependencies (#4742) ### Problem It is useful to combine multiple targets into a single plain old `target`. Also, `managed_jar_dependencies` did not identify addresses in the `artifacts` section that violated its assumptions. ### Solution * Teach managed_jar_dependencies to accept the addresses of targets that have jar_library targets as transitive dependencies. * Identify the bad address when an artfiact is not a jar() object, the address of a jar_library target, or a target with jar_library's as transitive dependencies. --- .../subsystems/jar_dependency_management.py | 14 ++-- .../test_jar_dependency_management_setup.py | 70 +++++++++++++++++++ 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/src/python/pants/backend/jvm/subsystems/jar_dependency_management.py b/src/python/pants/backend/jvm/subsystems/jar_dependency_management.py index 052d826a917..3b1b6252be0 100644 --- a/src/python/pants/backend/jvm/subsystems/jar_dependency_management.py +++ b/src/python/pants/backend/jvm/subsystems/jar_dependency_management.py @@ -14,6 +14,7 @@ from pants.base.exceptions import TargetDefinitionException, TaskError from pants.base.revision import Revision from pants.build_graph.address_lookup_error import AddressLookupError +from pants.build_graph.target import Target from pants.java.jar.jar_dependency_utils import M2Coordinate from pants.subsystem.subsystem import Subsystem from pants.task.task import Task @@ -359,9 +360,9 @@ def _compute_artifact_sets(self, targets): .format(target.id, dm._artifact_set_map[target.id])) def _library_targets(self, managed_jar_dependencies): - for spec in managed_jar_dependencies.library_specs: - for target in self.context.resolve(spec): - yield target # NB(gmalmquist): I don't think this needs to be transitive. + targets = [t for spec in managed_jar_dependencies.library_specs for t in self.context.resolve(spec)] + for t in Target.closure_for_targets(targets): + yield t def _jar_iterator(self, managed_jar_dependencies): for jar in managed_jar_dependencies.payload.artifacts: @@ -370,10 +371,13 @@ def _jar_iterator(self, managed_jar_dependencies): if isinstance(dep, JarLibrary): for jar in dep.jar_dependencies: yield jar + elif type(dep) == Target: + pass else: raise TargetDefinitionException(managed_jar_dependencies, - 'Artifacts must be jar() objects or the addresses of ' - 'jar_library objects.') + 'Artifacts must be jar() objects, the address of a jar_library(), ' + 'or the address of a target() with jar_library()\'s as dependencies. ' + 'Spec {} does not refer to any of those.'.format(dep.address.spec)) def _compute_artifact_set(self, management_target): """Computes the set of pinned artifacts specified by this target, and any of its dependencies. diff --git a/tests/python/pants_test/backend/jvm/tasks/test_jar_dependency_management_setup.py b/tests/python/pants_test/backend/jvm/tasks/test_jar_dependency_management_setup.py index dcea5383286..a40d67cc22b 100644 --- a/tests/python/pants_test/backend/jvm/tasks/test_jar_dependency_management_setup.py +++ b/tests/python/pants_test/backend/jvm/tasks/test_jar_dependency_management_setup.py @@ -11,6 +11,7 @@ from pants.backend.jvm.targets.managed_jar_dependencies import (ManagedJarDependencies, ManagedJarLibraries) from pants.backend.jvm.targets.unpacked_jars import UnpackedJars +from pants.base.exceptions import TargetDefinitionException from pants.build_graph.target import Target from pants.java.jar.jar_dependency import JarDependency from pants.java.jar.jar_dependency_utils import M2Coordinate @@ -403,3 +404,72 @@ def check_task_execution(manager): manager = self._init_manager(default_target='//foo:management') with self.assertRaises(JarDependencyManagementSetup.IllegalVersionOverride): check_task_execution(manager) + + def test_artifacts_indirection(self): + jar_library_unversioned = self.make_target( + target_type=JarLibrary, + spec='//foo:library-unversioned', + jars=[ + JarDependency(org='foobar', name='foobar'), + ], + ) + jar_library_versioned = self.make_target( + target_type=JarLibrary, + spec='//foo:library-versioned', + jars=[ + JarDependency(org='foobar', name='foobar', rev='2'), + ], + ) + indirect_target = self.make_target( + target_type=Target, + spec='//foo:indirect-deps', + dependencies=[ + jar_library_versioned, + ]) + management_target = self.make_target( + target_type=ManagedJarDependencies, + spec='//foo:management', + artifacts=[ + '//foo:indirect-deps', + ]) + context = self.context(target_roots=[ + management_target, + indirect_target, + jar_library_versioned, + jar_library_unversioned + ]) + manager = self._init_manager(default_target='//foo:management') + task = self.create_task(context) + task.execute() + artifact_set = self._single_artifact_set(manager, [jar_library_unversioned]) + self.assertFalse(artifact_set is None) + self.assertEquals('2', artifact_set[M2Coordinate('foobar', 'foobar')].rev) + + def test_invalid_artifacts_indirection(self): + class DummyTarget(Target): + pass + dummy_target = self.make_target( + target_type=DummyTarget, + spec='//foo:dummy', + ) + indirect_target = self.make_target( + target_type=Target, + spec='//foo:indirect', + dependencies=[ + dummy_target, + ]) + management_target = self.make_target( + target_type=ManagedJarDependencies, + spec='//foo:management', + artifacts=[ + '//foo:indirect', + ]) + context = self.context(target_roots=[ + management_target, + indirect_target, + dummy_target, + ]) + self._init_manager() + task = self.create_task(context) + with self.assertRaises(TargetDefinitionException): + task.execute() From 8700f7233041225f975d42b151261f2b0d1f6e67 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Tue, 26 Sep 2017 18:18:52 +0100 Subject: [PATCH 66/67] Remove obsolete target-specific scripts (#4903) --- build-support/bin/native/bootstrap.sh | 31 ++++++--------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/build-support/bin/native/bootstrap.sh b/build-support/bin/native/bootstrap.sh index fee43e09f3a..7a2c488cbf6 100644 --- a/build-support/bin/native/bootstrap.sh +++ b/build-support/bin/native/bootstrap.sh @@ -11,7 +11,7 @@ # Exposes: # + calculate_current_hash: Calculates the current native engine version hash and echoes it to # stdout. -# + bootstrap_native_code: Builds target-specific native engine binaries. +# + bootstrap_native_code: Builds native engine binaries. REPO_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd ../../.. && pwd -P) source ${REPO_ROOT}/build-support/common.sh @@ -66,10 +66,7 @@ function _ensure_cffi_sources() { } function _ensure_build_prerequisites() { - # Control a pants-specific rust toolchain, optionally ensuring the given target toolchain is - # installed. - local readonly target=$1 - + # Control a pants-specific rust toolchain. export CARGO_HOME=${CACHE_ROOT}/rust-toolchain export RUSTUP_HOME=${CARGO_HOME} @@ -83,32 +80,16 @@ function _ensure_build_prerequisites() { rm -f ${rustup} ${RUSTUP_HOME}/bin/rustup override set stable 1>&2 fi - - if [[ -n "${target}" ]] - then - if ! ${RUSTUP_HOME}/bin/rustup target list | grep -E "${target} \((default|installed)\)" &> /dev/null - then - ${RUSTUP_HOME}/bin/rustup target add ${target} - fi - fi } function _build_native_code() { - # Builds the native code, optionally taking an explicit target triple arg, and echos the path of - # the built binary. - local readonly target=$1 - _ensure_build_prerequisites ${target} + # Builds the native code, and echos the path of the built binary. + _ensure_build_prerequisites local readonly cargo="${CARGO_HOME}/bin/cargo" local readonly build_cmd="${cargo} build --manifest-path ${NATIVE_ROOT}/Cargo.toml ${MODE_FLAG}" - if [[ -z "${target}" ]] - then - ${build_cmd} || die - echo "${NATIVE_ROOT}/target/${MODE}/libengine.${LIB_EXTENSION}" - else - ${build_cmd} --target ${target} || echo "FAILED to build for target ${target}" - echo "${NATIVE_ROOT}/target/${target}/${MODE}/libengine.${LIB_EXTENSION}" - fi + ${build_cmd} || die + echo "${NATIVE_ROOT}/target/${MODE}/libengine.${LIB_EXTENSION}" } function bootstrap_native_code() { From a4b5793120e34aa93da2930a464c135c9de99b37 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 27 Sep 2017 17:21:47 +0100 Subject: [PATCH 67/67] Run rust tests on travis (#4899) Currently there are no tests, so it runs zero tests, but some will be coming soon. --- .travis.yml | 34 +++++++++++++++++---------- build-support/bin/ci.sh | 19 ++++++++++++++- build-support/bin/native/bootstrap.sh | 17 +++++++++----- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc4b67a30af..d9a934735ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -109,7 +109,7 @@ matrix: env: - SHARD="Various pants self checks and lint" script: - - ./build-support/bin/ci.sh -x -cjlpn 'Various pants self checks and lint' + - ./build-support/bin/ci.sh -x -cejlpn 'Various pants self checks and lint' - os: linux dist: trusty @@ -137,7 +137,7 @@ matrix: env: - SHARD="Unit tests for pants and pants-plugins - shard 1" script: - - ./build-support/bin/ci.sh -x -fkmsrcnt -u 0/2 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmsrcnt -u 0/2 "${SHARD}" - os: linux dist: trusty @@ -165,7 +165,7 @@ matrix: env: - SHARD="Unit tests for pants and pants-plugins - shard 2" script: - - ./build-support/bin/ci.sh -x -fkmsrcnt -u 1/2 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmsrcnt -u 1/2 "${SHARD}" - os: linux dist: trusty @@ -193,7 +193,7 @@ matrix: env: - SHARD="Python contrib tests - shard 1" script: - - ./build-support/bin/ci.sh -x -fkmsrcjlpt -y 0/2 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmsrcjlpt -y 0/2 "${SHARD}" - os: linux dist: trusty @@ -221,7 +221,7 @@ matrix: env: - SHARD="Python contrib tests - shard 2" script: - - ./build-support/bin/ci.sh -x -fkmsrcjlpt -y 1/2 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmsrcjlpt -y 1/2 "${SHARD}" - os: linux dist: trusty @@ -249,7 +249,7 @@ matrix: env: - SHARD="Python integration tests for pants - shard 1" script: - - ./build-support/bin/ci.sh -x -fkmsrjlpnt -i 0/7 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 0/7 "${SHARD}" - os: linux dist: trusty @@ -277,7 +277,7 @@ matrix: env: - SHARD="Python integration tests for pants - shard 2" script: - - ./build-support/bin/ci.sh -x -fkmsrjlpnt -i 1/7 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 1/7 "${SHARD}" - os: linux dist: trusty @@ -305,7 +305,7 @@ matrix: env: - SHARD="Python integration tests for pants - shard 3" script: - - ./build-support/bin/ci.sh -x -fkmsrjlpnt -i 2/7 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 2/7 "${SHARD}" - os: linux dist: trusty @@ -333,7 +333,7 @@ matrix: env: - SHARD="Python integration tests for pants - shard 4" script: - - ./build-support/bin/ci.sh -x -fkmsrjlpnt -i 3/7 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 3/7 "${SHARD}" - os: linux dist: trusty @@ -361,7 +361,7 @@ matrix: env: - SHARD="Python integration tests for pants - shard 5" script: - - ./build-support/bin/ci.sh -x -fkmsrjlpnt -i 4/7 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 4/7 "${SHARD}" - os: linux dist: trusty @@ -389,7 +389,7 @@ matrix: env: - SHARD="Python integration tests for pants - shard 6" script: - - ./build-support/bin/ci.sh -x -fkmsrjlpnt -i 5/7 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 5/7 "${SHARD}" - os: linux dist: trusty @@ -417,7 +417,17 @@ matrix: env: - SHARD="Python integration tests for pants - shard 7" script: - - ./build-support/bin/ci.sh -x -fkmsrjlpnt -i 6/7 "${SHARD}" + - ./build-support/bin/ci.sh -x -efkmsrjlpnt -i 6/7 "${SHARD}" + + # Rust tests + - os: linux + dist: trusty + language: python + python: "2.7.13" + env: + - SHARD="Rust tests" + script: + - ./build-support/bin/ci.sh -abcfjklmnprstx deploy: # See: https://docs.travis-ci.com/user/deployment/s3/ diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index 097f5e7d2fc..28cd2a2b4be 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -32,6 +32,7 @@ function usage() { echo " to run only even tests: '-u 0/2', odd: '-u 1/2'" echo " -a skip android targets when running tests" echo " -n skip contrib python tests" + echo " -e skip rust tests" echo " -y SHARD_NUMBER/TOTAL_SHARDS" echo " if running contrib python tests, divide them into" echo " TOTAL_SHARDS shards and just run those in SHARD_NUMBER" @@ -58,7 +59,7 @@ python_unit_shard="0/1" python_contrib_shard="0/1" python_intg_shard="0/1" -while getopts "hfxbkmsrjlpu:ny:ci:at" opt; do +while getopts "hfxbkmsrjlpeu:ny:ci:at" opt; do case ${opt} in h) usage ;; f) skip_pre_commit_checks="true" ;; @@ -72,6 +73,7 @@ while getopts "hfxbkmsrjlpu:ny:ci:at" opt; do l) skip_internal_backends="true" ;; p) skip_python="true" ;; u) python_unit_shard=${OPTARG} ;; + e) skip_rust_tests="true" ;; n) skip_contrib="true" ;; y) python_contrib_shard=${OPTARG} ;; c) skip_integration="true" ;; @@ -225,6 +227,21 @@ if [[ "${skip_contrib:-false}" == "false" ]]; then end_travis_section fi +if [[ "${skip_rust_tests:-false}" == "false" ]]; then + source "${REPO_ROOT}/build-support/bin/native/bootstrap.sh" + + start_travis_section "RustTests" "Running Pants rust tests" + ( + source "${REPO_ROOT}/build-support/pants_venv" + activate_pants_venv + PANTS_SRCPATH="${REPO_ROOT}/src/python" prepare_to_build_native_code + export RUST_BACKTRACE=1 + "${CARGO_HOME}/bin/cargo" test --all --manifest-path="${REPO_ROOT}/src/rust/engine/Cargo.toml" + ) || die "Pants rust test failure" + end_travis_section +fi + + if [[ "${skip_integration:-false}" == "false" ]]; then if [[ "0/1" != "${python_intg_shard}" ]]; then shard_desc=" [shard ${python_intg_shard}]" diff --git a/build-support/bin/native/bootstrap.sh b/build-support/bin/native/bootstrap.sh index 7a2c488cbf6..4794ef30edb 100644 --- a/build-support/bin/native/bootstrap.sh +++ b/build-support/bin/native/bootstrap.sh @@ -62,11 +62,12 @@ function calculate_current_hash() { function _ensure_cffi_sources() { # N.B. Here we assume that higher level callers have already setup the pants' venv and $PANTS_SRCPATH. - PYTHONPATH="${PANTS_SRCPATH}:${PYTHONPATH}" python "${CFFI_BOOTSTRAPPER}" "$@" + PYTHONPATH="${PANTS_SRCPATH}:${PYTHONPATH}" python "${CFFI_BOOTSTRAPPER}" "${NATIVE_ROOT}/src/cffi" >&2 } function _ensure_build_prerequisites() { # Control a pants-specific rust toolchain. + export CARGO_HOME=${CACHE_ROOT}/rust-toolchain export RUSTUP_HOME=${CARGO_HOME} @@ -82,9 +83,17 @@ function _ensure_build_prerequisites() { fi } +function prepare_to_build_native_code() { + # Must happen in the pants venv and have PANTS_SRCPATH set. + + _ensure_build_prerequisites + _ensure_cffi_sources +} + function _build_native_code() { # Builds the native code, and echos the path of the built binary. - _ensure_build_prerequisites + + prepare_to_build_native_code local readonly cargo="${CARGO_HOME}/bin/cargo" local readonly build_cmd="${cargo} build --manifest-path ${NATIVE_ROOT}/Cargo.toml ${MODE_FLAG}" @@ -97,12 +106,8 @@ function bootstrap_native_code() { # version if needed. local native_engine_version="$(calculate_current_hash)" local target_binary="${NATIVE_ENGINE_CACHE_TARGET_DIR}/${native_engine_version}/${NATIVE_ENGINE_BINARY}" - local cffi_output_dir="${NATIVE_ROOT}/src/cffi" - local cffi_env_script="${cffi_output_dir}/${NATIVE_ENGINE_MODULE}.cflags" if [ ! -f "${target_binary}" ] then - _ensure_cffi_sources "${cffi_output_dir}" - export CFLAGS="$(cat "${cffi_env_script}")" local readonly native_binary="$(_build_native_code)" # If bootstrapping the native engine fails, don't attempt to run pants