Add make targets for fast linting and testing

Fast linting only needs modified files with pylint and diff with
pycodestyle. It's good enough to detect most code errors very fast. It
typically takes less than 10 seconds. A complete full pylint run uses
all CPU cores for several minutes. PEP 8 violations are typically
reported after 30 minutes to several hours on Travis CI.

Fast lintings uses git diff and git merge-base to find all modified
files in a branch or working tree. There is no easy way to find the
branch source. On Travis the information is provided by Travis. For
local development it's a new variable IPA_GIT_BRANCH in VERSION.m4.

Fast testing execute all unit tests that do not depend on ipalib.api.

In total it takes about 30-40 seconds (!) to execute linting, PEP 8 checks
and unittests for both Python 2 and 3.

Signed-off-by: Christian Heimes <[email protected]>
Reviewed-By: Alexander Bokovoy <[email protected]>

.gitignore

@@ -112,3 +112,4 @@ freeipa2-dev-doc
 /ipapython/version.py
 /ipapython/.DEFAULT_PLUGINS
 
+/ipatests/.cache/

BUILD.txt

@@ -66,9 +66,9 @@ changes are required.
 Testing
 -------
 
-For more information, see http://www.freeipa.org/page/Testing
+For more information, see https://www.freeipa.org/page/Testing
 
-We use python nosetests to test for regressions in the management framework
+We use python pytest to test for regressions in the management framework
 and plugins. All test dependencies are required by the freeipa-tests package.
 
 To run all of the tests you will need 2 sessions, one to run the lite-server
@@ -82,6 +82,14 @@ Some tests may be skipped. For example, all the XML-RPC tests will be skipped
 if you haven't started the lite-server. The DNS tests will be skipped if
 the underlying IPA installation doesn't configure DNS, etc.
 
+To just execute fast unittest and code linters, use the fastcheck target.
+Fast tests only execute a subset of the test suite that does not depend on
+an initialized API and server instance. Fast linting just verifies modified
+files / lines.
+
+% make fastcheck
+
+
 API.txt
 -------
 The purpose of the file API.txt is to prevent accidental API changes. The

Makefile.am

@@ -169,20 +169,63 @@ if ! WITH_PYTHON2
 	@echo "ERROR: python2 not available"; exit 1
 endif
 	@ # run all linters, tests, and check with Python 2
-	PYTHONPATH=$(top_srcdir) $(PYTHON2) ipatests/ipa-run-tests \
+	PYTHONPATH=$(abspath $(top_srcdir)) $(PYTHON2) ipatests/ipa-run-tests \
 	    --ipaclient-unittests
 	$(MAKE) $(AM_MAKEFLAGS) acilint apilint polint jslint check
 	$(MAKE) $(AM_MAKEFLAGS) PYTHON=$(PYTHON2) pylint
 if WITH_PYTHON3
 	@ # just tests and pylint on Python 3
-	PYTHONPATH=$(top_srcdir) $(PYTHON3) ipatests/ipa-run-tests \
+	PYTHONPATH=$(abspath $(top_srcdir)) $(PYTHON3) ipatests/ipa-run-tests \
 	    --ipaclient-unittests
 	$(MAKE) $(AM_MAKEFLAGS) PYTHON=$(PYTHON3) pylint
 else
 	@echo "WARNING: python3 not available"
 endif
 	@echo "All tests passed."
 
+.PHONY: fastcheck fasttest fastlint
+fastcheck:
+if WITH_PYTHON2
+	@$(MAKE) -j1 $(AM_MAKEFLAGS) PYTHON=$(PYTHON2) fastlint fasttest
+endif
+if WITH_PYTHON3
+	@$(MAKE) -j1 $(AM_MAKEFLAGS) PYTHON=$(PYTHON3) fastlint fasttest
+endif
+
+fasttest: $(GENERATED_PYTHON_FILES) ipasetup.py
+	@ # --ignore doubles speed of total test run compared to pytest.skip()
+	@ # on module.
+	PYTHONPATH=$(abspath $(top_srcdir)) $(PYTHON3) ipatests/ipa-run-tests \
+	    --skip-ipaapi \
+	    --ignore $(abspath $(top_srcdir))/ipatests/test_integration \
+	    --ignore $(abspath $(top_srcdir))/ipatests/test_xmlrpc
+
+fastlint: $(GENERATED_PYTHON_FILES) ipasetup.py
+if ! WITH_PYLINT
+	@echo "ERROR: pylint not available"; exit 1
+endif
+	@echo "Fast linting with $(PYTHON) from branch '$(GIT_BRANCH)'"
+
+	@MERGEBASE=$$(git merge-base --fork-point $(GIT_BRANCH)); \
+	FILES=$$(git diff --name-only $${MERGEBASE} \
+	    | grep -E '\.py$$'); \
+	if [ -n "$${FILES}" ]; then \
+		echo "Fast linting files: $${FILES}"; \
+		echo "pylint"; \
+		echo "------"; \
+	    PYTHONPATH=$(abspath $(top_srcdir)) $(PYTHON) -m pylint \
+	        --rcfile=$(top_srcdir)/pylintrc \
+	        --load-plugins pylint_plugins \
+	        $${FILES} || exit $?; \
+	    echo "pycodestyle"; \
+	    echo "-----------"; \
+	    git diff $${MERGEBASE} | \
+	        $(PYTHON) -m pycodestyle --diff || exit $?; \
+	else \
+	    echo "No modified Python files found"; \
+	fi
+
+
 .PHONY: $(top_builddir)/ipaplatform/override.py
 $(top_builddir)/ipaplatform/override.py:
 	(cd $(top_builddir)/ipaplatform && make override.py)

VERSION.m4

@@ -48,6 +48,16 @@ define(IPA_VERSION_PRE_RELEASE, )
 ########################################################
 define(IPA_VERSION_IS_GIT_SNAPSHOT, yes)
 
+########################################################
+# git development branch:                              #
+#                                                      #
+# - master: define(IPA_GIT_BRANCH, master)             #
+# - ipa-X-X: define(IPA_GIT_BRANCH,                    #
+#       ipa-IPA_VERSION_MAJOR-IPA_VERSION_MINOR)       #
+########################################################
+define(IPA_GIT_BRANCH, master)
+dnl define(IPA_GIT_BRANCH, ipa-IPA_VERSION_MAJOR-IPA_VERSION_MINOR)
+
 ########################################################
 # The version of IPA data. This is used to identify    #
 # incompatibilities in data that could cause issues    #
@@ -128,6 +138,7 @@ NEWLINE)) dnl IPA_VERSION end
 dnl DEBUG: uncomment following lines and run command m4 VERSION.m4
 dnl `IPA_VERSION: ''IPA_VERSION'
 dnl `IPA_GIT_VERSION: ''IPA_GIT_VERSION'
+dnf `IPA_GIT_BRANCH: ''IPA_GIT_BRANCH'
 dnl `IPA_API_VERSION: ''IPA_API_VERSION'
 dnl `IPA_DATA_VERSION: ''IPA_DATA_VERSION'
 dnl `IPA_NUM_VERSION: ''IPA_NUM_VERSION'

configure.ac

@@ -364,6 +364,7 @@ AC_SUBST([NUM_VERSION], [IPA_NUM_VERSION])
 AC_SUBST(VENDOR_SUFFIX)
 AC_SUBST([VERSION], [IPA_VERSION])
 AC_SUBST([GIT_VERSION], [IPA_GIT_VERSION])
+AC_SUBST([GIT_BRANCH], [IPA_GIT_BRANCH])
 # used by Makefile.am for files depending on templates
 AC_SUBST([CONFIG_STATUS])
 

freeipa.spec.in

@@ -178,18 +178,20 @@ BuildRequires:  python3-wheel
 %endif # with_wheels
 
 #
-# Build dependencies for lint
+# Build dependencies for lint and fastcheck
 #
 %if 0%{?with_lint}
 BuildRequires:  python2-samba
 # 1.6: x509.Name.rdns (https://github.com/pyca/cryptography/issues/3199)
 BuildRequires:  python2-cryptography >= 1.6
 BuildRequires:  python2-gssapi >= 1.2.0-5
+BuildRequires:  softhsm
 %if 0%{?fedora} >= 26
 BuildRequires:  python2-pylint
 %else
 BuildRequires:  pylint >= 1.7
 %endif
+BuildRequires:  python2-pycodestyle
 # workaround for https://bugzilla.redhat.com/show_bug.cgi?id=1096506
 BuildRequires:  python2-polib
 BuildRequires:  python2-libipa_hbac
@@ -227,6 +229,7 @@ BuildRequires:  python3-samba
 BuildRequires:  python3-cryptography >= 1.6
 BuildRequires:  python3-gssapi >= 1.2.0
 BuildRequires:  python3-pylint >= 1.7
+BuildRequires:  python3-pycodestyle
 # workaround for https://bugzilla.redhat.com/show_bug.cgi?id=1096506
 BuildRequires:  python3-polib
 BuildRequires:  python3-libipa_hbac