From 5d825ab3efa45f44017df59a3d58e3efe40b8d44 Mon Sep 17 00:00:00 2001 From: "Mikhail I. Izmestev" Date: Wed, 5 Sep 2018 12:44:31 +0400 Subject: [PATCH 1/7] Migrate to MSVC 2015 compiler Prepare to add msbuild integration tests. MSVC 2015 support both v120 (MSVC 2013) and v140 toolchains. --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index ad5673a4..7dfcef7f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,8 +16,8 @@ install: - 7z x memcached.zip -y - ps: $Memcached = Start-Process memcached\memcached.exe -PassThru - # Make compiler available (use MSVC 2013, 32 bit) - - call "%ProgramFiles(x86)%\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" x86 + # Make compiler available (use MSVC 2015, 32 bit) + - call "%ProgramFiles(x86)%\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86 # Check compiler version - cl From 0485e901438114b2797fd8a2ddb6c9970be68b70 Mon Sep 17 00:00:00 2001 From: "Mikhail I. Izmestev" Date: Tue, 14 Aug 2018 09:46:02 +0400 Subject: [PATCH 2/7] Add msbuild integration tests * test incremental build with VS2013 * test incremental build with VS2015 * test clean target with VS2015 causing clcache files removed --- setup.py | 1 + tests/integrationtests/msbuild/another.cpp | 1 + tests/integrationtests/msbuild/test.vcxproj | 30 ++++++ tests/test_integration.py | 103 ++++++++++++++++++++ 4 files changed, 135 insertions(+) create mode 100644 tests/integrationtests/msbuild/another.cpp create mode 100644 tests/integrationtests/msbuild/test.vcxproj diff --git a/setup.py b/setup.py index 6804caf8..d5f661ca 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ entry_points={ 'console_scripts': [ 'clcache = clcache.__main__:main', + 'cl.cache = clcache.__main__:main', 'clcache-server = clcache.server.__main__:main', ] }, diff --git a/tests/integrationtests/msbuild/another.cpp b/tests/integrationtests/msbuild/another.cpp new file mode 100644 index 00000000..f96747dd --- /dev/null +++ b/tests/integrationtests/msbuild/another.cpp @@ -0,0 +1 @@ +int somefunc() { return 1; } diff --git a/tests/integrationtests/msbuild/test.vcxproj b/tests/integrationtests/msbuild/test.vcxproj new file mode 100644 index 00000000..cee1f3a9 --- /dev/null +++ b/tests/integrationtests/msbuild/test.vcxproj @@ -0,0 +1,30 @@ + + + + + Release + Win32 + + + + + Application + + + + + + + + + OldStyle + + + + + ProgramDatabase + + + + + diff --git a/tests/test_integration.py b/tests/test_integration.py index ced806ec..acb2d880 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1182,6 +1182,109 @@ def testEvictedManifest(self): self.assertEqual(subprocess.call(cmd, env=customEnv), 0) +@pytest.mark.skipif(os.environ["VisualStudioVersion"] < "14.0", reason="Require newer visual studio") +class TestMSBuildV140(unittest.TestCase): + def _clean(self): + cmd = self.getBuildCmd() + subprocess.check_call(cmd + ["/t:Clean"]) + + def setUp(self): + with cd(os.path.join(ASSETS_DIR, "msbuild")): + self._clean() + + def getBuildCmd(self): + return ["msbuild", "/p:Configuration=Release", "/nologo", "/verbosity:minimal", + "/p:PlatformToolset=v140", "/p:ClToolExe=clcache.exe"] + + def testClean(self): + with tempfile.TemporaryDirectory(dir=os.path.join(ASSETS_DIR, "msbuild")) as tempDir: + customEnv = dict(os.environ, CLCACHE_DIR=tempDir) + + with cd(os.path.join(ASSETS_DIR, "msbuild")): + cmd = self.getBuildCmd() + + # Compile once to insert the objects in the cache + subprocess.check_call(cmd, env=customEnv) + + # build Clean target + subprocess.check_call(cmd + ["/t:Clean"], env=customEnv) + + cache = clcache.Cache(tempDir) + with cache.statistics as stats: + self.assertEqual(stats.numCallsForExternalDebugInfo(), 1) + self.assertEqual(stats.numCacheEntries(), 2) + + def testIncrementalBuild(self): + with tempfile.TemporaryDirectory(dir=os.path.join(ASSETS_DIR, "msbuild")) as tempDir: + customEnv = dict(os.environ, CLCACHE_DIR=tempDir) + cmd = self.getBuildCmd() + + with cd(os.path.join(ASSETS_DIR, "msbuild")): + # Compile once to insert the objects in the cache + subprocess.check_call(cmd, env=customEnv) + + output = subprocess.check_output(cmd, env=customEnv, stderr=subprocess.STDOUT) + output = output.decode("utf-8") + + + self.assertTrue("another.cpp" not in output) + self.assertTrue("minimal.cpp" not in output) + self.assertTrue("fibonacci.cpp" not in output) + + +class TestMSBuildV120(unittest.TestCase): + def _clean(self): + cmd = self.getBuildCmd() + subprocess.check_call(cmd + ["/t:Clean"]) + + # workaround due to cl.cache.exe is not frozen it create no cl.read.1.tlog, but + # this file is important for v120 toolchain, see comment at getMSBuildCmd + try: + os.makedirs(os.path.join("Release", "test.tlog")) + except FileExistsError: + pass + with open(os.path.join("Release", "test.tlog", "cl.read.1.tlog"), "w"),\ + open(os.path.join("Release", "test.tlog", "cl.write.1.tlog"), "w"): + pass + + def setUp(self): + with cd(os.path.join(ASSETS_DIR, "msbuild")): + self._clean() + + def getBuildCmd(self): + # v120 toolchain hardcoded "cl.read.1.tlog" and "cl.*.read.1.tlog" + # file names to inspect as input dependency. + # The best way to use clcache with v120 toolchain is to froze clcache to cl.exe + # and then specify ClToolPath property. + + # There is no frozen cl.exe in tests available, as workaround we would use cl.cache.exe + # and manually create cl.read.1.tlog empty file, without this file msbuild think that + # FileTracker created wrong tlogs. + return ["msbuild", "/p:Configuration=Release", "/nologo", "/verbosity:minimal", + "/p:PlatformToolset=v120", "/p:ClToolExe=cl.cache.exe"] + + def testIncrementalBuild(self): + with tempfile.TemporaryDirectory(dir=os.path.join(ASSETS_DIR, "msbuild")) as tempDir: + customEnv = dict(os.environ, CLCACHE_DIR=tempDir) + cmd = self.getBuildCmd() + + with cd(os.path.join(ASSETS_DIR, "msbuild")): + # Compile once to insert the objects in the cache + subprocess.check_call(cmd, env=customEnv) + + self._clean() + + # Compile using cached objects + subprocess.check_call(cmd, env=customEnv) + + output = subprocess.check_output(cmd, env=customEnv, stderr=subprocess.STDOUT) + output = output.decode("utf-8") + + self.assertTrue("another.cpp" not in output) + self.assertTrue("minimal.cpp" not in output) + self.assertTrue("fibonacci.cpp" not in output) + + if __name__ == '__main__': unittest.TestCase.longMessage = True unittest.main() From 2e13e112e886b5c86da2f6dd1182e1452f7242bb Mon Sep 17 00:00:00 2001 From: "Mikhail I. Izmestev" Date: Tue, 14 Aug 2018 09:50:56 +0400 Subject: [PATCH 3/7] Add tracker suspend operations FileTracker.dll exports API SuspendTracking/ResumeTracking --- clcache/__main__.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/clcache/__main__.py b/clcache/__main__.py index daa77fe0..4eee9ac0 100644 --- a/clcache/__main__.py +++ b/clcache/__main__.py @@ -116,6 +116,44 @@ def normalizeBaseDir(baseDir): return None +class SuspendTracker(): + fileTracker = None + def __init__(self): + if not SuspendTracker.fileTracker: + if windll.kernel32.GetModuleHandleW("FileTracker.dll"): + SuspendTracker.fileTracker = windll.FileTracker + elif windll.kernel32.GetModuleHandleW("FileTracker32.dll"): + SuspendTracker.fileTracker = windll.FileTracker32 + + def __enter__(self): + SuspendTracker.suspend() + + def __exit__(self, typ, value, traceback): + SuspendTracker.resume() + + @staticmethod + def suspend(): + if SuspendTracker.fileTracker: + SuspendTracker.fileTracker.SuspendTracking() + + @staticmethod + def resume(): + if SuspendTracker.fileTracker: + SuspendTracker.fileTracker.ResumeTracking() + +def isTrackerEnabled(): + return 'TRACKER_ENABLED' in os.environ + +def untrackable(func): + if not isTrackerEnabled(): + return func + + def untrackedFunc(*args, **kwargs): + with SuspendTracker(): + return func(*args, **kwargs) + + return untrackedFunc + @contextlib.contextmanager def atomicWrite(fileName): tempFileName = fileName + '.new' @@ -193,6 +231,7 @@ def manifestPath(self, manifestHash): def manifestFiles(self): return filesBeneath(self.manifestSectionDir) + @untrackable def setManifest(self, manifestHash, manifest): manifestPath = self.manifestPath(manifestHash) printTraceStatement("Writing manifest with manifestHash = {} to {}".format(manifestHash, manifestPath)) @@ -203,6 +242,7 @@ def setManifest(self, manifestHash, manifest): jsonobject = {'entries': entries} json.dump(jsonobject, outFile, sort_keys=True, indent=2) + @untrackable def getManifest(self, manifestHash): fileName = self.manifestPath(manifestHash) if not os.path.exists(fileName): @@ -741,6 +781,7 @@ def __init__(self, statsFile): self._stats = None self.lock = CacheLock.forPath(self._statsFile) + @untrackable def __enter__(self): self._stats = PersistentJSONDict(self._statsFile) for k in Statistics.RESETTABLE_KEYS | Statistics.NON_RESETTABLE_KEYS: @@ -748,6 +789,7 @@ def __enter__(self): self._stats[k] = 0 return self + @untrackable def __exit__(self, typ, value, traceback): # Does not write to disc when unchanged self._stats.save() From 1aebba37d534db37ac78b45b4d36cdb71251f8e2 Mon Sep 17 00:00:00 2001 From: "Mikhail I. Izmestev" Date: Wed, 5 Sep 2018 13:42:27 +0400 Subject: [PATCH 4/7] Use process pool executor when tracker active v120 toolchain require that source files read operations will happen in main thread --- clcache/__main__.py | 8 +++++++- pyinstaller/clcache_main.py | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/clcache/__main__.py b/clcache/__main__.py index 4eee9ac0..22b83b8c 100644 --- a/clcache/__main__.py +++ b/clcache/__main__.py @@ -1687,7 +1687,13 @@ def scheduleJobs(cache: Any, compiler: str, cmdLine: List[str], environment: Any exitCode = 0 cleanupRequired = False - with concurrent.futures.ThreadPoolExecutor(max_workers=jobCount(cmdLine)) as executor: + + def poolExecutor(*args, **kwargs) -> concurrent.futures.Executor: + if isTrackerEnabled(): + return concurrent.futures.ProcessPoolExecutor(*args, **kwargs) + return concurrent.futures.ThreadPoolExecutor(*args, **kwargs) + + with poolExecutor(max_workers=min(jobCount(cmdLine), len(objectFiles))) as executor: jobs = [] for (srcFile, srcLanguage), objFile in zip(sourceFiles, objectFiles): jobCmdLine = baseCmdLine + [srcLanguage + srcFile] diff --git a/pyinstaller/clcache_main.py b/pyinstaller/clcache_main.py index 42327ad8..b9bddbc8 100644 --- a/pyinstaller/clcache_main.py +++ b/pyinstaller/clcache_main.py @@ -1,2 +1,4 @@ +import multiprocessing from clcache.__main__ import main +multiprocessing.freeze_support() main() From 5e3fbafb4aaf4d2996c0d0eb3643ae0c32bb4aea Mon Sep 17 00:00:00 2001 From: Tihomir Heidelberg Date: Tue, 8 Jan 2019 17:30:45 +0100 Subject: [PATCH 5/7] Detect if 64-bit file tracker (FileTracker64.dll) is used --- clcache/__main__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clcache/__main__.py b/clcache/__main__.py index 22b83b8c..a075afcd 100644 --- a/clcache/__main__.py +++ b/clcache/__main__.py @@ -124,6 +124,8 @@ def __init__(self): SuspendTracker.fileTracker = windll.FileTracker elif windll.kernel32.GetModuleHandleW("FileTracker32.dll"): SuspendTracker.fileTracker = windll.FileTracker32 + elif windll.kernel32.GetModuleHandleW("FileTracker64.dll"): + SuspendTracker.fileTracker = windll.FileTracker64 def __enter__(self): SuspendTracker.suspend() From ca27e392f9689c99bd3df52b135260e67bb90500 Mon Sep 17 00:00:00 2001 From: George Tattersall <5692370+Cascades@users.noreply.github.com> Date: Thu, 15 Aug 2019 13:56:44 +0100 Subject: [PATCH 6/7] Add check for toolsetVersion --- clcache/__main__.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/clcache/__main__.py b/clcache/__main__.py index 2622b082..34204065 100644 --- a/clcache/__main__.py +++ b/clcache/__main__.py @@ -71,6 +71,10 @@ NMPWAIT_WAIT_FOREVER = wintypes.DWORD(0xFFFFFFFF) ERROR_PIPE_BUSY = 231 +# Toolset version 140 +# https://devblogs.microsoft.com/cppblog/side-by-side-minor-version-msvc-toolsets-in-visual-studio-2017/ +TOOLSET_VERSION_140 = 140 + # ManifestEntry: an entry in a manifest file # `includeFiles`: list of paths to include files, which this source file uses # `includesContentsHash`: hash of the contents of the includeFiles @@ -1730,6 +1734,35 @@ def filterSourceFiles(cmdLine: List[str], sourceFiles: List[Tuple[str, str]]) -> if not (arg in setOfSources or arg.startswith(skippedArgs)) ) + +def findCompilerVersion(compiler: str) -> int: + compilerInfo = subprocess.Popen([compiler], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + compilerVersionLine = compilerInfo.communicate()[0].decode('utf-8').splitlines()[0] + compilerVersion = compilerVersionLine[compilerVersionLine.find("Version ") + 8: + compilerVersionLine.find(" for")] + return int(compilerVersion[:2] + compilerVersion[3:5]) + + +def findToolsetVersion(compilerVersion: int) -> int: + versionMap = {1400: 80, + 1500: 90, + 1600: 100, + 1700: 110, + 1800: 120, + 1900: 140} + + if compilerVersion in versionMap: + return versionMap[compilerVersion] + elif 1910 <= compilerVersion < 1920: + return 141 + elif 1920 <= compilerVersion < 1930: + return 142 + else: + raise LogicException('Bad cl.exe version: {}'.format(compilerVersion)) + + def scheduleJobs(cache: Any, compiler: str, cmdLine: List[str], environment: Any, sourceFiles: List[Tuple[str, str]], objectFiles: List[str]) -> int: # Filter out all source files from the command line to form baseCmdLine @@ -1740,7 +1773,8 @@ def scheduleJobs(cache: Any, compiler: str, cmdLine: List[str], environment: Any def poolExecutor(*args, **kwargs) -> concurrent.futures.Executor: if isTrackerEnabled(): - return concurrent.futures.ProcessPoolExecutor(*args, **kwargs) + if findToolsetVersion(findCompilerVersion(compiler)) < TOOLSET_VERSION_140: + return concurrent.futures.ProcessPoolExecutor(*args, **kwargs) return concurrent.futures.ThreadPoolExecutor(*args, **kwargs) with poolExecutor(max_workers=min(jobCount(cmdLine), len(objectFiles))) as executor: From 87f41b511cabe2beb4d0c35773cdb7d09614e85d Mon Sep 17 00:00:00 2001 From: George Tattersall <5692370+Cascades@users.noreply.github.com> Date: Fri, 16 Aug 2019 11:52:25 +0100 Subject: [PATCH 7/7] Added unit tests --- tests/test_unit.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_unit.py b/tests/test_unit.py index 88863b37..0861e985 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -1166,6 +1166,29 @@ def testDecompression(self): self.assertNotEqual(os.path.getsize(srcFilePath), os.path.getsize(tmpFilePath)) self.assertEqual(os.path.getsize(srcFilePath), os.path.getsize(dstFilePath)) +class TestToolsetVersion(unittest.TestCase): + def testCorrectMapping(self): + from clcache.__main__ import findToolsetVersion + + for i in range(0, 5): + compVer, toolVer = ((i * 100) + 1400), ((i * 10) + 80) + self.assertEqual(findToolsetVersion(compVer), toolVer) + self.assertEqual(findToolsetVersion(1900), 140) + for compVer in range(1910, 1920): + self.assertEqual(findToolsetVersion(compVer), 141) + for compVer in range(1920, 1930): + self.assertEqual(findToolsetVersion(compVer), 142) + + def testIncorrectMapping(self): + from clcache.__main__ import findToolsetVersion + from clcache.__main__ import LogicException + + with self.assertRaises(LogicException): + findToolsetVersion(100) + with self.assertRaises(LogicException): + findToolsetVersion(1456) + with self.assertRaises(LogicException): + findToolsetVersion(1930) if __name__ == '__main__': unittest.TestCase.longMessage = True