Add msbuild tracker support

appveyor.yml

@@ -16,8 +16,8 @@ install:
   - 7z x memcached.zip -y
   - ps: $Memcached = Start-Process memcached\memcached.exe -PassThru
 
-  # Make compiler available (use MSVC 2013, 32 bit)
-  - call "%ProgramFiles(x86)%\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" x86
+  # Make compiler available (use MSVC 2015, 32 bit)
+  - call "%ProgramFiles(x86)%\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86
 
   # Check compiler version
   - cl

clcache/__main__.py

@@ -71,6 +71,10 @@
 NMPWAIT_WAIT_FOREVER = wintypes.DWORD(0xFFFFFFFF)
 ERROR_PIPE_BUSY = 231
 
+# Toolset version 140
+# https://devblogs.microsoft.com/cppblog/side-by-side-minor-version-msvc-toolsets-in-visual-studio-2017/
+TOOLSET_VERSION_140 = 140
+
 # ManifestEntry: an entry in a manifest file
 # `includeFiles`: list of paths to include files, which this source file uses
 # `includesContentsHash`: hash of the contents of the includeFiles
@@ -119,6 +123,46 @@ def normalizeBaseDir(baseDir):
         return None
 
 
+class SuspendTracker():
+    fileTracker = None
+    def __init__(self):
+        if not SuspendTracker.fileTracker:
+            if windll.kernel32.GetModuleHandleW("FileTracker.dll"):
+                SuspendTracker.fileTracker = windll.FileTracker
+            elif windll.kernel32.GetModuleHandleW("FileTracker32.dll"):
+                SuspendTracker.fileTracker = windll.FileTracker32
+            elif windll.kernel32.GetModuleHandleW("FileTracker64.dll"):
+                SuspendTracker.fileTracker = windll.FileTracker64
+
+    def __enter__(self):
+        SuspendTracker.suspend()
+
+    def __exit__(self, typ, value, traceback):
+        SuspendTracker.resume()
+
+    @staticmethod
+    def suspend():
+        if SuspendTracker.fileTracker:
+            SuspendTracker.fileTracker.SuspendTracking()
+
+    @staticmethod
+    def resume():
+        if SuspendTracker.fileTracker:
+            SuspendTracker.fileTracker.ResumeTracking()
+
+def isTrackerEnabled():
+    return 'TRACKER_ENABLED' in os.environ
+
+def untrackable(func):
+    if not isTrackerEnabled():
+        return func
+
+    def untrackedFunc(*args, **kwargs):
+        with SuspendTracker():
+            return func(*args, **kwargs)
+
+    return untrackedFunc
+
 def getCachedCompilerConsoleOutput(path):
     try:
         with open(path, 'rb') as f:
@@ -188,6 +232,7 @@ def manifestPath(self, manifestHash):
     def manifestFiles(self):
         return filesBeneath(self.manifestSectionDir)
 
+    @untrackable
     def setManifest(self, manifestHash, manifest):
         manifestPath = self.manifestPath(manifestHash)
         printTraceStatement("Writing manifest with manifestHash = {} to {}".format(manifestHash, manifestPath))
@@ -198,6 +243,7 @@ def setManifest(self, manifestHash, manifest):
             jsonobject = {'entries': entries}
             json.dump(jsonobject, outFile, sort_keys=True, indent=2)
 
+    @untrackable
     def getManifest(self, manifestHash):
         fileName = self.manifestPath(manifestHash)
         if not os.path.exists(fileName):
@@ -738,13 +784,15 @@ def __init__(self, statsFile):
         self._stats = None
         self.lock = CacheLock.forPath(self._statsFile)
 
+    @untrackable
     def __enter__(self):
         self._stats = PersistentJSONDict(self._statsFile)
         for k in Statistics.RESETTABLE_KEYS | Statistics.NON_RESETTABLE_KEYS:
             if k not in self._stats:
                 self._stats[k] = 0
         return self
 
+    @untrackable
     def __exit__(self, typ, value, traceback):
         # Does not write to disc when unchanged
         self._stats.save()
@@ -1686,14 +1734,50 @@ def filterSourceFiles(cmdLine: List[str], sourceFiles: List[Tuple[str, str]]) ->
         if not (arg in setOfSources or arg.startswith(skippedArgs))
     )
 
+
+def findCompilerVersion(compiler: str) -> int:
+    compilerInfo = subprocess.Popen([compiler],
+                                    stdout=subprocess.PIPE,
+                                    stderr=subprocess.STDOUT)
+    compilerVersionLine = compilerInfo.communicate()[0].decode('utf-8').splitlines()[0]
+    compilerVersion = compilerVersionLine[compilerVersionLine.find("Version ") + 8:
+                                          compilerVersionLine.find(" for")]
+    return int(compilerVersion[:2] + compilerVersion[3:5])
+
+
+def findToolsetVersion(compilerVersion: int) -> int:
+    versionMap = {1400: 80,
+                  1500: 90,
+                  1600: 100,
+                  1700: 110,
+                  1800: 120,
+                  1900: 140}
+
+    if compilerVersion in versionMap:
+        return versionMap[compilerVersion]
+    elif 1910 <= compilerVersion < 1920:
+        return 141
+    elif 1920 <= compilerVersion < 1930:
+        return 142
+    else:
+        raise LogicException('Bad cl.exe version: {}'.format(compilerVersion))
+
+
 def scheduleJobs(cache: Any, compiler: str, cmdLine: List[str], environment: Any,
                  sourceFiles: List[Tuple[str, str]], objectFiles: List[str]) -> int:
     # Filter out all source files from the command line to form baseCmdLine
     baseCmdLine = [arg for arg in filterSourceFiles(cmdLine, sourceFiles) if not arg.startswith('/MP')]
 
     exitCode = 0
     cleanupRequired = False
-    with concurrent.futures.ThreadPoolExecutor(max_workers=jobCount(cmdLine)) as executor:
+
+    def poolExecutor(*args, **kwargs) -> concurrent.futures.Executor:
+        if isTrackerEnabled():
+            if findToolsetVersion(findCompilerVersion(compiler)) < TOOLSET_VERSION_140:
+                return concurrent.futures.ProcessPoolExecutor(*args, **kwargs)
+        return concurrent.futures.ThreadPoolExecutor(*args, **kwargs)
+
+    with poolExecutor(max_workers=min(jobCount(cmdLine), len(objectFiles))) as executor:
         jobs = []
         for (srcFile, srcLanguage), objFile in zip(sourceFiles, objectFiles):
             jobCmdLine = baseCmdLine + [srcLanguage + srcFile]

pyinstaller/clcache_main.py

@@ -1,2 +1,4 @@
+import multiprocessing
 from clcache.__main__ import main
+multiprocessing.freeze_support()
 main()

setup.py

@@ -23,6 +23,7 @@
     entry_points={
           'console_scripts': [
               'clcache = clcache.__main__:main',
+              'cl.cache = clcache.__main__:main',
               'clcache-server = clcache.server.__main__:main',
           ]
     },

tests/integrationtests/msbuild/another.cpp

@@ -0,0 +1 @@
+int somefunc() { return 1; }

tests/integrationtests/msbuild/test.vcxproj

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+
+  <PropertyGroup Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+  </PropertyGroup>
+
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <DebugInformationFormat>OldStyle</DebugInformationFormat>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="another.cpp">
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <ClCompile Include="../minimal.cpp" />
+    <ClCompile Include="../fibonacci.cpp" />
+  </ItemGroup>
+</Project>

tests/test_integration.py

@@ -1195,6 +1195,109 @@ def testEvictedManifest(self):
             self.assertEqual(subprocess.call(cmd, env=customEnv), 0)
 
 
+@pytest.mark.skipif(os.environ["VisualStudioVersion"] < "14.0", reason="Require newer visual studio")
+class TestMSBuildV140(unittest.TestCase):
+    def _clean(self):
+        cmd = self.getBuildCmd()
+        subprocess.check_call(cmd + ["/t:Clean"])
+
+    def setUp(self):
+        with cd(os.path.join(ASSETS_DIR, "msbuild")):
+            self._clean()
+
+    def getBuildCmd(self):
+        return ["msbuild", "/p:Configuration=Release", "/nologo", "/verbosity:minimal",
+                "/p:PlatformToolset=v140", "/p:ClToolExe=clcache.exe"]
+
+    def testClean(self):
+        with tempfile.TemporaryDirectory(dir=os.path.join(ASSETS_DIR, "msbuild")) as tempDir:
+            customEnv = dict(os.environ, CLCACHE_DIR=tempDir)
+
+            with cd(os.path.join(ASSETS_DIR, "msbuild")):
+                cmd = self.getBuildCmd()
+
+                # Compile once to insert the objects in the cache
+                subprocess.check_call(cmd, env=customEnv)
+
+                # build Clean target
+                subprocess.check_call(cmd + ["/t:Clean"], env=customEnv)
+
+            cache = clcache.Cache(tempDir)
+            with cache.statistics as stats:
+                self.assertEqual(stats.numCallsForExternalDebugInfo(), 1)
+                self.assertEqual(stats.numCacheEntries(), 2)
+
+    def testIncrementalBuild(self):
+        with tempfile.TemporaryDirectory(dir=os.path.join(ASSETS_DIR, "msbuild")) as tempDir:
+            customEnv = dict(os.environ, CLCACHE_DIR=tempDir)
+            cmd = self.getBuildCmd()
+
+            with cd(os.path.join(ASSETS_DIR, "msbuild")):
+                # Compile once to insert the objects in the cache
+                subprocess.check_call(cmd, env=customEnv)
+
+                output = subprocess.check_output(cmd, env=customEnv, stderr=subprocess.STDOUT)
+                output = output.decode("utf-8")
+
+
+            self.assertTrue("another.cpp" not in output)
+            self.assertTrue("minimal.cpp" not in output)
+            self.assertTrue("fibonacci.cpp" not in output)
+
+
+class TestMSBuildV120(unittest.TestCase):
+    def _clean(self):
+        cmd = self.getBuildCmd()
+        subprocess.check_call(cmd + ["/t:Clean"])
+
+        # workaround due to cl.cache.exe is not frozen it create no cl.read.1.tlog, but
+        # this file is important for v120 toolchain, see comment at getMSBuildCmd
+        try:
+            os.makedirs(os.path.join("Release", "test.tlog"))
+        except FileExistsError:
+            pass
+        with open(os.path.join("Release", "test.tlog", "cl.read.1.tlog"), "w"),\
+             open(os.path.join("Release", "test.tlog", "cl.write.1.tlog"), "w"):
+            pass
+
+    def setUp(self):
+        with cd(os.path.join(ASSETS_DIR, "msbuild")):
+            self._clean()
+
+    def getBuildCmd(self):
+        # v120 toolchain hardcoded "cl.read.1.tlog" and "cl.*.read.1.tlog"
+        # file names to inspect as input dependency.
+        # The best way to use clcache with v120 toolchain is to froze clcache to cl.exe
+        # and then specify ClToolPath property.
+
+        # There is no frozen cl.exe in tests available, as workaround we would use cl.cache.exe
+        # and manually create cl.read.1.tlog empty file, without this file msbuild think that
+        # FileTracker created wrong tlogs.
+        return ["msbuild", "/p:Configuration=Release", "/nologo", "/verbosity:minimal",
+                "/p:PlatformToolset=v120", "/p:ClToolExe=cl.cache.exe"]
+
+    def testIncrementalBuild(self):
+        with tempfile.TemporaryDirectory(dir=os.path.join(ASSETS_DIR, "msbuild")) as tempDir:
+            customEnv = dict(os.environ, CLCACHE_DIR=tempDir)
+            cmd = self.getBuildCmd()
+
+            with cd(os.path.join(ASSETS_DIR, "msbuild")):
+                # Compile once to insert the objects in the cache
+                subprocess.check_call(cmd, env=customEnv)
+
+                self._clean()
+
+                # Compile using cached objects
+                subprocess.check_call(cmd, env=customEnv)
+
+                output = subprocess.check_output(cmd, env=customEnv, stderr=subprocess.STDOUT)
+                output = output.decode("utf-8")
+
+            self.assertTrue("another.cpp" not in output)
+            self.assertTrue("minimal.cpp" not in output)
+            self.assertTrue("fibonacci.cpp" not in output)
+
+
 if __name__ == '__main__':
     unittest.TestCase.longMessage = True
     unittest.main()

tests/test_unit.py

@@ -1166,6 +1166,29 @@ def testDecompression(self):
             self.assertNotEqual(os.path.getsize(srcFilePath), os.path.getsize(tmpFilePath))
             self.assertEqual(os.path.getsize(srcFilePath), os.path.getsize(dstFilePath))
 
+class TestToolsetVersion(unittest.TestCase):
+    def testCorrectMapping(self):
+        from clcache.__main__ import findToolsetVersion
+
+        for i in range(0, 5):
+            compVer, toolVer = ((i * 100) + 1400), ((i * 10) + 80)
+            self.assertEqual(findToolsetVersion(compVer), toolVer)
+        self.assertEqual(findToolsetVersion(1900), 140)
+        for compVer in range(1910, 1920):
+            self.assertEqual(findToolsetVersion(compVer), 141)
+        for compVer in range(1920, 1930):
+            self.assertEqual(findToolsetVersion(compVer), 142)
+
+    def testIncorrectMapping(self):
+        from clcache.__main__ import findToolsetVersion
+        from clcache.__main__ import LogicException
+
+        with self.assertRaises(LogicException):
+            findToolsetVersion(100)
+        with self.assertRaises(LogicException):
+            findToolsetVersion(1456)
+        with self.assertRaises(LogicException):
+            findToolsetVersion(1930)
 
 if __name__ == '__main__':
     unittest.TestCase.longMessage = True