Add msbuild tracker support

appveyor.yml

@@ -16,8 +16,8 @@ install:
   - 7z x memcached.zip -y
   - ps: $Memcached = Start-Process memcached\memcached.exe -PassThru
 
-  # Make compiler available (use MSVC 2013, 32 bit)
-  - call "%ProgramFiles(x86)%\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" x86
+  # Make compiler available (use MSVC 2015, 32 bit)
+  - call "%ProgramFiles(x86)%\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86
 
   # Check compiler version
   - cl

clcache/__main__.py

@@ -116,6 +116,46 @@ def normalizeBaseDir(baseDir):
         return None
 
 
+class SuspendTracker():
+    fileTracker = None
+    def __init__(self):
+        if not SuspendTracker.fileTracker:
+            if windll.kernel32.GetModuleHandleW("FileTracker.dll"):
+                SuspendTracker.fileTracker = windll.FileTracker
+            elif windll.kernel32.GetModuleHandleW("FileTracker32.dll"):
+                SuspendTracker.fileTracker = windll.FileTracker32
+            elif windll.kernel32.GetModuleHandleW("FileTracker64.dll"):
+                SuspendTracker.fileTracker = windll.FileTracker64
+
+    def __enter__(self):
+        SuspendTracker.suspend()
+
+    def __exit__(self, typ, value, traceback):
+        SuspendTracker.resume()
+
+    @staticmethod
+    def suspend():
+        if SuspendTracker.fileTracker:
+            SuspendTracker.fileTracker.SuspendTracking()
+
+    @staticmethod
+    def resume():
+        if SuspendTracker.fileTracker:
+            SuspendTracker.fileTracker.ResumeTracking()
+
+def isTrackerEnabled():
+    return 'TRACKER_ENABLED' in os.environ
+
+def untrackable(func):
+    if not isTrackerEnabled():
+        return func
+
+    def untrackedFunc(*args, **kwargs):
+        with SuspendTracker():
+            return func(*args, **kwargs)
+
+    return untrackedFunc
+
 @contextlib.contextmanager
 def atomicWrite(fileName):
     tempFileName = fileName + '.new'
@@ -193,6 +233,7 @@ def manifestPath(self, manifestHash):
     def manifestFiles(self):
         return filesBeneath(self.manifestSectionDir)
 
+    @untrackable
     def setManifest(self, manifestHash, manifest):
         manifestPath = self.manifestPath(manifestHash)
         printTraceStatement("Writing manifest with manifestHash = {} to {}".format(manifestHash, manifestPath))
@@ -203,6 +244,7 @@ def setManifest(self, manifestHash, manifest):
             jsonobject = {'entries': entries}
             json.dump(jsonobject, outFile, sort_keys=True, indent=2)
 
+    @untrackable
     def getManifest(self, manifestHash):
         fileName = self.manifestPath(manifestHash)
         if not os.path.exists(fileName):
@@ -741,13 +783,15 @@ def __init__(self, statsFile):
         self._stats = None
         self.lock = CacheLock.forPath(self._statsFile)
 
+    @untrackable
     def __enter__(self):
         self._stats = PersistentJSONDict(self._statsFile)
         for k in Statistics.RESETTABLE_KEYS | Statistics.NON_RESETTABLE_KEYS:
             if k not in self._stats:
                 self._stats[k] = 0
         return self
 
+    @untrackable
     def __exit__(self, typ, value, traceback):
         # Does not write to disc when unchanged
         self._stats.save()
@@ -1645,7 +1689,13 @@ def scheduleJobs(cache: Any, compiler: str, cmdLine: List[str], environment: Any
 
     exitCode = 0
     cleanupRequired = False
-    with concurrent.futures.ThreadPoolExecutor(max_workers=jobCount(cmdLine)) as executor:
+
+    def poolExecutor(*args, **kwargs) -> concurrent.futures.Executor:
+        if isTrackerEnabled():
+            return concurrent.futures.ProcessPoolExecutor(*args, **kwargs)
+        return concurrent.futures.ThreadPoolExecutor(*args, **kwargs)
+
+    with poolExecutor(max_workers=min(jobCount(cmdLine), len(objectFiles))) as executor:
         jobs = []
         for (srcFile, srcLanguage), objFile in zip(sourceFiles, objectFiles):
             jobCmdLine = baseCmdLine + [srcLanguage + srcFile]

pyinstaller/clcache_main.py

@@ -1,2 +1,4 @@
+import multiprocessing
 from clcache.__main__ import main
+multiprocessing.freeze_support()
 main()

setup.py

@@ -22,6 +22,7 @@
     entry_points={
           'console_scripts': [
               'clcache = clcache.__main__:main',
+              'cl.cache = clcache.__main__:main',
               'clcache-server = clcache.server.__main__:main',
           ]
     },

tests/integrationtests/msbuild/another.cpp

@@ -0,0 +1 @@
+int somefunc() { return 1; }

tests/integrationtests/msbuild/test.vcxproj

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+
+  <PropertyGroup Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+  </PropertyGroup>
+
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <DebugInformationFormat>OldStyle</DebugInformationFormat>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="another.cpp">
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <ClCompile Include="../minimal.cpp" />
+    <ClCompile Include="../fibonacci.cpp" />
+  </ItemGroup>
+</Project>

tests/test_integration.py

@@ -1182,6 +1182,109 @@ def testEvictedManifest(self):
             self.assertEqual(subprocess.call(cmd, env=customEnv), 0)
 
 
+@pytest.mark.skipif(os.environ["VisualStudioVersion"] < "14.0", reason="Require newer visual studio")
+class TestMSBuildV140(unittest.TestCase):
+    def _clean(self):
+        cmd = self.getBuildCmd()
+        subprocess.check_call(cmd + ["/t:Clean"])
+
+    def setUp(self):
+        with cd(os.path.join(ASSETS_DIR, "msbuild")):
+            self._clean()
+
+    def getBuildCmd(self):
+        return ["msbuild", "/p:Configuration=Release", "/nologo", "/verbosity:minimal",
+                "/p:PlatformToolset=v140", "/p:ClToolExe=clcache.exe"]
+
+    def testClean(self):
+        with tempfile.TemporaryDirectory(dir=os.path.join(ASSETS_DIR, "msbuild")) as tempDir:
+            customEnv = dict(os.environ, CLCACHE_DIR=tempDir)
+
+            with cd(os.path.join(ASSETS_DIR, "msbuild")):
+                cmd = self.getBuildCmd()
+
+                # Compile once to insert the objects in the cache
+                subprocess.check_call(cmd, env=customEnv)
+
+                # build Clean target
+                subprocess.check_call(cmd + ["/t:Clean"], env=customEnv)
+
+            cache = clcache.Cache(tempDir)
+            with cache.statistics as stats:
+                self.assertEqual(stats.numCallsForExternalDebugInfo(), 1)
+                self.assertEqual(stats.numCacheEntries(), 2)
+
+    def testIncrementalBuild(self):
+        with tempfile.TemporaryDirectory(dir=os.path.join(ASSETS_DIR, "msbuild")) as tempDir:
+            customEnv = dict(os.environ, CLCACHE_DIR=tempDir)
+            cmd = self.getBuildCmd()
+
+            with cd(os.path.join(ASSETS_DIR, "msbuild")):
+                # Compile once to insert the objects in the cache
+                subprocess.check_call(cmd, env=customEnv)
+
+                output = subprocess.check_output(cmd, env=customEnv, stderr=subprocess.STDOUT)
+                output = output.decode("utf-8")
+
+
+            self.assertTrue("another.cpp" not in output)
+            self.assertTrue("minimal.cpp" not in output)
+            self.assertTrue("fibonacci.cpp" not in output)
+
+
+class TestMSBuildV120(unittest.TestCase):
+    def _clean(self):
+        cmd = self.getBuildCmd()
+        subprocess.check_call(cmd + ["/t:Clean"])
+
+        # workaround due to cl.cache.exe is not frozen it create no cl.read.1.tlog, but
+        # this file is important for v120 toolchain, see comment at getMSBuildCmd
+        try:
+            os.makedirs(os.path.join("Release", "test.tlog"))
+        except FileExistsError:
+            pass
+        with open(os.path.join("Release", "test.tlog", "cl.read.1.tlog"), "w"),\
+             open(os.path.join("Release", "test.tlog", "cl.write.1.tlog"), "w"):
+            pass
+
+    def setUp(self):
+        with cd(os.path.join(ASSETS_DIR, "msbuild")):
+            self._clean()
+
+    def getBuildCmd(self):
+        # v120 toolchain hardcoded "cl.read.1.tlog" and "cl.*.read.1.tlog"
+        # file names to inspect as input dependency.
+        # The best way to use clcache with v120 toolchain is to froze clcache to cl.exe
+        # and then specify ClToolPath property.
+
+        # There is no frozen cl.exe in tests available, as workaround we would use cl.cache.exe
+        # and manually create cl.read.1.tlog empty file, without this file msbuild think that
+        # FileTracker created wrong tlogs.
+        return ["msbuild", "/p:Configuration=Release", "/nologo", "/verbosity:minimal",
+                "/p:PlatformToolset=v120", "/p:ClToolExe=cl.cache.exe"]
+
+    def testIncrementalBuild(self):
+        with tempfile.TemporaryDirectory(dir=os.path.join(ASSETS_DIR, "msbuild")) as tempDir:
+            customEnv = dict(os.environ, CLCACHE_DIR=tempDir)
+            cmd = self.getBuildCmd()
+
+            with cd(os.path.join(ASSETS_DIR, "msbuild")):
+                # Compile once to insert the objects in the cache
+                subprocess.check_call(cmd, env=customEnv)
+
+                self._clean()
+
+                # Compile using cached objects
+                subprocess.check_call(cmd, env=customEnv)
+
+                output = subprocess.check_output(cmd, env=customEnv, stderr=subprocess.STDOUT)
+                output = output.decode("utf-8")
+
+            self.assertTrue("another.cpp" not in output)
+            self.assertTrue("minimal.cpp" not in output)
+            self.assertTrue("fibonacci.cpp" not in output)
+
+
 if __name__ == '__main__':
     unittest.TestCase.longMessage = True
     unittest.main()