Add support of precompiled headers

clcache/__main__.py

@@ -271,6 +271,14 @@ def allSectionsLocked(repository):
             section.lock.release()
 
 
+def readPchHash(pchFile):
+    with open(pchFile+".clcache", 'r') as f:
+        return f.read()
+
+def writePchHash(pchFile, hashSum):
+    with open(pchFile+".clcache", 'w') as f:
+        f.write('{}'.format(hashSum))
+
 class ManifestRepository:
     # Bump this counter whenever the current manifest file format changes.
     # E.g. changing the file format from {'oldkey': ...} to {'newkey': ...} requires
@@ -333,6 +341,10 @@ def getManifestHash(compilerBinary, commandLine, sourceFile):
 
         additionalData = "{}|{}|{}".format(
             compilerHash, commandLine, ManifestRepository.MANIFEST_FILE_FORMAT_VERSION)
+
+        if 'Yu' in arguments:
+            pchFile = CommandLineAnalyzer.getPchFileName(arguments, sourceFile)
+            additionalData += readPchHash(pchFile)
         return getFileHash(sourceFile, additionalData)
 
     @staticmethod
@@ -519,6 +531,10 @@ def computeKeyNodirect(compilerBinary, commandLine, environment):
         h.update(preprocessedSourceCode)
         return h.hexdigest()
 
+    @staticmethod
+    def computePchKey(cacheKey):
+        return cacheKey + "-pch"
+
     @staticmethod
     def _normalizedCommandLine(cmdline):
         # Remove all arguments from the command line which only influence the
@@ -745,7 +761,6 @@ class Statistics:
     CALLS_WITH_INVALID_ARGUMENT = "CallsWithInvalidArgument"
     CALLS_WITHOUT_SOURCE_FILE = "CallsWithoutSourceFile"
     CALLS_WITH_MULTIPLE_SOURCE_FILES = "CallsWithMultipleSourceFiles"
-    CALLS_WITH_PCH = "CallsWithPch"
     CALLS_FOR_LINKING = "CallsForLinking"
     CALLS_FOR_EXTERNAL_DEBUG_INFO = "CallsForExternalDebugInfo"
     CALLS_FOR_PREPROCESSING = "CallsForPreprocessing"
@@ -761,7 +776,6 @@ class Statistics:
         CALLS_WITH_INVALID_ARGUMENT,
         CALLS_WITHOUT_SOURCE_FILE,
         CALLS_WITH_MULTIPLE_SOURCE_FILES,
-        CALLS_WITH_PCH,
         CALLS_FOR_LINKING,
         CALLS_FOR_EXTERNAL_DEBUG_INFO,
         CALLS_FOR_PREPROCESSING,
@@ -815,12 +829,6 @@ def numCallsWithMultipleSourceFiles(self):
     def registerCallWithMultipleSourceFiles(self):
         self._stats[Statistics.CALLS_WITH_MULTIPLE_SOURCE_FILES] += 1
 
-    def numCallsWithPch(self):
-        return self._stats[Statistics.CALLS_WITH_PCH]
-
-    def registerCallWithPch(self):
-        self._stats[Statistics.CALLS_WITH_PCH] += 1
-
     def numCallsForLinking(self):
         return self._stats[Statistics.CALLS_FOR_LINKING]
 
@@ -913,10 +921,6 @@ class CalledForLinkError(AnalysisError):
     pass
 
 
-class CalledWithPchError(AnalysisError):
-    pass
-
-
 class ExternalDebugInfoError(AnalysisError):
     pass
 
@@ -1308,6 +1312,20 @@ def parseArgumentsAndInputFiles(cmdline):
 
         return dict(arguments), inputFiles
 
+    @staticmethod
+    def getPchFileName(options, inputFile):
+        if 'Fp' in options:
+            return options['Fp'][0]
+        if 'Yc' in options:
+            headerName = options['Yc'][0]
+        elif 'Yu' in options:
+            headerName = options['Yu'][0]
+        else:
+            return None
+        if not headerName:
+            headerName = inputFile
+        return basenameWithoutExtension(headerName) + '.pch'
+
     @staticmethod
     def analyze(cmdline: List[str]) -> Tuple[List[Tuple[str, str]], List[str]]:
         options, inputFiles = CommandLineAnalyzer.parseArgumentsAndInputFiles(cmdline)
@@ -1336,9 +1354,6 @@ def analyze(cmdline: List[str]) -> Tuple[List[Tuple[str, str]], List[str]]:
         if 'Zi' in options:
             raise ExternalDebugInfoError()
 
-        if 'Yc' in options or 'Yu' in options:
-            raise CalledWithPchError()
-
         if 'link' in options or 'c' not in options:
             raise CalledForLinkError()
 
@@ -1358,6 +1373,10 @@ def analyze(cmdline: List[str]) -> Tuple[List[Tuple[str, str]], List[str]]:
             # Generate from .c/.cpp filenames
             objectFiles = [os.path.join(prefix, basenameWithoutExtension(f)) + '.obj' for f, _ in inputFiles]
 
+        if 'Yc' in options:
+            objectFiles = [(objectFile, CommandLineAnalyzer.getPchFileName(options, inputFile[0]))
+                           for inputFile, objectFile in zip(inputFiles, objectFiles)]
+
         printTraceStatement("Compiler source files: {}".format(inputFiles))
         printTraceStatement("Compiler object file: {}".format(objectFiles))
         return inputFiles, objectFiles
@@ -1439,8 +1458,7 @@ def printStatistics(cache):
     called for linking         : {}
     called for external debug  : {}
     called w/o source          : {}
-    called w/ multiple sources : {}
-    called w/ PCH              : {}""".strip()
+    called w/ multiple sources : {}""".strip()
 
     with cache.statistics.lock, cache.statistics as stats, cache.configuration as cfg:
         print(template.format(
@@ -1459,7 +1477,6 @@ def printStatistics(cache):
             stats.numCallsForExternalDebugInfo(),
             stats.numCallsWithoutSourceFile(),
             stats.numCallsWithMultipleSourceFiles(),
-            stats.numCallsWithPch(),
         ))
 
 
@@ -1531,6 +1548,7 @@ def addObjectToCache(stats, cache, cachekey, artifacts):
 
 def processCacheHit(cache, objectFile, cachekey):
     printTraceStatement("Reusing cached object for key {} for object file {}".format(cachekey, objectFile))
+    objectFile, pchFile = objectFile if isinstance(objectFile, tuple) else (objectFile, None)
 
     with cache.lockFor(cachekey):
         with cache.statistics.lock, cache.statistics as stats:
@@ -1541,6 +1559,11 @@ def processCacheHit(cache, objectFile, cachekey):
 
         cachedArtifacts = cache.getEntry(cachekey)
         copyOrLink(cachedArtifacts.objectFilePath, objectFile)
+        if pchFile is not None:
+            pchCachedArtifacts = cache.getEntry(CompilerArtifactsRepository.computePchKey(cachekey))
+            copyOrLink(pchCachedArtifacts.objectFilePath, pchFile)
+            writePchHash(pchFile, cachekey)
+
         printTraceStatement("Finished. Exit code 0")
         return 0, cachedArtifacts.stdout, cachedArtifacts.stderr, False
 
@@ -1653,9 +1676,6 @@ def processCompileRequest(cache, compiler, args):
     except MultipleSourceFilesComplexError:
         printTraceStatement("Cannot cache invocation as {}: multiple source files found".format(cmdLine))
         updateCacheStatistics(cache, Statistics.registerCallWithMultipleSourceFiles)
-    except CalledWithPchError:
-        printTraceStatement("Cannot cache invocation as {}: precompiled headers in use".format(cmdLine))
-        updateCacheStatistics(cache, Statistics.registerCallWithPch)
     except CalledForLinkError:
         printTraceStatement("Cannot cache invocation as {}: called for linking".format(cmdLine))
         updateCacheStatistics(cache, Statistics.registerCallForLinking)
@@ -1804,6 +1824,8 @@ def processNoDirect(cache, objectFile, compiler, cmdLine, environment):
 def ensureArtifactsExist(cache, cachekey, reason, objectFile, compilerResult, extraCallable=None):
     cleanupRequired = False
     returnCode, compilerOutput, compilerStderr = compilerResult
+    objectFile, pchFile = objectFile if isinstance(objectFile, tuple) else (objectFile, None)
+
     correctCompiliation = (returnCode == 0 and os.path.exists(objectFile))
     with cache.lockFor(cachekey):
         if not cache.hasEntry(cachekey):
@@ -1814,6 +1836,17 @@ def ensureArtifactsExist(cache, cachekey, reason, objectFile, compilerResult, ex
                     cleanupRequired = addObjectToCache(stats, cache, cachekey, artifacts)
             if extraCallable and correctCompiliation:
                 extraCallable()
+
+    if pchFile:
+        writePchHash(pchFile, cachekey)
+        cachekey = CompilerArtifactsRepository.computePchKey(cachekey)
+        with cache.lockFor(cachekey):
+            if not cache.hasEntry(cachekey):
+                with cache.statistics.lock, cache.statistics as stats:
+                    if correctCompiliation:
+                        artifacts = CompilerArtifacts(pchFile, "", "")
+                        cleanupRequired = addObjectToCache(stats, cache, cachekey, artifacts) or cleanupRequired
+
     return returnCode, compilerOutput, compilerStderr, cleanupRequired
 
 

tests/test_integration.py

@@ -957,7 +957,6 @@ def testClearPostcondition(self):
             self.assertEqual(stats.numCacheEntries(), 0)
             self.assertEqual(stats.numCallsWithoutSourceFile(), oldStats.numCallsWithoutSourceFile())
             self.assertEqual(stats.numCallsWithMultipleSourceFiles(), oldStats.numCallsWithMultipleSourceFiles())
-            self.assertEqual(stats.numCallsWithPch(), oldStats.numCallsWithPch())
             self.assertEqual(stats.numCallsForLinking(), oldStats.numCallsForLinking())
             self.assertEqual(stats.numCallsForPreprocessing(), oldStats.numCallsForPreprocessing())
             self.assertEqual(stats.numCallsForExternalDebugInfo(), oldStats.numCallsForExternalDebugInfo())
@@ -995,8 +994,6 @@ def testAllKnownAnalysisErrors(self):
             subprocess.check_call(baseCmd + ['/c', '/Tcfibonacci.c', "minimal.cpp"])
             # CalledForLinkError
             subprocess.check_call(baseCmd + ["fibonacci.cpp"])
-            # CalledWithPchError
-            subprocess.check_call(baseCmd + ['/c', '/Yc', "minimal.cpp"])
             # ExternalDebugInfoError
             subprocess.check_call(baseCmd + ['/c', '/Zi', "minimal.cpp"])
             # CalledForPreprocessingError

tests/test_unit.py

@@ -172,7 +172,6 @@ def testHitCounts(self):
             self.assertEqual(s.numCallsWithInvalidArgument(), 0)
             self.assertEqual(s.numCallsWithoutSourceFile(), 0)
             self.assertEqual(s.numCallsWithMultipleSourceFiles(), 0)
-            self.assertEqual(s.numCallsWithPch(), 0)
             self.assertEqual(s.numCallsForLinking(), 0)
             self.assertEqual(s.numCallsForExternalDebugInfo(), 0)
             self.assertEqual(s.numEvictedMisses(), 0)
@@ -186,7 +185,6 @@ def testHitCounts(self):
             s.registerCallWithInvalidArgument()
             s.registerCallWithoutSourceFile()
             s.registerCallWithMultipleSourceFiles()
-            s.registerCallWithPch()
             s.registerCallForLinking()
             s.registerCallForExternalDebugInfo()
             s.registerEvictedMiss()
@@ -199,7 +197,6 @@ def testHitCounts(self):
             self.assertEqual(s.numCallsWithInvalidArgument(), 1)
             self.assertEqual(s.numCallsWithoutSourceFile(), 1)
             self.assertEqual(s.numCallsWithMultipleSourceFiles(), 1)
-            self.assertEqual(s.numCallsWithPch(), 1)
             self.assertEqual(s.numCallsForLinking(), 1)
             self.assertEqual(s.numCallsForExternalDebugInfo(), 1)
             self.assertEqual(s.numEvictedMisses(), 1)
@@ -576,6 +573,13 @@ def testOutputFileFromSourcefile(self):
                        [('main.cpp', '')], ['main.obj'])
         # For preprocessor file
         self._testFailure(['/c', '/P', 'main.cpp'], CalledForPreprocessingError)
+        # For precompiled header
+        self._testFull(['/c', '/Yc', 'stdafx.cpp'], [('stdafx.cpp', '')],
+                       [('stdafx.obj', 'stdafx.pch')])
+        self._testFull(['/c', '/Yccommon.h', 'stdafx.cpp'], [('stdafx.cpp', '')],
+                       [('stdafx.obj', 'common.pch')])
+        self._testFull(['/c', '/Yc', r'/Fpoutput\common.pch', 'stdafx.cpp'],
+                       [('stdafx.cpp', '')], [('stdafx.obj', r'output\common.pch')])
 
     def testPreprocessIgnoresOtherArguments(self):
         # All those inputs must ignore the /Fo, /Fa and /Fm argument according