From 2b54b8d616a2a8055b86a5a1bfa4f610b9ba577f Mon Sep 17 00:00:00 2001 From: "Mikhail I. Izmestev" Date: Tue, 24 Jul 2018 09:58:57 +0400 Subject: [PATCH 1/3] Add support of precompiled headers Creating PCH file (/Yc option) works in usual way as any other object, except that it will produce meta info (stdafx.pch.clcache file) contains it's cache key as identification of headers files and it's contents. Using PCH file (/Yu option) affects /showIncludes - it not contains PCH includes, so for generating manifest hash additionally used PCH cache key. So when PCH headers or it's content changed but no changes in source file or it's headers then it will produce different cache entry. --- clcache/__main__.py | 56 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/clcache/__main__.py b/clcache/__main__.py index 22b83b8c..d14cd59d 100644 --- a/clcache/__main__.py +++ b/clcache/__main__.py @@ -271,6 +271,14 @@ def allSectionsLocked(repository): section.lock.release() +def readPchHash(pchFile): + with open(pchFile+".clcache", 'r') as f: + return f.read() + +def writePchHash(pchFile, hashSum): + with open(pchFile+".clcache", 'w') as f: + f.write('{}'.format(hashSum)) + class ManifestRepository: # Bump this counter whenever the current manifest file format changes. # E.g. changing the file format from {'oldkey': ...} to {'newkey': ...} requires @@ -333,6 +341,10 @@ def getManifestHash(compilerBinary, commandLine, sourceFile): additionalData = "{}|{}|{}".format( compilerHash, commandLine, ManifestRepository.MANIFEST_FILE_FORMAT_VERSION) + + if 'Yu' in arguments: + pchFile = CommandLineAnalyzer.getPchFileName(arguments, sourceFile) + additionalData += readPchHash(pchFile) return getFileHash(sourceFile, additionalData) @staticmethod @@ -519,6 +531,10 @@ def computeKeyNodirect(compilerBinary, commandLine, environment): h.update(preprocessedSourceCode) return h.hexdigest() + @staticmethod + def computePchKey(cacheKey): + return cacheKey + "-pch" + @staticmethod def _normalizedCommandLine(cmdline): # Remove all arguments from the command line which only influence the @@ -1308,6 +1324,20 @@ def parseArgumentsAndInputFiles(cmdline): return dict(arguments), inputFiles + @staticmethod + def getPchFileName(options, inputFile): + if 'Fp' in options: + return options['Fp'][0] + if 'Yc' in options: + headerName = options['Yc'][0] + elif 'Yu' in options: + headerName = options['Yu'][0] + else: + return None + if not headerName: + headerName = inputFile + return basenameWithoutExtension(headerName) + '.pch' + @staticmethod def analyze(cmdline: List[str]) -> Tuple[List[Tuple[str, str]], List[str]]: options, inputFiles = CommandLineAnalyzer.parseArgumentsAndInputFiles(cmdline) @@ -1336,9 +1366,6 @@ def analyze(cmdline: List[str]) -> Tuple[List[Tuple[str, str]], List[str]]: if 'Zi' in options: raise ExternalDebugInfoError() - if 'Yc' in options or 'Yu' in options: - raise CalledWithPchError() - if 'link' in options or 'c' not in options: raise CalledForLinkError() @@ -1358,6 +1385,10 @@ def analyze(cmdline: List[str]) -> Tuple[List[Tuple[str, str]], List[str]]: # Generate from .c/.cpp filenames objectFiles = [os.path.join(prefix, basenameWithoutExtension(f)) + '.obj' for f, _ in inputFiles] + if 'Yc' in options: + objectFiles = [(objectFile, CommandLineAnalyzer.getPchFileName(options, inputFile[0])) + for inputFile, objectFile in zip(inputFiles, objectFiles)] + printTraceStatement("Compiler source files: {}".format(inputFiles)) printTraceStatement("Compiler object file: {}".format(objectFiles)) return inputFiles, objectFiles @@ -1531,6 +1562,7 @@ def addObjectToCache(stats, cache, cachekey, artifacts): def processCacheHit(cache, objectFile, cachekey): printTraceStatement("Reusing cached object for key {} for object file {}".format(cachekey, objectFile)) + objectFile, pchFile = objectFile if isinstance(objectFile, tuple) else (objectFile, None) with cache.lockFor(cachekey): with cache.statistics.lock, cache.statistics as stats: @@ -1541,6 +1573,11 @@ def processCacheHit(cache, objectFile, cachekey): cachedArtifacts = cache.getEntry(cachekey) copyOrLink(cachedArtifacts.objectFilePath, objectFile) + if pchFile is not None: + pchCachedArtifacts = cache.getEntry(CompilerArtifactsRepository.computePchKey(cachekey)) + copyOrLink(pchCachedArtifacts.objectFilePath, pchFile) + writePchHash(pchFile, cachekey) + printTraceStatement("Finished. Exit code 0") return 0, cachedArtifacts.stdout, cachedArtifacts.stderr, False @@ -1804,6 +1841,8 @@ def processNoDirect(cache, objectFile, compiler, cmdLine, environment): def ensureArtifactsExist(cache, cachekey, reason, objectFile, compilerResult, extraCallable=None): cleanupRequired = False returnCode, compilerOutput, compilerStderr = compilerResult + objectFile, pchFile = objectFile if isinstance(objectFile, tuple) else (objectFile, None) + correctCompiliation = (returnCode == 0 and os.path.exists(objectFile)) with cache.lockFor(cachekey): if not cache.hasEntry(cachekey): @@ -1814,6 +1853,17 @@ def ensureArtifactsExist(cache, cachekey, reason, objectFile, compilerResult, ex cleanupRequired = addObjectToCache(stats, cache, cachekey, artifacts) if extraCallable and correctCompiliation: extraCallable() + + if pchFile: + writePchHash(pchFile, cachekey) + cachekey = CompilerArtifactsRepository.computePchKey(cachekey) + with cache.lockFor(cachekey): + if not cache.hasEntry(cachekey): + with cache.statistics.lock, cache.statistics as stats: + if correctCompiliation: + artifacts = CompilerArtifacts(pchFile, "", "") + cleanupRequired = addObjectToCache(stats, cache, cachekey, artifacts) or cleanupRequired + return returnCode, compilerOutput, compilerStderr, cleanupRequired From f447905727d58089f7a40b0582601f78ea64b735 Mon Sep 17 00:00:00 2001 From: "Mikhail I. Izmestev" Date: Mon, 3 Sep 2018 14:04:07 +0400 Subject: [PATCH 2/3] Add tests for output files when PCH used --- tests/test_unit.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_unit.py b/tests/test_unit.py index d7e9544d..a37cc0a9 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -576,6 +576,13 @@ def testOutputFileFromSourcefile(self): [('main.cpp', '')], ['main.obj']) # For preprocessor file self._testFailure(['/c', '/P', 'main.cpp'], CalledForPreprocessingError) + # For precompiled header + self._testFull(['/c', '/Yc', 'stdafx.cpp'], [('stdafx.cpp', '')], + [('stdafx.obj', 'stdafx.pch')]) + self._testFull(['/c', '/Yccommon.h', 'stdafx.cpp'], [('stdafx.cpp', '')], + [('stdafx.obj', 'common.pch')]) + self._testFull(['/c', '/Yc', r'/Fpoutput\common.pch', 'stdafx.cpp'], + [('stdafx.cpp', '')], [('stdafx.obj', r'output\common.pch')]) def testPreprocessIgnoresOtherArguments(self): # All those inputs must ignore the /Fo, /Fa and /Fm argument according From 33b2ef2054334bca23323ac3d7480bd0550b5ab7 Mon Sep 17 00:00:00 2001 From: "Mikhail I. Izmestev" Date: Mon, 3 Sep 2018 14:05:05 +0400 Subject: [PATCH 3/3] Remove PCH error related code --- clcache/__main__.py | 19 +------------------ tests/test_integration.py | 3 --- tests/test_unit.py | 3 --- 3 files changed, 1 insertion(+), 24 deletions(-) diff --git a/clcache/__main__.py b/clcache/__main__.py index d14cd59d..085f862a 100644 --- a/clcache/__main__.py +++ b/clcache/__main__.py @@ -761,7 +761,6 @@ class Statistics: CALLS_WITH_INVALID_ARGUMENT = "CallsWithInvalidArgument" CALLS_WITHOUT_SOURCE_FILE = "CallsWithoutSourceFile" CALLS_WITH_MULTIPLE_SOURCE_FILES = "CallsWithMultipleSourceFiles" - CALLS_WITH_PCH = "CallsWithPch" CALLS_FOR_LINKING = "CallsForLinking" CALLS_FOR_EXTERNAL_DEBUG_INFO = "CallsForExternalDebugInfo" CALLS_FOR_PREPROCESSING = "CallsForPreprocessing" @@ -777,7 +776,6 @@ class Statistics: CALLS_WITH_INVALID_ARGUMENT, CALLS_WITHOUT_SOURCE_FILE, CALLS_WITH_MULTIPLE_SOURCE_FILES, - CALLS_WITH_PCH, CALLS_FOR_LINKING, CALLS_FOR_EXTERNAL_DEBUG_INFO, CALLS_FOR_PREPROCESSING, @@ -831,12 +829,6 @@ def numCallsWithMultipleSourceFiles(self): def registerCallWithMultipleSourceFiles(self): self._stats[Statistics.CALLS_WITH_MULTIPLE_SOURCE_FILES] += 1 - def numCallsWithPch(self): - return self._stats[Statistics.CALLS_WITH_PCH] - - def registerCallWithPch(self): - self._stats[Statistics.CALLS_WITH_PCH] += 1 - def numCallsForLinking(self): return self._stats[Statistics.CALLS_FOR_LINKING] @@ -929,10 +921,6 @@ class CalledForLinkError(AnalysisError): pass -class CalledWithPchError(AnalysisError): - pass - - class ExternalDebugInfoError(AnalysisError): pass @@ -1470,8 +1458,7 @@ def printStatistics(cache): called for linking : {} called for external debug : {} called w/o source : {} - called w/ multiple sources : {} - called w/ PCH : {}""".strip() + called w/ multiple sources : {}""".strip() with cache.statistics.lock, cache.statistics as stats, cache.configuration as cfg: print(template.format( @@ -1490,7 +1477,6 @@ def printStatistics(cache): stats.numCallsForExternalDebugInfo(), stats.numCallsWithoutSourceFile(), stats.numCallsWithMultipleSourceFiles(), - stats.numCallsWithPch(), )) @@ -1690,9 +1676,6 @@ def processCompileRequest(cache, compiler, args): except MultipleSourceFilesComplexError: printTraceStatement("Cannot cache invocation as {}: multiple source files found".format(cmdLine)) updateCacheStatistics(cache, Statistics.registerCallWithMultipleSourceFiles) - except CalledWithPchError: - printTraceStatement("Cannot cache invocation as {}: precompiled headers in use".format(cmdLine)) - updateCacheStatistics(cache, Statistics.registerCallWithPch) except CalledForLinkError: printTraceStatement("Cannot cache invocation as {}: called for linking".format(cmdLine)) updateCacheStatistics(cache, Statistics.registerCallForLinking) diff --git a/tests/test_integration.py b/tests/test_integration.py index acb2d880..3143dd23 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -957,7 +957,6 @@ def testClearPostcondition(self): self.assertEqual(stats.numCacheEntries(), 0) self.assertEqual(stats.numCallsWithoutSourceFile(), oldStats.numCallsWithoutSourceFile()) self.assertEqual(stats.numCallsWithMultipleSourceFiles(), oldStats.numCallsWithMultipleSourceFiles()) - self.assertEqual(stats.numCallsWithPch(), oldStats.numCallsWithPch()) self.assertEqual(stats.numCallsForLinking(), oldStats.numCallsForLinking()) self.assertEqual(stats.numCallsForPreprocessing(), oldStats.numCallsForPreprocessing()) self.assertEqual(stats.numCallsForExternalDebugInfo(), oldStats.numCallsForExternalDebugInfo()) @@ -995,8 +994,6 @@ def testAllKnownAnalysisErrors(self): subprocess.check_call(baseCmd + ['/c', '/Tcfibonacci.c', "minimal.cpp"]) # CalledForLinkError subprocess.check_call(baseCmd + ["fibonacci.cpp"]) - # CalledWithPchError - subprocess.check_call(baseCmd + ['/c', '/Yc', "minimal.cpp"]) # ExternalDebugInfoError subprocess.check_call(baseCmd + ['/c', '/Zi', "minimal.cpp"]) # CalledForPreprocessingError diff --git a/tests/test_unit.py b/tests/test_unit.py index a37cc0a9..8948a535 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -172,7 +172,6 @@ def testHitCounts(self): self.assertEqual(s.numCallsWithInvalidArgument(), 0) self.assertEqual(s.numCallsWithoutSourceFile(), 0) self.assertEqual(s.numCallsWithMultipleSourceFiles(), 0) - self.assertEqual(s.numCallsWithPch(), 0) self.assertEqual(s.numCallsForLinking(), 0) self.assertEqual(s.numCallsForExternalDebugInfo(), 0) self.assertEqual(s.numEvictedMisses(), 0) @@ -186,7 +185,6 @@ def testHitCounts(self): s.registerCallWithInvalidArgument() s.registerCallWithoutSourceFile() s.registerCallWithMultipleSourceFiles() - s.registerCallWithPch() s.registerCallForLinking() s.registerCallForExternalDebugInfo() s.registerEvictedMiss() @@ -199,7 +197,6 @@ def testHitCounts(self): self.assertEqual(s.numCallsWithInvalidArgument(), 1) self.assertEqual(s.numCallsWithoutSourceFile(), 1) self.assertEqual(s.numCallsWithMultipleSourceFiles(), 1) - self.assertEqual(s.numCallsWithPch(), 1) self.assertEqual(s.numCallsForLinking(), 1) self.assertEqual(s.numCallsForExternalDebugInfo(), 1) self.assertEqual(s.numEvictedMisses(), 1)