install_clcache_msbuild.py : helper for msbuild integration

install_for_msbuild/Readme.md

@@ -0,0 +1,137 @@
+# install_clcache_msbuild
+
+Since the integration of `clcache` with `msbuild` is rather cumbersome, 
+this script provide an helper in order to simplify the process.
+
+It is compatible with MSVC 2010, 2015 and 2017 (tested with MSVC 2015 and 2017).
+
+## What this script does:
+
+* Check that python3 and pip3 are installed and are in the PATH
+* Check that the pip installed scripts are in the PATH (PYTHONHOME\Scripts)
+* Call `pip install .` from the repo and check that clcache is then in the PATH.
+  `clcache` will subsequently be used from the PYTHONHOME\\Scripts directory.
+* Modify the user msbuild preference files inside `%AppData%\..\Local\Microsoft\MSBuild\v4.0`
+  so that clcache becomes the default compiler. (These prefs are shared between MSVC 2010 to 2017).
+* Find all cl.exes version on your computer (for MSVC 2010 to MSVC 2017), and allows you 
+  to select the correct one, by showing a detailed list of their version and target architecture.
+* Set the env variable `CLCACHE_CL` with the correct path to cl.exe
+
+As additional options, this script can also 
+* change the cache location
+* change the cache size
+* change the timeout CLCACHE_OBJECT_CACHE_TIMEOUT_MS
+
+## Caveat 
+Since the msbuild preference files inside `%AppData%\..\Local\Microsoft\MSBuild\v4.0` are shared
+between different MSVC installations, clcache will be activated for all instances of MSVC.
+
+## Note
+`vswhere.exe` is a tool provided by Microsoft in order to locate installations of MSVC >= 2017.
+
+## Usage
+
+````
+usage: install_clcache_msbuild.py [-h] [--cachedir CACHEDIR]
+                                  [--cache_size CACHE_SIZE]
+                                  [--clcache_timeout CLCACHE_TIMEOUT]
+                                  {status,install,enable,disable,enable_logs,disable_logs,show_cl_list,select_cl}
+````
+
+Sample usage session:
+
+````bash
+> python install_clcache_msbuild.py install
+Looking for python in PATH
+C:\Python36-32\python.exe
+Looking for pip in PATH
+C:\Python36-32\Scripts\pip.exe
+
+######################################################################
+Installing clcache (installClcache)
+######################################################################
+====> pip install .(in folder F:\dvp\OpenSource\clcache)
+Processing f:\dvp\opensource\clcache
+Requirement already satisfied: pymemcache in c:\python36-32\lib\site-packages (from clcache==4.1.1.dev65+g105e486.d20181119) (2.0.0)
+Requirement already satisfied: pyuv in c:\python36-32\lib\site-packages (from clcache==4.1.1.dev65+g105e486.d20181119) (1.4.0)
+Requirement already satisfied: six in c:\python36-32\lib\site-packages (from pymemcache->clcache==4.1.1.dev65+g105e486.d20181119) (1.11.0)
+Installing collected packages: clcache
+  Found existing installation: clcache 4.1.1.dev65+g105e486.d20181119
+    Uninstalling clcache-4.1.1.dev65+g105e486.d20181119:
+      Successfully uninstalled clcache-4.1.1.dev65+g105e486.d20181119
+  Running setup.py install for clcache ... done
+Successfully installed clcache-4.1.1.dev65+g105e486.d20181119
+You are using pip version 10.0.1, however version 18.1 is available.
+You should consider upgrading via the 'python -m pip install --upgrade pip' command.
+Looking for clcache in PATH
+C:\Python36-32\Scripts\clcache.exe
+
+######################################################################
+Force clcache via Msbbuild user settings (copyMsvcPrefClcache)
+######################################################################
+Wrote pref in C:\Users\pascal\AppData\Local\Microsoft\MSBuild\v4.0\Microsoft.Cpp.ARM.user.props
+Wrote pref in C:\Users\pascal\AppData\Local\Microsoft\MSBuild\v4.0\Microsoft.Cpp.Win32.user.props
+Wrote pref in C:\Users\pascal\AppData\Local\Microsoft\MSBuild\v4.0\Microsoft.Cpp.x64.user.props
+
+######################################################################
+Select cl compiler: (selectCl)
+######################################################################
+   #           version targetArch   hostArch                                                              folder (shortened)
+   1              14.0      amd64      amd64                                               C:\ProgX86\MSVC 14.0\vc\bin\amd64
+   2              14.0        arm      amd64                                           C:\ProgX86\MSVC 14.0\vc\bin\amd64_arm
+   3              14.0        x86      amd64                                           C:\ProgX86\MSVC 14.0\vc\bin\amd64_x86
+   4              14.0      amd64        x86                                           C:\ProgX86\MSVC 14.0\vc\bin\x86_amd64
+   5              14.0        arm        x86                                             C:\ProgX86\MSVC 14.0\vc\bin\x86_arm
+   6              14.0        x86        x86                                                     C:\ProgX86\MSVC 14.0\vc\bin
+   7   15.7.27703.2047        x64        x64       C:\ProgX86\MSVC\2017\Enterprise\VC\Tools\MSVC\14.14.26428\bin\Hostx64\x64
+   8   15.7.27703.2047        x86        x64       C:\ProgX86\MSVC\2017\Enterprise\VC\Tools\MSVC\14.14.26428\bin\Hostx64\x86
+   9   15.7.27703.2047        x64        x86       C:\ProgX86\MSVC\2017\Enterprise\VC\Tools\MSVC\14.14.26428\bin\Hostx86\x64
+  10   15.7.27703.2047        x86        x86       C:\ProgX86\MSVC\2017\Enterprise\VC\Tools\MSVC\14.14.26428\bin\Hostx86\x86
+Enter the number corresponding to the desired compiler: 6
+Selected : C:\Program Files (x86)\Microsoft Visual Studio 14.0\vc\bin\cl.exe
+====> SETX CLCACHE_CL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\vc\bin\cl.exe"
+
+SUCCESS: Specified value was saved.
+
+######################################################################
+Note about clcache usage: (showClCacheUsage)
+######################################################################
+====> clcache --help
+clcache.py v4.1.0-dev
+  --help    : show this help
+  -s        : print cache statistics
+  -c        : clean cache
+  -C        : clear cache
+  -z        : reset cache statistics
+  -M <size> : set maximum cache size (in bytes)
+````
+
+# Caveats with msbuild and clcache : 
+
+## incremental builds with clcache and msbuild
+
+clcache has serious isses with incremental builds. After a full build, you will always get a full rebuild even if you modify only one file !
+
+There is a pull request that succesfully correct this here: https://github.com/frerich/clcache/pull/319/commits
+
+
+
+## clcache is not compatible with `/Zi` debug information format : use `/Z7` instead.
+
+See 
+https://github.com/frerich/clcache/issues/30 
+and https://stackoverflow.com/questions/284778/what-are-the-implications-of-using-zi-vs-z7-for-visual-studio-c-projects
+
+With cmake, you can do the following:
+
+
+  ```cmake
+  message("msvc_clccache_force_z7_debug_format use /Z7 debug format")
+  if(MSVC)
+    string(REGEX REPLACE "/Z[iI7]" ""
+           CMAKE_CXX_FLAGS_DEBUG
+           "${CMAKE_CXX_FLAGS_DEBUG}")
+    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Z7")
+  endif()
+  ````
+

install_for_msbuild/env_utils.py

@@ -0,0 +1,196 @@
+import ctypes
+import winreg
+import os
+import os.path
+import sys
+import subprocess
+import platform
+# import winshell
+from win32com.client import Dispatch
+
+def createShortcut(src_file, dst_shortcut_path):
+    path = os.path.join(dst_shortcut_path)
+    wDir = os.path.dirname(src_file)
+    icon = src_file
+    shell = Dispatch('WScript.Shell')
+    shortcut = shell.CreateShortCut(path)
+    shortcut.Targetpath = src_file
+    shortcut.WorkingDirectory = wDir
+    shortcut.IconLocation = icon
+    shortcut.save()
+    print("Created shortcut : {} points to {}".format(dst_shortcut_path, src_file))
+
+
+def removeFile(filename):
+    if os.path.exists(filename):
+        os.remove(filename)
+        print("Removed file " +  filename)
+    else:
+        print("Cannot remove " +  filename + " (does not exist)")
+
+
+def runProcessDetached(command):
+    pid = subprocess.Popen(command).pid
+    print("Launched detached process with pid={} for command {}".format(pid, command))
+    return pid
+
+
+def shortDirectoryName(folder):
+    result = folder
+    result = result.replace("Program Files (x86)", "ProgX86")
+    result = result.replace("Program Files", "Prog")
+    result = result.replace("Microsoft Visual Studio", "MSVC")
+    return result
+
+
+def isOs64bit():
+    return platform.machine().endswith('64')
+
+def dirNameAbsolute(folder: str) -> str:
+    return os.path.abspath(os.path.realpath(folder))
+
+
+def fileDirNameAbsolute(file: str) -> str:
+    return dirNameAbsolute(os.path.dirname(file))
+
+
+def appDataPathLocal() -> str:
+    result = dirNameAbsolute(os.getenv('APPDATA') + "\\..\\Local")
+    return result
+
+def isAdmin():
+    return ctypes.windll.shell32.IsUserAnAdmin()
+
+
+def currentFuncName(n=0):
+    return sys._getframe(n + 1).f_code.co_name #pylint: disable=W0212
+
+
+def showFunctionIntro(details=""):
+    print()
+    print("######################################################################")
+    print(details + " (" + currentFuncName(1) + ")")
+    print("######################################################################")
+
+
+def hasProgramInPath(prog):
+    print("Looking for " + prog + " in PATH")
+    result = subprocess.call("where " + prog)
+    if result != 0:
+        print(prog + " not found in PATH")
+    return result == 0
+
+
+def whereProgram(prog: str) -> str:
+    allProgrs = subprocess.check_output("where " + prog).decode("utf-8")
+    firstProg = allProgrs.split("\r")[0]
+    return firstProg
+
+
+def whereProgramFolder(prog: str) -> str:
+    progLocation = whereProgram(prog)
+    return os.path.dirname(progLocation)
+
+
+def pipScriptsDir():
+    return whereProgramFolder("python") + "\\Scripts"
+
+
+def showCmd(cmd):
+    print("====> " + cmd)
+
+
+def callAndShowCmd(command: str, cwd: str = None) -> bool:
+    if cwd is not None:
+        print("====> " + command + "(in folder " + cwd +  ")")
+    else:
+        print("====> " + command)
+    if cwd is not None:
+        return subprocess.call(command, cwd=cwd) == 0
+    else:
+        return subprocess.call(command) == 0
+
+
+def callCmdGetOutput(command: str, cwd: str = None) -> str:
+    return subprocess.check_output(command, cwd=cwd).decode("utf-8")
+
+
+def implSetAndStoreEnvVariable(name, value, allUsers=False):
+    if allUsers:
+        cmd = "SETX {0} \"{1}\" /M".format(name, value)
+    else:
+        cmd = "SETX {0} \"{1}\"".format(name, value)
+    if not callAndShowCmd(cmd):
+        return False
+    os.environ[name] = value
+    return True
+
+
+def setAndStoreEnvVariable(name, value):
+    return implSetAndStoreEnvVariable(name, value, allUsers=isAdmin())
+
+
+def implRemoveEnvVariable(name, allUsers=False):
+    if allUsers:
+        command = "REG DELETE HKLM\\SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Environment /F /V " + name
+    else:
+        command = "REG DELETE HKCU\\Environment /F /V " + name
+    if not callAndShowCmd(command):
+        return False
+    if name is os.environ:
+        os.environ.pop(name, None)
+    return True
+
+
+def removeEnvVariable(name):
+    return implRemoveEnvVariable(name, allUsers=isAdmin())
+
+
+def implReadRegistryValue(registryKind: int, keyName: str, valueName: str) -> str:
+    reg = winreg.ConnectRegistry(None, registryKind)
+    try:
+        key = winreg.OpenKey(reg, keyName)
+        result = winreg.QueryValueEx(key, valueName)
+        return result[0]
+    except FileNotFoundError:
+        return None
+
+
+def readRegistryValueLocalMachine(keyName: str, valueName: str) -> str:
+    return implReadRegistryValue(winreg.HKEY_LOCAL_MACHINE, keyName, valueName)
+
+
+def implReadEnvVariableFromRegistry(valueName, allUsers=False) -> str:
+    if allUsers:
+        keyName = "SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Environment"
+        return implReadRegistryValue(winreg.HKEY_LOCAL_MACHINE, keyName, valueName)
+    else:
+        return implReadRegistryValue(winreg.HKEY_CURRENT_USER, "Environment", valueName)
+
+
+def readEnvVariableFromRegistry(name):
+    return implReadEnvVariableFromRegistry(name, allUsers=isAdmin())
+
+
+def readPathFromEnvironment() -> str:
+    return os.environ["PATH"]
+
+
+def readPathFromRegistry() -> str:
+    result = implReadEnvVariableFromRegistry("PATH", allUsers=False)
+    result = result + ";" + implReadEnvVariableFromRegistry("PATH", allUsers=True)
+    return result
+
+
+def listFiles(folder, appendFolder=True):
+    files = [f for f in os.listdir(folder) if os.path.isfile(os.path.join(folder, f))]
+    if appendFolder:
+        files = [os.path.join(folder, f) for f in files]
+    return files
+
+
+def listSubdirs(folder, appendFolder=True):
+    files = [f for f in os.listdir(folder) if os.path.isdir(os.path.join(folder, f))]
+    if appendFolder:
+        files = [os.path.join(folder, f) for f in files]
+    return files

install_for_msbuild/env_utils_tests.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+import os.path
+import time
+import unittest
+import winshell
+import env_utils
+import locate_cl_exe
+
+class EnvUtilsTests(unittest.TestCase):
+    def testEnvVars(self):
+        env_utils.setAndStoreEnvVariable("DUMMYVAR", "dummy")
+        result = env_utils.readEnvVariableFromRegistry("DUMMYVAR")
+        self.assertEqual(result, "dummy")
+        env_utils.removeEnvVariable("DUMMYVAR")
+        result = env_utils.readEnvVariableFromRegistry("DUMMYVAR")
+        self.assertTrue(result is None)
+    def testShortcut(self):
+        pythonProg = env_utils.whereProgram("python")
+        dstFolder = winshell.startup()
+        dst = dstFolder + "\\python.lnk"
+        env_utils.createShortcut(pythonProg, dst)
+        self.assertTrue(os.path.exists(dst))
+        os.remove(dst)
+        self.assertTrue(not os.path.exists(dst))
+    # def testRunProcessDetached(self):
+    #     prog = env_utils.whereProgram("Calc.exe")
+    #     pid = env_utils.runProcessDetached(prog)
+    #     time.sleep(1)
+    #     env_utils.callAndShowCmd("taskkill /PID {}".format(pid))
+
+
+
+class LocateClTests(unittest.TestCase):
+    def testFindfindMsvcUpTo2015(self):
+        result = locate_cl_exe.implFindMsvcUpTo2015()
+        self.assertTrue(len(result) > 0)
+    def testFindMsvc2017(self):
+        aux = locate_cl_exe.implFindMsvc2017()
+        self.assertTrue(len(aux) >= 0)
+    def testFindMsvc(self):
+        aux = locate_cl_exe.findMsvc()
+        self.assertTrue(len(aux) > 0)
+    def testFindCl(self):
+        aux = locate_cl_exe.findClExesList()
+        self.assertTrue(len(aux) > 0)
+
+
+if __name__ == '__main__':
+    unittest.TestCase.longMessage = True
+    unittest.main()