diff --git a/appveyor.yml b/appveyor.yml index ad5673a4..7dfcef7f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,8 +16,8 @@ install: - 7z x memcached.zip -y - ps: $Memcached = Start-Process memcached\memcached.exe -PassThru - # Make compiler available (use MSVC 2013, 32 bit) - - call "%ProgramFiles(x86)%\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" x86 + # Make compiler available (use MSVC 2015, 32 bit) + - call "%ProgramFiles(x86)%\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86 # Check compiler version - cl diff --git a/clcache/__main__.py b/clcache/__main__.py index 2689a928..34204065 100644 --- a/clcache/__main__.py +++ b/clcache/__main__.py @@ -71,6 +71,10 @@ NMPWAIT_WAIT_FOREVER = wintypes.DWORD(0xFFFFFFFF) ERROR_PIPE_BUSY = 231 +# Toolset version 140 +# https://devblogs.microsoft.com/cppblog/side-by-side-minor-version-msvc-toolsets-in-visual-studio-2017/ +TOOLSET_VERSION_140 = 140 + # ManifestEntry: an entry in a manifest file # `includeFiles`: list of paths to include files, which this source file uses # `includesContentsHash`: hash of the contents of the includeFiles @@ -119,6 +123,46 @@ def normalizeBaseDir(baseDir): return None +class SuspendTracker(): + fileTracker = None + def __init__(self): + if not SuspendTracker.fileTracker: + if windll.kernel32.GetModuleHandleW("FileTracker.dll"): + SuspendTracker.fileTracker = windll.FileTracker + elif windll.kernel32.GetModuleHandleW("FileTracker32.dll"): + SuspendTracker.fileTracker = windll.FileTracker32 + elif windll.kernel32.GetModuleHandleW("FileTracker64.dll"): + SuspendTracker.fileTracker = windll.FileTracker64 + + def __enter__(self): + SuspendTracker.suspend() + + def __exit__(self, typ, value, traceback): + SuspendTracker.resume() + + @staticmethod + def suspend(): + if SuspendTracker.fileTracker: + SuspendTracker.fileTracker.SuspendTracking() + + @staticmethod + def resume(): + if SuspendTracker.fileTracker: + SuspendTracker.fileTracker.ResumeTracking() + +def isTrackerEnabled(): + return 'TRACKER_ENABLED' in os.environ + +def untrackable(func): + if not isTrackerEnabled(): + return func + + def untrackedFunc(*args, **kwargs): + with SuspendTracker(): + return func(*args, **kwargs) + + return untrackedFunc + def getCachedCompilerConsoleOutput(path): try: with open(path, 'rb') as f: @@ -188,6 +232,7 @@ def manifestPath(self, manifestHash): def manifestFiles(self): return filesBeneath(self.manifestSectionDir) + @untrackable def setManifest(self, manifestHash, manifest): manifestPath = self.manifestPath(manifestHash) printTraceStatement("Writing manifest with manifestHash = {} to {}".format(manifestHash, manifestPath)) @@ -198,6 +243,7 @@ def setManifest(self, manifestHash, manifest): jsonobject = {'entries': entries} json.dump(jsonobject, outFile, sort_keys=True, indent=2) + @untrackable def getManifest(self, manifestHash): fileName = self.manifestPath(manifestHash) if not os.path.exists(fileName): @@ -738,6 +784,7 @@ def __init__(self, statsFile): self._stats = None self.lock = CacheLock.forPath(self._statsFile) + @untrackable def __enter__(self): self._stats = PersistentJSONDict(self._statsFile) for k in Statistics.RESETTABLE_KEYS | Statistics.NON_RESETTABLE_KEYS: @@ -745,6 +792,7 @@ def __enter__(self): self._stats[k] = 0 return self + @untrackable def __exit__(self, typ, value, traceback): # Does not write to disc when unchanged self._stats.save() @@ -1686,6 +1734,35 @@ def filterSourceFiles(cmdLine: List[str], sourceFiles: List[Tuple[str, str]]) -> if not (arg in setOfSources or arg.startswith(skippedArgs)) ) + +def findCompilerVersion(compiler: str) -> int: + compilerInfo = subprocess.Popen([compiler], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + compilerVersionLine = compilerInfo.communicate()[0].decode('utf-8').splitlines()[0] + compilerVersion = compilerVersionLine[compilerVersionLine.find("Version ") + 8: + compilerVersionLine.find(" for")] + return int(compilerVersion[:2] + compilerVersion[3:5]) + + +def findToolsetVersion(compilerVersion: int) -> int: + versionMap = {1400: 80, + 1500: 90, + 1600: 100, + 1700: 110, + 1800: 120, + 1900: 140} + + if compilerVersion in versionMap: + return versionMap[compilerVersion] + elif 1910 <= compilerVersion < 1920: + return 141 + elif 1920 <= compilerVersion < 1930: + return 142 + else: + raise LogicException('Bad cl.exe version: {}'.format(compilerVersion)) + + def scheduleJobs(cache: Any, compiler: str, cmdLine: List[str], environment: Any, sourceFiles: List[Tuple[str, str]], objectFiles: List[str]) -> int: # Filter out all source files from the command line to form baseCmdLine @@ -1693,7 +1770,14 @@ def scheduleJobs(cache: Any, compiler: str, cmdLine: List[str], environment: Any exitCode = 0 cleanupRequired = False - with concurrent.futures.ThreadPoolExecutor(max_workers=jobCount(cmdLine)) as executor: + + def poolExecutor(*args, **kwargs) -> concurrent.futures.Executor: + if isTrackerEnabled(): + if findToolsetVersion(findCompilerVersion(compiler)) < TOOLSET_VERSION_140: + return concurrent.futures.ProcessPoolExecutor(*args, **kwargs) + return concurrent.futures.ThreadPoolExecutor(*args, **kwargs) + + with poolExecutor(max_workers=min(jobCount(cmdLine), len(objectFiles))) as executor: jobs = [] for (srcFile, srcLanguage), objFile in zip(sourceFiles, objectFiles): jobCmdLine = baseCmdLine + [srcLanguage + srcFile] diff --git a/pyinstaller/clcache_main.py b/pyinstaller/clcache_main.py index 42327ad8..b9bddbc8 100644 --- a/pyinstaller/clcache_main.py +++ b/pyinstaller/clcache_main.py @@ -1,2 +1,4 @@ +import multiprocessing from clcache.__main__ import main +multiprocessing.freeze_support() main() diff --git a/setup.py b/setup.py index 890107bb..67a5022a 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ entry_points={ 'console_scripts': [ 'clcache = clcache.__main__:main', + 'cl.cache = clcache.__main__:main', 'clcache-server = clcache.server.__main__:main', ] }, diff --git a/tests/integrationtests/msbuild/another.cpp b/tests/integrationtests/msbuild/another.cpp new file mode 100644 index 00000000..f96747dd --- /dev/null +++ b/tests/integrationtests/msbuild/another.cpp @@ -0,0 +1 @@ +int somefunc() { return 1; } diff --git a/tests/integrationtests/msbuild/test.vcxproj b/tests/integrationtests/msbuild/test.vcxproj new file mode 100644 index 00000000..cee1f3a9 --- /dev/null +++ b/tests/integrationtests/msbuild/test.vcxproj @@ -0,0 +1,30 @@ + + + + + Release + Win32 + + + + + Application + + + + + + + + + OldStyle + + + + + ProgramDatabase + + + + + diff --git a/tests/test_integration.py b/tests/test_integration.py index a33d7272..7f67c104 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1195,6 +1195,109 @@ def testEvictedManifest(self): self.assertEqual(subprocess.call(cmd, env=customEnv), 0) +@pytest.mark.skipif(os.environ["VisualStudioVersion"] < "14.0", reason="Require newer visual studio") +class TestMSBuildV140(unittest.TestCase): + def _clean(self): + cmd = self.getBuildCmd() + subprocess.check_call(cmd + ["/t:Clean"]) + + def setUp(self): + with cd(os.path.join(ASSETS_DIR, "msbuild")): + self._clean() + + def getBuildCmd(self): + return ["msbuild", "/p:Configuration=Release", "/nologo", "/verbosity:minimal", + "/p:PlatformToolset=v140", "/p:ClToolExe=clcache.exe"] + + def testClean(self): + with tempfile.TemporaryDirectory(dir=os.path.join(ASSETS_DIR, "msbuild")) as tempDir: + customEnv = dict(os.environ, CLCACHE_DIR=tempDir) + + with cd(os.path.join(ASSETS_DIR, "msbuild")): + cmd = self.getBuildCmd() + + # Compile once to insert the objects in the cache + subprocess.check_call(cmd, env=customEnv) + + # build Clean target + subprocess.check_call(cmd + ["/t:Clean"], env=customEnv) + + cache = clcache.Cache(tempDir) + with cache.statistics as stats: + self.assertEqual(stats.numCallsForExternalDebugInfo(), 1) + self.assertEqual(stats.numCacheEntries(), 2) + + def testIncrementalBuild(self): + with tempfile.TemporaryDirectory(dir=os.path.join(ASSETS_DIR, "msbuild")) as tempDir: + customEnv = dict(os.environ, CLCACHE_DIR=tempDir) + cmd = self.getBuildCmd() + + with cd(os.path.join(ASSETS_DIR, "msbuild")): + # Compile once to insert the objects in the cache + subprocess.check_call(cmd, env=customEnv) + + output = subprocess.check_output(cmd, env=customEnv, stderr=subprocess.STDOUT) + output = output.decode("utf-8") + + + self.assertTrue("another.cpp" not in output) + self.assertTrue("minimal.cpp" not in output) + self.assertTrue("fibonacci.cpp" not in output) + + +class TestMSBuildV120(unittest.TestCase): + def _clean(self): + cmd = self.getBuildCmd() + subprocess.check_call(cmd + ["/t:Clean"]) + + # workaround due to cl.cache.exe is not frozen it create no cl.read.1.tlog, but + # this file is important for v120 toolchain, see comment at getMSBuildCmd + try: + os.makedirs(os.path.join("Release", "test.tlog")) + except FileExistsError: + pass + with open(os.path.join("Release", "test.tlog", "cl.read.1.tlog"), "w"),\ + open(os.path.join("Release", "test.tlog", "cl.write.1.tlog"), "w"): + pass + + def setUp(self): + with cd(os.path.join(ASSETS_DIR, "msbuild")): + self._clean() + + def getBuildCmd(self): + # v120 toolchain hardcoded "cl.read.1.tlog" and "cl.*.read.1.tlog" + # file names to inspect as input dependency. + # The best way to use clcache with v120 toolchain is to froze clcache to cl.exe + # and then specify ClToolPath property. + + # There is no frozen cl.exe in tests available, as workaround we would use cl.cache.exe + # and manually create cl.read.1.tlog empty file, without this file msbuild think that + # FileTracker created wrong tlogs. + return ["msbuild", "/p:Configuration=Release", "/nologo", "/verbosity:minimal", + "/p:PlatformToolset=v120", "/p:ClToolExe=cl.cache.exe"] + + def testIncrementalBuild(self): + with tempfile.TemporaryDirectory(dir=os.path.join(ASSETS_DIR, "msbuild")) as tempDir: + customEnv = dict(os.environ, CLCACHE_DIR=tempDir) + cmd = self.getBuildCmd() + + with cd(os.path.join(ASSETS_DIR, "msbuild")): + # Compile once to insert the objects in the cache + subprocess.check_call(cmd, env=customEnv) + + self._clean() + + # Compile using cached objects + subprocess.check_call(cmd, env=customEnv) + + output = subprocess.check_output(cmd, env=customEnv, stderr=subprocess.STDOUT) + output = output.decode("utf-8") + + self.assertTrue("another.cpp" not in output) + self.assertTrue("minimal.cpp" not in output) + self.assertTrue("fibonacci.cpp" not in output) + + if __name__ == '__main__': unittest.TestCase.longMessage = True unittest.main() diff --git a/tests/test_unit.py b/tests/test_unit.py index 88863b37..0861e985 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -1166,6 +1166,29 @@ def testDecompression(self): self.assertNotEqual(os.path.getsize(srcFilePath), os.path.getsize(tmpFilePath)) self.assertEqual(os.path.getsize(srcFilePath), os.path.getsize(dstFilePath)) +class TestToolsetVersion(unittest.TestCase): + def testCorrectMapping(self): + from clcache.__main__ import findToolsetVersion + + for i in range(0, 5): + compVer, toolVer = ((i * 100) + 1400), ((i * 10) + 80) + self.assertEqual(findToolsetVersion(compVer), toolVer) + self.assertEqual(findToolsetVersion(1900), 140) + for compVer in range(1910, 1920): + self.assertEqual(findToolsetVersion(compVer), 141) + for compVer in range(1920, 1930): + self.assertEqual(findToolsetVersion(compVer), 142) + + def testIncorrectMapping(self): + from clcache.__main__ import findToolsetVersion + from clcache.__main__ import LogicException + + with self.assertRaises(LogicException): + findToolsetVersion(100) + with self.assertRaises(LogicException): + findToolsetVersion(1456) + with self.assertRaises(LogicException): + findToolsetVersion(1930) if __name__ == '__main__': unittest.TestCase.longMessage = True