Rework/update toolchain script for vs2015
This is the other side of https://codereview.chromium.org/1163723003/
The changes here are to remove the use of 'vs2013_files' and 'win8sdk'
(as those will be different numbers soon enough) but still maintain
behaviour for the old "style" while in transition.
Secondarily, to remove the dependence of these two scripts on
'toolchain2013.py' as most of the script is now unused.
R=dpranke@chromium.org
BUG=440500, 492774
Review URL: https://codereview.chromium.org/1165563003
git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@295485 0039d316-1c4b-4281-b951-d872f2087c98
diff --git a/win_toolchain/get_toolchain_if_necessary.py b/win_toolchain/get_toolchain_if_necessary.py
index 7bb3eba..cb4b5da 100755
--- a/win_toolchain/get_toolchain_if_necessary.py
+++ b/win_toolchain/get_toolchain_if_necessary.py
@@ -33,13 +33,20 @@
import shutil
import subprocess
import sys
+import tempfile
import time
+import zipfile
BASEDIR = os.path.dirname(os.path.abspath(__file__))
DEPOT_TOOLS_PATH = os.path.join(BASEDIR, '..')
sys.path.append(DEPOT_TOOLS_PATH)
-import download_from_google_storage
+try:
+ import download_from_google_storage
+except ImportError:
+ # Allow use of utility functions in this script from package_from_installed
+ # on bare VM that doesn't have a full depot_tools.
+ pass
if sys.platform != 'cygwin':
import ctypes.wintypes
@@ -186,6 +193,32 @@
print
+def DownloadUsingGsutil(filename):
+ """Downloads the given file from Google Storage chrome-wintoolchain bucket."""
+ temp_dir = tempfile.mkdtemp()
+ assert os.path.basename(filename) == filename
+ target_path = os.path.join(temp_dir, filename)
+ gsutil = download_from_google_storage.Gsutil(
+ download_from_google_storage.GSUTIL_DEFAULT_PATH, boto_path=None)
+ code = gsutil.call('cp', 'gs://chrome-wintoolchain/' + filename, target_path)
+ if code != 0:
+ sys.exit('gsutil failed')
+ return temp_dir, target_path
+
+
+def DoTreeMirror(target_dir, tree_sha1):
+ """In order to save temporary space on bots that do not have enough space to
+ download ISOs, unpack them, and copy to the target location, the whole tree
+ is uploaded as a zip to internal storage, and then mirrored here."""
+ temp_dir, local_zip = DownloadUsingGsutil(tree_sha1 + '.zip')
+ sys.stdout.write('Extracting %s...\n' % local_zip)
+ sys.stdout.flush()
+ with zipfile.ZipFile(local_zip, 'r', zipfile.ZIP_DEFLATED, True) as zf:
+ zf.extractall(target_dir)
+ if temp_dir:
+ subprocess.check_call('rmdir /s/q "%s"' % temp_dir, shell=True)
+
+
def main():
if not sys.platform.startswith(('cygwin', 'win32')):
return 0
@@ -215,7 +248,13 @@
# the downloader script is.
os.chdir(os.path.normpath(os.path.join(BASEDIR)))
toolchain_dir = '.'
- target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files'))
+ if os.environ.get('GYP_MSVS_VERSION') == '2015':
+ target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs_files'))
+ else:
+ target_dir = os.path.normpath(os.path.join(toolchain_dir, 'vs2013_files'))
+ abs_target_dir = os.path.abspath(target_dir)
+
+ got_new_toolchain = False
# If the current hash doesn't match what we want in the file, nuke and pave.
# Typically this script is only run when the .sha1 one file is updated, but
@@ -224,8 +263,8 @@
current_hash = CalculateHash(target_dir)
if current_hash not in desired_hashes:
should_use_gs = False
- if (HaveSrcInternalAccess() or
- LooksLikeGoogler() or
+ if (HaveSrcInternalAccess() or
+ LooksLikeGoogler() or
CanAccessToolchainBucket()):
should_use_gs = True
if not CanAccessToolchainBucket():
@@ -246,12 +285,35 @@
stdin=nul, stdout=nul, stderr=nul)
if os.path.isdir(target_dir):
subprocess.check_call('rmdir /s/q "%s"' % target_dir, shell=True)
- args = [sys.executable,
- 'toolchain2013.py',
- '--targetdir', target_dir,
- '--sha1', desired_hashes[0],
- '--use-gs']
- subprocess.check_call(args)
+
+ DoTreeMirror(target_dir, desired_hashes[0])
+
+ got_new_toolchain = True
+
+ win_sdk = os.path.join(abs_target_dir, 'win_sdk')
+ try:
+ with open(os.path.join(target_dir, 'VS_VERSION'), 'rb') as f:
+ vs_version = f.read().strip()
+ except IOError:
+ # Older toolchains didn't have the VS_VERSION file, and used 'win8sdk'
+ # instead of just 'win_sdk'.
+ vs_version = '2013'
+ win_sdk = os.path.join(abs_target_dir, 'win8sdk')
+
+ data = {
+ 'path': abs_target_dir,
+ 'version': vs_version,
+ 'win_sdk': win_sdk,
+ 'wdk': os.path.join(abs_target_dir, 'wdk'),
+ 'runtime_dirs': [
+ os.path.join(abs_target_dir, 'sys64'),
+ os.path.join(abs_target_dir, 'sys32'),
+ ],
+ }
+ with open(os.path.join(target_dir, '..', 'data.json'), 'w') as f:
+ json.dump(data, f)
+
+ if got_new_toolchain:
current_hash = CalculateHash(target_dir)
if current_hash not in desired_hashes:
print >> sys.stderr, (
diff --git a/win_toolchain/package_from_installed.py b/win_toolchain/package_from_installed.py
index 22f52ac..6fe606f 100644
--- a/win_toolchain/package_from_installed.py
+++ b/win_toolchain/package_from_installed.py
@@ -28,15 +28,16 @@
import zipfile
import get_toolchain_if_necessary
-import toolchain2013 # pylint: disable=F0401
+
+
+VS_VERSION = None
def BuildFileList():
result = []
# Subset of VS corresponding roughly to VC.
- vs_path = r'C:\Program Files (x86)\Microsoft Visual Studio 12.0'
- for path in [
+ paths = [
'DIA SDK/bin',
'DIA SDK/idl',
'DIA SDK/include',
@@ -47,15 +48,39 @@
'VC/include',
'VC/lib',
'VC/redist',
- ('VC/redist/x86/Microsoft.VC120.CRT', 'sys32'),
- ('VC/redist/x86/Microsoft.VC120.MFC', 'sys32'),
- ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugCRT', 'sys32'),
- ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugMFC', 'sys32'),
- ('VC/redist/x64/Microsoft.VC120.CRT', 'sys64'),
- ('VC/redist/x64/Microsoft.VC120.MFC', 'sys64'),
- ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugCRT', 'sys64'),
- ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugMFC', 'sys64'),
- ]:
+ ]
+
+ if VS_VERSION == '2013':
+ paths += [
+ ('VC/redist/x86/Microsoft.VC120.CRT', 'sys32'),
+ ('VC/redist/x86/Microsoft.VC120.MFC', 'sys32'),
+ ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugCRT', 'sys32'),
+ ('VC/redist/Debug_NonRedist/x86/Microsoft.VC120.DebugMFC', 'sys32'),
+ ('VC/redist/x64/Microsoft.VC120.CRT', 'sys64'),
+ ('VC/redist/x64/Microsoft.VC120.MFC', 'sys64'),
+ ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugCRT', 'sys64'),
+ ('VC/redist/Debug_NonRedist/x64/Microsoft.VC120.DebugMFC', 'sys64'),
+ ]
+ elif VS_VERSION == '2015':
+ paths += [
+ ('VC/redist/x86/Microsoft.VC140.CRT', 'sys32'),
+ ('VC/redist/x86/Microsoft.VC140.MFC', 'sys32'),
+ ('VC/redist/debug_nonredist/x86/Microsoft.VC140.DebugCRT', 'sys32'),
+ ('VC/redist/debug_nonredist/x86/Microsoft.VC140.DebugMFC', 'sys32'),
+ ('VC/redist/x64/Microsoft.VC140.CRT', 'sys64'),
+ ('VC/redist/x64/Microsoft.VC140.MFC', 'sys64'),
+ ('VC/redist/debug_nonredist/x64/Microsoft.VC140.DebugCRT', 'sys64'),
+ ('VC/redist/debug_nonredist/x64/Microsoft.VC140.DebugMFC', 'sys64'),
+ ]
+ else:
+ raise ValueError('VS_VERSION %s' % VS_VERSION)
+
+ if VS_VERSION == '2013':
+ vs_path = r'C:\Program Files (x86)\Microsoft Visual Studio 12.0'
+ else:
+ vs_path = r'C:\Program Files (x86)\Microsoft Visual Studio 14.0'
+
+ for path in paths:
src = path[0] if isinstance(path, tuple) else path
combined = os.path.join(vs_path, src)
assert os.path.exists(combined) and os.path.isdir(combined)
@@ -68,8 +93,8 @@
else:
assert final_from.startswith(vs_path)
dest = final_from[len(vs_path) + 1:]
- if dest.lower().endswith('\\xtree'):
- # Patch for C4702 in xtree. http://crbug.com/346399.
+ if VS_VERSION == '2013' and dest.lower().endswith('\\xtree'):
+ # Patch for C4702 in xtree on VS2013. http://crbug.com/346399.
(handle, patched) = tempfile.mkstemp()
with open(final_from, 'rb') as unpatched_f:
unpatched_contents = unpatched_f.read()
@@ -85,21 +110,80 @@
for root, _, files in os.walk(sdk_path):
for f in files:
combined = os.path.normpath(os.path.join(root, f))
- to = os.path.join('win8sdk', combined[len(sdk_path) + 1:])
+ to = os.path.join('win_sdk', combined[len(sdk_path) + 1:])
result.append((combined, to))
+ if VS_VERSION == '2015':
+ for ucrt_path in (
+ (r'C:\Program Files (x86)\Windows Kits\10\Include', 'Include'),
+ (r'C:\Program Files (x86)\Windows Kits\10\Lib', 'Lib'),
+ (r'C:\Program Files (x86)\Windows Kits\10\Source', 'Source')):
+ src, target = ucrt_path
+ for root, _, files in os.walk(src):
+ for f in files:
+ combined = os.path.normpath(os.path.join(root, f))
+ to = os.path.join('ucrt', target, combined[len(src) + 1:])
+ result.append((combined, to))
+
# Generically drop all arm stuff that we don't need.
- return [(f, t) for f, t in result if 'arm\\' not in f.lower()]
+ return [(f, t) for f, t in result if 'arm\\' not in f.lower() and
+ 'arm64\\' not in f.lower()]
+
+
+def GenerateSetEnvCmd(target_dir):
+ """Generate a batch file that gyp expects to exist to set up the compiler
+ environment.
+
+ This is normally generated by a full install of the SDK, but we
+ do it here manually since we do not do a full install."""
+ with open(os.path.join(
+ target_dir, r'win_sdk\bin\SetEnv.cmd'), 'w') as f:
+ f.write('@echo off\n'
+ ':: Generated by win_toolchain\\package_from_installed.py.\n'
+ # Common to x86 and x64
+ 'set PATH=%~dp0..\\..\\Common7\\IDE;%PATH%\n'
+ 'set INCLUDE=%~dp0..\\..\\win_sdk\\Include\\um;'
+ '%~dp0..\\..\\win_sdk\\Include\\shared;'
+ '%~dp0..\\..\\win_sdk\\Include\\winrt;'
+ '%~dp0..\\..\\ucrt\\Include\\10.0.10056.0\\ucrt;'
+ '%~dp0..\\..\\VC\\include;'
+ '%~dp0..\\..\\VC\\atlmfc\\include\n'
+ 'if "%1"=="/x64" goto x64\n')
+
+ # x86. Always use amd64_x86 cross, not x86 on x86.
+ f.write('set PATH=%~dp0..\\..\\win_sdk\\bin\\x86;'
+ '%~dp0..\\..\\VC\\bin\\amd64_x86;'
+ '%~dp0..\\..\\VC\\bin\\amd64;' # Needed for mspdb1x0.dll.
+ '%PATH%\n')
+ f.write('set LIB=%~dp0..\\..\\VC\\lib;'
+ '%~dp0..\\..\\win_sdk\\Lib\\winv6.3\\um\\x86;'
+ '%~dp0..\\..\\ucrt\\Lib\\10.0.10056.0\\ucrt\\x86;'
+ '%~dp0..\\..\\VC\\atlmfc\\lib\n'
+ 'goto :EOF\n')
+
+ # x64.
+ f.write(':x64\n'
+ 'set PATH=%~dp0..\\..\\win_sdk\\bin\\x64;'
+ '%~dp0..\\..\\VC\\bin\\amd64;'
+ '%PATH%\n')
+ f.write('set LIB=%~dp0..\\..\\VC\\lib\\amd64;'
+ '%~dp0..\\..\\win_sdk\\Lib\\winv6.3\\um\\x64;'
+ '%~dp0..\\..\\ucrt\\Lib\\10.0.10056.0\\ucrt\\x64;'
+ '%~dp0..\\..\\VC\\atlmfc\\lib\\amd64\n')
def AddEnvSetup(files):
"""We need to generate this file in the same way that the "from pieces"
script does, so pull that in here."""
tempdir = tempfile.mkdtemp()
- os.makedirs(os.path.join(tempdir, 'win8sdk', 'bin'))
- toolchain2013.GenerateSetEnvCmd(tempdir, True)
- files.append((os.path.join(tempdir, 'win8sdk', 'bin', 'SetEnv.cmd'),
- 'win8sdk\\bin\\SetEnv.cmd'))
+ os.makedirs(os.path.join(tempdir, 'win_sdk', 'bin'))
+ GenerateSetEnvCmd(tempdir)
+ files.append((os.path.join(tempdir, 'win_sdk', 'bin', 'SetEnv.cmd'),
+ 'win_sdk\\bin\\SetEnv.cmd'))
+ vs_version_file = os.path.join(tempdir, 'VS_VERSION')
+ with open(vs_version_file, 'wb') as version:
+ print >>version, VS_VERSION
+ files.append((vs_version_file, 'VS_VERSION'))
def RenameToSha1(output):
@@ -109,7 +193,7 @@
tempdir = tempfile.mkdtemp()
old_dir = os.getcwd()
os.chdir(tempdir)
- rel_dir = 'vs2013_files'
+ rel_dir = 'vs_files'
with zipfile.ZipFile(
os.path.join(old_dir, output), 'r', zipfile.ZIP_DEFLATED, True) as zf:
zf.extractall(rel_dir)
@@ -123,6 +207,13 @@
def main():
+ if len(sys.argv) != 2 or sys.argv[1] not in ('2013', '2015'):
+ print 'Usage: package_from_installed.py 2013|2015'
+ return 1
+
+ global VS_VERSION
+ VS_VERSION = sys.argv[1]
+
print 'Building file list...'
files = BuildFileList()
