build/toolchain/win/midl.py - cobalt - Git at Google

 # Copyright 2017 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 from __future__ import division
 from __future__ import print_function

 import array
 import difflib
 import distutils.dir_util
 import filecmp
 import io
 import operator
 import os
 import posixpath
 import re
 import shutil
 import struct
 import subprocess
 import sys
 import tempfile
 import uuid

 from functools import reduce


 def ZapTimestamp(filename):
   contents = open(filename, 'rb').read()
   # midl.exe writes timestamp 2147483647 (2^31 - 1) as creation date into its
   # outputs, but using the local timezone.  To make the output timezone-
   # independent, replace that date with a fixed string of the same length.
   # Also blank out the minor version number.
   if filename.endswith('.tlb'):
     # See https://chromium-review.googlesource.com/c/chromium/src/+/693223 for
     # a fairly complete description of the .tlb binary format.
     # TLB files start with a 54 byte header. Offset 0x20 stores how many types
     # are defined in the file, and the header is followed by that many uint32s.
     # After that, 15 section headers appear.  Each section header is 16 bytes,
     # starting with offset and length uint32s.
     # Section 12 in the file contains custom() data. custom() data has a type
     # (int, string, etc).  Each custom data chunk starts with a uint16_t
     # describing its type.  Type 8 is string data, consisting of a uint32_t
     # len, followed by that many data bytes, followed by 'W' bytes to pad to a
     # 4 byte boundary.  Type 0x13 is uint32 data, followed by 4 data bytes,
     # followed by two 'W' to pad to a 4 byte boundary.
     # The custom block always starts with one string containing "Created by
     # MIDL version 8...", followed by one uint32 containing 0x7fffffff,
     # followed by another uint32 containing the MIDL compiler version (e.g.
     # 0x0801026e for v8.1.622 -- 0x26e == 622).  These 3 fields take 0x54 bytes.
     # There might be more custom data after that, but these 3 blocks are always
     # there for file-level metadata.
     # All data is little-endian in the file.
     assert contents[0:8] == b'MSFT\x02\x00\x01\x00'
     ntypes, = struct.unpack_from('<I', contents, 0x20)
     custom_off, custom_len = struct.unpack_from(
         '<II', contents, 0x54 + 4*ntypes + 11*16)
     assert custom_len >= 0x54
     # First: Type string (0x8), followed by 0x3e characters.
     assert contents[custom_off:custom_off + 6] == b'\x08\x00\x3e\x00\x00\x00'
     assert re.match(
         br'Created by MIDL version 8\.\d\d\.\d{4} '
         br'at ... Jan 1. ..:..:.. 2038\n',
         contents[custom_off + 6:custom_off + 6 + 0x3e])
     # Second: Type uint32 (0x13) storing 0x7fffffff (followed by WW / 0x57 pad)
     assert contents[custom_off+6+0x3e:custom_off+6+0x3e+8] == \
         b'\x13\x00\xff\xff\xff\x7f\x57\x57'
     # Third: Type uint32 (0x13) storing MIDL compiler version.
     assert contents[custom_off + 6 + 0x3e + 8:custom_off + 6 + 0x3e + 8 +
                     2] == b'\x13\x00'
     # Replace "Created by" string with fixed string, and fixed MIDL version with
     # 8.1.622 always.
     contents = (
         contents[0:custom_off + 6] +
         b'Created by MIDL version 8.xx.xxxx at a redacted point in time\n' +
         # uint32 (0x13) val 0x7fffffff, WW, uint32 (0x13), val 0x0801026e, WW
         b'\x13\x00\xff\xff\xff\x7f\x57\x57\x13\x00\x6e\x02\x01\x08\x57\x57' +
         contents[custom_off + 0x54:])
   else:
     contents = re.sub(
         br'File created by MIDL compiler version 8\.\d\d\.\d{4} \*/\r\n'
         br'/\* at ... Jan 1. ..:..:.. 2038',
         br'File created by MIDL compiler version 8.xx.xxxx */\r\n'
         br'/* at a redacted point in time', contents)
     contents = re.sub(
         br'    Oicf, W1, Zp8, env=(.....) \(32b run\), '
         br'target_arch=(AMD64|X86) 8\.\d\d\.\d{4}',
         br'    Oicf, W1, Zp8, env=\1 (32b run), target_arch=\2 8.xx.xxxx',
         contents)
     # TODO(thakis): If we need more hacks than these, try to verify checked-in
     # outputs when we're using the hermetic toolchain.
     # midl.exe older than 8.1.622 omit '//' after #endif, fix that:
     contents = contents.replace(b'#endif !_MIDL_USE_GUIDDEF_',
                                 b'#endif // !_MIDL_USE_GUIDDEF_')
     # midl.exe puts the midl version into code in one place.  To have
     # predictable output, lie about the midl version if it's not 8.1.622.
     # This is unfortunate, but remember that there's beauty too in imperfection.
     contents = contents.replace(b'0x801026c, /* MIDL Version 8.1.620 */',
                                 b'0x801026e, /* MIDL Version 8.1.622 */')
   open(filename, 'wb').write(contents)


 def get_tlb_contents(tlb_file):
   # See ZapTimestamp() for a short overview of the .tlb format.
   contents = open(tlb_file, 'rb').read()
   assert contents[0:8] == b'MSFT\x02\x00\x01\x00'
   ntypes, = struct.unpack_from('<I', contents, 0x20)
   type_off, type_len = struct.unpack_from('<II', contents, 0x54 + 4*ntypes)

   guid_off, guid_len = struct.unpack_from(
       '<II', contents, 0x54 + 4*ntypes + 5*16)
   assert guid_len % 24 == 0

   contents = array.array('B', contents)

   return contents, ntypes, type_off, guid_off, guid_len


 def recreate_guid_hashtable(contents, ntypes, guid_off, guid_len):
   # This function is called after changing guids in section 6 (the "guid"
   # section). This function recreates the GUID hashtable in section 5. Since the
   # hash table uses chaining, it's easiest to recompute it from scratch rather
   # than trying to patch it up.
   hashtab = [0xffffffff] * (0x80 // 4)
   for guidind in range(guid_off, guid_off + guid_len, 24):
     guidbytes, typeoff, nextguid = struct.unpack_from(
         '<16sII', contents, guidind)
     words = struct.unpack('<8H', guidbytes)
     # midl seems to use the following simple hash function for GUIDs:
     guidhash = reduce(operator.xor, [w for w in words]) % (0x80 // 4)
     nextguid = hashtab[guidhash]
     struct.pack_into('<I', contents, guidind + 0x14, nextguid)
     hashtab[guidhash] = guidind - guid_off
   hash_off, hash_len = struct.unpack_from(
       '<II', contents, 0x54 + 4*ntypes + 4*16)
   for i, hashval in enumerate(hashtab):
     struct.pack_into('<I', contents, hash_off + 4*i, hashval)


 def overwrite_guids_h(h_file, dynamic_guids):
   contents = open(h_file, 'rb').read()
   for key in dynamic_guids:
     contents = re.sub(key, dynamic_guids[key], contents, flags=re.I)
   open(h_file, 'wb').write(contents)


 def get_uuid_format(guid, prefix):
   formatted_uuid = '0x%s,0x%s,0x%s,' % (guid[0:8], guid[9:13], guid[14:18])
   formatted_uuid += '%s0x%s,0x%s' % (prefix, guid[19:21], guid[21:23])
   for i in range(24, len(guid), 2):
     formatted_uuid += ',0x' + guid[i:i + 2]
   return formatted_uuid


 def get_uuid_format_iid_file(guid):
   # Convert from "D0E1CACC-C63C-4192-94AB-BF8EAD0E3B83" to
   # 0xD0E1CACC,0xC63C,0x4192,0x94,0xAB,0xBF,0x8E,0xAD,0x0E,0x3B,0x83.
   return get_uuid_format(guid, '')


 def overwrite_guids_iid(iid_file, dynamic_guids):
   contents = open(iid_file, 'rb').read()
   for key in dynamic_guids:
     contents = re.sub(get_uuid_format_iid_file(key),
                       get_uuid_format_iid_file(dynamic_guids[key]),
                       contents,
                       flags=re.I)
   open(iid_file, 'wb').write(contents)


 def get_uuid_format_proxy_file(guid):
   # Convert from "D0E1CACC-C63C-4192-94AB-BF8EAD0E3B83" to
   # {0xD0E1CACC,0xC63C,0x4192,{0x94,0xAB,0xBF,0x8E,0xAD,0x0E,0x3B,0x83}}.
   return get_uuid_format(guid, '{')


 def overwrite_guids_proxy(proxy_file, dynamic_guids):
   contents = open(proxy_file, 'rb').read()
   for key in dynamic_guids:
     contents = re.sub(get_uuid_format_proxy_file(key),
                       get_uuid_format_proxy_file(dynamic_guids[key]),
                       contents,
                       flags=re.I)
   open(proxy_file, 'wb').write(contents)


 def getguid(contents, offset):
   # Returns a guid string of the form "D0E1CACC-C63C-4192-94AB-BF8EAD0E3B83".
   g0, g1, g2, g3 = struct.unpack_from('<IHH8s', contents, offset)
   g3 = ''.join(['%02X' % ord(g) for g in g3])
   return '%08X-%04X-%04X-%s-%s' % (g0, g1, g2, g3[0:4], g3[4:])


 def setguid(contents, offset, guid):
   guid = uuid.UUID(guid)
   struct.pack_into('<IHH8s', contents, offset,
                    *(guid.fields[0:3] + (guid.bytes[8:], )))


 def overwrite_guids_tlb(tlb_file, dynamic_guids):
   contents, ntypes, type_off, guid_off, guid_len = get_tlb_contents(tlb_file)

   for i in range(0, guid_len, 24):
     current_guid = getguid(contents, guid_off + i)
     for key in dynamic_guids:
       if key.lower() == current_guid.lower():
         setguid(contents, guid_off + i, dynamic_guids[key])

   recreate_guid_hashtable(contents, ntypes, guid_off, guid_len)
   open(tlb_file, 'wb').write(contents)


 # Handle multiple guid substitutions, where |dynamic_guids| is of the form
 # "PLACEHOLDER-GUID-158428a4-6014-4978-83ba-9fad0dabe791="
 # "3d852661-c795-4d20-9b95-5561e9a1d2d9,"
 # "PLACEHOLDER-GUID-63B8FFB1-5314-48C9-9C57-93EC8BC6184B="
 # "D0E1CACC-C63C-4192-94AB-BF8EAD0E3B83".
 #
 # Before specifying |dynamic_guids| in the build, the IDL file is first compiled
 # with "158428a4-6014-4978-83ba-9fad0dabe791" and
 # "63B8FFB1-5314-48C9-9C57-93EC8BC6184B". These are the "replaceable" guids,
 # i.e., guids that can be replaced in future builds. The resulting MIDL outputs
 # are copied over to src\third_party\win_build_output\.
 #
 # Then, in the future, any changes to these guids can be accomplished by
 # providing |dynamic_guids| of the format above in the build file. These
 # "dynamic" guid changes by themselves will not require the MIDL compiler and
 # therefore will not require copying output over to
 # src\third_party\win_build_output\.
 #
 # The pre-generated src\third_party\win_build_output\ files are used for
 # cross-compiling on other platforms, since the MIDL compiler is Windows-only.
 def overwrite_guids(h_file, iid_file, proxy_file, tlb_file, dynamic_guids):
   # Fix up GUIDs in .h, _i.c, _p.c, and .tlb.
   overwrite_guids_h(h_file, dynamic_guids)
   overwrite_guids_iid(iid_file, dynamic_guids)
   overwrite_guids_proxy(proxy_file, dynamic_guids)
   if tlb_file:
     overwrite_guids_tlb(tlb_file, dynamic_guids)


 # This function removes all occurrences of 'PLACEHOLDER-GUID-' from the
 # template, and if |dynamic_guids| is specified, also replaces the guids within
 # the file. Finally, it writes the resultant output to the |idl| file.
 def generate_idl_from_template(idl_template, dynamic_guids, idl):
   contents = open(idl_template, 'rb').read()
   contents = re.sub('PLACEHOLDER-GUID-', '', contents, flags=re.I)
   if dynamic_guids:
     for key in dynamic_guids:
       contents = re.sub(key, dynamic_guids[key], contents, flags=re.I)
   open(idl, 'wb').write(contents)


 # This function runs the MIDL compiler with the provided arguments. It creates
 # and returns a tuple of |0,midl_output_dir| on success.
 def run_midl(args, env_dict):
   midl_output_dir = tempfile.mkdtemp()
   delete_midl_output_dir = True

   try:
     popen = subprocess.Popen(args + ['/out', midl_output_dir],
                              shell=True,
                              env=env_dict,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.STDOUT)
     out, _ = popen.communicate()
     if popen.returncode != 0:
       return popen.returncode, midl_output_dir

     # Filter junk out of stdout, and write filtered versions. Output we want
     # to filter is pairs of lines that look like this:
     # Processing C:\Program Files (x86)\Microsoft SDKs\...\include\objidl.idl
     # objidl.idl
     lines = out.decode('utf-8').splitlines()
     prefixes = ('Processing ', '64 bit Processing ')
     processing = set(
         os.path.basename(x) for x in lines if x.startswith(prefixes))
     for line in lines:
       if not line.startswith(prefixes) and line not in processing:
         print(line)

     for f in os.listdir(midl_output_dir):
       ZapTimestamp(os.path.join(midl_output_dir, f))

     delete_midl_output_dir = False
   finally:
     if os.path.exists(midl_output_dir) and delete_midl_output_dir:
       shutil.rmtree(midl_output_dir)

   return 0, midl_output_dir


 # This function adds support for dynamic generation of guids: when values are
 # specified as 'uuid5:name', this function will substitute the values with
 # generated dynamic guids using the uuid5 function. The uuid5 function generates
 # a guid based on the SHA-1 hash of a namespace identifier (which is the guid
 # that comes after 'PLACEHOLDER-GUID-') and a name (which is a string, such as a
 # version string "87.1.2.3").
 #
 # For instance, when |dynamic_guid| is of the form:
 # "PLACEHOLDER-GUID-158428a4-6014-4978-83ba-9fad0dabe791=uuid5:88.0.4307.0
 # ,"
 # "PLACEHOLDER-GUID-63B8FFB1-5314-48C9-9C57-93EC8BC6184B=uuid5:88.0.4307.0
 # "
 #
 # "PLACEHOLDER-GUID-158428a4-6014-4978-83ba-9fad0dabe791" would be substituted
 # with uuid5("158428a4-6014-4978-83ba-9fad0dabe791", "88.0.4307.0"), which is
 # "64700170-AD80-5DE3-924E-2F39D862CFD5". And
 # "PLACEHOLDER-GUID-63B8FFB1-5314-48C9-9C57-93EC8BC6184B" would be
 # substituted with uuid5("63B8FFB1-5314-48C9-9C57-93EC8BC6184B", "88.0.4307.0"),
 # which is "7B6E7538-3C38-5565-BC92-42BCEE268D76".
 def uuid5_substitutions(dynamic_guids):
   for key, value in dynamic_guids.items():
     if value.startswith("uuid5:"):
       name = value.split("uuid5:", 1)[1]
       assert name
       dynamic_guids[key] = str(uuid.uuid5(uuid.UUID(key), name)).upper()


 def main(arch, gendir, outdir, dynamic_guids, tlb, h, dlldata, iid, proxy,
          clang, idl, *flags):
   # Copy checked-in outputs to final location.
   source = gendir
   if os.path.isdir(os.path.join(source, os.path.basename(idl))):
     source = os.path.join(source, os.path.basename(idl))
   source = os.path.join(source, arch.split('.')[1])  # Append 'x86' or 'x64'.
   source = os.path.normpath(source)

   source_exists = True
   if not os.path.isdir(source):
     source_exists = False
     if sys.platform != 'win32':
       print('Directory %s needs to be populated from Windows first' % source)
       return 1

     # This is a brand new IDL file that does not have outputs under
     # third_party\win_build_output\midl. We create an empty directory for now.
     os.makedirs(source)

   common_files = [h, iid]
   if tlb != 'none':
     # Not all projects use tlb files.
     common_files += [tlb]
   else:
     tlb = None

   if dlldata != 'none':
     # Not all projects use dlldta files.
     common_files += [dlldata]
   else:
     dlldata = None

   # Not all projects use proxy files
   if proxy != 'none':
     # Not all projects use proxy files.
     common_files += [proxy]
   else:
     proxy = None

   for source_file in common_files:
     file_path = os.path.join(source, source_file)
     if not os.path.isfile(file_path):
       source_exists = False
       if sys.platform != 'win32':
         print('File %s needs to be generated from Windows first' % file_path)
         return 1

       # Either this is a brand new IDL file that does not have outputs under
       # third_party\win_build_output\midl or the file is (unexpectedly) missing.
       # We create an empty file for now. The rest of the machinery below will
       # then generate the correctly populated file using the MIDL compiler and
       # instruct the developer to copy that file under
       # third_party\win_build_output\midl.
       open(file_path, 'wb').close()
     shutil.copy(file_path, outdir)

   if dynamic_guids != 'none':
     assert '=' in dynamic_guids
     if dynamic_guids.startswith("ignore_proxy_stub,"):
       # TODO(ganesh): The custom proxy/stub file ("_p.c") is not generated
       # correctly for dynamic IIDs (but correctly if there are only dynamic
       # CLSIDs). The proxy/stub lookup functions generated by MIDL.exe within
       # "_p.c" rely on a sorted set of vtable lists, which we are not currently
       # regenerating. At the moment, no project in Chromium that uses dynamic
       # IIDs is relying on the custom proxy/stub file. So for now, if
       # |dynamic_guids| is prefixed with "ignore_proxy_stub,", we exclude the
       # custom proxy/stub file from the directory comparisons.
       common_files.remove(proxy)
       dynamic_guids = dynamic_guids.split("ignore_proxy_stub,", 1)[1]
     dynamic_guids = re.sub('PLACEHOLDER-GUID-', '', dynamic_guids, flags=re.I)
     dynamic_guids = dynamic_guids.split(',')
     dynamic_guids = dict(s.split('=') for s in dynamic_guids)
     uuid5_substitutions(dynamic_guids)
     if source_exists:
       overwrite_guids(*(os.path.join(outdir, file) if file else None
                         for file in [h, iid, proxy, tlb]),
                       dynamic_guids=dynamic_guids)
   else:
     dynamic_guids = None

   # On non-Windows, that's all we can do.
   if sys.platform != 'win32':
     return 0

   idl_template = None
   if dynamic_guids:
     idl_template = idl

     # posixpath is used here to keep the MIDL-generated files with a uniform
     # separator of '/' instead of mixed '/' and '\\'.
     idl = posixpath.join(
         outdir,
         os.path.splitext(os.path.basename(idl_template))[0] + '.idl')

     # |idl_template| can contain one or more occurrences of guids that are
     # substituted with |dynamic_guids|, and then MIDL is run on the substituted
     # IDL file.
     generate_idl_from_template(idl_template, dynamic_guids, idl)

   # On Windows, run midl.exe on the input and check that its outputs are
   # identical to the checked-in outputs (after replacing guids if
   # |dynamic_guids| is specified).

   # Read the environment block from the file. This is stored in the format used
   # by CreateProcess. Drop last 2 NULs, one for list terminator, one for
   # trailing vs. separator.
   env_pairs = open(arch).read()[:-2].split('\0')
   env_dict = dict([item.split('=', 1) for item in env_pairs])

   # Extract the /D options and send them to the preprocessor.
   preprocessor_options = '-E -nologo -Wno-nonportable-include-path'
   preprocessor_options += ''.join(
       [' ' + flag for flag in flags if flag.startswith('/D')])
   args = ['midl', '/nologo'] + list(flags) + (['/tlb', tlb] if tlb else []) + [
       '/h', h
   ] + (['/dlldata', dlldata] if dlldata else []) + ['/iid', iid] + (
       ['/proxy', proxy] if proxy else
       []) + ['/cpp_cmd', clang, '/cpp_opt', preprocessor_options, idl]

   returncode, midl_output_dir = run_midl(args, env_dict)
   if returncode != 0:
     return returncode

   # Now compare the output in midl_output_dir to the copied-over outputs.
   _, mismatch, errors = filecmp.cmpfiles(midl_output_dir, outdir, common_files)
   assert not errors

   if mismatch:
     print('midl.exe output different from files in %s, see %s' %
           (outdir, midl_output_dir))
     for f in mismatch:
       if f.endswith('.tlb'): continue
       fromfile = os.path.join(outdir, f)
       tofile = os.path.join(midl_output_dir, f)
       print(''.join(
           difflib.unified_diff(
               io.open(fromfile).readlines(),
               io.open(tofile).readlines(), fromfile, tofile)))

     if dynamic_guids:
       # |idl_template| can contain one or more occurrences of guids prefixed
       # with 'PLACEHOLDER-GUID-'. We first remove the extraneous
       # 'PLACEHOLDER-GUID-' prefix and then run MIDL on the substituted IDL
       # file.
       # No guid substitutions are done at this point, because we want to compile
       # with the placeholder guids and then instruct the user to copy the output
       # over to |source| which is typically src\third_party\win_build_output\.
       # In future runs, the placeholder guids in |source| are replaced with the
       # guids specified in |dynamic_guids|.
       generate_idl_from_template(idl_template, None, idl)
       returncode, midl_output_dir = run_midl(args, env_dict)
       if returncode != 0:
         return returncode

     print('To rebaseline:')
     print(r'  copy /y %s\* %s' % (midl_output_dir, source))
     return 1

   return 0


 if __name__ == '__main__':
   sys.exit(main(*sys.argv[1:]))
	# Copyright 2017 The Chromium Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	from __future__ import division
	from __future__ import print_function

	import array
	import difflib
	import distutils.dir_util
	import filecmp
	import io
	import operator
	import os
	import posixpath
	import re
	import shutil
	import struct
	import subprocess
	import sys
	import tempfile
	import uuid

	from functools import reduce


	def ZapTimestamp(filename):
	contents = open(filename, 'rb').read()
	# midl.exe writes timestamp 2147483647 (2^31 - 1) as creation date into its
	# outputs, but using the local timezone. To make the output timezone-
	# independent, replace that date with a fixed string of the same length.
	# Also blank out the minor version number.
	if filename.endswith('.tlb'):
	# See https://chromium-review.googlesource.com/c/chromium/src/+/693223 for
	# a fairly complete description of the .tlb binary format.
	# TLB files start with a 54 byte header. Offset 0x20 stores how many types
	# are defined in the file, and the header is followed by that many uint32s.
	# After that, 15 section headers appear. Each section header is 16 bytes,
	# starting with offset and length uint32s.
	# Section 12 in the file contains custom() data. custom() data has a type
	# (int, string, etc). Each custom data chunk starts with a uint16_t
	# describing its type. Type 8 is string data, consisting of a uint32_t
	# len, followed by that many data bytes, followed by 'W' bytes to pad to a
	# 4 byte boundary. Type 0x13 is uint32 data, followed by 4 data bytes,
	# followed by two 'W' to pad to a 4 byte boundary.
	# The custom block always starts with one string containing "Created by
	# MIDL version 8...", followed by one uint32 containing 0x7fffffff,
	# followed by another uint32 containing the MIDL compiler version (e.g.
	# 0x0801026e for v8.1.622 -- 0x26e == 622). These 3 fields take 0x54 bytes.
	# There might be more custom data after that, but these 3 blocks are always
	# there for file-level metadata.
	# All data is little-endian in the file.
	assert contents[0:8] == b'MSFT\x02\x00\x01\x00'
	ntypes, = struct.unpack_from('<I', contents, 0x20)
	custom_off, custom_len = struct.unpack_from(
	'<II', contents, 0x54 + 4ntypes + 1116)
	assert custom_len >= 0x54
	# First: Type string (0x8), followed by 0x3e characters.
	assert contents[custom_off:custom_off + 6] == b'\x08\x00\x3e\x00\x00\x00'
	assert re.match(
	br'Created by MIDL version 8\.\d\d\.\d{4} '
	br'at ... Jan 1. ..:..:.. 2038\n',
	contents[custom_off + 6:custom_off + 6 + 0x3e])
	# Second: Type uint32 (0x13) storing 0x7fffffff (followed by WW / 0x57 pad)
	assert contents[custom_off+6+0x3e:custom_off+6+0x3e+8] == \
	b'\x13\x00\xff\xff\xff\x7f\x57\x57'
	# Third: Type uint32 (0x13) storing MIDL compiler version.
	assert contents[custom_off + 6 + 0x3e + 8:custom_off + 6 + 0x3e + 8 +
	2] == b'\x13\x00'
	# Replace "Created by" string with fixed string, and fixed MIDL version with
	# 8.1.622 always.
	contents = (
	contents[0:custom_off + 6] +
	b'Created by MIDL version 8.xx.xxxx at a redacted point in time\n' +
	# uint32 (0x13) val 0x7fffffff, WW, uint32 (0x13), val 0x0801026e, WW
	b'\x13\x00\xff\xff\xff\x7f\x57\x57\x13\x00\x6e\x02\x01\x08\x57\x57' +
	contents[custom_off + 0x54:])
	else:
	contents = re.sub(
	br'File created by MIDL compiler version 8\.\d\d\.\d{4} \*/\r\n'
	br'/\* at ... Jan 1. ..:..:.. 2038',
	br'File created by MIDL compiler version 8.xx.xxxx */\r\n'
	br'/* at a redacted point in time', contents)
	contents = re.sub(
	br' Oicf, W1, Zp8, env=(.....) \(32b run\), '
	br'target_arch=(AMD64\|X86) 8\.\d\d\.\d{4}',
	br' Oicf, W1, Zp8, env=\1 (32b run), target_arch=\2 8.xx.xxxx',
	contents)
	# TODO(thakis): If we need more hacks than these, try to verify checked-in
	# outputs when we're using the hermetic toolchain.
	# midl.exe older than 8.1.622 omit '//' after #endif, fix that:
	contents = contents.replace(b'#endif !_MIDL_USE_GUIDDEF_',
	b'#endif // !_MIDL_USE_GUIDDEF_')
	# midl.exe puts the midl version into code in one place. To have
	# predictable output, lie about the midl version if it's not 8.1.622.
	# This is unfortunate, but remember that there's beauty too in imperfection.
	contents = contents.replace(b'0x801026c, /* MIDL Version 8.1.620 */',
	b'0x801026e, /* MIDL Version 8.1.622 */')
	open(filename, 'wb').write(contents)


	def get_tlb_contents(tlb_file):
	# See ZapTimestamp() for a short overview of the .tlb format.
	contents = open(tlb_file, 'rb').read()
	assert contents[0:8] == b'MSFT\x02\x00\x01\x00'
	ntypes, = struct.unpack_from('<I', contents, 0x20)
	type_off, type_len = struct.unpack_from('<II', contents, 0x54 + 4*ntypes)

	guid_off, guid_len = struct.unpack_from(
	'<II', contents, 0x54 + 4ntypes + 516)
	assert guid_len % 24 == 0

	contents = array.array('B', contents)

	return contents, ntypes, type_off, guid_off, guid_len


	def recreate_guid_hashtable(contents, ntypes, guid_off, guid_len):
	# This function is called after changing guids in section 6 (the "guid"
	# section). This function recreates the GUID hashtable in section 5. Since the
	# hash table uses chaining, it's easiest to recompute it from scratch rather
	# than trying to patch it up.
	hashtab = [0xffffffff] * (0x80 // 4)
	for guidind in range(guid_off, guid_off + guid_len, 24):
	guidbytes, typeoff, nextguid = struct.unpack_from(
	'<16sII', contents, guidind)
	words = struct.unpack('<8H', guidbytes)
	# midl seems to use the following simple hash function for GUIDs:
	guidhash = reduce(operator.xor, [w for w in words]) % (0x80 // 4)
	nextguid = hashtab[guidhash]
	struct.pack_into('<I', contents, guidind + 0x14, nextguid)
	hashtab[guidhash] = guidind - guid_off
	hash_off, hash_len = struct.unpack_from(
	'<II', contents, 0x54 + 4ntypes + 416)
	for i, hashval in enumerate(hashtab):
	struct.pack_into('<I', contents, hash_off + 4*i, hashval)


	def overwrite_guids_h(h_file, dynamic_guids):
	contents = open(h_file, 'rb').read()
	for key in dynamic_guids:
	contents = re.sub(key, dynamic_guids[key], contents, flags=re.I)
	open(h_file, 'wb').write(contents)


	def get_uuid_format(guid, prefix):
	formatted_uuid = '0x%s,0x%s,0x%s,' % (guid[0:8], guid[9:13], guid[14:18])
	formatted_uuid += '%s0x%s,0x%s' % (prefix, guid[19:21], guid[21:23])
	for i in range(24, len(guid), 2):
	formatted_uuid += ',0x' + guid[i:i + 2]
	return formatted_uuid


	def get_uuid_format_iid_file(guid):
	# Convert from "D0E1CACC-C63C-4192-94AB-BF8EAD0E3B83" to
	# 0xD0E1CACC,0xC63C,0x4192,0x94,0xAB,0xBF,0x8E,0xAD,0x0E,0x3B,0x83.
	return get_uuid_format(guid, '')


	def overwrite_guids_iid(iid_file, dynamic_guids):
	contents = open(iid_file, 'rb').read()
	for key in dynamic_guids:
	contents = re.sub(get_uuid_format_iid_file(key),
	get_uuid_format_iid_file(dynamic_guids[key]),
	contents,
	flags=re.I)
	open(iid_file, 'wb').write(contents)


	def get_uuid_format_proxy_file(guid):
	# Convert from "D0E1CACC-C63C-4192-94AB-BF8EAD0E3B83" to
	# {0xD0E1CACC,0xC63C,0x4192,{0x94,0xAB,0xBF,0x8E,0xAD,0x0E,0x3B,0x83}}.
	return get_uuid_format(guid, '{')


	def overwrite_guids_proxy(proxy_file, dynamic_guids):
	contents = open(proxy_file, 'rb').read()
	for key in dynamic_guids:
	contents = re.sub(get_uuid_format_proxy_file(key),
	get_uuid_format_proxy_file(dynamic_guids[key]),
	contents,
	flags=re.I)
	open(proxy_file, 'wb').write(contents)


	def getguid(contents, offset):
	# Returns a guid string of the form "D0E1CACC-C63C-4192-94AB-BF8EAD0E3B83".
	g0, g1, g2, g3 = struct.unpack_from('<IHH8s', contents, offset)
	g3 = ''.join(['%02X' % ord(g) for g in g3])
	return '%08X-%04X-%04X-%s-%s' % (g0, g1, g2, g3[0:4], g3[4:])


	def setguid(contents, offset, guid):
	guid = uuid.UUID(guid)
	struct.pack_into('<IHH8s', contents, offset,
	*(guid.fields[0:3] + (guid.bytes[8:], )))


	def overwrite_guids_tlb(tlb_file, dynamic_guids):
	contents, ntypes, type_off, guid_off, guid_len = get_tlb_contents(tlb_file)

	for i in range(0, guid_len, 24):
	current_guid = getguid(contents, guid_off + i)
	for key in dynamic_guids:
	if key.lower() == current_guid.lower():
	setguid(contents, guid_off + i, dynamic_guids[key])

	recreate_guid_hashtable(contents, ntypes, guid_off, guid_len)
	open(tlb_file, 'wb').write(contents)


	# Handle multiple guid substitutions, where \|dynamic_guids\| is of the form
	# "PLACEHOLDER-GUID-158428a4-6014-4978-83ba-9fad0dabe791="
	# "3d852661-c795-4d20-9b95-5561e9a1d2d9,"
	# "PLACEHOLDER-GUID-63B8FFB1-5314-48C9-9C57-93EC8BC6184B="
	# "D0E1CACC-C63C-4192-94AB-BF8EAD0E3B83".
	#
	# Before specifying \|dynamic_guids\| in the build, the IDL file is first compiled
	# with "158428a4-6014-4978-83ba-9fad0dabe791" and
	# "63B8FFB1-5314-48C9-9C57-93EC8BC6184B". These are the "replaceable" guids,
	# i.e., guids that can be replaced in future builds. The resulting MIDL outputs
	# are copied over to src\third_party\win_build_output\.
	#
	# Then, in the future, any changes to these guids can be accomplished by
	# providing \|dynamic_guids\| of the format above in the build file. These
	# "dynamic" guid changes by themselves will not require the MIDL compiler and
	# therefore will not require copying output over to
	# src\third_party\win_build_output\.
	#
	# The pre-generated src\third_party\win_build_output\ files are used for
	# cross-compiling on other platforms, since the MIDL compiler is Windows-only.
	def overwrite_guids(h_file, iid_file, proxy_file, tlb_file, dynamic_guids):
	# Fix up GUIDs in .h, _i.c, _p.c, and .tlb.
	overwrite_guids_h(h_file, dynamic_guids)
	overwrite_guids_iid(iid_file, dynamic_guids)
	overwrite_guids_proxy(proxy_file, dynamic_guids)
	if tlb_file:
	overwrite_guids_tlb(tlb_file, dynamic_guids)


	# This function removes all occurrences of 'PLACEHOLDER-GUID-' from the
	# template, and if \|dynamic_guids\| is specified, also replaces the guids within
	# the file. Finally, it writes the resultant output to the \|idl\| file.
	def generate_idl_from_template(idl_template, dynamic_guids, idl):
	contents = open(idl_template, 'rb').read()
	contents = re.sub('PLACEHOLDER-GUID-', '', contents, flags=re.I)
	if dynamic_guids:
	for key in dynamic_guids:
	contents = re.sub(key, dynamic_guids[key], contents, flags=re.I)
	open(idl, 'wb').write(contents)


	# This function runs the MIDL compiler with the provided arguments. It creates
	# and returns a tuple of \|0,midl_output_dir\| on success.
	def run_midl(args, env_dict):
	midl_output_dir = tempfile.mkdtemp()
	delete_midl_output_dir = True

	try:
	popen = subprocess.Popen(args + ['/out', midl_output_dir],
	shell=True,
	env=env_dict,
	stdout=subprocess.PIPE,
	stderr=subprocess.STDOUT)
	out, _ = popen.communicate()
	if popen.returncode != 0:
	return popen.returncode, midl_output_dir

	# Filter junk out of stdout, and write filtered versions. Output we want
	# to filter is pairs of lines that look like this:
	# Processing C:\Program Files (x86)\Microsoft SDKs\...\include\objidl.idl
	# objidl.idl
	lines = out.decode('utf-8').splitlines()
	prefixes = ('Processing ', '64 bit Processing ')
	processing = set(
	os.path.basename(x) for x in lines if x.startswith(prefixes))
	for line in lines:
	if not line.startswith(prefixes) and line not in processing:
	print(line)

	for f in os.listdir(midl_output_dir):
	ZapTimestamp(os.path.join(midl_output_dir, f))

	delete_midl_output_dir = False
	finally:
	if os.path.exists(midl_output_dir) and delete_midl_output_dir:
	shutil.rmtree(midl_output_dir)

	return 0, midl_output_dir


	# This function adds support for dynamic generation of guids: when values are
	# specified as 'uuid5:name', this function will substitute the values with
	# generated dynamic guids using the uuid5 function. The uuid5 function generates
	# a guid based on the SHA-1 hash of a namespace identifier (which is the guid
	# that comes after 'PLACEHOLDER-GUID-') and a name (which is a string, such as a
	# version string "87.1.2.3").
	#
	# For instance, when \|dynamic_guid\| is of the form:
	# "PLACEHOLDER-GUID-158428a4-6014-4978-83ba-9fad0dabe791=uuid5:88.0.4307.0
	# ,"
	# "PLACEHOLDER-GUID-63B8FFB1-5314-48C9-9C57-93EC8BC6184B=uuid5:88.0.4307.0
	# "
	#
	# "PLACEHOLDER-GUID-158428a4-6014-4978-83ba-9fad0dabe791" would be substituted
	# with uuid5("158428a4-6014-4978-83ba-9fad0dabe791", "88.0.4307.0"), which is
	# "64700170-AD80-5DE3-924E-2F39D862CFD5". And
	# "PLACEHOLDER-GUID-63B8FFB1-5314-48C9-9C57-93EC8BC6184B" would be
	# substituted with uuid5("63B8FFB1-5314-48C9-9C57-93EC8BC6184B", "88.0.4307.0"),
	# which is "7B6E7538-3C38-5565-BC92-42BCEE268D76".
	def uuid5_substitutions(dynamic_guids):
	for key, value in dynamic_guids.items():
	if value.startswith("uuid5:"):
	name = value.split("uuid5:", 1)[1]
	assert name
	dynamic_guids[key] = str(uuid.uuid5(uuid.UUID(key), name)).upper()


	def main(arch, gendir, outdir, dynamic_guids, tlb, h, dlldata, iid, proxy,
	clang, idl, *flags):
	# Copy checked-in outputs to final location.
	source = gendir
	if os.path.isdir(os.path.join(source, os.path.basename(idl))):
	source = os.path.join(source, os.path.basename(idl))
	source = os.path.join(source, arch.split('.')[1]) # Append 'x86' or 'x64'.
	source = os.path.normpath(source)

	source_exists = True
	if not os.path.isdir(source):
	source_exists = False
	if sys.platform != 'win32':
	print('Directory %s needs to be populated from Windows first' % source)
	return 1

	# This is a brand new IDL file that does not have outputs under
	# third_party\win_build_output\midl. We create an empty directory for now.
	os.makedirs(source)

	common_files = [h, iid]
	if tlb != 'none':
	# Not all projects use tlb files.
	common_files += [tlb]
	else:
	tlb = None

	if dlldata != 'none':
	# Not all projects use dlldta files.
	common_files += [dlldata]
	else:
	dlldata = None

	# Not all projects use proxy files
	if proxy != 'none':
	# Not all projects use proxy files.
	common_files += [proxy]
	else:
	proxy = None

	for source_file in common_files:
	file_path = os.path.join(source, source_file)
	if not os.path.isfile(file_path):
	source_exists = False
	if sys.platform != 'win32':
	print('File %s needs to be generated from Windows first' % file_path)
	return 1

	# Either this is a brand new IDL file that does not have outputs under
	# third_party\win_build_output\midl or the file is (unexpectedly) missing.
	# We create an empty file for now. The rest of the machinery below will
	# then generate the correctly populated file using the MIDL compiler and
	# instruct the developer to copy that file under
	# third_party\win_build_output\midl.
	open(file_path, 'wb').close()
	shutil.copy(file_path, outdir)

	if dynamic_guids != 'none':
	assert '=' in dynamic_guids
	if dynamic_guids.startswith("ignore_proxy_stub,"):
	# TODO(ganesh): The custom proxy/stub file ("_p.c") is not generated
	# correctly for dynamic IIDs (but correctly if there are only dynamic
	# CLSIDs). The proxy/stub lookup functions generated by MIDL.exe within
	# "_p.c" rely on a sorted set of vtable lists, which we are not currently
	# regenerating. At the moment, no project in Chromium that uses dynamic
	# IIDs is relying on the custom proxy/stub file. So for now, if
	# \|dynamic_guids\| is prefixed with "ignore_proxy_stub,", we exclude the
	# custom proxy/stub file from the directory comparisons.
	common_files.remove(proxy)
	dynamic_guids = dynamic_guids.split("ignore_proxy_stub,", 1)[1]
	dynamic_guids = re.sub('PLACEHOLDER-GUID-', '', dynamic_guids, flags=re.I)
	dynamic_guids = dynamic_guids.split(',')
	dynamic_guids = dict(s.split('=') for s in dynamic_guids)
	uuid5_substitutions(dynamic_guids)
	if source_exists:
	overwrite_guids(*(os.path.join(outdir, file) if file else None
	for file in [h, iid, proxy, tlb]),
	dynamic_guids=dynamic_guids)
	else:
	dynamic_guids = None

	# On non-Windows, that's all we can do.
	if sys.platform != 'win32':
	return 0

	idl_template = None
	if dynamic_guids:
	idl_template = idl

	# posixpath is used here to keep the MIDL-generated files with a uniform
	# separator of '/' instead of mixed '/' and '\\'.
	idl = posixpath.join(
	outdir,
	os.path.splitext(os.path.basename(idl_template))[0] + '.idl')

	# \|idl_template\| can contain one or more occurrences of guids that are
	# substituted with \|dynamic_guids\|, and then MIDL is run on the substituted
	# IDL file.
	generate_idl_from_template(idl_template, dynamic_guids, idl)

	# On Windows, run midl.exe on the input and check that its outputs are
	# identical to the checked-in outputs (after replacing guids if
	# \|dynamic_guids\| is specified).

	# Read the environment block from the file. This is stored in the format used
	# by CreateProcess. Drop last 2 NULs, one for list terminator, one for
	# trailing vs. separator.
	env_pairs = open(arch).read()[:-2].split('\0')
	env_dict = dict([item.split('=', 1) for item in env_pairs])

	# Extract the /D options and send them to the preprocessor.
	preprocessor_options = '-E -nologo -Wno-nonportable-include-path'
	preprocessor_options += ''.join(
	[' ' + flag for flag in flags if flag.startswith('/D')])
	args = ['midl', '/nologo'] + list(flags) + (['/tlb', tlb] if tlb else []) + [
	'/h', h
	] + (['/dlldata', dlldata] if dlldata else []) + ['/iid', iid] + (
	['/proxy', proxy] if proxy else
	[]) + ['/cpp_cmd', clang, '/cpp_opt', preprocessor_options, idl]

	returncode, midl_output_dir = run_midl(args, env_dict)
	if returncode != 0:
	return returncode

	# Now compare the output in midl_output_dir to the copied-over outputs.
	_, mismatch, errors = filecmp.cmpfiles(midl_output_dir, outdir, common_files)
	assert not errors

	if mismatch:
	print('midl.exe output different from files in %s, see %s' %
	(outdir, midl_output_dir))
	for f in mismatch:
	if f.endswith('.tlb'): continue
	fromfile = os.path.join(outdir, f)
	tofile = os.path.join(midl_output_dir, f)
	print(''.join(
	difflib.unified_diff(
	io.open(fromfile).readlines(),
	io.open(tofile).readlines(), fromfile, tofile)))

	if dynamic_guids:
	# \|idl_template\| can contain one or more occurrences of guids prefixed
	# with 'PLACEHOLDER-GUID-'. We first remove the extraneous
	# 'PLACEHOLDER-GUID-' prefix and then run MIDL on the substituted IDL
	# file.
	# No guid substitutions are done at this point, because we want to compile
	# with the placeholder guids and then instruct the user to copy the output
	# over to \|source\| which is typically src\third_party\win_build_output\.
	# In future runs, the placeholder guids in \|source\| are replaced with the
	# guids specified in \|dynamic_guids\|.
	generate_idl_from_template(idl_template, None, idl)
	returncode, midl_output_dir = run_midl(args, env_dict)
	if returncode != 0:
	return returncode

	print('To rebaseline:')
	print(r' copy /y %s\* %s' % (midl_output_dir, source))
	return 1

	return 0


	if __name__ == '__main__':
	sys.exit(main(*sys.argv[1:]))