Fix Python 3 incompatibilities

After running gyptest.py and fixing issues found in Python 3,
I've modified some of the code to support both Python 2 and 3.
I've tested this using Python 3.7.3 and Python 2.7.15

Change-Id: I3f4a0f05821b76edcf5b8827a7931a629b799aa6
Reviewed-on: https://chromium-review.googlesource.com/c/external/gyp/+/1639564
Reviewed-by: Dirk Pranke <[email protected]>

AUTHORS

@@ -14,3 +14,4 @@ Tom Freudenberg <[email protected]>
 Julien Brianceau <[email protected]>
 Refael Ackermann <[email protected]>
 Jiajie Hu <[email protected]>
+Philip Nery <[email protected]>

pylib/gyp/generator/make.py

@@ -1773,7 +1773,8 @@ def WriteMakeRule(self, outputs, inputs, actions=None, comment=None,
       # - The multi-output rule will have an do-nothing recipe.
 
       # Hash the target name to avoid generating overlong filenames.
-      cmddigest = hashlib.sha1(command if command else self.target).hexdigest()
+      cmdstring = (command if command else self.target).encode('utf-8')
+      cmddigest = hashlib.sha1(cmdstring).hexdigest()
       intermediate = "%s.intermediate" % (cmddigest)
       self.WriteLn('%s: %s' % (' '.join(outputs), intermediate))
       self.WriteLn('\t%s' % '@:');

pylib/gyp/input.py

@@ -902,7 +902,8 @@ def ExpandVariables(input, phase, variables, build_file):
           p_stdout, p_stderr = p.communicate('')
 
           if p.wait() != 0 or p_stderr:
-            sys.stderr.write(p_stderr)
+            p_stderr_decoded = p_stderr.decode('utf-8')
+            sys.stderr.write(p_stderr_decoded)
             # Simulate check_call behavior, since check_call only exists
             # in python 2.5 and later.
             raise GypError("Call to '%s' returned exit status %d while in %s." %
@@ -948,7 +949,7 @@ def ExpandVariables(input, phase, variables, build_file):
       ProcessVariablesAndConditionsInList(replacement, phase, variables,
                                           build_file)
     elif type(replacement) not in (str, int):
-          raise GypError('Variable ' + contents +
+          raise GypError('Variable ' + str(contents) +
                          ' must expand to a string or list of strings; ' +
                          'found a ' + replacement.__class__.__name__)
 

pylib/gyp/mac_tool.py

@@ -113,13 +113,14 @@ def _CopyXIBFile(self, source, dest):
       raise
     current_section_header = None
     for line in stdout.splitlines():
-      if ibtool_section_re.match(line):
-        current_section_header = line
-      elif not ibtool_re.match(line):
+      line_decoded = line.decode('utf-8')
+      if ibtool_section_re.match(line_decoded):
+        current_section_header = line_decoded
+      elif not ibtool_re.match(line_decoded):
         if current_section_header:
           print(current_section_header)
           current_section_header = None
-        print(line)
+        print(line_decoded)
     return 0
 
   def _ConvertToBinary(self, dest):
@@ -270,8 +271,9 @@ def ExecFilterLibtool(self, *cmd_list):
     libtoolout = subprocess.Popen(cmd_list, stderr=subprocess.PIPE, env=env)
     _, err = libtoolout.communicate()
     for line in err.splitlines():
-      if not libtool_re.match(line) and not libtool_re5.match(line):
-        print(line, file=sys.stderr)
+      line_decoded = line.decode('utf-8')
+      if not libtool_re.match(line_decoded) and not libtool_re5.match(line_decoded):
+        print(line_decoded, file=sys.stderr)
     # Unconditionally touch the output .a file on the command line if present
     # and the command succeeded. A bit hacky.
     if not libtoolout.returncode:

pylib/gyp/ninja_syntax.py

@@ -149,6 +149,14 @@ def _as_list(self, input):
             return []
         if isinstance(input, list):
             return input
+
+        # map is not a class in Python 2
+        try:
+            if isinstance(input, map):
+                return list(input)
+        except TypeError:
+            pass
+
         return [input]
 
 

pylib/gyp/xcode_emulation.py

@@ -526,7 +526,7 @@ def _XcodeSdkPath(self, sdk_root):
       XcodeSettings._sdk_path_cache[sdk_root] = sdk_path
       if sdk_root:
         XcodeSettings._sdk_root_cache[sdk_path] = sdk_root
-    return XcodeSettings._sdk_path_cache[sdk_root].decode()
+    return XcodeSettings._sdk_path_cache[sdk_root]
 
   def _AppendPlatformVersionMinFlags(self, lst):
     self._Appendf(lst, 'MACOSX_DEPLOYMENT_TARGET', '-mmacosx-version-min=%s')
@@ -926,7 +926,7 @@ def GetLdflags(self, configname, product_dir, gyp_to_build_path, arch=None):
       # extensions and provide loader and main function.
       # These flags reflect the compilation options used by xcode to compile
       # extensions.
-      if XcodeVersion() < '0900':
+      if XcodeVersion()[0] < '0900':
         ldflags.append('-lpkstart')
         ldflags.append(sdk_root +
             '/System/Library/PrivateFrameworks/PlugInKit.framework/PlugInKit')
@@ -1133,8 +1133,9 @@ def _GetIOSCodeSignIdentityKey(self, settings):
       output = subprocess.check_output(
           ['security', 'find-identity', '-p', 'codesigning', '-v'])
       for line in output.splitlines():
-        if identity in line:
-          fingerprint = line.split()[1]
+        line_decoded = line.decode('utf-8')
+        if identity in line_decoded:
+          fingerprint = line_decoded.split()[1]
           cache = XcodeSettings._codesigning_key_cache
           assert identity not in cache or fingerprint == cache[identity], (
               "Multiple codesigning fingerprints for identity: %s" % identity)
@@ -1413,7 +1414,7 @@ def XcodeVersion():
   version = version_list[0]
   build = version_list[-1]
   # Be careful to convert "4.2" to "0420":
-  version = version.split()[-1].decode().replace('.', '')
+  version = version.split()[-1].replace('.', '')
   version = (version + '0' * (3 - len(version))).zfill(4)
   if build:
     build = build.split()[-1]
@@ -1452,7 +1453,7 @@ def GetStdout(cmdlist):
   if job.returncode != 0:
     sys.stderr.write(out + b'\n')
     raise GypError('Error %d running %s' % (job.returncode, cmdlist[0]))
-  return out.rstrip(b'\n')
+  return out.rstrip(b'\n').decode('utf-8')
 
 
 def MergeGlobalXcodeSettingsToSpec(global_dict, spec):
@@ -1660,7 +1661,7 @@ def _GetXcodeEnv(xcode_settings, built_products_dir, srcroot, configuration,
   install_name_base = xcode_settings.GetInstallNameBase()
   if install_name_base:
     env['DYLIB_INSTALL_NAME_BASE'] = install_name_base
-  if XcodeVersion() >= '0500' and not env.get('SDKROOT'):
+  if XcodeVersion()[0] >= '0500' and not env.get('SDKROOT'):
     sdk_root = xcode_settings._SdkRoot(configuration)
     if not sdk_root:
       sdk_root = xcode_settings._XcodeSdkPath('')

pylib/gyp/xcodeproj_file.py

@@ -137,6 +137,7 @@
 a project file is output.
 """
 
+import functools
 import gyp.common
 import posixpath
 import re
@@ -430,7 +431,7 @@ def _HashUpdate(hash, data):
       """
 
       hash.update(struct.pack('>i', len(data)))
-      hash.update(data)
+      hash.update(data.encode('utf-8'))
 
     if seed_hash is None:
       seed_hash = _new_sha1()
@@ -1417,7 +1418,8 @@ def TakeOverOnlyChild(self, recurse=False):
 
   def SortGroup(self):
     self._properties['children'] = \
-        sorted(self._properties['children'], cmp=lambda x,y: x.Compare(y))
+        sorted(self._properties['children'],
+               key=functools.cmp_to_key(XCHierarchicalElement.Compare))
 
     # Recurse.
     for child in self._properties['children']:
@@ -2721,7 +2723,7 @@ def SortGroups(self):
     # according to their defined order.
     self._properties['mainGroup']._properties['children'] = \
         sorted(self._properties['mainGroup']._properties['children'],
-               cmp=lambda x,y: x.CompareRootGroup(y))
+               key=functools.cmp_to_key(XCHierarchicalElement.CompareRootGroup))
 
     # Sort everything else by putting group before files, and going
     # alphabetically by name within sections of groups and files.  SortGroup
@@ -2812,9 +2814,8 @@ def AddOrGetProjectReference(self, other_pbxproject):
 
       # Xcode seems to sort this list case-insensitively
       self._properties['projectReferences'] = \
-          sorted(self._properties['projectReferences'], cmp=lambda x,y:
-                 cmp(x['ProjectRef'].Name().lower(),
-                     y['ProjectRef'].Name().lower()))
+          sorted(self._properties['projectReferences'],
+                 key=lambda x: x['ProjectRef'].Name().lower())
     else:
       # The link already exists.  Pull out the relevnt data.
       project_ref_dict = self._other_pbxprojects[other_pbxproject]
@@ -2911,19 +2912,6 @@ def SortRemoteProductReferences(self):
     # same order that the targets are sorted in the remote project file.  This
     # is the sort order used by Xcode.
 
-    def CompareProducts(x, y, remote_products):
-      # x and y are PBXReferenceProxy objects.  Go through their associated
-      # PBXContainerItem to get the remote PBXFileReference, which will be
-      # present in the remote_products list.
-      x_remote = x._properties['remoteRef']._properties['remoteGlobalIDString']
-      y_remote = y._properties['remoteRef']._properties['remoteGlobalIDString']
-      x_index = remote_products.index(x_remote)
-      y_index = remote_products.index(y_remote)
-
-      # Use the order of each remote PBXFileReference in remote_products to
-      # determine the sort order.
-      return cmp(x_index, y_index)
-
     for other_pbxproject, ref_dict in self._other_pbxprojects.items():
       # Build up a list of products in the remote project file, ordered the
       # same as the targets that produce them.
@@ -2938,7 +2926,7 @@ def CompareProducts(x, y, remote_products):
       product_group = ref_dict['ProductGroup']
       product_group._properties['children'] = sorted(
           product_group._properties['children'],
-          cmp=lambda x, y, rp=remote_products: CompareProducts(x, y, rp))
+          key=lambda x: remote_products.index(x._properties['remoteRef']._properties['remoteGlobalIDString']))
 
 
 class XCProjectFile(XCObject):
@@ -2969,8 +2957,7 @@ def Print(self, file=sys.stdout):
       self._XCPrint(file, 0, '{ ')
     else:
       self._XCPrint(file, 0, '{\n')
-    for property, value in sorted(self._properties.iteritems(),
-                                  cmp=lambda x, y: cmp(x, y)):
+    for property, value in sorted(self._properties.items()):
       if property == 'objects':
         self._PrintObjects(file)
       else:
@@ -2997,7 +2984,7 @@ def _PrintObjects(self, file):
       self._XCPrint(file, 0, '\n')
       self._XCPrint(file, 0, '/* Begin ' + class_name + ' section */\n')
       for object in sorted(objects_by_class[class_name],
-                           cmp=lambda x, y: cmp(x.id, y.id)):
+                           key=lambda x: x.id):
         object.Print(file)
       self._XCPrint(file, 0, '/* End ' + class_name + ' section */\n')