1
    2
    3
    4
    5
    6
    7
    8
    9
   10
   11
   12
   13
   14
   15
   16
   17
   18
   19
   20
   21
   22
   23
   24
   25
   26
   27
   28
   29
   30
   31
   32
   33
   34
   35
   36
   37
   38
   39
   40
   41
   42
   43
   44
   45
   46
   47
   48
   49
   50
   51
   52
   53
   54
   55
   56
   57
   58
   59
   60
   61
   62
   63
   64
   65
   66
   67
   68
   69
   70
   71
   72
   73
   74
   75
   76
   77
   78
   79
   80
   81
   82
   83
   84
   85
   86
   87
   88
   89
   90
   91
   92
   93
   94
   95
   96
   97
   98
   99
  100
  101
  102
  103
  104
  105
  106
  107
  108
  109
  110
  111
  112
  113
  114
  115
  116
  117
  118
  119
  120
  121
  122
  123
  124
  125
  126
  127
  128
  129
  130
  131
  132
  133
  134
  135
  136
  137
  138
  139
  140
  141
  142
  143
  144
  145
  146
  147
  148
  149
  150
  151
  152
  153
  154
  155
  156
  157
  158
  159
  160
  161
  162
  163
  164
  165
  166
  167
  168
  169
  170
  171
  172
  173
  174
  175
  176
  177
  178
  179
  180
  181
  182
  183
  184
  185
  186
  187
  188
  189
  190
  191
  192
  193
  194
  195
  196
  197
  198
  199
  200
  201
  202
  203
  204
  205
  206
  207
  208
  209
  210
  211
  212
  213
  214
  215
  216
  217
  218
  219
  220
  221
  222
  223
  224
  225
  226
  227
  228
  229
  230
  231
  232
  233
  234
  235
  236
  237
  238
  239
  240
  241
  242
  243
  244
  245
  246
  247
  248
  249
  250
  251
  252
  253
  254
  255
  256
  257
  258
  259
  260
  261
  262
  263
  264
  265
  266
  267
  268
  269
  270
  271
  272
  273
  274
  275
  276
  277
  278
  279
  280
  281
  282
  283
  284
  285
  286
  287
  288
  289
  290
  291
  292
  293
  294
  295
  296
  297
  298
  299
  300
  301
  302
  303
  304
  305
  306
  307
  308
  309
  310
  311
  312
  313
  314
  315
  316
  317
  318
  319
  320
  321
  322
  323

build / ios / update_bundle_filelist.py [blame]

#!/usr/bin/env python3
# Copyright 2023 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Updates .filelist files using data from corresponding .globlist files (or
checks whether they are up to date).

bundle_data targets require an explicit source list, but maintaining these large
lists can be cumbersome. This script aims to simplify the process of updating
these lists by either expanding globs to update file lists or check that an
existing file list matches such an expansion (i.e., checking during presubmit).

The .globlist file contains a list of globs that will be expanded to either
compare or replace a corresponding .filelist. It is possible to exclude items
from the file list with globs as well. These lines are prefixed with '-' and are
processed in order, so be sure that exclusions succeed inclusions in the list of
globs. Comments and empty lines are permitted in .globfiles; comments are
prefixed with '#'.

By convention, the base name of the .globlist and .filelist files matches the
label of their corresponding bundle_data from the .gn file. In order to ensure
that these filelists don't get stale, there should also be a PRESUBMIT.py
which uses this script to check that list is up to date.

By default, the script will update the file list to match the expanded globs.
"""

import argparse
import datetime
import difflib
import glob
import os.path
import re
import subprocess
import sys

# Character to set colors in terminal. Taken, along with the printing routine
# below, from update_deps.py.
TERMINAL_ERROR_COLOR = '\033[91m'
TERMINAL_RESET_COLOR = '\033[0m'

_HEADER = """# Copyright %d The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# NOTE: this file is generated by build/ios/update_bundle_filelist.py
#       If it requires updating, you should get a presubmit error with
#       instructions on how to regenerate. Otherwise, do not edit.
""" % (datetime.datetime.now().year)

_HEADER_PATTERN = re.compile(r"""# Copyright [0-9]+ The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# NOTE: this file is generated by build/ios/update_bundle_filelist.py
#       If it requires updating, you should get a presubmit error with
#       instructions on how to regenerate. Otherwise, do not edit.
""")

_HEADER_HEIGHT = 6

_START_IGNORE_EXPANSIONS_OUTSIDE_GLOBLIST_DIR = '# push(ignore-relative)'
_STOP_IGNORE_EXPANSIONS_OUTSIDE_GLOBLIST_DIR = '# pop(ignore-relative)'


def parse_filelist(filelist_name):
  try:
    with open(filelist_name) as filelist:
      unfiltered = [l for l in filelist]
      header = ''.join(unfiltered[:_HEADER_HEIGHT])
      files = sorted(l.strip() for l in unfiltered[_HEADER_HEIGHT:])
      return (files, header)
  except Exception as e:
    print_error(f'Could not read file list: {filelist_name}', f'{type(e)}: {e}')
    return []


def get_git_command_name():
  if sys.platform.startswith('win'):
    return 'git.bat'
  return 'git'


def get_tracked_files(directory, globroot, repository_root_relative, verbose):
  try:
    if os.getcwd().startswith('/google/cog/cloud'):
      files = []
      for root, _, filenames in os.walk(directory):
        files.extend([os.path.join(root, f) for f in filenames])
      return set(files)
    cmd = [get_git_command_name(), 'ls-files', '--error-unmatch', directory]
    with subprocess.Popen(cmd,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE,
                          cwd=globroot) as p:
      output = p.communicate()
      if p.returncode != 0:
        if verbose:
          print_error(
              f'Could not gather a list of tracked files in {directory}',
              f'{output[1]}')
        return set()

      files = [f.decode('utf-8') for f in output[0].splitlines()]

      # Need paths to be relative to directory in order to match expansions.
      # This should happen naturally due to cwd above, but we need to take
      # special care if relative to the repository root.
      if repository_root_relative:
        files = ['//' + f for f in files]

      # Handle Windows backslashes
      files = [f.replace('\\', '/') for f in files]

      return set(files)

  except Exception as e:
    if verbose:
      print_error(f'Could not gather a list of tracked files in {directory}',
                  f'{type(e)}: {e}')
    return set()


def combine_potentially_repository_root_relative_paths(a, b):
  if b.startswith('//'):
    # If b is relative to the repository root, os.path will consider it absolute
    # and os.path.join will fail. In this case, we can simply concatenate the
    # paths.
    return (a + b, True)
  else:
    return (os.path.join(a, b), False)


def parse_and_expand_globlist(globlist_name, glob_root):
  # The following expects glob_root not to end in a trailing slash.
  if glob_root.endswith('/'):
    glob_root = glob_root[:-1]

  check_expansions_outside_globlist_dir = True
  globlist_dir = os.path.dirname(globlist_name)

  with open(globlist_name) as globlist:
    # Paths in |files| and |to_check| must use unix separators. Using a set
    # ensures no unwanted duplicates. The files in |to_check| must be in the
    # globroot or a subdirectory.
    files = set()
    to_check = set()
    for g in globlist:
      g = g.strip()

      # Ignore blank lines
      if not g:
        continue

      # Toggle error checking.
      if g == _START_IGNORE_EXPANSIONS_OUTSIDE_GLOBLIST_DIR:
        check_expansions_outside_globlist_dir = False
      elif g == _STOP_IGNORE_EXPANSIONS_OUTSIDE_GLOBLIST_DIR:
        check_expansions_outside_globlist_dir = True

      # Ignore comments.
      if not g or g.startswith('#'):
        continue

      # Exclusions are prefixed with '-'.
      is_exclusion = g.startswith('-')
      if is_exclusion:
        g = g[1:]

      (combined,
       root_relative) = combine_potentially_repository_root_relative_paths(
           glob_root, g)

      prefix_size = len(glob_root)
      if not root_relative:
        # We need to account for the separator.
        prefix_size += 1

      expansion = glob.glob(combined, recursive=True)

      # Filter out directories.
      expansion = [f for f in expansion if os.path.isfile(f)]

      if check_expansions_outside_globlist_dir:
        for f in expansion:
          relative = os.path.relpath(f, globlist_dir)
          if relative.startswith('..'):
            raise Exception(f'Globlist expansion outside globlist dir: {f}')

      # Make relative to |glob_root|.
      expansion = [f[prefix_size:] for f in expansion]

      # Handle Windows backslashes
      expansion = [f.replace('\\', '/') for f in expansion]

      # Since paths in |expansion| only use unix separators, it is safe to
      # compare for both the purpose of exclusion and addition.
      if is_exclusion:
        files = files.difference(expansion)
      else:
        files = files.union(expansion)

    # Return a sorted list.
    return sorted(files)


def compare_lists(a, b):
  differ = difflib.Differ()
  full_diff = differ.compare(a, b)
  lines = [d for d in full_diff if not d.startswith('  ')]
  additions = [l[2:] for l in lines if l.startswith('+ ')]
  removals = [l[2:] for l in lines if l.startswith('- ')]
  return (additions, removals)


def write_filelist(filelist_name, files, header):
  try:
    with open(filelist_name, 'w', encoding='utf-8', newline='') as filelist:
      if not _HEADER_PATTERN.search(header):
        header = _HEADER
      filelist.write(header)
      for file in files:
        filelist.write(f'{file}\n')
  except Exception as e:
    print_error(f'Could not write file list: {filelist_name}',
                f'{type(e)}: {e}')
    return []


def process_filelist(filelist, globlist, globroot, check=False, verbose=False):
  files_from_globlist = []
  try:
    files_from_globlist = parse_and_expand_globlist(globlist, globroot)
  except Exception as e:
    if verbose:
      print_error(f'Could not read glob list: {globlist}', f'{type(e)}: {e}')
    return 1

  (files, header) = parse_filelist(filelist)

  (additions, removals) = compare_lists(files, files_from_globlist)
  to_ignore = set()

  # Ignore additions of untracked files.
  if additions:
    directories = set([os.path.dirname(f) for f in additions])
    tracked_files = set()
    for d in directories:
      (combined,
       root_relative) = combine_potentially_repository_root_relative_paths(
           globroot, d)
      relative = os.path.relpath(combined, globroot)
      tracked_files = tracked_files.union(
          get_tracked_files(relative, globroot, root_relative, verbose))
    to_ignore = set(additions).difference(tracked_files)
    additions = [f for f in additions if f in tracked_files]

  files_from_globlist = [f for f in files_from_globlist if f not in to_ignore]

  if check:
    if not _HEADER_PATTERN.search(header):
      if verbose:
        print_error(f'Unexpected header for {filelist}', f'{header}')
      return 1
    if not additions and not removals:
      return 0
    if verbose:
      pretty_additions = ['+ ' + f for f in additions]
      pretty_removals = ['- ' + f for f in removals]
      pretty_diff = '\n'.join(pretty_additions + pretty_removals)
      print_error('File list does not match glob expansion', f'{pretty_diff}')
    return 1
  else:
    write_filelist(filelist, files_from_globlist, header)
    return 0


def main(args):
  parser = argparse.ArgumentParser(
      description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
  parser.add_argument('filelist', help='Contains one file per line')
  parser.add_argument('globlist',
                      help='Contains globs that, when expanded, '
                      'should match the filelist. Use '
                      '--help for details on syntax')
  parser.add_argument('globroot',
                      help='Directory from which globs are relative')
  parser.add_argument('-c',
                      '--check',
                      action='store_true',
                      help='Prevents modifying the file list')
  parser.add_argument('-v',
                      '--verbose',
                      action='store_true',
                      help='Use this to print details on differences')
  args = parser.parse_args()
  return process_filelist(args.filelist,
                          args.globlist,
                          args.globroot,
                          check=args.check,
                          verbose=args.verbose)


def print_error(error_message, error_info):
  """ Print the `error_message` with additional `error_info` """
  color_start, color_end = adapted_color_for_output(TERMINAL_ERROR_COLOR,
                                                    TERMINAL_RESET_COLOR)

  error_message = color_start + 'ERROR: ' + error_message + color_end
  if len(error_info) > 0:
    error_message = error_message + '\n' + error_info
  print(error_message, file=sys.stderr)


def adapted_color_for_output(color_start, color_end):
  """ Returns a the `color_start`, `color_end` tuple if the output is a
    terminal, or empty strings otherwise """
  if not sys.stdout.isatty():
    return '', ''
  return color_start, color_end


if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))