1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
build / ios / update_bundle_filelist.py [blame]
#!/usr/bin/env python3
# Copyright 2023 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Updates .filelist files using data from corresponding .globlist files (or
checks whether they are up to date).
bundle_data targets require an explicit source list, but maintaining these large
lists can be cumbersome. This script aims to simplify the process of updating
these lists by either expanding globs to update file lists or check that an
existing file list matches such an expansion (i.e., checking during presubmit).
The .globlist file contains a list of globs that will be expanded to either
compare or replace a corresponding .filelist. It is possible to exclude items
from the file list with globs as well. These lines are prefixed with '-' and are
processed in order, so be sure that exclusions succeed inclusions in the list of
globs. Comments and empty lines are permitted in .globfiles; comments are
prefixed with '#'.
By convention, the base name of the .globlist and .filelist files matches the
label of their corresponding bundle_data from the .gn file. In order to ensure
that these filelists don't get stale, there should also be a PRESUBMIT.py
which uses this script to check that list is up to date.
By default, the script will update the file list to match the expanded globs.
"""
import argparse
import datetime
import difflib
import glob
import os.path
import re
import subprocess
import sys
# Character to set colors in terminal. Taken, along with the printing routine
# below, from update_deps.py.
TERMINAL_ERROR_COLOR = '\033[91m'
TERMINAL_RESET_COLOR = '\033[0m'
_HEADER = """# Copyright %d The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# NOTE: this file is generated by build/ios/update_bundle_filelist.py
# If it requires updating, you should get a presubmit error with
# instructions on how to regenerate. Otherwise, do not edit.
""" % (datetime.datetime.now().year)
_HEADER_PATTERN = re.compile(r"""# Copyright [0-9]+ The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# NOTE: this file is generated by build/ios/update_bundle_filelist.py
# If it requires updating, you should get a presubmit error with
# instructions on how to regenerate. Otherwise, do not edit.
""")
_HEADER_HEIGHT = 6
_START_IGNORE_EXPANSIONS_OUTSIDE_GLOBLIST_DIR = '# push(ignore-relative)'
_STOP_IGNORE_EXPANSIONS_OUTSIDE_GLOBLIST_DIR = '# pop(ignore-relative)'
def parse_filelist(filelist_name):
try:
with open(filelist_name) as filelist:
unfiltered = [l for l in filelist]
header = ''.join(unfiltered[:_HEADER_HEIGHT])
files = sorted(l.strip() for l in unfiltered[_HEADER_HEIGHT:])
return (files, header)
except Exception as e:
print_error(f'Could not read file list: {filelist_name}', f'{type(e)}: {e}')
return []
def get_git_command_name():
if sys.platform.startswith('win'):
return 'git.bat'
return 'git'
def get_tracked_files(directory, globroot, repository_root_relative, verbose):
try:
if os.getcwd().startswith('/google/cog/cloud'):
files = []
for root, _, filenames in os.walk(directory):
files.extend([os.path.join(root, f) for f in filenames])
return set(files)
cmd = [get_git_command_name(), 'ls-files', '--error-unmatch', directory]
with subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=globroot) as p:
output = p.communicate()
if p.returncode != 0:
if verbose:
print_error(
f'Could not gather a list of tracked files in {directory}',
f'{output[1]}')
return set()
files = [f.decode('utf-8') for f in output[0].splitlines()]
# Need paths to be relative to directory in order to match expansions.
# This should happen naturally due to cwd above, but we need to take
# special care if relative to the repository root.
if repository_root_relative:
files = ['//' + f for f in files]
# Handle Windows backslashes
files = [f.replace('\\', '/') for f in files]
return set(files)
except Exception as e:
if verbose:
print_error(f'Could not gather a list of tracked files in {directory}',
f'{type(e)}: {e}')
return set()
def combine_potentially_repository_root_relative_paths(a, b):
if b.startswith('//'):
# If b is relative to the repository root, os.path will consider it absolute
# and os.path.join will fail. In this case, we can simply concatenate the
# paths.
return (a + b, True)
else:
return (os.path.join(a, b), False)
def parse_and_expand_globlist(globlist_name, glob_root):
# The following expects glob_root not to end in a trailing slash.
if glob_root.endswith('/'):
glob_root = glob_root[:-1]
check_expansions_outside_globlist_dir = True
globlist_dir = os.path.dirname(globlist_name)
with open(globlist_name) as globlist:
# Paths in |files| and |to_check| must use unix separators. Using a set
# ensures no unwanted duplicates. The files in |to_check| must be in the
# globroot or a subdirectory.
files = set()
to_check = set()
for g in globlist:
g = g.strip()
# Ignore blank lines
if not g:
continue
# Toggle error checking.
if g == _START_IGNORE_EXPANSIONS_OUTSIDE_GLOBLIST_DIR:
check_expansions_outside_globlist_dir = False
elif g == _STOP_IGNORE_EXPANSIONS_OUTSIDE_GLOBLIST_DIR:
check_expansions_outside_globlist_dir = True
# Ignore comments.
if not g or g.startswith('#'):
continue
# Exclusions are prefixed with '-'.
is_exclusion = g.startswith('-')
if is_exclusion:
g = g[1:]
(combined,
root_relative) = combine_potentially_repository_root_relative_paths(
glob_root, g)
prefix_size = len(glob_root)
if not root_relative:
# We need to account for the separator.
prefix_size += 1
expansion = glob.glob(combined, recursive=True)
# Filter out directories.
expansion = [f for f in expansion if os.path.isfile(f)]
if check_expansions_outside_globlist_dir:
for f in expansion:
relative = os.path.relpath(f, globlist_dir)
if relative.startswith('..'):
raise Exception(f'Globlist expansion outside globlist dir: {f}')
# Make relative to |glob_root|.
expansion = [f[prefix_size:] for f in expansion]
# Handle Windows backslashes
expansion = [f.replace('\\', '/') for f in expansion]
# Since paths in |expansion| only use unix separators, it is safe to
# compare for both the purpose of exclusion and addition.
if is_exclusion:
files = files.difference(expansion)
else:
files = files.union(expansion)
# Return a sorted list.
return sorted(files)
def compare_lists(a, b):
differ = difflib.Differ()
full_diff = differ.compare(a, b)
lines = [d for d in full_diff if not d.startswith(' ')]
additions = [l[2:] for l in lines if l.startswith('+ ')]
removals = [l[2:] for l in lines if l.startswith('- ')]
return (additions, removals)
def write_filelist(filelist_name, files, header):
try:
with open(filelist_name, 'w', encoding='utf-8', newline='') as filelist:
if not _HEADER_PATTERN.search(header):
header = _HEADER
filelist.write(header)
for file in files:
filelist.write(f'{file}\n')
except Exception as e:
print_error(f'Could not write file list: {filelist_name}',
f'{type(e)}: {e}')
return []
def process_filelist(filelist, globlist, globroot, check=False, verbose=False):
files_from_globlist = []
try:
files_from_globlist = parse_and_expand_globlist(globlist, globroot)
except Exception as e:
if verbose:
print_error(f'Could not read glob list: {globlist}', f'{type(e)}: {e}')
return 1
(files, header) = parse_filelist(filelist)
(additions, removals) = compare_lists(files, files_from_globlist)
to_ignore = set()
# Ignore additions of untracked files.
if additions:
directories = set([os.path.dirname(f) for f in additions])
tracked_files = set()
for d in directories:
(combined,
root_relative) = combine_potentially_repository_root_relative_paths(
globroot, d)
relative = os.path.relpath(combined, globroot)
tracked_files = tracked_files.union(
get_tracked_files(relative, globroot, root_relative, verbose))
to_ignore = set(additions).difference(tracked_files)
additions = [f for f in additions if f in tracked_files]
files_from_globlist = [f for f in files_from_globlist if f not in to_ignore]
if check:
if not _HEADER_PATTERN.search(header):
if verbose:
print_error(f'Unexpected header for {filelist}', f'{header}')
return 1
if not additions and not removals:
return 0
if verbose:
pretty_additions = ['+ ' + f for f in additions]
pretty_removals = ['- ' + f for f in removals]
pretty_diff = '\n'.join(pretty_additions + pretty_removals)
print_error('File list does not match glob expansion', f'{pretty_diff}')
return 1
else:
write_filelist(filelist, files_from_globlist, header)
return 0
def main(args):
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('filelist', help='Contains one file per line')
parser.add_argument('globlist',
help='Contains globs that, when expanded, '
'should match the filelist. Use '
'--help for details on syntax')
parser.add_argument('globroot',
help='Directory from which globs are relative')
parser.add_argument('-c',
'--check',
action='store_true',
help='Prevents modifying the file list')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Use this to print details on differences')
args = parser.parse_args()
return process_filelist(args.filelist,
args.globlist,
args.globroot,
check=args.check,
verbose=args.verbose)
def print_error(error_message, error_info):
""" Print the `error_message` with additional `error_info` """
color_start, color_end = adapted_color_for_output(TERMINAL_ERROR_COLOR,
TERMINAL_RESET_COLOR)
error_message = color_start + 'ERROR: ' + error_message + color_end
if len(error_info) > 0:
error_message = error_message + '\n' + error_info
print(error_message, file=sys.stderr)
def adapted_color_for_output(color_start, color_end):
""" Returns a the `color_start`, `color_end` tuple if the output is a
terminal, or empty strings otherwise """
if not sys.stdout.isatty():
return '', ''
return color_start, color_end
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))