1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
content / test / gpu / unexpected_pass_finder.py [blame]
#!/usr/bin/env vpython3
# Copyright 2020 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Script for determining which GPU tests are unexpectedly passing.
This script depends on the `bb` tool, which is available as part of depot tools,
and the `bq` tool, which is available as part of the Google Cloud SDK
https://cloud.google.com/sdk/docs/quickstarts.
Example usage:
unexpected_pass_finder.py \
--project <BigQuery billing project> \
--suite <test suite to check> \
Concrete example:
unexpected_pass_finder.py \
--project luci-resultdb-dev \
--suite pixel
You would typically want to pass in --remove-stale-expectations as well in order
to have the script automatically remove any expectations it determines are no
longer necessary. If a particular expectation proves to be erroneously flagged
and removed (e.g. due to a very low flake rate that doesn't get caught
consistently by the script), expectations can be omitted from automatic removal
using an inline `# finder:disable` comment for a single expectation or a pair of
`# finder:disable`/`# finder:enable` comments for a block of expectations.
General disables can be handled via `finder:disable-general` and
`finder:enable-general`. Disabling removal only if the expectation is found to
be unused can be handled via `finder:disable-unused` and `finder:enable-unused`.
Disabling removal only if the expectation is found to be stale can be handled
via `finder:disable-stale` and `finder:enable-stale`.
"""
import argparse
import datetime
import os
from gpu_path_util import setup_telemetry_paths # pylint: disable=unused-import
from gpu_path_util import setup_testing_paths # pylint: disable=unused-import
from gpu_tests import gpu_integration_test
from unexpected_passes import gpu_builders
from unexpected_passes import gpu_expectations
from unexpected_passes import gpu_queries
from unexpected_passes_common import argument_parsing
from unexpected_passes_common import builders
from unexpected_passes_common import expectations
from unexpected_passes_common import result_output
def ParseArgs() -> argparse.Namespace:
name_mapping = gpu_integration_test.GenerateTestNameMapping()
test_suites = list(name_mapping.keys())
test_suites.sort()
parser = argparse.ArgumentParser(
description=('Script for finding cases of stale expectations that can '
'be removed/modified.'))
argument_parsing.AddCommonArguments(parser)
input_group = parser.add_mutually_exclusive_group()
input_group.add_argument(
'--expectation-file',
help='A path to an expectation file to read from. If not specified and '
'--test is not used, will automatically determine based off the '
'provided suite.')
input_group.add_argument(
'--test',
action='append',
dest='tests',
default=[],
help='The name of a test to check for unexpected passes. Can be passed '
'multiple times to specify multiple tests. Will be treated as if it was '
'expected to be flaky on all configurations.')
parser.add_argument('--suite',
required=True,
choices=test_suites,
help='The test suite being checked.')
args = parser.parse_args()
argument_parsing.PerformCommonPostParseSetup(args)
suite_class = name_mapping[args.suite]
if not (args.tests or args.expectation_file):
expectation_files = suite_class.ExpectationsFiles()
if not expectation_files:
raise RuntimeError(
'Suite %s does not specify an expectation file and is thus not '
'compatible with this script.' % args.suite)
if len(expectation_files) > 1:
raise RuntimeError(
'Suite %s specifies %d expectation files when only 1 is supported.' %
len(expectation_files))
args.expectation_file = expectation_files[0]
if args.remove_stale_expectations and not args.expectation_file:
parser.error(
'--remove-stale-expectations can only be used with expectation files')
# Change to whatever repo the test suite claims the expectation file lives in.
# This allows the script to work for most suites if run from outside of
# chromium/src. Similarly, it allows suites such as WebGPU CTS that have
# expectation files in a different repo to be work when run from chromium/src.
os.chdir(suite_class.GetExpectationsFilesRepoPath())
return args
# pylint: disable=too-many-locals
def main() -> None:
args = ParseArgs()
builders_instance = gpu_builders.GpuBuilders(args.suite,
args.include_internal_builders)
builders.RegisterInstance(builders_instance)
expectations_instance = gpu_expectations.GpuExpectations()
expectations.RegisterInstance(expectations_instance)
test_expectation_map = expectations_instance.CreateTestExpectationMap(
args.expectation_file, args.tests,
datetime.timedelta(days=args.expectation_grace_period))
ci_builders = builders_instance.GetCiBuilders()
querier = gpu_queries.GpuBigQueryQuerier(args.suite, args.project,
args.num_samples,
args.keep_unmatched_results)
# Unmatched results are mainly useful for script maintainers, as they don't
# provide any additional information for the purposes of finding unexpectedly
# passing tests or unused expectations.
unmatched = querier.FillExpectationMapForBuilders(test_expectation_map,
ci_builders)
try_builders = builders_instance.GetTryBuilders(ci_builders)
unmatched.update(
querier.FillExpectationMapForBuilders(test_expectation_map, try_builders))
unused_expectations = test_expectation_map.FilterOutUnusedExpectations()
stale, semi_stale, active = test_expectation_map.SplitByStaleness()
if args.result_output_file:
with open(args.result_output_file, 'w') as outfile:
result_output.OutputResults(stale, semi_stale, active, unmatched,
unused_expectations, args.output_format,
outfile)
else:
result_output.OutputResults(stale, semi_stale, active, unmatched,
unused_expectations, args.output_format)
affected_urls = set()
stale_message = ''
if args.remove_stale_expectations:
for expectation_file, expectation_map in stale.items():
affected_urls |= expectations_instance.RemoveExpectationsFromFile(
expectation_map.keys(), expectation_file,
expectations.RemovalType.STALE)
stale_message += ('Stale expectations removed from %s. Stale comments, '
'etc. may still need to be removed.\n' %
expectation_file)
for expectation_file, unused_list in unused_expectations.items():
affected_urls |= expectations_instance.RemoveExpectationsFromFile(
unused_list, expectation_file, expectations.RemovalType.UNUSED)
stale_message += ('Unused expectations removed from %s. Stale comments, '
'etc. may still need to be removed.\n' %
expectation_file)
if args.narrow_semi_stale_expectation_scope:
affected_urls |= expectations_instance.NarrowSemiStaleExpectationScope(
semi_stale)
stale_message += ('Semi-stale expectations narrowed in %s. Stale comments, '
'etc. may still need still need to be removed.\n' %
args.expectation_file)
if stale_message:
print(stale_message)
if affected_urls:
orphaned_urls = expectations_instance.FindOrphanedBugs(affected_urls)
if args.bug_output_file:
with open(args.bug_output_file, 'w') as bug_outfile:
result_output.OutputAffectedUrls(affected_urls,
orphaned_urls,
bug_outfile,
auto_close_bugs=args.auto_close_bugs)
else:
result_output.OutputAffectedUrls(affected_urls,
orphaned_urls,
auto_close_bugs=args.auto_close_bugs)
# pylint: enable=too-many-locals
if __name__ == '__main__':
main()