1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
infra / config / luci-analysis-dev.cfg [blame]
# Schema for this config file: ProjectConfig in:
# https://luci-config.appspot.com/schemas/projects:luci-analysis.cfg
bug_filing_thresholds {
metric_id: "critical-failures-exonerated"
threshold {
three_day: 75
}
}
bug_filing_thresholds {
metric_id: "builds-failed-due-to-flaky-tests"
threshold {
seven_day: 10
}
}
clustering {
test_name_rules {
name: "Blink Web Tests"
# To match blink_web_tests as well as webgpu_blink_web_tests and any others.
pattern: "^ninja://:(?P<target>\\w*blink_web_tests)/(virtual/[^/]+/)?(?P<test>([^/]+/)+[^/]+\\.[a-zA-Z]+).*$"
like_template: "ninja://:${target}/%${test}%"
}
test_name_rules {
name: "Google Test (Value-parameterized)"
pattern: "^ninja:(?P<target>[\\w/]+:\\w+)/(\\w+/)?(?P<suite>\\w+)\\.(?P<test>\\w+)/[\\w.]+$"
like_template: "ninja:${target}/%${suite}.${test}%"
}
test_name_rules {
name: "Google Test (Type-parameterized)"
pattern: "^ninja:(?P<target>[\\w/]+:\\w+)/(\\w+/)?(?P<suite>\\w+)/\\w+\\.(?P<test>\\w+)$"
like_template: "ninja:${target}/%${suite}/%.${test}"
}
test_name_rules {
name: "JUnit Test (Parameterized)"
# Matches parameterized JUnit tests like:
# ninja://android_webview/test:webview_instrumentation_test_apk/org.chromium.android_webview.test.MyTest#testFoo__parameter1
# Also matches tests parameterized with different command line flags,
# as constructed by https://source.chromium.org/chromium/chromium/src/+/main:build/android/pylib/instrumentation/instrumentation_test_instance.py?q=%22def%20GetUniqueTestName(%22
# E.g. ninja://chrome/android:chrome_public_test_apk/org.chromium.chrome.browser.omnibox.OmniboxTest#testDefaultText_with___disable_features=SpannableInlineAutocomplete
pattern: "^ninja:(?P<target>[\\w/]+:\\w+)/(?P<class>[\\w$.]+)#(?P<test>\\w+?)(?P<sep>__|_with_)[\\w.=,]+$"
like_template: "ninja:${target}/${class}#${test}${sep}%"
}
}
metrics {
overrides {
metric_id: "builds-failed-due-to-flaky-tests"
is_default: true
}
}
bug_management {
policies {
id: "exonerations"
owners: "kimstephanie@google.com"
owners: "chrome-browser-infra-team@google.com"
human_readable_name: "test variant(s) are being exonerated (ignored) in presubmit"
priority: P2
metrics {
metric_id: "critical-failures-exonerated"
activation_threshold {
three_day: 30
}
deactivation_threshold {
seven_day: 1
}
}
explanation {
problem_html: "Test variant(s) in this cluster are being exonerated (ignored) in presubmit because they are too flaky or failing to gate new code changes. This means the test variant(s) are unable to prevent new breakages from landing in the tree."
action_html: "<ul><li>View the exonerations tab to see the test variant(s) still being exonerated.</li><li>View the recent failures tab to see example failures for those test variant(s).</li><li>If the failure cannot be fixed in a timely way, consider disabling the test.</ul>"
}
bug_template {
}
}
policies {
id: "builds-failed-due-to-flaky-tests"
owners: "kuanhuang@google.com"
owners: "chrome-browser-engprod@google.com"
human_readable_name: "many gardened builders are failing"
priority: P1
metrics {
metric_id: "builds-failed-due-to-flaky-tests"
activation_threshold {
seven_day: 10
}
deactivation_threshold {
seven_day: 1
}
}
explanation {
problem_html: "This cluster is making a large number of gardened builders fail."
action_html: "Please consider the following strategies to mitigate the impact from these issues which are rated in order of resolution preference:<ol><li>Resolve the underlying test issue</li><li>Move the flaky test from critical builders to FYI builder</li><li>Disable test (least desirable as it reduces test coverage) and add a Test-Disabled label to the bug.</li></ol>"
}
bug_template{
comment_template: "This bug been identified as part of Chrome's flake reduction initiative. Resolution of this issue is critical because it has a large impact on the stability of Chrome CI/CQ. Resolving these issues will be essential to meet Chrome’s flake reduction goals.\n\nThis bug has led to a large number of gardened builds failing in the last 7 days. List of gardened build failures caused by this cluster can be viewed at:\nhttps://data.corp.google.com/sites/chrome_generic_flakiness_dashboard_datasite/build_failures/?av=rpqxzw:ktu56&fb=is_sheriff_or_cq_builder:eq:true&f=cluster_name:in:{{if .BugID.IsBuganizer}}b%2F{{.BugID.BuganizerBugID}}{{end}}{{if .BugID.IsMonorail}}crbug%2F{{.BugID.MonorailBugID}}{{end}}\nor in LUCI Analysis and grouped by builders:\n{{.RuleURL}}?tab=recent-failures&filterToMetric=builds-failed-due-to-flaky-tests&groupBy=builder&orderBy=invocationFailures\n\nPlease consider the following strategies to mitigate the impact from this issue which are rated in order of resolution preference:\n1. Resolve the underlying test issue.\n2. Move the flaky test from Critical Builders to FYI Builder\n3. Disable test (least desirable as it reduces test coverage) and add a Test-Disabled label to this issue. The disabled tests might continue running in reviver builders (go/test-reviver), see config [1] for a list of supported builders.\n\nhttp://go/resolve-chrome-top-flakes provides more information on the tools available for resolving flaky tests.\n\nWhen investigating this failure, you may identify this bug is too broad (encompasses multiple different issues) or too narrow (only captures one part of a larger issue). If this applies, you can combine issues[2] or split issues[3].\n\nLinks:\n[1] https://source.chromium.org/chromium/chromium/src/+/main:infra/config/subprojects/reviver/reviver.star\n[2] https://luci-analysis.appspot.com/help#combining-issues\n[3] https://luci-analysis.appspot.com/help#splitting-issues"
buganizer {
hotlists: 5432827 # Top-Flakes
hotlists: 5433124 # Test-Flaky
}
}
}
policies {
id: "cq-rejects"
owners: "kimstephanie@google.com"
owners: "chrome-browser-infra-team@google.com"
human_readable_name: "many CLs are being blocked from submission"
priority: P0
metrics {
metric_id: "human-cls-failed-presubmit"
activation_threshold {
one_day: 10
}
deactivation_threshold {
one_day: 3
}
}
explanation {
problem_html: "Ten or more developer changelists have been blocked from submission in the last day due to failures in this cluster. This is usually because the cluster's failures are so widespread or the tree is so broken that exoneration can no longer mitigate the impact. Urgent action should be taken to mitigate the impact to developer productivity."
action_html: "<ul><li>View the recent failures tab to see example failures for those test variant(s).</li><li>If the failure cannot be fixed in a timely way, consider disabling the test.</ul>"
}
bug_template {
comment_template: "This cluster has blocked 10 or more CLs from submission in the last day. This cluster is having a major impact on the ability of chromium developers to get work done. Please disable the test(s) and priorise immediate action to resolve the test failures."
}
}
}
test_stability_criteria {
failure_rate {
failure_threshold: 6
consecutive_failure_threshold: 3
}
flake_rate {
min_window: 0
flake_threshold: 3
flake_rate_threshold: 0
flake_threshold_1wd: 1
}
}