Add chrome_wpt_tests to CI/CQ for experimenting
Data has show that ~2% WPTs produce different results between
Chrome and Content_Shell. We plan to add chrome_wpt_tests
to CI/CQ to detect Chrome specific regressions.
Per the plan (go/wpt-in-chromium), we will start to run WPTs
against Chrome on Linux. Add this to `linux-rel`/`Linux Tests`
to start experiment for two weeks. We do not plan to add this
suite to other Linux variants unless there have data to show
that is useful.
Changed step name to chrome_wpt_tests, to align with current
blink_wpt_tests.
Resource request is approved in crbug.com/1382931
Bug: 1339410, 1382931
Change-Id: I443258abda1e28498d5eb6b9e11b988c78d49ba4
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5005207
Reviewed-by: Rakib Hasan <rmhasan@google.com>
Reviewed-by: Ben Pastene <bpastene@chromium.org>
Commit-Queue: Weizhong Xia <weizhong@google.com>
Cr-Commit-Position: refs/heads/main@{#1224606}
diff --git a/infra/config/generated/testing/test_suites.pyl b/infra/config/generated/testing/test_suites.pyl
index e4825fc..7cec711 100644
--- a/infra/config/generated/testing/test_suites.pyl
+++ b/infra/config/generated/testing/test_suites.pyl
@@ -1462,6 +1462,29 @@
},
},
+ 'chromium_wpt_tests_isolated_scripts': {
+ 'chrome_wpt_tests': {
+ 'test': 'chrome_wpt_tests',
+ 'results_handler': 'layout tests',
+ 'args': [
+ '--test-type',
+ 'testharness',
+ 'reftest',
+ 'crashtest',
+ 'print-reftest',
+ ],
+ 'swarming': {
+ 'shards': 15,
+ },
+ 'merge': {
+ 'script': '//third_party/blink/tools/merge_web_test_results.py',
+ 'args': [
+ '--verbose',
+ ],
+ },
+ },
+ },
+
'clang_tot_gtests': {
'base_unittests': {},
},
@@ -5565,29 +5588,6 @@
},
},
- 'wpt_web_tests': {
- 'chrome_wpt_tests': {
- 'test': 'chrome_wpt_tests',
- 'results_handler': 'layout tests',
- 'args': [
- '--test-type',
- 'testharness',
- 'reftest',
- 'crashtest',
- 'print-reftest',
- ],
- 'swarming': {
- 'shards': 15,
- },
- 'merge': {
- 'script': '//third_party/blink/tools/merge_web_test_results.py',
- 'args': [
- '--verbose',
- ],
- },
- },
- },
-
'wpt_web_tests_android': {
'chrome_public_wpt': {
'results_handler': 'layout tests',
@@ -5961,6 +5961,7 @@
'vulkan_swiftshader_isolated_scripts',
'chromium_web_tests_high_dpi_isolated_scripts',
'chromium_web_tests_brfetch_isolated_scripts',
+ 'chromium_wpt_tests_isolated_scripts',
],
'chromium_mac_gtests': [
diff --git a/infra/config/targets/basic_suites.star b/infra/config/targets/basic_suites.star
index ed2a258..1f96d18 100644
--- a/infra/config/targets/basic_suites.star
+++ b/infra/config/targets/basic_suites.star
@@ -1743,6 +1743,32 @@
)
targets.legacy_basic_suite(
+ name = "chromium_wpt_tests_isolated_scripts",
+ tests = {
+ "chrome_wpt_tests": targets.legacy_test_config(
+ test = "chrome_wpt_tests",
+ results_handler = "layout tests",
+ args = [
+ "--test-type",
+ "testharness",
+ "reftest",
+ "crashtest",
+ "print-reftest",
+ ],
+ swarming = targets.swarming(
+ shards = 15,
+ ),
+ merge = targets.merge(
+ script = "//third_party/blink/tools/merge_web_test_results.py",
+ args = [
+ "--verbose",
+ ],
+ ),
+ ),
+ },
+)
+
+targets.legacy_basic_suite(
name = "clang_tot_gtests",
tests = {
"base_unittests": None,
@@ -6596,32 +6622,6 @@
)
targets.legacy_basic_suite(
- name = "wpt_web_tests",
- tests = {
- "chrome_wpt_tests": targets.legacy_test_config(
- test = "chrome_wpt_tests",
- results_handler = "layout tests",
- args = [
- "--test-type",
- "testharness",
- "reftest",
- "crashtest",
- "print-reftest",
- ],
- swarming = targets.swarming(
- shards = 15,
- ),
- merge = targets.merge(
- script = "//third_party/blink/tools/merge_web_test_results.py",
- args = [
- "--verbose",
- ],
- ),
- ),
- },
-)
-
-targets.legacy_basic_suite(
name = "wpt_web_tests_android",
tests = {
"chrome_public_wpt": targets.legacy_test_config(
diff --git a/infra/config/targets/compound_suites.star b/infra/config/targets/compound_suites.star
index f017a7e..9bede8a 100644
--- a/infra/config/targets/compound_suites.star
+++ b/infra/config/targets/compound_suites.star
@@ -362,6 +362,10 @@
"telemetry_perf_unittests_isolated_scripts",
"vulkan_swiftshader_isolated_scripts",
"chromium_web_tests_high_dpi_isolated_scripts",
+ # TODO(weizhong): we should eventually run chrome_wpt_tests where
+ # blink_wpt_tests runs on Linux. There should not have any resource
+ # concern on this because those are all CI builders.
+ #"chromium_wpt_tests_isolated_scripts",
],
)
@@ -400,6 +404,7 @@
# TODO(crbug.com/1498364): Remove this once the BackgroundResourceFetch
# feature launches.
"chromium_web_tests_brfetch_isolated_scripts",
+ "chromium_wpt_tests_isolated_scripts",
],
)
diff --git a/testing/buildbot/chromium.linux.json b/testing/buildbot/chromium.linux.json
index 1112eb3..521cbd4 100644
--- a/testing/buildbot/chromium.linux.json
+++ b/testing/buildbot/chromium.linux.json
@@ -3131,6 +3131,34 @@
},
{
"args": [
+ "--test-type",
+ "testharness",
+ "reftest",
+ "crashtest",
+ "print-reftest"
+ ],
+ "experiment_percentage": 10,
+ "isolate_profile_data": true,
+ "merge": {
+ "args": [
+ "--verbose"
+ ],
+ "script": "//third_party/blink/tools/merge_web_test_results.py"
+ },
+ "name": "chrome_wpt_tests",
+ "results_handler": "layout tests",
+ "swarming": {
+ "dimensions": {
+ "os": "Ubuntu-22.04"
+ },
+ "service_account": "chromium-tester@chops-service-accounts.iam.gserviceaccount.com",
+ "shards": 15
+ },
+ "test": "chrome_wpt_tests",
+ "test_id_prefix": "ninja://:chrome_wpt_tests/"
+ },
+ {
+ "args": [
"--test-type=integration"
],
"isolate_profile_data": true,
diff --git a/testing/buildbot/test_suite_exceptions.pyl b/testing/buildbot/test_suite_exceptions.pyl
index 905c1d2..9528821 100644
--- a/testing/buildbot/test_suite_exceptions.pyl
+++ b/testing/buildbot/test_suite_exceptions.pyl
@@ -1670,6 +1670,14 @@
},
},
},
+ # TODO(weizhong): Remove this once experiment completes
+ 'chrome_wpt_tests': {
+ 'modifications': {
+ 'Linux Tests': {
+ 'experiment_percentage': 10,
+ },
+ },
+ },
'chromedriver_py_tests': {
'modifications': {
'Win10 Tests x64': {
diff --git a/testing/buildbot/test_suites.pyl b/testing/buildbot/test_suites.pyl
index e4825fc..7cec711 100644
--- a/testing/buildbot/test_suites.pyl
+++ b/testing/buildbot/test_suites.pyl
@@ -1462,6 +1462,29 @@
},
},
+ 'chromium_wpt_tests_isolated_scripts': {
+ 'chrome_wpt_tests': {
+ 'test': 'chrome_wpt_tests',
+ 'results_handler': 'layout tests',
+ 'args': [
+ '--test-type',
+ 'testharness',
+ 'reftest',
+ 'crashtest',
+ 'print-reftest',
+ ],
+ 'swarming': {
+ 'shards': 15,
+ },
+ 'merge': {
+ 'script': '//third_party/blink/tools/merge_web_test_results.py',
+ 'args': [
+ '--verbose',
+ ],
+ },
+ },
+ },
+
'clang_tot_gtests': {
'base_unittests': {},
},
@@ -5565,29 +5588,6 @@
},
},
- 'wpt_web_tests': {
- 'chrome_wpt_tests': {
- 'test': 'chrome_wpt_tests',
- 'results_handler': 'layout tests',
- 'args': [
- '--test-type',
- 'testharness',
- 'reftest',
- 'crashtest',
- 'print-reftest',
- ],
- 'swarming': {
- 'shards': 15,
- },
- 'merge': {
- 'script': '//third_party/blink/tools/merge_web_test_results.py',
- 'args': [
- '--verbose',
- ],
- },
- },
- },
-
'wpt_web_tests_android': {
'chrome_public_wpt': {
'results_handler': 'layout tests',
@@ -5961,6 +5961,7 @@
'vulkan_swiftshader_isolated_scripts',
'chromium_web_tests_high_dpi_isolated_scripts',
'chromium_web_tests_brfetch_isolated_scripts',
+ 'chromium_wpt_tests_isolated_scripts',
],
'chromium_mac_gtests': [
diff --git a/testing/buildbot/waterfalls.pyl b/testing/buildbot/waterfalls.pyl
index 5cb20eb..03ddab7 100644
--- a/testing/buildbot/waterfalls.pyl
+++ b/testing/buildbot/waterfalls.pyl
@@ -3811,7 +3811,7 @@
'linux-jammy',
],
'test_suites': {
- 'isolated_scripts': 'wpt_web_tests',
+ 'isolated_scripts': 'chromium_wpt_tests_isolated_scripts',
},
},
'mac-fieldtrial-tester': {