[go: nahoru, domu]
Skip to content

Commit

Permalink
Add Firebase tests to flutter dashboard. (flutter#66677)
Browse files Browse the repository at this point in the history 
This PR also enables firebase tests on presubmit and removes code that
won't be used anymore from test runner.

Bug:
  https://github.com/flutter/infra/pull/225
  • Loading branch information
@godofredoc
godofredoc committed Sep 25, 2020
1 parent e6b9b5e commit fd0e223
Show file tree
Hide file tree
Showing 4 changed files with 36 additions and 95 deletions.
21 changes: 0 additions & 21 deletions .cirrus.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,6 @@
# YAML anchors used to share fields between tasks.
# See https://confluence.atlassian.com/bitbucket/yaml-anchors-960154027.html

firebase_shard_template: &FIREBASE_SHARD_TEMPLATE
environment:
# Empirically, this shard runs in 20-25 minutes with just one CPU and 4G of RAM, as of
# October 2019. It does not seem to be sensitive to the number of CPUs or amount of RAM;
# doubling CPUs had no effect (mere seconds under 20 minutes), increasing RAM to 24G left it
# on the high end of the 20-25 minute range. (This makes sense, as it's just driving the
# Firebase test lab remotely.) Less than 4G of RAM made it go OOM.
CLOUDSDK_CORE_DISABLE_PROMPTS: 1
GCLOUD_FIREBASE_TESTLAB_KEY: ENCRYPTED[1c140257edc48f5578fa5a0e5038b84c8e53270c405efa5a8e35ea303a4e0d135853989f448f72136206de854d17fbec]
script:
- dart --enable-asserts ./dev/bots/test.dart

use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'

Expand Down Expand Up @@ -189,16 +178,6 @@ task:
- git clone https://github.com/flutter/tests.git bin/cache/pkg/tests
- dart --enable-asserts dev/customer_testing/run_tests.dart --skip-on-fetch-failure --skip-template bin/cache/pkg/tests/registry/*.test

# firebase_test_lab_tests are linux-only
- name: firebase_test_lab_tests-0-linux
<<: *FIREBASE_SHARD_TEMPLATE

- name: firebase_test_lab_tests-1-linux
<<: *FIREBASE_SHARD_TEMPLATE

- name: firebase_test_lab_tests-2_last-linux
<<: *FIREBASE_SHARD_TEMPLATE

- name: web_smoke_test
only_if: "changesInclude('.cirrus.yml', 'examples/hello_world/**' ,'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/**') && $CIRRUS_PR != ''"
environment:
Expand Down
74 changes: 0 additions & 74 deletions dev/bots/test.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,6 @@ final List<String> flutterTestArgs = <String>[];

final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTER'] == 'true';

/// The number of Cirrus jobs that run host-only devicelab tests in parallel.
///
/// WARNING: if you change this number, also change .cirrus.yml
/// and make sure it runs _all_ shards.
const int kDeviceLabShardCount = 2;

/// The number of Cirrus jobs that run build tests in parallel.
///
Expand Down Expand Up @@ -121,10 +116,8 @@ Future<void> main(List<String> args) async {
await selectShard(const <String, ShardRunner>{
'add_to_app_life_cycle_tests': _runAddToAppLifeCycleTests,
'build_tests': _runBuildTests,
'firebase_test_lab_tests': _runFirebaseTestLabTests,
'framework_coverage': _runFrameworkCoverage,
'framework_tests': _runFrameworkTests,
'hostonly_devicelab_tests': _runHostOnlyDeviceLabTests,
'tool_coverage': _runToolCoverage,
'tool_tests': _runToolTests,
'web_tests': _runWebUnitTests,
Expand Down Expand Up @@ -669,28 +662,6 @@ Future<void> _runFrameworkTests() async {
});
}

Future<void> _runFirebaseTestLabTests() async {
// Firebase Lab tests take ~20 minutes per integration test,
// so only one test is run per shard. Therefore, there are as
// many shards available as there are integration tests in this list.
// If you add a new test, add a corresponding firebase_test_lab-#-linux
// to .cirrus.yml
final List<String> integrationTests = <String>[
'release_smoke_test',
'abstract_method_smoke_test',
'android_embedding_v2_smoke_test',
];

final String firebaseScript = path.join(flutterRoot, 'dev', 'bots', 'firebase_testlab.sh');
final String integrationTestDirectory = path.join(flutterRoot, 'dev', 'integration_tests');

final List<ShardRunner> tests = integrationTests.map((String integrationTest) =>
() => runCommand(firebaseScript, <String>[ path.join(integrationTestDirectory, integrationTest) ])
).toList();

await _selectIndexedSubshard(tests, integrationTests.length);
}

Future<void> _runFrameworkCoverage() async {
final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info'));
if (!coverageFile.existsSync()) {
Expand Down Expand Up @@ -1215,51 +1186,6 @@ Map<String, String> _initGradleEnvironment() {

final Map<String, String> gradleEnvironment = _initGradleEnvironment();

Future<void> _runHostOnlyDeviceLabTests() async {
// Please don't add more tests here. We should not be using the devicelab
// logic to run tests outside devicelab, that's just confusing.
// Instead, create tests that are not devicelab tests, and run those.

// TODO(ianh): Move the tests that are not running on devicelab any more out
// of the device lab directory.

// List the tests to run.
// We split these into subshards. The tests are randomly distributed into
// those subshards so as to get a uniform distribution of costs, but the
// seed is fixed so that issues are reproducible.
final List<ShardRunner> tests = <ShardRunner>[
// Keep this in alphabetical order.
() => _runDevicelabTest('build_aar_module_test', environment: gradleEnvironment),
() => _runDevicelabTest('gradle_jetifier_test', environment: gradleEnvironment),
() => _runDevicelabTest('gradle_plugin_light_apk_test', environment: gradleEnvironment),
() => _runDevicelabTest('module_custom_host_app_name_test', environment: gradleEnvironment),
() => _runDevicelabTest('module_test', environment: gradleEnvironment, testEmbeddingV2: true),
() => _runDevicelabTest('plugin_dependencies_test', environment: gradleEnvironment),
]..shuffle(math.Random(0));

await _selectIndexedSubshard(tests, kDeviceLabShardCount);
}

Future<void> _runDevicelabTest(String testName, {
Map<String, String> environment,
// testEmbeddingV2 is only supported by certain specific devicelab tests.
// Don't use it unless you're sure the test actually supports it.
// You can check by looking to see if the test examines the environment
// for the ENABLE_ANDROID_EMBEDDING_V2 variable.
bool testEmbeddingV2 = false,
}) async {
await runCommand(
dart,
<String>['bin/run.dart', '-t', testName],
workingDirectory: path.join(flutterRoot, 'dev', 'devicelab'),
environment: <String, String>{
...?environment,
if (testEmbeddingV2)
'ENABLE_ANDROID_EMBEDDING_V2': 'true',
},
);
}

void deleteFile(String path) {
// This is technically a race condition but nobody else should be running
// while this script runs, so we should be ok. (Sadly recursive:true does not
Expand Down
18 changes: 18 additions & 0 deletions dev/prod_builders.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,24 @@
"task_name": "linux_build_tests",
"flaky": false
},
{
"name": "Linux firebase_abstract_method_smoke_test",
"repo": "flutter",
"task_name": "linux_firebase_abstract_method_smoke_test",
"flaky": false
},
{
"name": "Linux firebase_android_embedding_v2_smoke_test",
"repo": "flutter",
"task_name": "linux_firebase_android_embedding_v2_smoke_test",
"flaky": false
},
{
"name": "Linux firebase_release_smoke_test",
"repo": "flutter",
"task_name": "linux_firebase_release_smoke_test",
"flaky": false
},
{
"name": "Linux framework_tests",
"repo": "flutter",
Expand Down
18 changes: 18 additions & 0 deletions dev/try_builders.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,24 @@
"enabled":true,
"run_if":["dev/", "packages/flutter/", "packages/flutter_test/", "packages/flutter_goldens/", "packages/flutter_tools/lib/src/test/", "bin/"]
},
{
"name":"Linux firebase_abstract_method_smoke_test",
"repo":"flutter",
"task_name":"linux_firebase_abstract_method_smoke_test",
"enabled":true
},
{
"name":"Linux firebase_android_embedding_v2_smoke_test",
"repo":"flutter",
"task_name":"linux_firebase_android_embedding_v2_smoke_test",
"enabled":true
},
{
"name":"Linux firebase_release_smoke_test",
"repo":"flutter",
"task_name":"linux_firebase_release_smoke_test",
"enabled":true
},
{
"name":"Linux fuchsia_precache",
"repo":"flutter",
Expand Down

0 comments on commit fd0e223

Please sign in to comment.