diff --git a/.ci/Dockerfile b/.ci/Dockerfile index 09db02e8967e..324a2b97615f 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -5,7 +5,6 @@ RUN yes | sdkmanager \ "platforms;android-27" \ "build-tools;27.0.3" \ "extras;google;m2repository" \ - "extras;android;m2repository" \ - "system-images;android-21;default;armeabi-v7a" + "extras;android;m2repository" RUN yes | sdkmanager --licenses diff --git a/.cirrus.yml b/.cirrus.yml index 6735d0b62e7d..02c7551b3500 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -30,11 +30,6 @@ task: PLUGIN_SHARDING: "--shardIndex 0 --shardCount 2" PLUGIN_SHARDING: "--shardIndex 1 --shardCount 2" MAPS_API_KEY: ENCRYPTED[596a9f6bca436694625ac50851dc5da6b4d34cba8025f7db5bc9465142e8cd44e15f69e3507787753accebfc4910d550] - create_device_script: - echo no | avdmanager -v create avd -n test -k "system-images;android-21;default;armeabi-v7a" - start_emulator_background_script: - - $ANDROID_HOME/emulator/emulator -avd test -no-audio -no-window - wait_for_emulator_script: adb wait-for-device script: # Unsetting CIRRUS_CHANGE_MESSAGE and CIRRUS_COMMIT_MESSAGE as they # might include non-ASCII characters which makes Gradle crash. @@ -48,9 +43,6 @@ task: - export CIRRUS_COMMIT_MESSAGE="" - ./script/incremental_build.sh build-examples --apk - ./script/incremental_build.sh java-test # must come after apk build - # TODO(jackson): Re-enable once Android emulators support Firebase - # https://github.com/flutter/flutter/issues/29571 - # - ./script/incremental_build.sh drive-examples - export CIRRUS_CHANGE_MESSAGE=`cat /tmp/cirrus_change_message.txt` - export CIRRUS_COMMIT_MESSAGE=`cat /tmp/cirrus_commit_message.txt` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 936b5b921ebf..e6e3e957af29 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,13 +50,20 @@ To run the unit tests: flutter test test/_test.dart ``` -To run the integration tests: +To run the integration tests using Flutter driver: ``` cd example flutter drive test/.dart ``` +To run integration tests as instrumentation tests on a local Android device: + +``` +cd example +(cd android && ./gradlew -Ptarget=$(pwd)/../test_live/_test.dart connectedAndroidTest) +``` + ## Contributing code We gladly accept contributions via GitHub pull requests. diff --git a/packages/instrumentation_adapter/.gitignore b/packages/instrumentation_adapter/.gitignore new file mode 100644 index 000000000000..e9dc58d3d6e2 --- /dev/null +++ b/packages/instrumentation_adapter/.gitignore @@ -0,0 +1,7 @@ +.DS_Store +.dart_tool/ + +.packages +.pub/ + +build/ diff --git a/packages/instrumentation_adapter/.metadata b/packages/instrumentation_adapter/.metadata new file mode 100644 index 000000000000..2b43f17e4faa --- /dev/null +++ b/packages/instrumentation_adapter/.metadata @@ -0,0 +1,10 @@ +# This file tracks properties of this Flutter project. +# Used by Flutter tool to assess capabilities and perform upgrades etc. +# +# This file should be version controlled and should not be manually edited. + +version: + revision: 3374ee380b499d99c50ed6dfdd45510aa8318741 + channel: master + +project_type: plugin diff --git a/packages/instrumentation_adapter/CHANGELOG.md b/packages/instrumentation_adapter/CHANGELOG.md new file mode 100644 index 000000000000..29f648fd17f1 --- /dev/null +++ b/packages/instrumentation_adapter/CHANGELOG.md @@ -0,0 +1,3 @@ +## 0.0.1 + +* Initial release diff --git a/packages/instrumentation_adapter/LICENSE b/packages/instrumentation_adapter/LICENSE new file mode 100644 index 000000000000..0c382ce171cc --- /dev/null +++ b/packages/instrumentation_adapter/LICENSE @@ -0,0 +1,27 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/packages/instrumentation_adapter/README.md b/packages/instrumentation_adapter/README.md new file mode 100644 index 000000000000..b2a65bd84a46 --- /dev/null +++ b/packages/instrumentation_adapter/README.md @@ -0,0 +1,45 @@ +# instrumentation_adapter + +Adapts flutter_test results as Android instrumentation tests, making them usable for Firebase Test Lab and other Android CI providers. + +iOS support is not available yet, but is planned in the future. + +## Usage + +Add a dependency on the `instrumentation_adapter` package in the `dev_dependencies` section of pubspec.yaml. For plugins, do this in the pubspec.yaml of the example app. + +Invoke `InstrumentationAdapterFlutterBinding.ensureInitialized()` at the start of a test file. + +```dart +import 'package:instrumentation_adapter/instrumentation_adapter.dart'; +import '../test/package_info.dart' as test; + +void main() { + InstrumentationAdapterFlutterBinding.ensureInitialized(); + testWidgets("failing test example", (WidgetTester tester) async { + expect(2 + 2, equals(5)); + }); +} +``` + +Use gradle commands to build an instrumentation test for Android. + +``` +pushd android +./gradlew assembleAndroidTest +./gradlew assembleDebug -Ptarget=.dart +popd +``` + +Upload to Firebase Test Lab, making sure to replace , , , and with your values. + +``` +gcloud auth activate-service-account --key-file= +gcloud --quiet config set project +gcloud firebase test android run --type instrumentation \ + --app build/app/outputs/apk/debug/app-debug.apk \ + --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk\ + --timeout 2m \ + --results-bucket= \ + --results-dir= +``` diff --git a/packages/instrumentation_adapter/android/.gitignore b/packages/instrumentation_adapter/android/.gitignore new file mode 100644 index 000000000000..c6cbe562a427 --- /dev/null +++ b/packages/instrumentation_adapter/android/.gitignore @@ -0,0 +1,8 @@ +*.iml +.gradle +/local.properties +/.idea/workspace.xml +/.idea/libraries +.DS_Store +/build +/captures diff --git a/packages/instrumentation_adapter/android/build.gradle b/packages/instrumentation_adapter/android/build.gradle new file mode 100644 index 000000000000..a117873b3560 --- /dev/null +++ b/packages/instrumentation_adapter/android/build.gradle @@ -0,0 +1,41 @@ +group 'com.example.instrumentation_adapter' +version '1.0-SNAPSHOT' + +buildscript { + repositories { + google() + jcenter() + } + + dependencies { + classpath 'com.android.tools.build:gradle:3.2.1' + } +} + +rootProject.allprojects { + repositories { + google() + jcenter() + } +} + +apply plugin: 'com.android.library' + +android { + compileSdkVersion 28 + + defaultConfig { + minSdkVersion 16 + testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" + } + lintOptions { + disable 'InvalidPackage' + } + dependencies { + api 'junit:junit:4.12' + api 'androidx.test:core:1.0.0' + api 'androidx.test:runner:1.1.1' + api 'androidx.test:rules:1.1.1' + api 'androidx.test.espresso:espresso-core:3.1.1' + } +} diff --git a/packages/instrumentation_adapter/android/gradle.properties b/packages/instrumentation_adapter/android/gradle.properties new file mode 100644 index 000000000000..2bd6f4fda009 --- /dev/null +++ b/packages/instrumentation_adapter/android/gradle.properties @@ -0,0 +1,2 @@ +org.gradle.jvmargs=-Xmx1536M + diff --git a/packages/instrumentation_adapter/android/settings.gradle b/packages/instrumentation_adapter/android/settings.gradle new file mode 100644 index 000000000000..ed03d0eb2a5e --- /dev/null +++ b/packages/instrumentation_adapter/android/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'instrumentation_adapter' diff --git a/packages/instrumentation_adapter/android/src/main/AndroidManifest.xml b/packages/instrumentation_adapter/android/src/main/AndroidManifest.xml new file mode 100644 index 000000000000..3b424b6fad67 --- /dev/null +++ b/packages/instrumentation_adapter/android/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + diff --git a/packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/FlutterRunner.java b/packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/FlutterRunner.java new file mode 100644 index 000000000000..bc635ea080de --- /dev/null +++ b/packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/FlutterRunner.java @@ -0,0 +1,53 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package dev.flutter.plugins.instrumentationadapter; + +import java.util.Map; +import java.util.concurrent.ExecutionException; +import org.junit.runner.Description; +import org.junit.runner.Runner; +import org.junit.runner.notification.Failure; +import org.junit.runner.notification.RunNotifier; + +public class FlutterRunner extends Runner { + + final Class testClass; + + public FlutterRunner(Class testClass) { + super(); + this.testClass = testClass; + try { + testClass.newInstance().launchActivity(); + } catch (InstantiationException | IllegalAccessException e) { + throw new IllegalThreadStateException("Unable to launch test"); + } + } + + @Override + public Description getDescription() { + return Description.createTestDescription(testClass, "Flutter Tests"); + } + + @Override + public void run(RunNotifier notifier) { + Map results = null; + try { + results = InstrumentationAdapterPlugin.testResults.get(); + } catch (ExecutionException | InterruptedException e) { + throw new IllegalThreadStateException("Unable to get test results"); + } + + for (String name : results.keySet()) { + Description d = Description.createTestDescription(testClass, name); + notifier.fireTestStarted(d); + String outcome = results.get(name); + if (outcome.equals("failed")) { + Exception dummyException = new Exception(outcome); + notifier.fireTestFailure(new Failure(d, dummyException)); + } + notifier.fireTestFinished(d); + } + } +} diff --git a/packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/FlutterTest.java b/packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/FlutterTest.java new file mode 100644 index 000000000000..df6b398bddd6 --- /dev/null +++ b/packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/FlutterTest.java @@ -0,0 +1,9 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package dev.flutter.plugins.instrumentationadapter; + +public abstract class FlutterTest { + public abstract void launchActivity(); +} diff --git a/packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/InstrumentationAdapterPlugin.java b/packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/InstrumentationAdapterPlugin.java new file mode 100644 index 000000000000..c77a94e89f91 --- /dev/null +++ b/packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/InstrumentationAdapterPlugin.java @@ -0,0 +1,38 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package dev.flutter.plugins.instrumentationadapter; + +import io.flutter.plugin.common.MethodCall; +import io.flutter.plugin.common.MethodChannel; +import io.flutter.plugin.common.MethodChannel.MethodCallHandler; +import io.flutter.plugin.common.MethodChannel.Result; +import io.flutter.plugin.common.PluginRegistry.Registrar; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +/** InstrumentationAdapterPlugin */ +public class InstrumentationAdapterPlugin implements MethodCallHandler { + + public static CompletableFuture> testResults = new CompletableFuture<>(); + + private static final String CHANNEL = "dev.flutter/InstrumentationAdapterFlutterBinding"; + + /** Plugin registration. */ + public static void registerWith(Registrar registrar) { + final MethodChannel channel = new MethodChannel(registrar.messenger(), CHANNEL); + channel.setMethodCallHandler(new InstrumentationAdapterPlugin()); + } + + @Override + public void onMethodCall(MethodCall call, Result result) { + if (call.method.equals("allTestsFinished")) { + Map results = call.argument("results"); + testResults.complete(results); + result.success(null); + } else { + result.notImplemented(); + } + } +} diff --git a/packages/instrumentation_adapter/lib/instrumentation_adapter.dart b/packages/instrumentation_adapter/lib/instrumentation_adapter.dart new file mode 100644 index 000000000000..81f81872d950 --- /dev/null +++ b/packages/instrumentation_adapter/lib/instrumentation_adapter.dart @@ -0,0 +1,48 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter/widgets.dart'; + +/// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results +/// on a channel to adapt them to native instrumentation test format. +class InstrumentationAdapterFlutterBinding + extends LiveTestWidgetsFlutterBinding { + InstrumentationAdapterFlutterBinding() { + // TODO(jackson): Report test results as they arrive + tearDownAll(() async { + await _channel.invokeMethod( + 'allTestsFinished', {'results': _results}); + }); + } + + static WidgetsBinding ensureInitialized() { + if (WidgetsBinding.instance == null) { + InstrumentationAdapterFlutterBinding(); + } + assert(WidgetsBinding.instance is InstrumentationAdapterFlutterBinding); + return WidgetsBinding.instance; + } + + static const MethodChannel _channel = + MethodChannel('dev.flutter/InstrumentationAdapterFlutterBinding'); + + static Map _results = {}; + + @override + Future runTest(Future testBody(), VoidCallback invariantTester, + {String description = '', Duration timeout}) async { + // TODO(jackson): Report the results individually instead of all at once + // See https://github.com/flutter/flutter/issues/38985 + reportTestException = + (FlutterErrorDetails details, String testDescription) { + _results[description] = 'failed'; + }; + await super.runTest(testBody, invariantTester, + description: description, timeout: timeout); + _results[description] ??= 'success'; + } +} diff --git a/packages/instrumentation_adapter/pubspec.yaml b/packages/instrumentation_adapter/pubspec.yaml new file mode 100644 index 000000000000..103a43628024 --- /dev/null +++ b/packages/instrumentation_adapter/pubspec.yaml @@ -0,0 +1,19 @@ +name: instrumentation_adapter +description: Runs tests that use the flutter_test API as platform native instrumentation tests. +version: 0.0.1 +author: Flutter Team +homepage: https://github.com/flutter/plugins/tree/master/packages/instrumentation_adapter + +environment: + sdk: ">=2.1.0 <3.0.0" + +dependencies: + flutter: + sdk: flutter + flutter_test: + sdk: flutter + +flutter: + plugin: + androidPackage: dev.flutter.plugins.instrumentationadapter + pluginClass: InstrumentationAdapterPlugin \ No newline at end of file diff --git a/script/build_all_plugins_app.sh b/script/build_all_plugins_app.sh index 4d30ad73e8a1..4bf59d2e8a80 100755 --- a/script/build_all_plugins_app.sh +++ b/script/build_all_plugins_app.sh @@ -12,7 +12,7 @@ check_changed_packages > /dev/null cd $REPO_DIR/examples/all_plugins flutter clean > /dev/null -(cd "$REPO_DIR" && pub global run flutter_plugin_tools gen-pubspec) +(cd "$REPO_DIR" && pub global run flutter_plugin_tools gen-pubspec --exclude instrumentation_adapter) function error() { echo "$@" 1>&2