Skip to content

Commit

Permalink
Improve Android testing scripts
Browse files Browse the repository at this point in the history
Summary:
The goal of this pull request is to make it easier for contributors to run Android tests locally, specifically the unit tests and integration tests. I added a bunch of checks to the local testing scripts that will warn you if your environment is misconfigured, and tell you how to fix it. I also updated the testing docs, so that the regular "Testing" page should be a decent resource to point people to when you are telling them "hey this pull request needs a test." Just Android, though, I haven't gotten to the iOS parts yet.

I also disabled a couple tests that seemed quite flaky while running on a local machine, and don't seem to be providing much value. In particular, the `TestId` test just hangs on my emulator a lot and has been flaky on CI in the past, so I removed about half of its test cases to make the sample app smaller. The testMetions test appears to be dependent on screen size so I commented it out.
Closes facebook#11442

Differential Revision: D4323569

Pulled By: bestander

fbshipit-source-id: 9c869f3915d5c7cee438615f37986b07ab251f8c
  • Loading branch information
lacker authored and Facebook Github Bot committed Dec 14, 2016
1 parent 3e6d762 commit affd5ac
Show file tree
Hide file tree
Showing 11 changed files with 249 additions and 93 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,4 @@ node_modules

# Test generated files
/ReactAndroid/src/androidTest/assets/AndroidTestBundle.js
*.js.meta
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import com.facebook.react.testing.ReactInstanceSpecForTest;
import com.facebook.react.testing.ReactTestHelper;


/**
* Tests that the 'testID' property can be set on various views.
* The 'testID' property is used to locate views in UI tests.
Expand All @@ -32,22 +31,13 @@ protected String getReactApplicationKeyUnderTest() {

private final List<String> viewTags = Arrays.asList(
"Image",
"ProgressBar",
"ScrollView",
"Horizontal ScrollView",
"Dropdown Picker",
"Dialog Picker",
"Switch",
"Text",
"TouchableBounce",
"TouchableHighlight",
"TouchableOpacity",
"TouchableWithoutFeedback",
"Toolbar",
"TextInput",
"View",
// "WebView", TODO t11449130
"ScrollView Item (same id used for all items)"
"View"
);

public void testPropertyIsSetForViews() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ public void testTextInputColors() throws Throwable {

/**
* Test that the mentions input has colors displayed correctly.
*/
* Removed for being flaky in open source, December 2016
public void testMetionsInputColors() throws Throwable {
EventDispatcher eventDispatcher =
getReactContext().getNativeModule(UIManagerModule.class).getEventDispatcher();
Expand Down Expand Up @@ -202,6 +202,7 @@ public void testMetionsInputColors() throws Throwable {
assertEquals(newText.length() - 25, reactEditText.getText().getSpanStart(spans[0]));
assertEquals(newText.length() - 11, reactEditText.getText().getSpanEnd(spans[0]));
}
*/

@Override
protected ReactInstanceSpecForTest createReactInstanceSpecForTest() {
Expand Down
54 changes: 0 additions & 54 deletions ReactAndroid/src/androidTest/js/TestIdTestModule.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,16 @@
'use strict';

var Image = require('Image');
var Picker = require('Picker');
var ProgressBarAndroid = require('ProgressBarAndroid');
var React = require('React');
var ScrollView = require('ScrollView');
var StyleSheet = require('StyleSheet');
var Switch = require('Switch');
var Text = require('Text');
var TextInput = require('TextInput');
var ToolbarAndroid = require('ToolbarAndroid');
var TouchableBounce = require('TouchableBounce');
var TouchableHighlight = require('TouchableHighlight');
var TouchableOpacity = require('TouchableOpacity');
var TouchableWithoutFeedback = require('TouchableWithoutFeedback');
var View = require('View');
var WebView = require('WebView');

/**
* All the views implemented on Android, each with the testID property set.
Expand All @@ -49,46 +44,8 @@ class TestIdTestApp extends React.Component {
'riy/Xl4/f1cf5VWzXyym7PHhhx4dbgYKAAA7'}}
style={styles.base} />

<ProgressBarAndroid
testID="ProgressBar"
styleAttr="Horizontal"
style={styles.base} />

<ScrollView
testID="ScrollView"
style={styles.base}>
<Text testID="ScrollView Item (same id used for all items)">Item 1</Text>
<Text testID="ScrollView Item (same id used for all items)">Item 2</Text>
</ScrollView>

<ScrollView
testID="Horizontal ScrollView"
horizontal={true}
style={styles.base}>
<Text testID="ScrollView Item (same id used for all items)">Item 1</Text>
<Text testID="ScrollView Item (same id used for all items)">Item 2</Text>
</ScrollView>

<Picker
testID="Dropdown Picker"
mode={Picker.MODE_DROPDOWN}
style={styles.base}>
<Picker.Item label="Dropdown picker" value="key0" />
</Picker>

<Picker
testID="Dialog Picker"
mode={Picker.MODE_DIALOG}
style={styles.base}>
<Picker.Item label="Dialog picker" value="key0" />
</Picker>

<Switch testID="Switch" value={true} />

<Text testID="Text">text</Text>

<ToolbarAndroid testID="Toolbar" style={styles.base} subtitle="toolbar" />

<TextInput testID="TextInput" value="Text input" />

<TouchableBounce testID="TouchableBounce">
Expand All @@ -111,17 +68,6 @@ class TestIdTestApp extends React.Component {

<View testID="View" />

{/*
Webview gets tests crashing or stalling occasionally
e.g. https://circleci.com/gh/facebook/react-native/7054
TODO t11449130
<WebView
testID="WebView"
url={'http://newsroom.fb.com'}
renderError={() => <View /> }
style={styles.base}
/>*/}

</View>
);
}
Expand Down
51 changes: 34 additions & 17 deletions docs/Testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,48 +10,65 @@ previous: debugging

## Running Tests and Contributing

The React Native repo has several tests you can run to verify you haven't caused a regression with your PR. These tests are run with the [Travis](http://docs.travis-ci.com/) and [CircleCI](https://circleci.com/) continuous integration systems, and will automatically post the results to your PR.
This document is about running tests on React Native itself. If you're interested in testing a React Native app, check out the [React Native Tutorial](http://facebook.github.io/jest/docs/tutorial-react-native.html) on the Jest website.

We don't have perfect test coverage of course, especially for complex end-to-end interactions with the user, so many changes will still require significant manual verification, but we would love it if you want to help us increase our test coverage and add more tests and test cases!
The React Native repo has several tests you can run to verify you haven't caused a regression with your PR. These tests are run with the [Travis](http://docs.travis-ci.com/) and [CircleCI](https://circleci.com/) continuous integration systems, which will automatically annotate pull requests with the test results.

Whenever you are fixing a bug or adding new functionality to React Native, you should add a test that covers it. Depending on the change you're making, there are different types of tests that may be appropriate.

## Jest Tests

[Jest](http://facebook.github.io/jest/) tests are JS-only tests run on the command line with node. The tests themselves live in the `__tests__` directories of the files they test, and there is a large emphasis on aggressively mocking out functionality that is not under test for failure isolation and maximum speed. You can run the existing React Native jest tests with
Jest tests are JavaScript-only tests run on the command line with node. You can run the existing React Native jest tests with:

$ cd react-native
$ npm test

It's a good idea to add a Jest test when you are working on a change that only modifies JavaScript code.

```
npm test
```
The tests themselves live in the `__tests__` directories of the files they test. See [`TouchableHighlight-test.js`](https://github.com/facebook/react-native/blob/master/Libraries/Components/Touchable/__tests__/TouchableHighlight-test.js) for a basic example.

from the react-native root, and we encourage you to add your own tests for any components you want to contribute to. See [`getImageSource-test.js`](https://github.com/facebook/react-native/blob/master/Examples/Movies/__tests__/getImageSource-test.js) for a basic example.
## Android Unit Tests

To use Jest for your react-native projects we recommend following the [React-Native Tutorial](http://facebook.github.io/jest/docs/tutorial-react-native.html) on the Jest website.
The Android unit tests do not run in an emulator. They just use a normal Java installation. You do need to install Java 8. In particular, the default OS X Java install is insufficient.

## Unit tests (Android)
You also need to install the [Buck build tool](https://buckbuild.com/setup/install.html).

React Native uses the [Buck build tool](https://buckbuild.com/setup/install.html) to run tests. Unit tests run locally on your machine, no emulator is needed. To run the tests:
To run the Android unit tests:

$ cd react-native
$ ./scripts/run-android-local-unit-tests.sh

## Integration tests (Android)
It's a good idea to add an Android unit test whenever you are working on code that can be tested by Java code alone. The Android unit tests live under [`ReactAndroid/src/tests`](https://github.com/facebook/react-native/tree/master/ReactAndroid/src/test/java/com/facebook/react), so you can browse through that directory for good examples of tests.

## Android Integration Tests

To run the integration tests, you need to install the Android NDK. See [Prerequisites](/react-native/docs/android-building-from-source.html#prerequisites).

React Native uses the [Buck build tool](https://buckbuild.com/setup/install.html) to run tests. Integration tests run on an emulator / device and verify that modules and components, as well as the core parts of React Native (such as the bridge) work well end-to-end.
You also need to install the [Buck build tool](https://buckbuild.com/setup/install.html).

Make sure you have the path to the Android NDK set up, see [Prerequisites](/react-native/docs/android-building-from-source.html#prerequisites).
We recommend running the Android integration tests in an emulator, although you can also use a real Android device. It's a good idea to keep the emulator running with a visible window. That way if your tests stall, you can look at the emulator to debug.

To run the tests:
Some devices and some emulator configurations may not work with the tests. We do maintain an emulator configuration that works, as the standard for testing. To run this emulator config:

$ cd react-native
$ ./scripts/run-android-emulator.sh

Once you have an emulator running, to run the integration tests:

$ cd react-native
$ npm install
$ ./scripts/run-android-local-integration-tests.sh

## Integration Tests (iOS)
The integration tests should only take a few minutes to run on a modern developer machine.

It's a good idea to add an Android integration test whenever you are working on code that needs both JavaScript and Java to be tested in conjunction. The Android integration tests live under [`ReactAndroid/src/androidTest`](https://github.com/facebook/react-native/tree/master/ReactAndroid/src/androidTest/java/com/facebook/react/tests), so you can browse through that directory for good examples of tests.

## iOS Integration Tests

React Native provides facilities to make it easier to test integrated components that require both native and JS components to communicate across the bridge. The two main components are `RCTTestRunner` and `RCTTestModule`. `RCTTestRunner` sets up the ReactNative environment and provides facilities to run the tests as `XCTestCase`s in Xcode (`runTest:module` is the simplest method). `RCTTestModule` is exported to JS as `NativeModules.TestModule`. The tests themselves are written in JS, and must call `TestModule.markTestCompleted()` when they are done, otherwise the test will timeout and fail. Test failures are primarily indicated by throwing a JS exception. It is also possible to test error conditions with `runTest:module:initialProps:expectErrorRegex:` or `runTest:module:initialProps:expectErrorBlock:` which will expect an error to be thrown and verify the error matches the provided criteria. See [`IntegrationTestHarnessTest.js`](https://github.com/facebook/react-native/blob/master/IntegrationTests/IntegrationTestHarnessTest.js), [`UIExplorerIntegrationTests.m`](https://github.com/facebook/react-native/blob/master/Examples/UIExplorer/UIExplorerIntegrationTests/UIExplorerIntegrationTests.m), and [IntegrationTestsApp.js](https://github.com/facebook/react-native/blob/master/IntegrationTests/IntegrationTestsApp.js) for example usage and integration points.

You can run integration tests locally with cmd+U in the IntegrationTest and UIExplorer apps in Xcode.

## Screenshot/Snapshot Tests (iOS)
## iOS Screenshot/Snapshot Tests

A common type of integration test is the snapshot test. These tests render a component, and verify snapshots of the screen against reference images using `TestModule.verifySnapshot()`, using the [`FBSnapshotTestCase`](https://github.com/facebook/ios-snapshot-test-case) library behind the scenes. Reference images are recorded by setting `recordMode = YES` on the `RCTTestRunner`, then running the tests. Snapshots will differ slightly between 32 and 64 bit, and various OS versions, so it's recommended that you enforce tests are run with the correct configuration. It's also highly recommended that all network data be mocked out, along with other potentially troublesome dependencies. See [`SimpleSnapshotTest`](https://github.com/facebook/react-native/blob/master/IntegrationTests/SimpleSnapshotTest.js) for a basic example.

Expand Down
19 changes: 19 additions & 0 deletions scripts/run-android-emulator.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/bin/bash

# Runs an Android emulator locally.
# If there already is a running emulator, this just uses that.
# The only reason to use this config is that it represents a known-good
# virtual device configuration.
# This is useful for running integration tests on a local machine.
# TODO: make continuous integration use the precise same setup

STATE=`adb get-state`

if [ -n "$STATE" ]; then
echo "An emulator is already running."
exit 1
fi

echo "Creating virtual device..."
echo no | android create avd -n testAVD -f -t android-23 --abi default/x86
emulator -avd testAVD
12 changes: 7 additions & 5 deletions scripts/run-android-local-integration-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,21 @@
# Runs all Android integration tests locally.
# See http://facebook.github.io/react-native/docs/testing.html

set -e
source $(dirname $0)/validate-android-sdk.sh
source $(dirname $0)/validate-android-test-env.sh
source $(dirname $0)/validate-android-device-env.sh

which buck > /dev/null || {
echo "React Native uses the Buck build tool to run tests. Please install Buck: https://buckbuild.com/setup/install.html";
exit 1;
}
set -e

echo "Compiling native code..."
./gradlew :ReactAndroid:packageReactNdkLibsForBuck

echo "Building JS bundle..."
node local-cli/cli.js bundle --platform android --dev true --entry-file ReactAndroid/src/androidTest/js/TestBundle.js --bundle-output ReactAndroid/src/androidTest/assets/AndroidTestBundle.js

echo "Installing test app on the device..."
buck fetch ReactAndroid/src/androidTest/buck-runner:instrumentation-tests
buck install ReactAndroid/src/androidTest/buck-runner:instrumentation-tests

echo "Running integration tests..."
adb shell am instrument -w com.facebook.react.tests/android.support.test.runner.AndroidJUnitRunner
9 changes: 4 additions & 5 deletions scripts/run-android-local-unit-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,13 @@
# Runs all Android unit tests locally.
# See http://facebook.github.io/react-native/docs/testing.html

set -e
source $(dirname $0)/validate-android-sdk.sh
source $(dirname $0)/validate-android-test-env.sh

which buck > /dev/null || {
echo "React Native uses the Buck build tool to run tests. Please install Buck: https://buckbuild.com/setup/install.html";
exit 1;
}
set -e

echo "Fetching dependencies..."
buck fetch ReactAndroid/src/test/...

echo "Running unit tests..."
buck test ReactAndroid/src/test/...
33 changes: 33 additions & 0 deletions scripts/validate-android-device-env.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/bin/bash

# This script validates that the Android environment is set up to run
# tests on a device or emulator (as opposed to a plain Java environment).

# This requires that the Android NDK is set up correctly and it also
# requires that you are currently either running an emulator or have
# an Android device plugged in.

if [ -z "$ANDROID_NDK" ]; then
echo "Error: \$ANDROID_NDK is not configured."
echo "You must first install the Android NDK and then set \$ANDROID_NDK."
echo "If you already installed the Android SDK, well, the NDK is a different thing that you also need to install."
echo "See https://facebook.github.io/react-native/docs/android-building-from-source.html for instructions."
exit 1
fi

if [ -z "$(adb get-state)" ]; then
echo "Error: you must either run an emulator or connect a device."
echo "You can check what devices are running with 'adb get-state'."
echo "You can run scripts/run-android-emulator.sh to get a known-good emulator config."
exit 1
fi

while :
do
BOOTANIM=`adb -e shell getprop init.svc.bootanim`
if [ -n `echo $BOOTANIM | grep stopped` ]; then
break
fi
echo "Waiting for the emulator to finish booting..."
sleep 3
done
51 changes: 51 additions & 0 deletions scripts/validate-android-sdk.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/bin/bash

# This script validates that the Android SDK is installed correctly.
# This means setting ANDROID_HOME and adding its subdirectories to PATH.
# If the Android SDK is not installed correctly, this script exits
# with an error and a helpful message is printed.

if [ -z "$ANDROID_HOME" ]; then
echo "Error: \$ANDROID_HOME is not configured."
echo "You must first install the Android SDK and then set \$ANDROID_HOME."
echo "If you already installed the Android SDK, the problem is that you need to export ANDROID_HOME from your .bashrc or equivalent."
echo "See https://facebook.github.io/react-native/docs/getting-started.html for instructions."
exit 1
fi

if [ ! -d "$ANDROID_HOME" ]; then
echo "Error: \$ANDROID_HOME = $ANDROID_HOME but that directory does not exist."
echo "It is possible that you installed then uninstalled the Android SDK."
echo "In that case, you should reinstall it."
echo "See https://facebook.github.io/react-native/docs/getting-started.html for instructions."
exit 1
fi

if [ ! -e "$ANDROID_HOME/tools/emulator" ]; then
echo "Error: could not find an emulator at \$ANDROID_HOME/tools/emulator."
echo "Specifically, $ANDROID_HOME/tools/emulator does not exist."
echo "This indicates something is borked with your Android SDK install."
echo "One possibility is that you have \$ANDROID_HOME set to the wrong value."
echo "If that seems correct, you might want to try reinstalling the Android SDK."
echo "See https://facebook.github.io/react-native/docs/getting-started.html for instructions."
exit 1
fi

if [ -z `which emulator` ]; then
echo "Error: could not find 'emulator'. Specifically, 'which emulator' was empty."
echo "However, the emulator seems to be installed at \$ANDROID_HOME/tools/emulator already."
echo "This means that the problem is that you are not adding \$ANDROID_HOME/tools to your \$PATH."
echo "You should do that, and then rerun this command."
echo "Sorry for not fixing this automatically - we just didn't want to mess with your \$PATH automatically because that can break things."
exit 1
fi

if [ -z `which adb` ]; then
echo "Error: could not find 'adb'. Specifically, 'which adb' was empty."
echo "This indicates something is borked with your Android SDK install."
echo "The most likely problem is that you are not adding \$ANDROID_HOME/platform-tools to your \$PATH."
echo "If all else fails, try reinstalling the Android SDK."
echo "See https://facebook.github.io/react-native/docs/getting-started.html for instructions."
exit 1
fi

Loading

0 comments on commit affd5ac

Please sign in to comment.