Skip to content

Commit

Permalink
Optimise e2e tests retries
Browse files Browse the repository at this point in the history 
Summary:Implemented smarter retries for e2e tests

E2e tests consist of two expensive and flaky steps: npm installation and tests execution.
Our CI is configured to retry a command 3 times before terminating the test.
In current setup if one of the steps fail the whole test is restarted.

This change adds ad-hoc ability to retry flaky bits of e2e script independently.
This will make tests fail faster when code gets broken while increasing the chances to succeed in case of random false errors.
Closes facebook#7184

Differential Revision: D3218927

fb-gh-sync-id: 9be8343484bb28aa3601b651db70fc55aa840e61
fbshipit-source-id: 9be8343484bb28aa3601b651db70fc55aa840e61
  • Loading branch information
@bestander
bestander authored and Facebook Github Bot 9 committed Apr 25, 2016
1 parent a2ee5bd commit a8d2079
Show file tree
Hide file tree
Showing 3 changed files with 200 additions and 139 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ install:

script:
- if [[ "$TEST_TYPE" = objc ]]; then travis_retry ./scripts/objc-test.sh; fi
- if [[ "$TEST_TYPE" = e2e-objc ]]; then travis_retry node ./scripts/run-ci-e2e-tests.js --ios --js; fi
- if [[ "$TEST_TYPE" = e2e-objc ]]; then node ./scripts/run-ci-e2e-tests.js --ios --js --retries 3; fi
- if [[ "$TEST_TYPE" = js ]]; then npm run flow check; fi
- if [[ "$TEST_TYPE" = js ]]; then npm test -- --maxWorkers=1; fi

Expand Down
4 changes: 1 addition & 3 deletions circle.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,7 @@ test:
- ./gradlew :ReactAndroid:assembleDebugAndroidTest

# Android e2e test
# TODO t10955118, temporary until master is fixed
#- source scripts/circle-ci-android-setup.sh && retry3 node ./scripts/run-ci-e2e-tests.js --android --js
- node ./scripts/run-ci-e2e-tests.js --android --js
- node ./scripts/run-ci-e2e-tests.js --android --js --retries 3

# testing docs generation is not broken
- cd website && node ./server/generate.js
Expand Down
333 changes: 198 additions & 135 deletions scripts/run-ci-e2e-tests.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,173 +16,236 @@
* --android - to test only android application end to end
* --js - to test that JS in the application is compilable
* --skip-cli-install - to skip react-native-cli global installation (for local debugging)
* --retries [num] - how many times to retry possible flaky commands: npm install and running tests, default 1
*/
/*eslint-disable no-undef */
require('shelljs/global');
var spawn = require('child_process').spawn;

const spawn = require('child_process').spawn;
const argv = require('yargs').argv;
const path = require('path');

const SCRIPTS = __dirname;
const ROOT = path.normalize(path.join(__dirname, '..'));

const TEMP=exec('mktemp -d /tmp/react-native-XXXXXXXX').stdout.trim();
const TEMP = exec('mktemp -d /tmp/react-native-XXXXXXXX').stdout.trim();
// To make sure we actually installed the local version
// of react-native, we will create a temp file inside the template
// and check that it exists after `react-native init
const MARKER_IOS = exec(`mktemp ${ROOT}/local-cli/generator-ios/templates/app/XXXXXXXX`).stdout.trim();
const MARKER_ANDROID = exec(`mktemp ${ROOT}/local-cli/generator-android/templates/src/XXXXXXXX`).stdout.trim();

const numberOfRetries = argv.retries || 1;
let SERVER_PID;
let APPIUM_PID;
let exitCode;

const args = process.argv.slice(2);

function cleanup(errorCode) {
if (errorCode !== 0) {
cat(`${TEMP}/server.log`);
cat(`/usr/local/Cellar/watchman/3.1/var/run/watchman/${process.env.USER}.log`);
}
rm(MARKER_IOS);
rm(MARKER_ANDROID);

if(SERVER_PID) {
echo(`Killing packager ${SERVER_PID}`);
exec(`kill -9 ${SERVER_PID}`);
// this is quite drastic but packager starts a daemon that we can't kill by killing the parent process
// it will be fixed in April (quote David Aurelio), so until then we will kill the zombie by the port number
exec("lsof -i tcp:8081 | awk 'NR!=1 {print $2}' | xargs kill");
}
if(APPIUM_PID) {
echo(`Killing appium ${APPIUM_PID}`);
exec(`kill -9 ${APPIUM_PID}`);
/**
* Try executing a function n times recursively.
* Return 0 the first time it succeeds
* Return code of the last failed commands if not more retries left
* @funcToRetry - function that gets retried
* @retriesLeft - number of retries to execute funcToRetry
* @onEveryError - func to execute if funcToRetry returns non 0
*/
function tryExecNTimes(funcToRetry, retriesLeft, onEveryError) {
const exitCode = funcToRetry();
if (exitCode === 0) {
return exitCode;
} else {
if (onEveryError) {
onEveryError();
}
retriesLeft--;
echo(`Command failed, ${retriesLeft} retries left`);
if (retriesLeft === 0) {
return exitCode;
} else {
return tryExecNTimes(funcToRetry, retriesLeft, onEveryError);
}
}
return errorCode;
}

// install CLI
cd('react-native-cli');
exec('npm pack');
const CLI_PACKAGE = path.join(ROOT, 'react-native-cli', 'react-native-cli-*.tgz');
cd('..');

// can skip cli install for non sudo mode
if(args.indexOf('--skip-cli-install') === -1) {
if(exec(`npm install -g ${CLI_PACKAGE}`).code) {
echo('Could not install react-native-cli globally, please run in su mode');
echo('Or with --skip-cli-install to skip this step');
exit(cleanup(1));
try {
// install CLI
cd('react-native-cli');
exec('npm pack');
const CLI_PACKAGE = path.join(ROOT, 'react-native-cli', 'react-native-cli-*.tgz');
cd('..');

// can skip cli install for non sudo mode
if (!argv['skip-cli-install']) {
if (exec(`npm install -g ${CLI_PACKAGE}`).code) {
echo('Could not install react-native-cli globally, please run in su mode');
echo('Or with --skip-cli-install to skip this step');
exitCode = 1;
throw Error(exitCode);
}
}
}

if (args.indexOf('--android') !== -1) {
if (exec('./gradlew :ReactAndroid:installArchives -Pjobs=1 -Dorg.gradle.jvmargs="-Xmx512m -XX:+HeapDumpOnOutOfMemoryError"').code) {
echo('Failed to compile Android binaries');
exit(cleanup(1));
if (argv['android']) {
if (exec('./gradlew :ReactAndroid:installArchives -Pjobs=1 -Dorg.gradle.jvmargs="-Xmx512m -XX:+HeapDumpOnOutOfMemoryError"').code) {
echo('Failed to compile Android binaries');
exitCode = 1;
throw Error(exitCode);
}
}
}

if (exec('npm pack').code) {
echo('Failed to pack react-native');
exit(cleanup(1));
}
if (exec('npm pack').code) {
echo('Failed to pack react-native');
exitCode = 1;
throw Error(exitCode);
}

// test begins
const PACKAGE = path.join(ROOT, 'react-native-*.tgz');
cd(TEMP);
if (exec(`react-native init EndToEndTest --version ${PACKAGE}`).code) {
echo('Failed to execute react-native init');
echo('Most common reason is npm registry connectivity, try again');
exit(cleanup(1));
}
cd('EndToEndTest');

if (args.indexOf('--android') !== -1) {
echo('Running an Android e2e test');
echo('Installing e2e framework');
if(exec('npm install --save-dev [email protected] [email protected] [email protected] [email protected] [email protected]', {silent: true}).code) {
echo('Failed to install appium');
exit(cleanup(1));
const PACKAGE = path.join(ROOT, 'react-native-*.tgz');
cd(TEMP);
if (tryExecNTimes(
() => {
exec('sleep 10s');
return exec(`react-native init EndToEndTest --version ${PACKAGE}`).code;
},
numberOfRetries,
() => rm('-rf', 'EndToEndTest'))) {
echo('Failed to execute react-native init');
echo('Most common reason is npm registry connectivity, try again');
exitCode = 1;
throw Error(exitCode);
}
cp(`${SCRIPTS}/android-e2e-test.js`, 'android-e2e-test.js');
cd('android');
echo('Downloading Maven deps');
exec('./gradlew :app:copyDownloadableDepsToLibs');
// Make sure we installed local version of react-native
if (!test('-e', path.basename(MARKER_ANDROID))) {
echo('Android marker was not found, react native init command failed?');
exit(cleanup(1));

cd('EndToEndTest');

if (argv['android']) {
echo('Running an Android e2e test');
echo('Installing e2e framework');
if (tryExecNTimes(
() => exec('npm install --save-dev [email protected] [email protected] [email protected] [email protected] [email protected]', { silent: true }).code,
numberOfRetries)) {
echo('Failed to install appium');
echo('Most common reason is npm registry connectivity, try again');
exitCode = 1;
throw Error(exitCode);
}
cp(`${SCRIPTS}/android-e2e-test.js`, 'android-e2e-test.js');
cd('android');
echo('Downloading Maven deps');
exec('./gradlew :app:copyDownloadableDepsToLibs');
// Make sure we installed local version of react-native
if (!test('-e', path.basename(MARKER_ANDROID))) {
echo('Android marker was not found, react native init command failed?');
exitCode = 1;
throw Error(exitCode);
}
cd('..');
exec('keytool -genkey -v -keystore android/keystores/debug.keystore -storepass android -alias androiddebugkey -keypass android -dname "CN=Android Debug,O=Android,C=US"');

echo(`Starting packager server, ${SERVER_PID}`);
const appiumProcess = spawn('node', ['./node_modules/.bin/appium']);
APPIUM_PID = appiumProcess.pid;
echo(`Starting appium server, ${APPIUM_PID}`);
echo('Building app');
if (exec('buck build android/app').code) {
echo('could not execute Buck build, is it installed and in PATH?');
exitCode = 1;
throw Error(exitCode);
}
let packagerEnv = Object.create(process.env);
packagerEnv.REACT_NATIVE_MAX_WORKERS = 1;
// shelljs exec('', {async: true}) does not emit stdout events, so we rely on good old spawn
const packagerProcess = spawn('npm', ['start'], {
// stdio: 'inherit',
env: packagerEnv
});
SERVER_PID = packagerProcess.pid;
// wait a bit to allow packager to startup
exec('sleep 5s');
echo('Executing android e2e test');
if (tryExecNTimes(
() => {
exec('sleep 10s');
return exec('node node_modules/.bin/_mocha android-e2e-test.js').code;
},
numberOfRetries)) {
echo('Failed to run Android e2e tests');
echo('Most likely the code is broken');
exitCode = 1;
throw Error(exitCode);
}
}
cd('..');
exec('keytool -genkey -v -keystore android/keystores/debug.keystore -storepass android -alias androiddebugkey -keypass android -dname "CN=Android Debug,O=Android,C=US"');

echo(`Starting packager server, ${SERVER_PID}`);
const appiumProcess = spawn('node', ['./node_modules/.bin/appium']);
APPIUM_PID = appiumProcess.pid;
echo(`Starting appium server, ${APPIUM_PID}`);
echo('Building app');
if (exec('buck build android/app').code) {
echo('could not execute Buck build, is it installed and in PATH?');
exit(cleanup(1));

if (argv['ios']) {
echo('Running an iOS app');
cd('ios');
// Make sure we installed local version of react-native
if (!test('-e', path.join('EndToEndTest', path.basename(MARKER_IOS)))) {
echo('iOS marker was not found, `react-native init` command failed?');
exitCode = 1;
throw Error(exitCode);
}
// shelljs exec('', {async: true}) does not emit stdout events, so we rely on good old spawn
let packagerEnv = Object.create(process.env);
packagerEnv.REACT_NATIVE_MAX_WORKERS = 1;
const packagerProcess = spawn('npm', ['start', '--', '--non-persistent'],
{
stdio: 'inherit',
env: packagerEnv
});
SERVER_PID = packagerProcess.pid;
exec('sleep 15s');
// prepare cache to reduce chances of possible red screen "Can't fibd variable __fbBatchedBridge..."
exec('response=$(curl --write-out %{http_code} --silent --output /dev/null localhost:8081/index.ios.bundle?platform=ios&dev=true)');
echo(`Starting packager server, ${SERVER_PID}`);
echo('Executing ios e2e test');
if (tryExecNTimes(
() => {
exec('sleep 10s');
return exec('xcodebuild -scheme EndToEndTest -sdk iphonesimulator test | xcpretty && exit ${PIPESTATUS[0]}').code;
},
numberOfRetries)) {
echo('Failed to run iOS e2e tests');
echo('Most likely the code is broken');
exitCode = 1;
throw Error(exitCode);
}
cd('..');
}
let packagerEnv = Object.create(process.env);
packagerEnv.REACT_NATIVE_MAX_WORKERS = 1;
// shelljs exec('', {async: true}) does not emit stdout events, so we rely on good old spawn
const packagerProcess = spawn('npm', ['start'], {
// stdio: 'inherit',
env: packagerEnv
});
SERVER_PID = packagerProcess.pid;
// wait a bit to allow packager to startup
exec('sleep 5s');
echo('Executing android e2e test');
if(exec('node node_modules/.bin/_mocha android-e2e-test.js').code) {
exit(cleanup(1));

if (argv['js']) {
// Check the packager produces a bundle (doesn't throw an error)
if (exec('react-native bundle --platform android --dev true --entry-file index.android.js --bundle-output android-bundle.js').code) {
echo('Could not build android package');
exitCode = 1;
throw Error(exitCode);
}
if (exec('react-native bundle --platform ios --dev true --entry-file index.ios.js --bundle-output ios-bundle.js').code) {
echo('Could not build ios package');
exitCode = 1;
throw Error(exitCode);
}
if (exec(`${ROOT}/node_modules/.bin/flow check`).code) {
echo('Flow check does not pass');
exitCode = 1;
throw Error(exitCode);
}
}
}
exitCode = 0;

} finally {
cd(ROOT);
rm(MARKER_IOS);
rm(MARKER_ANDROID);

if (args.indexOf('--ios') !== -1) {
echo('Running an iOS app');
cd('ios');
// Make sure we installed local version of react-native
if (!test('-e', path.join('EndToEndTest', path.basename(MARKER_IOS)))) {
echo('iOS marker was not found, `react-native init` command failed?');
exit(cleanup(1));
if (SERVER_PID) {
echo(`Killing packager ${SERVER_PID}`);
exec(`kill -9 ${SERVER_PID}`);
// this is quite drastic but packager starts a daemon that we can't kill by killing the parent process
// it will be fixed in April (quote David Aurelio), so until then we will kill the zombie by the port number
exec("lsof -i tcp:8081 | awk 'NR!=1 {print $2}' | xargs kill");
}
// shelljs exec('', {async: true}) does not emit stdout events, so we rely on good old spawn
let packagerEnv = Object.create(process.env);
packagerEnv.REACT_NATIVE_MAX_WORKERS = 1;
const packagerProcess = spawn('npm', ['start', '--', '--non-persistent'],
{
stdio: 'inherit',
env: packagerEnv
});
SERVER_PID = packagerProcess.pid;
exec('sleep 15s');
// prepare cache to reduce chances of possible red screen "Can't fibd variable __fbBatchedBridge..."
exec('response=$(curl --write-out %{http_code} --silent --output /dev/null localhost:8081/index.ios.bundle?platform=ios&dev=true)');
echo(`Starting packager server, ${SERVER_PID}`);
echo('Executing ios e2e test');
if (exec('xcodebuild -scheme EndToEndTest -sdk iphonesimulator test | xcpretty && exit ${PIPESTATUS[0]}').code) {
exit(cleanup(1));
if (APPIUM_PID) {
echo(`Killing appium ${APPIUM_PID}`);
exec(`kill -9 ${APPIUM_PID}`);
}
cd('..');
}

if (args.indexOf('--js') !== -1) {
// Check the packager produces a bundle (doesn't throw an error)
if (exec('react-native bundle --platform android --dev true --entry-file index.android.js --bundle-output android-bundle.js').code) {
echo('Could not build android package');
exit(cleanup(1));
}
if (exec('react-native bundle --platform ios --dev true --entry-file index.ios.js --bundle-output ios-bundle.js').code) {
echo('Could not build ios package');
exit(cleanup(1));
}
if (exec(`${ROOT}/node_modules/.bin/flow check`).code) {
echo('Flow check does not pass');
exit(cleanup(1));
}
}
exit(exitCode);

exit(cleanup(0));
/*eslint-enable no-undef */

0 comments on commit a8d2079

Please sign in to comment.