Skip to content

Commit

Permalink
Merge pull request appium#2667 from sebv/master
Browse files Browse the repository at this point in the history
Boosting ci
  • Loading branch information
sebv committed May 25, 2014
2 parents 8371ff0 + 112073d commit 1e9b7b0
Show file tree
Hide file tree
Showing 37 changed files with 814 additions and 750 deletions.
6 changes: 5 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ env:
- SAUCE_USERNAME=appium
- secure: GXGFbQptLEsKlaWnpOL2EB9hqQ4XBT6tqg7REt5j1h1jYBsfHQHSgSLt1jpzOF5kooXVbw2K7IrZyNVBXEUHR00Hp6O72FBOd+SSj8uPCJBLjAOaJWs0ZtujqGHwHpe6gadFfn0L7q0ffoV5OJVeNHHcuw4f9BaA78fTFNfm1jE=
- secure: WLRP9RJIHWPer189swCqMCnexS8XpVQ8FRMSXNxLXp5fUaERVzU7/ncGFeqcM8GWGIU12PP8TJUaMRa58vTGgApW6POMdAPiauepfzoru+n/S4Xrw64hmd4oK0GEZI3wgnkPofUn6TzQeXRmVG496PNzbV8KfqF4mTAesZOLSFA=
- HTTP_RETRIES=5
- HTTP_RETRY_DELAY=5000
- DEBUG_CONNECTION=1
- MOCHA_INIT_TIMEOUT=600000
- LAUNCH_TIMEOUT='{"global":90000,"afterSimLaunch":30000}'
matrix:
- CI_CONFIG=unit
- CI_CONFIG=build_ios
Expand All @@ -25,7 +30,6 @@ env:
- CI_CONFIG=build_gappium
before_script:
- gem install --no-rdoc --no-ri appium_doc_lint
- "./ci/git-config.sh"
- "./ci/installers/install-node.sh"
- "echo Node.js version: `node -v`"
- npm install -g jshint grunt-cli mocha
Expand Down
9 changes: 0 additions & 9 deletions ci/git-config.sh

This file was deleted.

17 changes: 0 additions & 17 deletions ci/git-push.sh

This file was deleted.

57 changes: 19 additions & 38 deletions ci/script.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
#!/bin/bash
set -e

BZ2_FILE=appium-ci-${TRAVIS_BRANCH}-${TRAVIS_JOB_NUMBER}-${TRAVIS_COMMIT:0:10}.tar.bz2

if [[ $CI_CONFIG == 'unit' ]]; then
cd docs
appium_doc_lint || exit 1
Expand All @@ -13,11 +16,10 @@ elif [[ $CI_CONFIG == 'build_ios' ]]; then
./reset.sh --hardcore --no-npmlink --dev --ios
if [[ $TRAVIS_SECURE_ENV_VARS == true ]]; then
./ci/upload_build_to_sauce.sh
GLOB_PATTERNS='test/functional/common/**/*-specs.js'
GLOB_PATTERNS+=',test/functional/ios/**/*-specs.js'
node ci/tools/testfiles-tool.js split "${GLOB_PATTERNS}" > ci/test-split.json
cp ci/mochas/ios71-mocha ci/mocha
BRANCH_CAT=ios ./ci/git-push.sh
TARBALL=sauce-storage:$BZ2_FILE \
node ./ci/tools/parallel-mocha.js \
-p 30 \
-c ios
fi
elif [[ $CI_CONFIG == 'build_android' ]]; then
source ./ci/android_env
Expand All @@ -27,11 +29,10 @@ elif [[ $CI_CONFIG == 'build_android' ]]; then
rm sample-code/apps/ApiDemos
mv submodules/ApiDemos sample-code/apps/
./ci/upload_build_to_sauce.sh
GLOB_PATTERNS='test/functional/common/**/*-specs.js'
GLOB_PATTERNS+=',test/functional/android/**/*-specs.js'
node ci/tools/testfiles-tool.js split "${GLOB_PATTERNS}" > ci/test-split.json
cp ci/mochas/android-mocha ci/mocha
BRANCH_CAT=android ./ci/git-push.sh
TARBALL=sauce-storage:$BZ2_FILE \
node ./ci/tools/parallel-mocha.js \
-p 30 \
-c android
fi
elif [[ $CI_CONFIG == 'build_selendroid' ]]; then
source ./ci/android_env
Expand All @@ -44,10 +45,10 @@ elif [[ $CI_CONFIG == 'build_selendroid' ]]; then
mv submodules/selendroid/selendroid-test-app/target/selendroid-test-app-0.10.0.apk \
sample-code/apps/selendroid-test-app.apk
./ci/upload_build_to_sauce.sh
GLOB_PATTERNS='test/functional/selendroid/**/*-specs.js'
node ci/tools/testfiles-tool.js split "${GLOB_PATTERNS}" > ci/test-split.json
cp ci/mochas/selendroid-mocha ci/mocha
BRANCH_CAT=selendroid ./ci/git-push.sh
TARBALL=sauce-storage:$BZ2_FILE \
node ./ci/tools/parallel-mocha.js \
-p 30 \
-c selendroid
fi
elif [[ $CI_CONFIG == 'build_gappium' ]]; then
source ./ci/android_env
Expand All @@ -61,29 +62,9 @@ elif [[ $CI_CONFIG == 'build_gappium' ]]; then
rm sample-code/apps/io.appium.gappium.sampleapp
mv submodules/io.appium.gappium.sampleapp sample-code/apps/
./ci/upload_build_to_sauce.sh
sed -i.bak s/CI_CONFIG=functional/CI_CONFIG=run_gappium/g ci/travis-functional.yml
rm ci/*.bak
BRANCH_CAT=gappium ./ci/git-push.sh
fi
elif [[ $CI_CONFIG == 'functional' ]]; then
TARBALL=sauce-storage:$(node ./ci/tools/build-upload-tool.js \
./ci/build-upload-info.json filename)
echo node ci/tools/testfiles-tool.js list ci/test-split.json "${TEST_GROUP}"
TEST_FILES=$(node ci/tools/testfiles-tool.js list ci/test-split.json "${TEST_GROUP}")
echo "TEST_FILES --> ${TEST_FILES}"
if [[ -n "$TEST_FILES" ]]; then
TARBALL=$TARBALL ci/mocha ${TEST_FILES}
fi
elif [[ $CI_CONFIG == 'run_gappium' ]]; then
TARBALL=sauce-storage:$(node ./ci/tools/build-upload-tool.js \
./ci/build-upload-info.json filename)
if [[ $TEST_GROUP == 'group 1' ]]; then
# TODO: investigate WEBVIEW context issue
echo 'skipping android gappium test'
# TARBALL=$TARBALL ci/mochas/android-mocha test/functional/gappium/**/*-specs.js
elif [[ $TEST_GROUP == 'group 2' ]]; then
TARBALL=$TARBALL ci/mochas/ios71-mocha test/functional/gappium/**/*-specs.js
elif [[ $TEST_GROUP == 'group 3' ]]; then
TARBALL=$TARBALL ci/mochas/selendroid-mocha test/functional/gappium/**/*-specs.js
TARBALL=sauce-storage:$BZ2_FILE \
node ./ci/tools/parallel-mocha.js \
-p 30 \
-c gappium
fi
fi
43 changes: 43 additions & 0 deletions ci/test-config.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
module.exports = {
'ios':{
'mocha-bin': 'ci/mochas/ios71-mocha',
'glob-patterns': [
'test/functional/common/**/*-specs.js',
'test/functional/ios/**/*-specs.js'
]
},
'android':{
'mocha-bin': 'ci/mochas/android-mocha',
'glob-patterns': [
'test/functional/common/**/*-specs.js',
'test/functional/android/**/*-specs.js'
]
},
'selendroid':{
'mocha-bin': 'ci/mochas/selendroid-mocha',
'glob-patterns': [
'test/functional/selendroid/**/*-specs.js'
]
},
'gappium':[
{
'mocha-bin': 'ci/mochas/ios71-mocha',
'glob-patterns': [
'test/functional/gappium/**/*-specs.js'
]
},
// TODO: gappium/android hangs on sauce.
// {
// 'mocha-bin': 'ci/mochas/android-mocha',
// 'glob-patterns': [
// 'test/functional/gappium/**/*-specs.js'
// ]
// },
{
'mocha-bin': 'ci/mochas/selendroid-mocha',
'glob-patterns': [
'test/functional/gappium/**/*-specs.js'
]
}
]
};
126 changes: 126 additions & 0 deletions ci/tools/parallel-mocha.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
'use strict';

var args = process.argv.slice(2),
_ = require('underscore'),
glob = require('glob'),
async = require('async'),
Q = require("q"),
spawn = require('child_process').spawn,
argparse = require('argparse');


var parser = new argparse.ArgumentParser({
description: 'Parallel Mocha'
});
parser.addArgument(
[ '-p' ],
{
help: 'number of processes',
type: 'int',
required: true,
dest: 'numOfWorkers'
}
);
parser.addArgument(
[ '-c' ],
{
help: 'test config',
required: true,
dest: 'config'
}
);
var args = parser.parseArgs();

var config = require('../test-config');

var fileInfos = config[args.config];
if (!(fileInfos instanceof Array)) {
fileInfos = [fileInfos];
}

var allGood = true;

function runOne(mochaBin, filepath) {
var deferred = Q.defer();
var out = '';
var child = spawn(mochaBin, [filepath]);
child.on('error', function (err) {
deferred.reject(err);
});

child.stdout.setEncoding('utf8');
child.stdout.on('data', function (data) {
out += data;
});
child.stderr.setEncoding('utf8');
child.stderr.on('data', function (data) {
out += data;
});
child.on('close', function (code) {
if (code !== 0) {
var err = new Error(
"Mocha process terminated with code: " + code + '.');
err.out = out;
return deferred.reject(err);
}
deferred.resolve(out);
});
return deferred.promise;
}

var queue = async.queue(function (task, done) {
var startMs = Date.now();
console.log('running ' + task.filepath);

var interval = setInterval(function () {
console.log(task.filepath, 'has been running for',
Math.round((Date.now() - startMs)/1000), 'seconds.');
}, 300000);

runOne(task.mochaBin, task.filepath)
.then(function (out) {
console.log('finished to run', task.filepath,'\n', out);
})
.catch(function (err) {
allGood = false;
console.error(err + (err.out ? '\n' + err.out : ''));
}).fin(function () { clearInterval(interval); })
.nodeify(done);
}, args.numOfWorkers);

queue.drain = function () {
if (allGood) {
console.log('\n\nALL GOOD\n\n');
process.exit(0);
}
else {
throw new Error('FAILING TESTS!');
}
};

function fillQueue(fileInfo) {
var deferred = Q.defer();
async.eachSeries(
fileInfo['glob-patterns'],
function (globPattern, done) {
glob(globPattern, function (err, _files) {
if (err) return done(err);
_(_files).map(function (file) {
queue.push({mochaBin: fileInfo['mocha-bin'], filepath: file});
});
done();
});
},
function (err) {
if (err) {
return deferred.reject(err);
}
deferred.resolve();
}
);
return deferred.promise;
}

Q.all(_(fileInfos).map(function (fileInfo) {
return fillQueue(fileInfo);
})).done();
61 changes: 0 additions & 61 deletions ci/tools/testfiles-tool.js

This file was deleted.

Loading

0 comments on commit 1e9b7b0

Please sign in to comment.