add retries option and change travis systest flag to fvtest (hyperled…

…ger-archives#2764)

Signed-off-by: Nick Lincoln <[email protected]>

.travis.yml

@@ -7,10 +7,10 @@ matrix:
         - env: DOCS=full FC_TASK=docs
         - env: SYSTEST=embedded,proxy,web FC_TASK=systest
 #        - env: SYSTEST=hlfv1_tls FC_TASK=systest
-        - env: SYSTEST=hlfv1-1_tls FC_TASK=systest
-        - env: SYSTEST=hlfv1-2_tls FC_TASK=systest
+        - env: FVTEST=hlfv1-1_tls FC_TASK=systest
+        - env: FVTEST=hlfv1-2_tls FC_TASK=systest
         - env: INTEST=hlfv1 FC_TASK=systest
-        - env: SYSTEST=e2e FC_TASK=systest
+        - env: INTEST=e2e FC_TASK=systest
 dist: trusty
 addons:
     apt:

.travis/before-install.sh

@@ -31,14 +31,14 @@ npm install -g lerna@2 @alrra/travis-scripts asciify gnomon
 echo "ABORT_BUILD=false" > ${DIR}/build.cfg
 echo "ABORT_CODE=0" >> ${DIR}/build.cfg
 
-# Abort the systest/integration if this is a merge build
+# Abort the fv/integration if this is a merge build
 # Check for the FC_TASK that is set in travis.yml, also the pull request is false => merge build
 # and that the TRAVIS_TAG is empty meaning this is not a release build
 if [ "${FC_TASK}" = "systest" ] && [ "${TRAVIS_PULL_REQUEST}" = "false" ] && [ -z "${TRAVIS_TAG}" ]; then
   if [[ "${TRAVIS_REPO_SLUG}" = hyperledger* ]]; then
     echo "ABORT_BUILD=true" > ${DIR}/build.cfg
     echo "ABORT_CODE=0" >> ${DIR}/build.cfg
-    echo Merge build from non release PR: ergo not running systest
+    echo Merge build from non release PR: ergo not running fv/integration tests
     exit 0
   fi
 fi

.travis/deploy.sh

@@ -22,9 +22,9 @@ if [ "${TRAVIS_NODE_VERSION}" != "" -a "${TRAVIS_NODE_VERSION}" != "8" ]; then
     exit 0
 fi
 
-# Check that this is not the system tests.
-if [ "${SYSTEST}" != "" ]; then
-    echo Not executing as running system tests.
+# Check that this is not the functional verification tests.
+if [ "${FVTEST}" != "" ]; then
+    echo Not executing as running fv tests.
     exit 0
 fi
 

.travis/script.sh

@@ -45,25 +45,27 @@ if [ "${DOCS}" != "" ]; then
        npm run linkcheck:unstable
     fi
 
+# Are we running functional verification tests?
+elif [ "${FVTEST}" != "" ]; then
+
+    # Run the fv tests.
+    ${DIR}/packages/composer-tests-functional/scripts/run-system-tests.sh 
+    # append to the previous line to get duration timestamps....  | gnomon --real-time=false 
+
 # Are we running playground e2e tests?
-elif [ "${SYSTEST}" = "e2e" ]; then
+elif [ "${INTEST}" = "e2e" ]; then
 
     # Run the playground e2e tests.
     cd "${DIR}/packages/composer-playground"
     npm run e2e:main
 
-# Are we running system tests?
-elif [ "${SYSTEST}" != "" ]; then
-
-    # Run the system tests.
-    ${DIR}/packages/composer-tests-functional/scripts/run-system-tests.sh 
-    # append to the previous line to get duration timestamps....  | gnomon --real-time=false 
 # Are we running integration tests?
 elif [ "${INTEST}" != "" ]; then
 
     # Run the integration tests.
     ${DIR}/packages/composer-tests-integration/scripts/run-integration-tests.sh 
-    # append to the previous line to get duration timestamps....  | gnomon --real-time=false 
+    # append to the previous line to get duration timestamps....  | gnomon --real-time=false
+
 # We must be running unit tests.
 else
 

packages/composer-tests-functional/scripts/run-system-tests.sh

@@ -14,17 +14,17 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
 cd "${DIR}"
 
 # Barf if we don't recognize this test suite.
-if [ "${SYSTEST}" = "" ]; then
-    echo You must set SYSTEST to 'embedded', 'hlfv1', 'proxy', or 'web', or a comma
+if [ "${FVTEST}" = "" ]; then
+    echo You must set FVTEST to 'embedded', 'hlfv1', 'proxy', or 'web', or a comma
     echo separated list of a set of system test configurations to run.
     echo For example:
-    echo  export SYSTEST=hlfv1
-    echo  export SYSTEST=embedded,proxy,web
+    echo  export FVTEST=hlfv1
+    echo  export FVTEST=embedded,proxy,web
     exit 1
 fi
 
 # Run for all specified configurations.
-for SYSTEST in $(echo ${SYSTEST} | tr "," " "); do
+for FVTEST in $(echo ${FVTEST} | tr "," " "); do
 
     # Set default timeouts
     export COMPOSER_PORT_WAIT_SECS=30
@@ -36,8 +36,8 @@ for SYSTEST in $(echo ${SYSTEST} | tr "," " "); do
     rm -rf ${HOME}/.composer-credentials/composer-systests*
 
     # Pull any required Docker images.
-    if [[ ${SYSTEST} == hlfv1* ]]; then
-        if [[ ${SYSTEST} == *tls ]]; then
+    if [[ ${FVTEST} == hlfv1* ]]; then
+        if [[ ${FVTEST} == *tls ]]; then
             DOCKER_FILE=${DIR}/hlfv1/docker-compose.tls.yml
         else
             DOCKER_FILE=${DIR}/hlfv1/docker-compose.yml
@@ -68,9 +68,9 @@ for SYSTEST in $(echo ${SYSTEST} | tr "," " "); do
     fi
 
     # configure v1 to run the tests
-    if [[ ${SYSTEST} == hlfv1* ]]; then
+    if [[ ${FVTEST} == hlfv1* ]]; then
         sleep 10
-        if [[ ${SYSTEST} == *tls ]]; then
+        if [[ ${FVTEST} == *tls ]]; then
             # Create the channel
             docker exec -e "CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/msp/users/[email protected]/msp" peer0.org1.example.com peer channel create -o orderer.example.com:7050 -c composerchannel -f /etc/hyperledger/configtx/composer-channel.tx --tls true --cafile /etc/hyperledger/orderer/tls/ca.crt
             # Join peer0 to the channel.
@@ -92,7 +92,7 @@ for SYSTEST in $(echo ${SYSTEST} | tr "," " "); do
     fi
 
     # Run the system tests.
-    npm run systest:${SYSTEST} 2>&1 | tee
+    npm run systest:${FVTEST} 2>&1 | tee
 
     # Kill and remove any started Docker images.
     if [ "${DOCKER_FILE}" != "" ]; then

packages/composer-tests-functional/systest/accesscontrols.js

@@ -29,15 +29,13 @@ chai.use(require('chai-as-promised'));
 
 process.setMaxListeners(Infinity);
 
+describe('Access control system tests', function() {
 
+    this.retries(TestUtil.retries());
 
-
-
-
-describe('Access control system tests', () => {
     let bnID;
     beforeEach(() => {
-        return TestUtil.resetBusinessNetwork(bnID);
+        return TestUtil.resetBusinessNetwork(bnID, 0);
     });
 
     let businessNetworkDefinition;

packages/composer-tests-functional/systest/assets.js

@@ -26,10 +26,13 @@ chai.should();
 chai.use(require('chai-as-promised'));
 chai.use(require('chai-subset'));
 
-describe('Asset system tests', function () {
+describe('Asset system tests', function() {
+
+    this.retries(TestUtil.retries());
+
     let bnID;
     beforeEach(() => {
-        return TestUtil.resetBusinessNetwork(bnID);
+        return TestUtil.resetBusinessNetwork(bnID, 0);
     });
 
     let businessNetworkDefinition;

packages/composer-tests-functional/systest/events.js

@@ -26,10 +26,13 @@ chai.should();
 chai.use(require('chai-as-promised'));
 chai.use(require('chai-subset'));
 
-describe('Event system tests', function () {
+describe('Event system tests', function() {
+
+    this.retries(TestUtil.retries());
+
     let bnID;
     beforeEach(() => {
-        return TestUtil.resetBusinessNetwork(bnID);
+        return TestUtil.resetBusinessNetwork(bnID, 0);
     });
     let businessNetworkDefinition;
     let client;

packages/composer-tests-functional/systest/historian.js

@@ -135,12 +135,12 @@ let deployCommon =  ()=> {
 };
 
 
-describe('Historian', () => {
-
+describe('Historian', function() {
 
+    this.retries(TestUtil.retries());
 
     beforeEach(() => {
-        return TestUtil.resetBusinessNetwork(bnID);
+        return TestUtil.resetBusinessNetwork(bnID, 0);
     });
 
     describe('CRUD Asset', () => {

packages/composer-tests-functional/systest/identities.js

@@ -30,10 +30,13 @@ chai.use(require('chai-as-promised'));
 
 process.setMaxListeners(Infinity);
 
-describe('Identity system tests', () => {
+describe('Identity system tests', function() {
+
+    this.retries(TestUtil.retries());
+
     let bnID;
     beforeEach(() => {
-        return TestUtil.resetBusinessNetwork(bnID);
+        return TestUtil.resetBusinessNetwork(bnID, 0);
     });
     let businessNetworkDefinition;
     let client;

packages/composer-tests-functional/systest/participants.js

@@ -26,10 +26,13 @@ chai.should();
 chai.use(require('chai-subset'));
 chai.use(require('chai-as-promised'));
 
-describe('Participant system tests', function () {
+describe('Participant system tests', function() {
+
+    this.retries(TestUtil.retries());
+
     let bnID;
     beforeEach(() => {
-        return TestUtil.resetBusinessNetwork(bnID);
+        return TestUtil.resetBusinessNetwork(bnID, 0);
     });
     let businessNetworkDefinition;
     let client;

packages/composer-tests-functional/systest/post.js

@@ -24,10 +24,13 @@ const chai = require('chai');
 chai.should();
 chai.use(require('chai-as-promised'));
 
-describe('HTTP POST system tests', () => {
+describe('HTTP POST system tests', function() {
+
+    this.retries(TestUtil.retries());
+
     let bnID;
     beforeEach(() => {
-        return TestUtil.resetBusinessNetwork(bnID);
+        return TestUtil.resetBusinessNetwork(bnID, 0);
     });
     let businessNetworkDefinition;
     let client;

packages/composer-tests-functional/systest/queries.js

@@ -25,7 +25,9 @@ const chai = require('chai');
 chai.should();
 chai.use(require('chai-as-promised'));
 
-describe('Query system tests', () => {
+describe('Query system tests', function() {
+
+    this.retries(TestUtil.retries());
 
     let businessNetworkDefinition;
     let client;

packages/composer-tests-functional/systest/testutil.js

@@ -23,12 +23,11 @@ const mkdirp = require('mkdirp');
 const net = require('net');
 const path = require('path');
 const sleep = require('sleep-promise');
-// const Util = require('composer-common').Util;
-
 
 let client;
 let docker = new Docker();
 let forceDeploy = false;
+let testRetries = 4;
 
 /**
  * Trick browserify by making the ID parameter to require dynamic.
@@ -77,15 +76,15 @@ class TestUtil {
      * @return {boolean} True if running in Hyperledger Fabric mode, false if not.
      */
     static isHyperledgerFabric() {
-        return process.env.SYSTEST && process.env.SYSTEST.match('^hlf.*');
+        return process.env.FVTEST && process.env.FVTEST.match('^hlf.*');
     }
 
     /**
      * Check to see if running in Hyperledger Fabric mode.
      * @return {boolean} True if running in Hyperledger Fabric mode, false if not.
      */
     static isHyperledgerFabricV1() {
-        return process.env.SYSTEST && process.env.SYSTEST.match('^hlfv1.*');
+        return process.env.FVTEST && process.env.FVTEST.match('^hlfv1.*');
     }
 
     /**
@@ -138,7 +137,7 @@ class TestUtil {
             return Promise.resolve();
         }
         // startsWith not available in browser test environment
-        if (process.env.SYSTEST.match('^hlfv1')) {
+        if (process.env.FVTEST.match('^hlfv1')) {
             return Promise.resolve();
         }
         return TestUtil.waitForPort('localhost', 7050)
@@ -227,7 +226,7 @@ class TestUtil {
                     const keyValStoreOrg2 = path.resolve(homedir(), '.composer-credentials', 'composer-systests-org2');
                     mkdirp.sync(keyValStoreOrg2);
                     let connectionProfileOrg1, connectionProfileOrg2;
-                    if (process.env.SYSTEST.match('tls$')) {
+                    if (process.env.FVTEST.match('tls$')) {
                         console.log('setting up TLS Connection Profile for HLF V1');
                         connectionProfileOrg1 = {
                             type: 'hlfv1',
@@ -597,30 +596,45 @@ class TestUtil {
     /**
      * Reset the business network to its initial state.
      * @param {String} identifier, business network identifier to reset
+     * @param {int} retryCount, current retry number
      * @return {Promise} - a promise that will be resolved when complete.
      */
-    static resetBusinessNetwork(identifier) {
+    static resetBusinessNetwork(identifier, retryCount) {
         if (!client) {
             return Promise.resolve();
         }
 
         if (TestUtil.isHyperledgerFabricV1() && !forceDeploy){
             const adminConnection = new AdminConnection();
-            return adminConnection.connectWithDetails('composer-systests-org1', 'admin', 'NOTNEEDED',identifier)
+            return adminConnection.connectWithDetails('composer-systests-org1', 'admin', 'NOTNEEDED', identifier)
             .then(() => {
                 return adminConnection.reset(identifier);
             })
             .then(() => {
                 return adminConnection.disconnect();
+            })
+            .catch((err) => {
+                if (retryCount >= this.retries) {
+                    throw(err);
+                } else {
+                    this.resetBusinessNetwork(identifier, retryCount++);
+                }
             });
         } else if(TestUtil.isHyperledgerFabricV1() && forceDeploy){
             const adminConnection = new AdminConnection();
-            return adminConnection.connectWithDetails('composer-systests-org1-solo', 'admin', 'NOTNEEDED',identifier)
+            return adminConnection.connectWithDetails('composer-systests-org1-solo', 'admin', 'NOTNEEDED', identifier)
             .then(() => {
                 return adminConnection.reset(identifier);
             })
             .then(() => {
                 return adminConnection.disconnect();
+            })
+            .catch((err) => {
+                if (retryCount >= this.retries) {
+                    throw(err);
+                } else {
+                    this.resetBusinessNetwork(identifier, retryCount++);
+                }
             });
         } else {
 
@@ -637,9 +651,12 @@ class TestUtil {
     }
 
 
-    /** Deploy the common systest business network
-     *  @return {Promise} - a promise that will be resolved when complete.
+    /** Return an integer for use as a number of retries
+     *  @return {int} - an integer
      */
+    static retries() {
+        return testRetries;
+    }
 
 }
 

packages/composer-tests-functional/systest/transactions.assets.js

@@ -26,10 +26,13 @@ chai.should();
 chai.use(require('chai-as-promised'));
 
 
-describe('Transaction (asset specific) system tests', () => {
+describe('Transaction (asset specific) system tests', function() {
+
+    this.retries(TestUtil.retries());
+
     let bnID;
     beforeEach(() => {
-        return TestUtil.resetBusinessNetwork(bnID);
+        return TestUtil.resetBusinessNetwork(bnID, 0);
     });
     let businessNetworkDefinition;
     let client;