Functional tests report (Dash-Industry-Forum#3206)

* set tests output directory for circle-ci * functional tests: update config * [tmp] circleci test * [tmp] circleci test * [tmp] circleci test * circleci: set output functional tests results filename * circleci: set output functional tests results filename * circleci: set output functional tests results filename * circleci: set output functional tests results filename * circleci: set output functional tests results filename * circleci: add tests results as artifact * circleci: add tests results as artifact * [tmp] circleci test * [tmp] circleci test * functional tests: complete tests suites names for report * [tmp] circleci test * functional tests: run also tests for http streams * [tmp] circleci test * functional tests: run also tests for http streams * circleci: set output functional tests results filename * [tmp] circleci test * functional tests: update readme * circleci: set output functional tests results filename * [tmp] circleci test * circleci: run multiple tests sequences * [tmp] circleci test * [tmp] circleci test * [tmp] circleci test * Revert "[tmp] circleci test" This reverts commit bf22b03. * Revert "[tmp] circleci test" This reverts commit 1121d75. * Revert "[tmp] circleci test"
dsparacio · Mar 26, 2020 · d17a44c · d17a44c
1 parent 04499cb
commit d17a44c
Show file tree

Hide file tree

Showing 12 changed files with 60 additions and 31 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,9 +1,9 @@
 version: 2
 
 _defaults: &defaults
-    working_directory: ~/repo
-    docker:
-      - image: circleci/node:7.10
+  working_directory: ~/repo
+  docker:
+    - image: circleci/node:7.10
 
 _steps:
   restore_cache: &restore_cache
@@ -55,7 +55,6 @@ jobs:
       - restore_cache: *restore_cache
       - run: *install_dependencies
       - save_cache: *save_cache
-
       # run functional tests
       - run:
           # Download the browserstack binary file to create a tunnel
@@ -74,9 +73,20 @@ jobs:
           background: true
 
       - run:
-          # Run functional tests
-          name: Run functional tests
-          command: node node_modules/intern/runner.js config=test/functional/config.js selenium=remote app=remote browsers=chrome
+          # Run functional tests for chrome browser for https
+          name: Run functional tests (chrome / https)
+          command:
+            node node_modules/intern/runner.js config=test/functional/config.js selenium=remote app=remote browsers=chrome protocol=https
+
+      - run:
+          # Run functional tests for chrome browser for http
+          name: Run functional tests (chrome / http)
+          when: always # run tests even if some previous tests failed
+          command:
+            node node_modules/intern/runner.js config=test/functional/config.js selenium=remote app=remote browsers=chrome protocol=http
+
+      - store_test_results:
+          path: test/functional/reports
 
 workflows:
   version: 2

diff --git a/test/functional/config.js b/test/functional/config.js
@@ -21,7 +21,7 @@ define(function(require) {
         defaultTimeout: 30000,
 
         // A regular expression matching URLs to files that should not be included in code coverage analysis
-        excludeInstrumentation: /^tests|bower_components|node_modules|testIntern/,
+        excludeInstrumentation: /^test|node_modules/,
 
         // to keep browser opened at the end of the test
         leaveRemoteOpen: false
@@ -79,7 +79,9 @@ define(function(require) {
     }
 
     // Set application protocol
-    conf.testPage = conf.protocol + '://' + conf.testPage
+    if (!conf.testPage.startsWith('http')) {
+        conf.testPage = conf.protocol + '://' + conf.testPage
+    };
 
     console.log('conf.testPage: ' + conf.testPage);
 
@@ -99,4 +101,4 @@ define(function(require) {
     // console.log(JSON.stringify(conf, null, '  '));
 
     return conf;
-});
+});
diff --git a/test/functional/config/applications.js b/test/functional/config/applications.js
@@ -1,4 +1,4 @@
 define({
-    local:'localhost:3000/samples/functional-tests/index.html',
+    local:'http://localhost:3000/samples/functional-tests/index.html',
     remote:'reference.dashif.org/dash.js/nightly/samples/functional-tests/index.html',
 });
diff --git a/test/functional/config/selenium.js b/test/functional/config/selenium.js
@@ -9,7 +9,13 @@ define({
             port: '4444',
             verbose: true
         },
-        reporters: ['Runner'],
+        reporters: [
+            'Runner',
+            {
+                id: 'JUnit',
+                filename: 'test/functional/reports/junit/results_' + (new Date().getFullYear())+'-'+(new Date().getMonth()+1)+'-'+(new Date().getDate())+'_'+(new Date().getHours())+'-'+(new Date().getMinutes()) + '.xml'
+            }
+        ],
         capabilities: {
             'selenium-version': '3.4.0'
         },
@@ -31,6 +37,12 @@ define({
             username: process.env.BROWSERSTACK_USER || 'BROWSERSTACK_USER',
             accessKey: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY'
         },
-        reporters: ['Runner', {id: 'JUnit', filename: 'test/functional/test-reports/' + (new Date().getFullYear())+'-'+(new Date().getMonth()+1)+'-'+(new Date().getDate())+'_'+(new Date().getHours())+'-'+(new Date().getMinutes())+'-'+(new Date().getSeconds()) + '_report.xml'}]
+        reporters: [
+            'Runner',
+            {
+                id: 'JUnit',
+                filename: 'test/functional/reports/junit/results_' + (new Date().getFullYear())+'-'+(new Date().getMonth()+1)+'-'+(new Date().getDate())+'_'+(new Date().getHours())+'-'+(new Date().getMinutes()) + '.xml'
+            }
+        ]
     }
-});
+});
diff --git a/test/functional/readme.md b/test/functional/readme.md
@@ -122,11 +122,11 @@ cd test/functional
 
 As for Windows, the script can be modified to setup the tests.
 
-####Troubleshooting
+#### Troubleshooting
 Question: The localhost:3000 webserver is not available and all tests fail.  
 Answer: A local webserver needs to run on port 3000. In order to launch a debug server launch the following command in the root folder of the project:
 ```
-npm dev
+npm run dev
 ```
 
 Question: The webserver responds with "cannot establish a secure connection"  

diff --git a/test/functional/tests/ended.js b/test/functional/tests/ended.js
@@ -27,7 +27,7 @@ define([
 
     var load = function(stream) {
         registerSuite({
-            name: NAME,
+            name: utils.testName(NAME, stream),
 
             load: function() {
                 if (!stream.available) this.skip();
@@ -49,7 +49,7 @@ define([
 
     var seek = function(stream) {
         registerSuite({
-            name: NAME,
+            name: utils.testName(NAME, stream),
 
             seek: function() {
                 if (!stream.available) this.skip();
@@ -66,7 +66,7 @@ define([
 
     var ended = function(stream) {
         registerSuite({
-            name: NAME,
+            name: utils.testName(NAME, stream),
 
             ended: function() {
                 if (!stream.available) this.skip();

diff --git a/test/functional/tests/pause.js b/test/functional/tests/pause.js
@@ -34,7 +34,7 @@ define([
 
     var load = function(stream) {
         registerSuite({
-            name: NAME,
+            name: utils.testName(NAME, stream),
 
             load: function() {
                 if (!stream.available) this.skip();
@@ -55,7 +55,7 @@ define([
     var pause = function(stream) {
 
         registerSuite({
-            name: NAME,
+            name: utils.testName(NAME, stream),
 
             pause: function() {
                 if (!stream.available || stream.duration < 60) this.skip();

diff --git a/test/functional/tests/play.js b/test/functional/tests/play.js
@@ -27,7 +27,7 @@ define([
 
     var load = function(stream) {
         registerSuite({
-            name: NAME,
+            name: utils.testName(NAME, stream),
 
             load: function() {
                 if (!stream.available) this.skip();
@@ -40,7 +40,7 @@ define([
 
     var play = function(stream) {
         registerSuite({
-            name: NAME,
+            name: utils.testName(NAME, stream),
 
             play: function() {
                 if (!stream.available) this.skip();
@@ -66,7 +66,7 @@ define([
 
     var getInfos = function(stream) {
         registerSuite({
-            name: NAME,
+            name: utils.testName(NAME, stream),
 
             isDynamic: function() {
                 if (!stream.available) this.skip();

diff --git a/test/functional/tests/playFromTime.js b/test/functional/tests/playFromTime.js
@@ -30,7 +30,7 @@ define([
 
     var loadAtRValue = function(stream) {
         registerSuite({
-            name: NAME,
+            name: utils.testName(NAME, stream),
 
             load: function() {
                 if (!stream.available) this.skip();
@@ -53,7 +53,7 @@ define([
 
     var play = function(stream) {
         registerSuite({
-            name: NAME,
+            name: utils.testName(NAME, stream),
 
             play: function() {
                 if (!stream.available) this.skip();

diff --git a/test/functional/tests/scripts/utils.js b/test/functional/tests/scripts/utils.js
@@ -3,9 +3,14 @@ define([
     ], function (intern) {
 
     var defaultTimeout = intern.config.defaultTimeout;
-    
+
     return {
 
+        testName: function (name, stream) {
+            return stream.name + ' # ' + name;
+        },
+
+
         info: function (tag, message) {
             console.info('[' + tag + ']', message);
         },

diff --git a/test/functional/tests/seek.js b/test/functional/tests/seek.js
@@ -46,10 +46,10 @@ define([
         }
         return pos;
     };
-    
+
     var load = function(stream) {
         registerSuite({
-            name: NAME,
+            name: utils.testName(NAME, stream),
 
             load: function() {
                 if (!stream.available) this.skip();
@@ -71,7 +71,7 @@ define([
 
     var seek = function(stream) {
         registerSuite({
-            name: NAME,
+            name: utils.testName(NAME, stream),
 
             seek: function() {
                 if (!stream.available) this.skip();

diff --git a/test/functional/tests/setup.js b/test/functional/tests/setup.js
@@ -15,7 +15,7 @@ define([
 
     var setup = function (stream) {
         registerSuite({
-            name: NAME,
+            name: utils.testName(NAME, stream),
 
             setup: function () {
                 utils.info(NAME, 'Setup stream: ' + stream.name);