Unified Test Runner

Summary: this adds a single command that can be used to run all tests and receive a single result, both in the VM and in TravisCI

Test Plan:
run `bin/tests` to install the prereqs and run all the tests

run `bin/tests help` to see the command usage

run `bin/tests install` to install the prereqs then `bin/tests run --no-install` to run without installing (which is what TravisCI will do using `before_script` & `script`)

try running from any current working directory... the script should work from anywhere

I tried using https://github.com/jolicode/JoliCi to test the travis script, but couldn't get it to run properly. Both travis commands work when tested locally, and the exit codes are correct, so the travis build should work but will need to be tested live.

Reviewers: rjmackay, vladimir, shadowhand

Reviewed By: shadowhand

Differential Revision: https://phabricator.ushahidi.com/D459

.gitignore

@@ -5,6 +5,7 @@
 composer.phar
 vendor/*
 bin/*
+!bin/tests
 .vagrant/*
 .floo
 .flooignore

.travis.yml

@@ -8,22 +8,9 @@ php:
 - '5.5'
 services:
 - mysql
-env:
-- KOHANA_ENV=testing
 install:
 - composer install --no-interaction
 before_script:
-- chmod 0777 application/cache application/logs
-- cp application/tests/behat.template application/tests/behat.yml
-- mysql -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES'";
-- mysql -e 'create database platform_test;'
-- bin/phinx migrate -c application/phinx.php
-- mysql -e 'create database zombie2x;'
-- curl https://72c9192a7b87de5fc63a-f9fe2e6be12470a7bff22b7693bc7329.ssl.cf1.rackcdn.com/lamu-ci-zombie2x.sql
-  | mysql zombie2x
-- php -S localhost:8000 httpdocs/index.php &
-- sleep 3
+- ./bin/tests install
 script:
-- ./bin/phpspec run --format dot
-- ./bin/behat --config application/tests/behat.yml --format progress
-- ./bin/phpunit -c application/tests/phpunit.xml
+- ./bin/tests run --no-install

bin/tests

@@ -0,0 +1,275 @@
+#!/usr/bin/env php
+<?php
+
+require __DIR__ . '/../vendor/autoload.php';
+
+use Symfony\Component\Console\Application;
+use Symfony\Component\Console\Command\Command;
+
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Formatter\OutputFormatterStyle;
+use Symfony\Component\Console\Helper\ProgressBar;
+
+abstract class UshahidiTestsCommand extends Command
+{
+    protected $dir;
+    protected $input;
+    protected $output;
+
+    private $test_server_pid;
+    private $test_database_sql_url = 'https://72c9192a7b87de5fc63a-f9fe2e6be12470a7bff22b7693bc7329.ssl.cf1.rackcdn.com/lamu-ci-zombie2x.sql';
+
+    private $env_variables = [
+        'KOHANA_ENV' => 'testing',
+    ];
+
+    public function __construct()
+    {
+        parent::__construct();
+        $this->dir = realpath(__DIR__ . '/../');
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $this->input = $input;
+        $this->output = $output;
+        $this->addOutputFormatting();
+    }
+
+    protected function install()
+    {
+        // change permissions
+        foreach ([
+            "{$this->dir}/application/cache",
+            "{$this->dir}/application/logs",
+        ] as $file) {
+            chmod($file, 0777);
+        }
+
+        // move the behat config into place
+        copy(
+            "{$this->dir}/application/tests/behat.template",
+            "{$this->dir}/application/tests/behat.yml"
+        );
+
+        // configure mysql, set up testing databases
+        $this->output->writeln("Configuring databases");
+        $this->mysqlCommand('SET GLOBAL sql_mode = "STRICT_ALL_TABLES"');
+        $this->mysqlCommand('DROP DATABASE IF EXISTS platform_test');
+        $this->mysqlCommand('CREATE DATABASE platform_test');
+        $this->mysqlCommand('DROP DATABASE IF EXISTS zombie2x');
+        $this->mysqlCommand('CREATE DATABASE zombie2x');
+        $this->execEnv("{$this->dir}/bin/phinx migrate -c {$this->dir}/application/phinx.php");
+
+        // if zombie2x.sql does not exist, download it
+        $zombie_sql_file = '/tmp/ushahidi-zombie2x.sql';
+        if (!file_exists($zombie_sql_file))
+        {
+            $this->downloadFile($this->test_database_sql_url, $zombie_sql_file);
+        }
+
+        // import the zombies
+        $this->output->writeln("Importing test data");
+        shell_exec("mysql -u root zombie2x < $zombie_sql_file");
+    }
+
+    protected function downloadFile($remote_file, $local_file)
+    {
+        $this->output->writeln("Downloading $remote_file");
+
+        $outfile = fopen($local_file, 'wb');
+
+        $progress = new ProgressBar($this->output, 100);
+        $progress->setFormat('[%bar%] %percent:3s%% (%elapsed% / %estimated:-6s%)');
+
+        $ch = curl_init();
+
+        curl_setopt($ch, CURLOPT_FILE, $outfile);
+        curl_setopt($ch, CURLOPT_HEADER, 0);
+        curl_setopt($ch, CURLOPT_URL, $remote_file);
+        curl_setopt($ch, CURLOPT_NOPROGRESS, false);
+        curl_setopt($ch, CURLOPT_PROGRESSFUNCTION, function() use ($progress) {
+            // curl versions 7.32 and below do not pass the
+            // $ch handler as the first argument to this function
+            $args = func_get_args();
+            if (count($args) >= 5) { array_shift($args); }
+
+            $total_down   = $args[0];
+            $current_down = $args[1];
+
+            if ($total_down === 0) { return; }
+            $progress->setCurrent(intval(($current_down / $total_down) * 100));
+        });
+
+        curl_exec($ch);
+        $progress->finish();
+        $this->output->writeln('');
+        curl_close($ch);
+        fclose($outfile);
+    }
+
+    protected function testServer($state)
+    {
+        if ($state === 'up' && !$this->test_server_pid)
+        {
+            $command = "php -S localhost:8000 -t {$this->dir}"
+                     . " {$this->dir}/httpdocs/index.php"
+                     . " > /dev/null 2>&1 & echo $!;"
+                     ;
+
+            $this->test_server_pid = $this->execEnv($command);
+            $this->output->writeln("PHP server started (pid: {$this->test_server_pid})");
+            sleep(3);
+        }
+        else if ($state === 'down' && $this->test_server_pid)
+        {
+            $this->output->writeln("Terminating PHP Server (pid: {$this->test_server_pid})");
+            posix_kill($this->test_server_pid, SIGTERM);
+            $this->test_server_pid = null;
+        }
+    }
+
+    protected function execEnv($command, $return_exit_code = false, $env_vars = array())
+    {
+        $env_vars = $env_vars ?: $this->env_variables;
+        array_walk($env_vars, function(&$value, $key) {
+            $value = "export $key=$value";
+        });
+
+        $command = implode(';', $env_vars) . ";$command";
+
+        if ($return_exit_code)
+        {
+            passthru($command, $exit_code);
+            return $exit_code;
+        }
+        else
+        {
+            return exec($command);
+        }
+    }
+
+    protected function mysqlCommand($command)
+    {
+        $command = escapeshellarg($command);
+        shell_exec("mysql -u root -e $command;");
+    }
+
+    protected function addOutputFormatting()
+    {
+        foreach ([ // custom output formatting
+            'pass' => new OutputFormatterStyle('white', 'green', ['bold']),
+            'fail' => new OutputFormatterStyle('white', 'red',   ['bold']),
+            'info' => new OutputFormatterStyle('white', 'cyan',  ['bold']),
+        ] as $tag => $style) {
+            $this->output->getFormatter()->setStyle($tag, $style);
+        }
+    }
+}
+
+class UshahidiTestsRunCommand extends UshahidiTestsCommand
+{
+    protected function configure()
+    {
+        $this
+            ->setName('run')
+            ->setDescription('Runs all the tests and returns a single pass/fail result & exit code (default command)')
+            ->addOption(
+                'no-install',
+                null,
+                InputOption::VALUE_NONE,
+                'Don\'t install the prerequisites'
+            )
+            ;
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        parent::execute($input, $output);
+
+        $this->output->writeln(
+            "\n<pass> Ushahidi Platform Tests </pass>\n"
+        );
+
+        $no_install = $input->getOption('no-install');
+        if (!$no_install) { $this->install(); }
+
+        // start the local test server
+        $this->testServer('up');
+
+        // we're optimistic
+        $tests_pass = true;
+        $test_start_time = time();
+
+        // run phpspec
+        chdir($this->dir); // phpspec won't find tests otherwise
+        $output->writeln("\n<info> - Running Phpspec tests - </info>");
+        $phpspec_command = "{$this->dir}/bin/phpspec run --format dot --verbose";
+        $tests_pass = $tests_pass && ($this->execEnv($phpspec_command, true) === 0);
+
+        // run behat
+        $output->writeln("\n<info> - Running Behat tests - </info>\n");
+        $behat_command = "{$this->dir}/bin/behat"
+                       . " --config {$this->dir}/application/tests/behat.yml"
+                       . " --format progress"
+                       . " --strict"
+                       ;
+        $tests_pass = $tests_pass && ($this->execEnv($behat_command, true) === 0);
+
+        // run phpunit
+        $output->writeln("\n<info> - Running PHPUnit tests - </info>\n");
+        $phpunit_command = "{$this->dir}/bin/phpunit"
+                         . " -c {$this->dir}/application/tests/phpunit.xml"
+                         . " --strict"
+                         ;
+        $tests_pass = $tests_pass && ($this->execEnv($phpunit_command, true) === 0);
+
+        // bring down the test server
+        $this->testServer('down');
+
+        $elapsed_time = time() - $test_start_time;
+
+        // output a single pass/fail message
+        if ($tests_pass)
+        {
+            $output->writeln(
+                "\n<pass> All tests ran successfully! </pass> ($elapsed_time seconds)\n"
+            );
+        }
+        else
+        {
+            $output->writeln(
+                "\n<fail> Some tests failed! </fail>\n"
+            );
+        }
+
+        // return a single exit code from the tests above
+        exit($tests_pass ? 0 : 1);
+    }
+}
+
+class UshahidiTestsInstallCommand extends UshahidiTestsCommand
+{
+    protected function configure()
+    {
+        $this
+            ->setName('install')
+            ->setDescription('Install necessary prerequisites for running tests')
+            ;
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        parent::execute($input, $output);
+        $this->install();
+    }
+}
+
+$test_runner = new Application();
+$test_runner->add(new UshahidiTestsRunCommand);
+$test_runner->add(new UshahidiTestsInstallCommand);
+$test_runner->setDefaultCommand('run');
+$test_runner->run();

composer.json

@@ -73,7 +73,7 @@
         "phpunit/phpunit": "~3.7.37",
         "phpunit/dbunit": "~1.2.3",
         "phpspec/phpspec": "~2.0.0",
-        "symfony/console": "~2.4.5",
+        "symfony/console": "~2.5",
         "squizlabs/php_codesniffer": "1.5.*"
     },
     "minimum-stability": "dev",