add the ability to run parallel rspec processes in container

refs DE-225
flag=none

By specifying RSPEC_PROCESSES, we can now run multiple
rspec processes in a single container. This change modifies
the existing database setup to create 1 database per rspec
process. This also impacts results.xml using a 1 results.xml
per rspec process.

Rename database key to db for redis configurations to match
current version of redis.

test plan:
- rspec runs as expected in single threaded mode
- rspec runs as expected in multi threaded mode
- results.xml contains valid results with no duplciate tests
- reruns only run for failing threads
- reruns work in single and multithreaded mode

Change-Id: Ib2e549d467e8a6d8fef9914f2733d9ddfa460e99
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/255120
Tested-by: Service Cloud Jenkins <[email protected]>
Reviewed-by: Aaron Ogata <[email protected]>
QA-Review: Kyle Rosenbaum <[email protected]>
Product-Review: Kyle Rosenbaum <[email protected]>

Jenkinsfile

@@ -509,7 +509,7 @@ pipeline {
 
             timedStage('Run Migrations') {
               timeout(time: 10) {
-                def cacheLoadScope = configuration.isChangeMerged() ? '' : env.IMAGE_CACHE_MERGE_SCOPE
+                def cacheLoadScope = configuration.isChangeMerged() || configuration.getBoolean('skip-cache') ? '' : env.IMAGE_CACHE_MERGE_SCOPE
                 def cacheSaveScope = configuration.isChangeMerged() ? env.IMAGE_CACHE_MERGE_SCOPE : ''
 
                 libraryScript.load('bash/docker-tag-remote.sh', './build/new-jenkins/docker-tag-remote.sh')
@@ -525,7 +525,8 @@ pipeline {
                   "DYNAMODB_PREFIX=${env.DYNAMODB_PREFIX}",
                   "POSTGRES_IMAGE_TAG=${imageTag.postgres()}",
                   "POSTGRES_PREFIX=${env.POSTGRES_PREFIX}",
-                  "POSTGRES_PASSWORD=sekret"
+                  "POSTGRES_PASSWORD=sekret",
+                  "RSPEC_PROCESSES=4"
                 ]) {
                   credentials.withStarlordCredentials({ ->
                     sh """

Jenkinsfile.contract-tests

@@ -41,7 +41,7 @@ def runAndCopyResults(database_name, consumer_name, script_name='build/new-jenki
     try {
       sh script_name
     } finally {
-      sh 'build/new-jenkins/docker-copy-files.sh /usr/src/app/log/results.xml tmp/spec_results/${DATABASE_NAME} ${DATABASE_NAME} --allow-error --clean-dir'
+      sh 'build/new-jenkins/docker-copy-files.sh /usr/src/app/log/results/ tmp/spec_results/${DATABASE_NAME} ${DATABASE_NAME} --allow-error --clean-dir'
     }
   }
 }

build/new-jenkins/docker-compose-rspec-parallel.sh

@@ -4,4 +4,4 @@ set -o errexit -o errtrace -o pipefail -o xtrace
 parallel --will-cite ::: :
 
 # Run each group of tests in separate docker container
-seq 0 $((DOCKER_PROCESSES-1)) | parallel "docker-compose --project-name canvas-lms{} exec -T -e TEST_PROCESS={} canvas bash -c 'build/new-jenkins/rspec-with-retries.sh'"
+seq 0 $((DOCKER_PROCESSES-1)) | parallel "docker-compose --project-name canvas-lms{} exec -T -e RSPEC_PROCESSES canvas bash -c 'build/new-jenkins/rspec-with-retries.sh'"

build/new-jenkins/docker-compose-setup-databases.sh

@@ -5,12 +5,24 @@ set -o errexit -o errtrace -o nounset -o pipefail -o xtrace
 parallel --will-cite ::: :
 
 PROCESSES=$((${DOCKER_PROCESSES:=1}-1))
+DATABASE_PROCESSES=$((${RSPEC_PROCESSES:=1}-1))
 
 create_cmd=""
 for keyspace in auditors global_lookups page_views; do
   create_cmd+="CREATE KEYSPACE IF NOT EXISTS ${keyspace} WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };"
 done
+
 seq 0 $PROCESSES | parallel "docker-compose --project-name canvas-lms{} exec -T cassandra cqlsh -e \"${create_cmd[@]}\""
 
 seq 0 $PROCESSES | parallel "docker-compose --project-name canvas-lms{} exec -T canvas bundle exec rails db:migrate >> ./migrate-{}.log"
 seq 0 $PROCESSES | parallel "docker-compose --project-name canvas-lms{} exec -T canvas bundle exec rails runner \"require 'switchman/test_helper'; Switchman::TestHelper.recreate_persistent_test_shards\""
+
+for i in $(seq 0 $PROCESSES); do
+  for keyspace in auditors global_lookups page_views; do
+    seq 0 $DATABASE_PROCESSES | parallel "docker-compose --project-name canvas-lms${i} exec -T cassandra bash -c 'cqlsh -e \"DESCRIBE KEYSPACE ${keyspace}\" | sed \"s/CREATE KEYSPACE ${keyspace}/CREATE KEYSPACE ${keyspace}{}/g ; s/CREATE TABLE ${keyspace}/CREATE TABLE ${keyspace}{}/g\" > ${keyspace}{} && cqlsh -f \"${keyspace}{}\"'"
+  done
+done
+
+for i in $(seq 0 $PROCESSES); do
+  seq 0 $DATABASE_PROCESSES | parallel "docker-compose --project-name canvas-lms${i} exec -T postgres sh -c 'createdb -U postgres -T canvas_test canvas_test_{}'"
+done

build/new-jenkins/groovy/rspec.groovy

@@ -37,6 +37,7 @@ def runSeleniumSuite(total, index) {
       config.reruns_retry,
       '^./(spec|gems/plugins/.*/spec_canvas)/selenium',
       '.*/performance',
+      '1',
       '3',
       config.force_failure,
       config.patchsetTag
@@ -64,6 +65,7 @@ def runRSpecSuite(total, index) {
       config.reruns_retry,
       '^./(spec|gems/plugins/.*/spec_canvas)/',
       '.*/(selenium|contracts)',
+      '1',
       '4',
       config.force_failure,
       config.patchsetTag
@@ -80,6 +82,7 @@ def _runRspecTestSuite(
     test_file_pattern,
     exclude_regex,
     docker_processes,
+    rspec_processes,
     force_failure,
     patchsetTag
 ) {
@@ -92,6 +95,7 @@ def _runRspecTestSuite(
       "EXCLUDE_TESTS=$exclude_regex",
       "CI_NODE_TOTAL=$total",
       "DOCKER_PROCESSES=$docker_processes",
+      "RSPEC_PROCESSES=$rspec_processes",
       "FORCE_FAILURE=$force_failure",
       "POSTGRES_PASSWORD=sekret",
       "SELENIUM_VERSION=3.141.59-20201119",
@@ -112,7 +116,7 @@ def _runRspecTestSuite(
     } finally {
       // copy spec failures to local
       sh "build/new-jenkins/docker-copy-files.sh /usr/src/app/log/spec_failures/ tmp/spec_failures/$prefix canvas_ --allow-error --clean-dir"
-      sh 'build/new-jenkins/docker-copy-files.sh /usr/src/app/log/results.xml tmp/rspec_results canvas_ --allow-error --clean-dir'
+      sh 'build/new-jenkins/docker-copy-files.sh /usr/src/app/log/results tmp/rspec_results canvas_ --allow-error --clean-dir'
 
       if(configuration.getBoolean('upload-docker-logs', 'false')) {
         sh "docker ps -aq | xargs -I{} -n1 -P1 docker logs --timestamps --details {} 2>&1 > tmp/docker-${prefix}-${index}.log"

build/new-jenkins/rspec-tests.sh

@@ -2,17 +2,21 @@
 
 set -o nounset -o errexit -o errtrace -o pipefail -o xtrace
 
+# required script parameters
+parallel_index=$1
+only_failures=${2-}
+
 # calculate which group to run
-max=$((CI_NODE_TOTAL * DOCKER_PROCESSES))
-group=$(((max-CI_NODE_TOTAL * TEST_PROCESS) - CI_NODE_INDEX))
+max=$((CI_NODE_TOTAL * (DOCKER_PROCESSES * RSPEC_PROCESSES)))
+group=$(((max-CI_NODE_TOTAL * $parallel_index) - CI_NODE_INDEX))
 maybeOnlyFailures=()
-if [ "${1-}" = 'only-failures' ] && [ ! "${RSPEC_LOG:-}" == "1" ]; then
+if [ "${only_failures}" = 'only-failures' ] && [ ! "${RSPEC_LOG:-}" == "1" ]; then
   maybeOnlyFailures=("--test-options" "'--only-failures'")
 fi
 
 # we want actual globbing of individual elements for passing argument literals
 # shellcheck disable=SC2068
-bundle exec parallel_rspec . \
+PARALLEL_INDEX=$parallel_index RAILS_DB_NAME_TEST="canvas_test_$parallel_index" bundle exec parallel_rspec . \
   --pattern "$TEST_PATTERN" \
   --exclude-pattern "$EXCLUDE_TESTS" \
   -n "$max" \

build/new-jenkins/rspec-with-retries.sh

@@ -4,54 +4,69 @@
 # has a lot of unset variables and needs to be addressed independently
 set -o errexit -o errtrace -o xtrace
 
+PROCESSES=$((${RSPEC_PROCESSES:=1}-1))
 export ERROR_CONTEXT_BASE_PATH="/usr/src/app/log/spec_failures/Initial"
 
 success_status=0
 test_failure_status=1
-
-runs_remaining=$((1+${RERUNS_RETRY:=2}))
+runs_remaining=${RERUNS_RETRY:=2}
 
 echo "STARTING"
 while true; do
-  set +e
+  last_statuses=()
+  command_pids=()
+
   if [[ $reruns_started ]]; then
     if [ $1 ] && [ $1 = 'performance' ]; then
-      build/new-jenkins/rspec-with-wait.sh "docker-compose --project-name canvas-lms0 exec -T canvas bundle exec rspec --options spec/spec.opts spec/selenium/performance/ --only-failures --failure-exit-code 99"
-    else
-      build/new-jenkins/rspec-with-wait.sh "build/new-jenkins/rspec-tests.sh only-failures"
+      commands+=("docker-compose --project-name canvas-lms0 exec -T canvas bundle exec rspec --options spec/spec.opts spec/selenium/performance/ --only-failures --failure-exit-code 99")
     fi
   else
     if [ $1 ] && [ $1 = 'performance' ]; then
-      build/new-jenkins/rspec-with-wait.sh "docker-compose --project-name canvas-lms0 exec -T canvas bundle exec rspec --options spec/spec.opts spec/selenium/performance/ --failure-exit-code 99"
+      commands+=("docker-compose --project-name canvas-lms0 exec -T canvas bundle exec rspec --options spec/spec.opts spec/selenium/performance/ --failure-exit-code 99")
     else
-      build/new-jenkins/rspec-with-wait.sh "build/new-jenkins/rspec-tests.sh"
+      for i in $(seq 0 $PROCESSES); do
+        commands+=("build/new-jenkins/rspec-tests.sh $i")
+      done
     fi
   fi
-  last_status=$?
-  set -e
 
-  [[ ! $reruns_started ]] && echo "FINISHED"
-  [[ $last_status == $success_status ]] && break
+  for command in "${commands[@]}"; do
+    ./build/new-jenkins/linters/run-and-collect-output.sh "$command" 2>&1 &
+    command_pids[$!]=$command
+  done
 
-  if [[ $last_status != $success_status && $last_status != $test_failure_status ]]; then
-    echo "unexpected exit code $last_status! perhaps the code is horribly broken :("
-    break
-  fi
+  for command_pid in "${!command_pids[@]}"; do
+    wait $command_pid || last_statuses[$command_pid]=$?
+  done
 
-  if [[ $last_status == $test_failure_status ]]; then
-    runs_remaining=$((runs_remaining-1))
+  exit_code=0
+  commands=()
+  for command_pid in ${!last_statuses[@]}; do
+    last_status="${last_statuses[$command_pid]}"
+    if [[ $last_status == $success_status ]]; then
+      continue
+    elif [[ $last_status == $test_failure_status ]]; then
+      export ERROR_CONTEXT_BASE_PATH="/usr/src/app/log/spec_failures/Rerun_$runs_remaining"
 
-    [[ $runs_remaining == 0 ]] && { echo "reruns failed $num_failures failure(s)"; break; }
-    export ERROR_CONTEXT_BASE_PATH="/usr/src/app/log/spec_failures/Rerun_$runs_remaining"
+      if [[ $runs_remaining == 0 ]]; then
+        exit_code=$last_status
+        break 2
+      fi
 
-    echo -e "failed, re-trying, $runs_remaining attempt(s) left\n\n\n"
-    if [[ ! $reruns_started ]]; then
       reruns_started=1
-      echo "RERUN STARTING"
+      echo -e "RERUN STARTING: failed in running command: ${command_pids[$command_pid]}, $runs_remaining attempt(s) left\n\n\n"
+      commands+=("${command_pids[$command_pid]} only-failures")
+      exit_code=$last_status
+    else
+      echo "unexpected exit code $last_status! perhaps the code is horribly broken :("
+      exit_code=$last_status
+      break 2
     fi
-  fi
+  done
+
+  [[ $exit_code == 0 ]] && break
+  runs_remaining=$((runs_remaining-1))
 done
 
-[[ $reruns_started ]] && echo " FINISHED"
-echo "rspec-queue-with-retries exiting with $last_status"
-exit $last_status
+echo "FINISHED: rspec-queue-with-retries exiting with $exit_code"
+exit $exit_code

config/cache_store.yml.example

@@ -11,6 +11,6 @@ development:
   #
   # url:
   # - localhost
-  # database: 0
+  # db: 0
   #
   #

config/redis.yml.example

@@ -22,4 +22,4 @@ test:
   # warning: the redis database will get cleared before each test, so if you
   # use this server for anything else, make sure to set aside a database id for
   # these tests to use.
-  database: 1
+  db: 1

docker-compose/config/new-jenkins/cassandra.yml

@@ -2,23 +2,23 @@ test:
    page_views:
      servers:
        - cassandra:9160
-     keyspace: page_views
+     keyspace: page_views<%= ENV.fetch('PARALLEL_INDEX', '') %>
      retries: 10
      connect_timeout: 15
      timeout: 15
 
    auditors:
      servers:
        - cassandra:9160
-     keyspace: auditors
+     keyspace: auditors<%= ENV.fetch('PARALLEL_INDEX', '') %>
      retries: 10
      connect_timeout: 15
      timeout: 15
 
    global_lookups:
      servers:
        - cassandra:9160
-     keyspace: global_lookups
+     keyspace: global_lookups<%= ENV.fetch('PARALLEL_INDEX', '') %>
      retries: 10
      connect_timeout: 15
      auto_snapshot: false

docker-compose/config/new-jenkins/dynamodb.yml

@@ -4,4 +4,4 @@ test:
     access_key_id: ASDFASDF
     secret_access_key: ZXCVZXCV
     endpoint: http://dynamodb:8000
-    table_prefix: canvas_auditors
+    table_prefix: canvas_auditors<%= ENV.fetch('PARALLEL_INDEX', '') %>

docker-compose/config/redis.yml

@@ -13,4 +13,4 @@ test:
   # warning: the redis database will get cleared before each test, so if you
   # use this server for anything else, make sure to set aside a database id for
   # these tests to use.
-  database: 1
+  db: <%= ENV.fetch('PARALLEL_INDEX', '1') %>

spec/spec_helper.rb

@@ -351,8 +351,9 @@ def assert_require_login
 
   # DOCKER_PROCESSES is only used on Jenkins and we only care to have RspecJunitFormatter on Jenkins.
   if ENV['DOCKER_PROCESSES']
+    file = "log/results/results-#{ENV.fetch('PARALLEL_INDEX', '0').to_i}.xml"
     # if file already exists this is a rerun of a failed spec, don't generate new xml.
-    config.add_formatter "RspecJunitFormatter", "log/results.xml" unless File.file?("log/results.xml")
+    config.add_formatter "RspecJunitFormatter", file unless File.file?(file)
   end
 
   if ENV['RSPEC_LOG']