use shared library for configuration

fixes: CCI-389 Test-Plan: we need to run all the builds that we can for this: - the current build needs to pass - main-for-plugins needs to be ran - master-bouncer - dive? - translations Change-Id: Iae7f9dad7668b90653a0b9a8edc20ec69fa109c3 Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/240345 Tested-by: Service Cloud Jenkins <[email protected]> QA-Review: Ryan Norton <[email protected]> Reviewed-by: James Butters <[email protected]> Reviewed-by: Aaron Ogata <[email protected]> Product-Review: Rex Fleischer <[email protected]>
em3ndez · Jun 18, 2020 · c5c3820 · c5c3820
1 parent 1dbee48
commit c5c3820
Show file tree

Hide file tree

Showing 21 changed files with 85 additions and 652 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -35,22 +35,7 @@ def buildParameters = [
   string(name: 'MASTER_BOUNCER_RUN', value: "${env.MASTER_BOUNCER_RUN}")
 ]
 
-def getImageTagVersion() {
-  def flags = load('build/new-jenkins/groovy/commit-flags.groovy')
-  flags.getImageTagVersion()
-}
-
-def getPublishableTagSuffix() {
-  load('build/new-jenkins/groovy/configuration.groovy').publishableTagSuffix()
-}
-
-def getRubyPassenger() {
-  load('build/new-jenkins/groovy/configuration.groovy').rubyPassenger()
-}
-
-def getPostgres() {
-  load('build/new-jenkins/groovy/configuration.groovy').postgres()
-}
+library "canvas-builds-library"
 
 def runDatadogMetric(name, body) {
   def dd = load('build/new-jenkins/groovy/datadog.groovy')
@@ -98,10 +83,6 @@ def ignoreBuildNeverStartedError(block) {
   }
 }
 
-def buildRegistryFQDN() {
-  load('build/new-jenkins/groovy/configuration.groovy').buildRegistryFQDN()
-}
-
 // ignore builds where the current patchset tag doesn't match the
 // mainline publishable tag. i.e. ignore ruby-passenger-2.6/pg-12
 // upgrade builds
@@ -123,18 +104,18 @@ pipeline {
   environment {
     GERRIT_PORT = '29418'
     GERRIT_URL = "$GERRIT_HOST:$GERRIT_PORT"
-    NAME = getImageTagVersion()
+    NAME = imageTagVersion()
     CANVAS_LMS_IMAGE = "$DOCKER_REGISTRY_FQDN/jenkins/canvas-lms"
-    BUILD_REGISTRY_FQDN = buildRegistryFQDN()
+    BUILD_REGISTRY_FQDN = configuration.buildRegistryFQDN()
     BUILD_IMAGE = "$BUILD_REGISTRY_FQDN/jenkins/canvas-lms"
-    POSTGRES = getPostgres()
-    RUBY_PASSENGER = getRubyPassenger()
+    POSTGRES = configuration.postgres()
+    RUBY_PASSENGER = configuration.rubyPassenger()
 
     // e.g. postgres-9.5-ruby-passenger-2.6
     TAG_SUFFIX = "postgres-$POSTGRES-ruby-passenger-$RUBY_PASSENGER"
 
     // this is found in the PUBLISHABLE_TAG_SUFFIX config file on jenkins
-    PUBLISHABLE_TAG_SUFFIX = getPublishableTagSuffix()
+    PUBLISHABLE_TAG_SUFFIX = configuration.publishableTagSuffix()
 
     // e.g. canvas-lms:01.123456.78-postgres-12-ruby-passenger-2.6
     PATCHSET_TAG = "$BUILD_IMAGE:$NAME-$TAG_SUFFIX"
@@ -155,8 +136,7 @@ pipeline {
         timeout(time: 5) {
           script {
             runDatadogMetric("Setup") {
-              sh 'build/new-jenkins/print-env-excluding-secrets.sh'
-              sh 'build/new-jenkins/docker-cleanup.sh'
+              cleanAndSetup()
 
               buildParameters += string(name: 'PATCHSET_TAG', value: "${env.PATCHSET_TAG}")
               buildParameters += string(name: 'POSTGRES', value: "${env.POSTGRES}")
@@ -168,25 +148,23 @@ pipeline {
                 buildParameters += string(name: 'CANVAS_LMS_REFSPEC', value: env.CANVAS_LMS_REFSPEC)
               }
 
-              def credentials = load ('build/new-jenkins/groovy/credentials.groovy')
-
-              credentials.fetchFromGerrit('gerrit_builder', '.', '', 'canvas-lms/config')
+              pullGerritRepo('gerrit_builder', 'master', '.')
               gems = readFile('gerrit_builder/canvas-lms/config/plugins_list').split()
               echo "Plugin list: ${gems}"
               /* fetch plugins */
               gems.each { gem ->
                 if (env.GERRIT_PROJECT == gem) {
                   /* this is the commit we're testing */
-                  credentials.fetchFromGerrit(gem, 'gems/plugins', null, null, env.GERRIT_REFSPEC)
+                  pullGerritRepo(gem, env.GERRIT_REFSPEC, 'gems/plugins')
                 } else {
-                  credentials.fetchFromGerrit(gem, 'gems/plugins')
+                  pullGerritRepo(gem, 'master', 'gems/plugins')
                 }
               }
-              credentials.fetchFromGerrit('qti_migration_tool', 'vendor', 'QTIMigrationTool')
+              pullGerritRepo("qti_migration_tool", "master", "vendor")
 
               sh 'mv -v gerrit_builder/canvas-lms/config/* config/'
               sh 'rm -v config/cache_store.yml'
-              sh 'rmdir -p gerrit_builder/canvas-lms/config'
+              sh 'rm -vr gerrit_builder'
               sh 'rm -v config/database.yml'
               sh 'rm -v config/security.yml'
               sh 'rm -v config/selenium.yml'
@@ -209,7 +187,6 @@ pipeline {
         timeout(time: 2) {
           script {
             runDatadogMetric("Rebase") {
-              def credentials = load('build/new-jenkins/groovy/credentials.groovy')
               credentials.withGerritCredentials({ ->
                 sh '''#!/bin/bash
                   set -o errexit -o errtrace -o nounset -o pipefail -o xtrace
@@ -249,13 +226,12 @@ pipeline {
         timeout(time: 36) { /* this timeout is `2 * average build time` which currently: 18m * 2 = 36m */
           skipIfPreviouslySuccessful('docker-build-and-push') {
             script {
-              def flags = load('build/new-jenkins/groovy/commit-flags.groovy')
-              if (flags.hasFlag('skip-docker-build')) {
+              if (configuration.getBoolean('skip-docker-build')) {
                 sh 'docker pull $MERGE_TAG'
                 sh 'docker tag $MERGE_TAG $PATCHSET_TAG'
               }
               else {
-                if (!flags.hasFlag('skip-cache')) {
+                if (!configuration.getBoolean('skip-cache')) {
                   sh 'docker pull $MERGE_TAG || true'
                 }
                 sh """
@@ -285,7 +261,6 @@ pipeline {
             echo 'adding Linters'
             stages['Linters'] = {
               skipIfPreviouslySuccessful("linters") {
-                def credentials = load 'build/new-jenkins/groovy/credentials.groovy'
                 credentials.withGerritCredentials {
                   sh 'build/new-jenkins/linters/run-gergich.sh'
                 }
@@ -330,7 +305,7 @@ pipeline {
             echo 'adding Flakey Spec Catcher'
             stages['Flakey Spec Catcher'] = {
               skipIfPreviouslySuccessful("flakey-spec-catcher") {
-                def propagate = load('build/new-jenkins/groovy/configuration.groovy').fscPropagate()
+                def propagate = configuration.fscPropagate()
                 echo "fsc propagation: $propagate"
                 wrapBuildExecution('/Canvas/test-suites/flakey-spec-catcher', buildParameters, propagate, "")
               }
@@ -447,7 +422,7 @@ pipeline {
     }
     cleanup {
       ignoreBuildNeverStartedError {
-        sh 'build/new-jenkins/docker-cleanup.sh --allow-failure'
+        execute 'bash/docker-cleanup.sh --allow-failure'
       }
     }
   }

diff --git a/Jenkinsfile.contract-tests b/Jenkinsfile.contract-tests
@@ -18,6 +18,8 @@
  * with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+library "canvas-builds-library"
+
 pipeline {
   agent { label 'canvas-docker' }
   options {
@@ -37,9 +39,7 @@ pipeline {
   stages {
     stage ('Pre-Cleanup') {
       steps {
-        timeout(time: 2) {
-          sh 'build/new-jenkins/docker-cleanup.sh'
-        }
+        cleanAndSetup()
       }
     }
 
@@ -157,7 +157,7 @@ pipeline {
     }
     cleanup {
       sh 'rm -vrf spec_results/'
-      sh 'build/new-jenkins/docker-cleanup.sh --allow-failure'
+      execute 'bash/docker-cleanup.sh --allow-failure'
     }
   }
 }
diff --git a/Jenkinsfile.dive b/Jenkinsfile.dive
@@ -18,6 +18,8 @@
  * with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+library "canvas-builds-library"
+
 pipeline {
   agent { label 'canvas-docker' }
   options {
@@ -29,10 +31,7 @@ pipeline {
   stages {
     stage('Setup') {
       steps {
-        timeout(time: 2) {
-          sh 'build/new-jenkins/print-env-excluding-secrets.sh'
-          sh 'build/new-jenkins/docker-cleanup.sh'
-        }
+        cleanAndSetup()
       }
     }
 
@@ -49,7 +48,7 @@ pipeline {
 
   post {
     cleanup {
-      sh 'build/new-jenkins/docker-cleanup.sh --allow-failure'
+      execute 'bash/docker-cleanup.sh --allow-failure'
     }
   }
 }
diff --git a/Jenkinsfile.js b/Jenkinsfile.js
@@ -18,57 +18,20 @@
  * with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-def runCoverage() {
-  def flags = load 'build/new-jenkins/groovy/commit-flags.groovy'
-  return env.RUN_COVERAGE == '1' || flags.forceRunCoverage() ? '1' : ''
-}
-
-def isForceFailure() {
-  def flags = load 'build/new-jenkins/groovy/commit-flags.groovy'
-  return flags.isForceFailureJS() ? "1" : ''
-}
-
-def getImageTagVersion() {
-  def flags = load 'build/new-jenkins/groovy/commit-flags.groovy'
-  // total hack, we shouldnt do it like this. but until there is a better
-  // way of passing info across builds, this is path of least resistance..
-  // also, it didnt seem to work with multiple return statements, so i'll
-  // just go ahead and leave this monstrosity here.
-  return env.RUN_COVERAGE == '1' ? 'master' : flags.getImageTagVersion()
-}
+library "canvas-builds-library"
 
 def copyFiles(dockerName, dockerPath, hostPath) {
   sh "mkdir -vp ./$hostPath"
   sh "docker cp \$(docker ps -qa -f name=$dockerName):/usr/src/app/$dockerPath ./$hostPath"
 }
 
-def withSentry(block) {
-  def credentials = load 'build/new-jenkins/groovy/credentials.groovy'
-  credentials.withSentryCredentials(block)
-}
-
-def runInSeriesOrParallel(isSeries, stagesMap) {
-  if (isSeries) {
-    echo "running tests in series: ${stagesMap.keys}"
-    stagesMap.each { name, block ->
-      stage(name) {
-        block()
-      }
-    }
-  } else {
-    echo "running tests in parallel: ${stagesMap.keys}"
-    parallel(stagesMap)
-  }
-}
-
 pipeline {
   agent { label 'canvas-docker' }
   options { ansiColor('xterm') }
 
   environment {
     COMPOSE_FILE = 'docker-compose.new-jenkins.canvas.yml:docker-compose.new-jenkins-karma.yml'
-    COVERAGE = runCoverage()
-    FORCE_FAILURE = isForceFailure()
+    FORCE_FAILURE = configuration.forceFailureJS()
     SENTRY_URL="https://sentry.insops.net"
     SENTRY_ORG="instructure"
     SENTRY_PROJECT="master-javascript-build"
@@ -77,9 +40,8 @@ pipeline {
   stages {
     stage('Setup') {
       steps {
+        cleanAndSetup()
         timeout(time: 10) {
-          sh 'build/new-jenkins/print-env-excluding-secrets.sh'
-          sh 'build/new-jenkins/docker-cleanup.sh'
           sh 'rm -vrf ./tmp/*'
           sh 'build/new-jenkins/docker-compose-pull.sh'
           sh 'docker-compose build'
@@ -96,10 +58,8 @@ pipeline {
             tests['Jest'] = {
               withEnv(['CONTAINER_NAME=tests-jest']) {
                 try {
-                  withSentry { sh 'build/new-jenkins/js/tests-jest.sh'
-                  }
-                  if (env.COVERAGE == '1') {
-                    copyFiles(env.CONTAINER_NAME, 'coverage-jest', "./tmp/${env.CONTAINER_NAME}-coverage")
+                  credentials.withSentryCredentials {
+                    sh 'build/new-jenkins/js/tests-jest.sh'
                   }
                 } finally {
                   copyFiles(env.CONTAINER_NAME, 'coverage-js', "./tmp/${env.CONTAINER_NAME}")
@@ -110,7 +70,9 @@ pipeline {
             tests['Packages'] = {
               withEnv(['CONTAINER_NAME=tests-packages']) {
                 try {
-                  withSentry { sh 'build/new-jenkins/js/tests-packages.sh' }
+                  credentials.withSentryCredentials {
+                    sh 'build/new-jenkins/js/tests-packages.sh'
+                  }
                 } finally {
                   copyFiles(env.CONTAINER_NAME, 'packages', "./tmp/${env.CONTAINER_NAME}")
                 }
@@ -125,9 +87,8 @@ pipeline {
               tests["Karma - Spec Group - ${group}"] = {
                 withEnv(["CONTAINER_NAME=tests-karma-${group}", "JSPEC_GROUP=${group}"]) {
                   try {
-                    withSentry { sh 'build/new-jenkins/js/tests-karma.sh' }
-                    if (env.COVERAGE == '1') {
-                      copyFiles(env.CONTAINER_NAME, 'coverage-karma', "./tmp/${env.CONTAINER_NAME}-coverage")
+                    credentials.withSentryCredentials {
+                      sh 'build/new-jenkins/js/tests-karma.sh'
                     }
                   } finally {
                     copyFiles(env.CONTAINER_NAME, 'coverage-js', "./tmp/${env.CONTAINER_NAME}")
@@ -136,26 +97,11 @@ pipeline {
               }
             }
 
-            runInSeriesOrParallel(env.COVERAGE == '1', tests)
+            parallel(tests)
           }
         }
       }
     }
-
-
-    stage('Upload Coverage') {
-      when { expression { env.COVERAGE == '1' } }
-      steps {
-        timeout(time: 10) {
-          sh 'build/new-jenkins/js/coverage-report.sh'
-          archiveArtifacts(artifacts: 'coverage-report-js/**/*')
-          uploadCoverage([
-              uploadSource: "/coverage-report-js/report-html",
-              uploadDest: "canvas-lms-js/coverage"
-          ])
-        }
-      }
-    }
   }
 
   post {
@@ -166,7 +112,7 @@ pipeline {
       }
     }
     cleanup {
-      sh 'build/new-jenkins/docker-cleanup.sh --allow-failure'
+      execute 'bash/docker-cleanup.sh --allow-failure'
     }
   }
 }