Skip to content

Commit

Permalink
[Jenkins] Use ECR in downstream builds (keybase#23679)
Browse files Browse the repository at this point in the history 
* don't boot up gregor

* fix dependency on kbweb

* run systests in parallel

* use tmpfs for mysql

* fix docker-compose version

* prep for ECR

* fix

* fix

* fix

* fix tag

* actually fix tag

* missing build context

* fix creds for downstream build

* testing wrong branch

* speed up docker builds a bit more

* use docker buildkit

* don't kick off kbfs-server

* Revert "don't kick off kbfs-server"

This reverts commit 8089ce9.

* no gregor

* fix race with kbfs-server

* try to open ports on localhost

* skip broken tests

* chown files to keybase

* try to pass the cert var here

* temporarily make iteration faster

* revert to previous way of building kbfsfuse image

* Revert "revert to previous way of building kbfsfuse image"

This reverts commit 353a05b.

* pull images more sanely

* pass in the real kbfs cert instead of b64, so git can work

* pass in correct docker cert arg

* reintroduce skipped test

* run tests again

* Remove temporary downstream builds, switch back to master

* jenkins: Fix citogo_extra

* remove extraneous withRegistry command

* disable parallel systests
  • Loading branch information
@jzila
jzila authored Apr 28, 2020
1 parent db6e330 commit 87690c0
Show file tree
Hide file tree
Showing 5 changed files with 91 additions and 261 deletions.
246 changes: 70 additions & 176 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ def withKbweb(closure) {
sh "docker ps -a"
sh "docker-compose stop"
helpers.logContainer('docker-compose', 'mysql')
helpers.logContainer('docker-compose', 'gregor')
logKbwebServices(kbwebName)
throw ex
} finally {
Expand Down Expand Up @@ -56,101 +55,63 @@ helpers.rootLinuxNode(env, {
defaultValue: '',
description: 'The project name of the upstream kbweb build',
),
string(
name: 'gregorProjectName',
defaultValue: '',
description: 'The project name of the upstream gregor build',
),
]),
])

def kbwebProjectName = env.kbwebProjectName
def cause = helpers.getCauseString(currentBuild)
println "Cause: ${cause}"
println "Pull Request ID: ${env.CHANGE_ID}"

env.BASEDIR=pwd()
env.GOPATH="${env.BASEDIR}/go"
def mysqlImage = docker.image("keybaseprivate/mysql")
def gregorImage = docker.image("keybaseprivate/kbgregor")
def kbwebImage = docker.image("keybaseprivate/kbweb")
def glibcImage = docker.image("keybaseprivate/glibc")
def clientImage = null
def kbwebTag = cause == 'upstream' && kbwebProjectName != '' ? kbwebProjectName : 'master'
def images = [
docker.image("897413463132.dkr.ecr.us-east-1.amazonaws.com/glibc"),
docker.image("897413463132.dkr.ecr.us-east-1.amazonaws.com/mysql"),
docker.image("897413463132.dkr.ecr.us-east-1.amazonaws.com/sqsd"),
docker.image("897413463132.dkr.ecr.us-east-1.amazonaws.com/kbweb:${kbwebTag}"),
]
def kbfsfuseImage

def kbwebNodePrivateIP = httpRequest("http://169.254.169.254/latest/meta-data/local-ipv4").content

println "Running on host $kbwebNodePrivateIP"
println "Setting up build: ${env.BUILD_TAG}"

def cause = helpers.getCauseString(currentBuild)
println "Cause: ${cause}"
println "Pull Request ID: ${env.CHANGE_ID}"

ws("${env.GOPATH}/src/github.com/keybase/client") {

stage("Setup") {
sh "docker rmi keybaseprivate/mysql || echo 'No mysql image to remove'"
docker.withRegistry("", "docker-hub-creds") {
parallel (
checkout: {
retry(3) {
checkout scm
sh 'echo -n $(git --no-pager show -s --format="%an" HEAD) > .author_name'
sh 'echo -n $(git --no-pager show -s --format="%ae" HEAD) > .author_email'
env.AUTHOR_NAME = readFile('.author_name')
env.AUTHOR_EMAIL = readFile('.author_email')
sh 'rm .author_name .author_email'
sh 'echo -n $(git rev-parse HEAD) > go/revision'
sh "git add go/revision"
env.GIT_COMMITTER_NAME = 'Jenkins'
env.GIT_COMMITTER_EMAIL = '[email protected]'
sh 'git commit --author="Jenkins <[email protected]>" -am "revision file added"'
env.COMMIT_HASH = readFile('go/revision')
}
},
pull_glibc: {
glibcImage.pull()
},
pull_mysql: {
mysqlImage.pull()
},
pull_gregor: {
if (cause == "upstream" && kbwebProjectName != '') {
retry(3) {
step([$class: 'CopyArtifact',
projectName: "${kbwebProjectName}",
filter: 'kbgregor.tar.gz',
fingerprintArtifacts: true,
selector: [$class: 'TriggeredBuildSelector',
allowUpstreamDependencies: false,
fallbackToLastSuccessful: false,
upstreamFilterStrategy: 'UseGlobalSetting'],
target: '.'])
sh "gunzip -c kbgregor.tar.gz | docker load"
}
} else {
gregorImage.pull()
}
},
pull_kbweb: {
if (cause == "upstream" && kbwebProjectName != '') {
retry(3) {
step([$class: 'CopyArtifact',
projectName: "${kbwebProjectName}",
filter: 'kbweb.tar.gz',
fingerprintArtifacts: true,
selector: [$class: 'TriggeredBuildSelector',
allowUpstreamDependencies: false,
fallbackToLastSuccessful: false,
upstreamFilterStrategy: 'UseGlobalSetting'],
target: '.'])
sh "gunzip -c kbweb.tar.gz | docker load"
}
} else {
kbwebImage.pull()
parallel (
checkout: {
retry(3) {
checkout scm
sh 'echo -n $(git --no-pager show -s --format="%an" HEAD) > .author_name'
sh 'echo -n $(git --no-pager show -s --format="%ae" HEAD) > .author_email'
env.AUTHOR_NAME = readFile('.author_name')
env.AUTHOR_EMAIL = readFile('.author_email')
sh 'rm .author_name .author_email'
sh 'echo -n $(git rev-parse HEAD) > go/revision'
sh "git add go/revision"
env.GIT_COMMITTER_NAME = 'Jenkins'
env.GIT_COMMITTER_EMAIL = '[email protected]'
sh 'git commit --author="Jenkins <[email protected]>" -am "revision file added"'
env.COMMIT_HASH = readFile('go/revision')
}
},
pull_images: {
docker.withRegistry('https://897413463132.dkr.ecr.us-east-1.amazonaws.com', 'ecr:us-east-1:aws-ecr-user') {
for (i in images) {
i.pull()
i.tag('latest')
}
},
remove_dockers: {
sh 'docker stop $(docker ps -q) || echo "nothing to stop"'
sh 'docker rm $(docker ps -aq) || echo "nothing to remove"'
},
)
}
}
},
remove_dockers: {
sh 'docker stop $(docker ps -q) || echo "nothing to stop"'
sh 'docker rm $(docker ps -aq) || echo "nothing to remove"'
},
)
}

def hasJenkinsfileChanges = helpers.getChanges(env.COMMIT_HASH, env.CHANGE_TARGET).findIndexOf{ name -> name =~ /Jenkinsfile/ } >= 0
Expand Down Expand Up @@ -204,7 +165,7 @@ helpers.rootLinuxNode(env, {
for (platform in platforms) {
withEnv(["GOOS=${platform}"]) {
println "Testing compilation on ${platform}"
sh "go build -tags production github.com/keybase/client/go/keybase"
sh "go build -tags production -o keybase_${platform} github.com/keybase/client/go/keybase"
println "End testing compilation on ${platform}"
}
}
Expand Down Expand Up @@ -233,51 +194,51 @@ helpers.rootLinuxNode(env, {
}
}
}},
integrate: {
integrate: { withEnv([
"DOCKER_BUILDKIT=1",
]) {
// Build the client docker first so we can immediately kick off KBFS
def hasKBFSChanges = packagesToTest.keySet().findIndexOf { key -> key =~ /^github.com\/keybase\/client\/go\/kbfs/ } >= 0
if (hasGoChanges && hasKBFSChanges) {
println "We have KBFS changes, so we are building kbfs-server."
dir('go') {
sh "go install -ldflags \"-s -w\" -buildmode=pie github.com/keybase/client/go/keybase"
sh "cp ${env.GOPATH}/bin/keybase ./keybase/keybase"
clientImage = docker.build("keybaseprivate/kbclient")
// TODO: only do this when we need to run at least one KBFS test.
docker.build("kbclient")
dir('kbfs') {
sh "go install -ldflags \"-s -w\" -buildmode=pie github.com/keybase/client/go/kbfs/kbfsfuse"
sh "cp ${env.GOPATH}/bin/kbfsfuse ./kbfsfuse/kbfsfuse"
sh "go install -ldflags \"-s -w\" -buildmode=pie github.com/keybase/client/go/kbfs/kbfsgit/git-remote-keybase"
sh "cp ${env.GOPATH}/bin/git-remote-keybase ./kbfsgit/git-remote-keybase/git-remote-keybase"
withCredentials([[$class: 'StringBinding', credentialsId: 'kbfs-docker-cert-b64-new', variable: 'KBFS_DOCKER_CERT_B64']]) {
println "Building Docker"
sh '''
set +x
KBFS_DOCKER_CERT="$(echo $KBFS_DOCKER_CERT_B64 | sed 's/ //g' | base64 -d)"
docker build -t keybaseprivate/kbfsfuse \
--build-arg KEYBASE_TEST_ROOT_CERT_PEM="$KBFS_DOCKER_CERT" \
--build-arg KEYBASE_TEST_ROOT_CERT_PEM_B64="$KBFS_DOCKER_CERT_B64" .
'''
def kbfsCert = sh(returnStdout: true, script: "echo \"$KBFS_DOCKER_CERT_B64\" | sed 's/ //g' | base64 -d")
kbfsfuseImage = docker.build('897413463132.dkr.ecr.us-east-1.amazonaws.com/client', "--build-arg KEYBASE_TEST_ROOT_CERT_PEM=\"$kbfsCert\" .")
}
docker.withRegistry('https://897413463132.dkr.ecr.us-east-1.amazonaws.com', 'ecr:us-east-1:aws-ecr-user') {
kbfsfuseImage.push(env.BUILD_TAG)
}
sh "docker save keybaseprivate/kbfsfuse | gzip > kbfsfuse.tar.gz"
archive("kbfsfuse.tar.gz")
build([
if (env.BRANCH_NAME == "master" && cause != "upstream") {
build([
job: "/kbfs-server/master",
parameters: [
string(
name: 'kbfsProjectName',
value: env.JOB_NAME,
value: env.BUILD_TAG,
),
string(
name: 'kbwebProjectName',
value: kbwebTag,
),
]
])
])
}
}
}
}
},
}},
)
},
test_windows: {
// TODO: If we re-enable tests other than Go tests on
// Windows, this check should go away.
if (hasGoChanges) {
helpers.nodeWithCleanup('windows-ssh', {}, {}) {
def BASEDIR="${pwd()}"
Expand Down Expand Up @@ -307,65 +268,14 @@ helpers.rootLinuxNode(env, {
}
}
},
test_macos: {
// TODO: remove once macos runners are back up
if (false) {
def mountDir='/Volumes/untitled/client'
helpers.nodeWithCleanup('macstadium', {}, {
sh "rm -rf ${mountDir} || echo 'Something went wrong with cleanup.'"
}) {
def BASEDIR="${pwd()}/${env.BUILD_NUMBER}"
def GOPATH="${BASEDIR}/go"
dir(mountDir) {
// Ensure that the mountDir exists
sh "touch test.txt"
}
withEnv([
"GOPATH=${GOPATH}",
"NODE_PATH=${env.HOME}/.node/lib/node_modules:${env.NODE_PATH}",
"PATH=${env.PATH}:${GOPATH}/bin:${env.HOME}/.node/bin",
"KEYBASE_SERVER_URI=http://${kbwebNodePrivateIP}:3000",
"KEYBASE_PUSH_SERVER_URI=fmprpc://${kbwebNodePrivateIP}:9911",
"TMPDIR=${mountDir}",
]) {
ws("$GOPATH/src/github.com/keybase/client") {
println "Checkout OS X"
retry(3) {
checkout scm
}

parallel (
//test_react_native: {
// println "Test React Native"
// dir("react-native") {
// sh "npm i"
// lock("iossimulator_${env.NODE_NAME}") {
// sh "npm run test-ios"
// }
// }
//},
test_macos_go: {
if (hasGoChanges) {
dir("go/keybase") {
sh "go build -ldflags \"-s -w\" --tags=production"
}
testGo("test_macos_go_", getPackagesToTest(dependencyFiles))
}
}
)
}}
}
}
},
)
}
}

stage("Push") {
if (env.BRANCH_NAME == "master" && cause != "upstream") {
docker.withRegistry("https://docker.io", "docker-hub-creds") {
clientImage.push()
sh "docker push keybaseprivate/kbfsfuse"
docker.withRegistry('https://897413463132.dkr.ecr.us-east-1.amazonaws.com', 'ecr:us-east-1:aws-ecr-user') {
kbfsfuseImage.push('master')
}
} else {
println "Not pushing docker"
Expand Down Expand Up @@ -468,14 +378,14 @@ def testGo(prefix, packagesToTest) {
def testGoBuilds(prefix, packagesToTest) {
if (prefix == "test_linux_go_") {
dir("keybase") {
sh "go build -ldflags \"-s -w\" -buildmode=pie --tags=production"
sh "go build -o keybase_production -ldflags \"-s -w\" -buildmode=pie --tags=production"
}
dir("fuzz") {
sh "go build -tags gofuzz ./..."
}
} else if (prefix == "test_windows_go_") {
dir("keybase") {
sh "go build -ldflags \"-s -w\" --tags=production"
sh "go build -o keybase_production -ldflags \"-s -w\" --tags=production"
}
}

Expand Down Expand Up @@ -581,25 +491,6 @@ def testGoTestSuite(prefix, packagesToTest) {
'ephemeral'
]
def testSpecMap = [
test_macos_go_: [
'github.com/keybase/client/go/kbfs/test': [
name: 'kbfs_test_fuse',
flags: '-tags fuse',
timeout: '15m',
],
'github.com/keybase/client/go/kbfs/libfuse': [
timeout: '3m',
],
'github.com/keybase/client/go/libkb': [
timeout: '5m',
],
'github.com/keybase/client/go/install': [
timeout: '30s',
],
'github.com/keybase/client/go/launchd': [
timeout: '30s',
],
],
test_linux_go_: [
'*': [],
'github.com/keybase/client/go/kbfs/test': [
Expand Down Expand Up @@ -714,6 +605,9 @@ def testGoTestSuite(prefix, packagesToTest) {
flags: '-race',
timeout: '30s',
],
'github.com/keybase/client/go/systests': [
// citogo_extra: '-parallel=8', TODO: re-enable later
],
],
test_windows_go_: [
'*': [],
Expand Down Expand Up @@ -783,7 +677,7 @@ def testGoTestSuite(prefix, packagesToTest) {
if (testSpec.no_citogo) {
sh "./${testBinary} -test.timeout ${testSpec.timeout}"
} else {
sh "citogo --flakes 3 --fails 3 --build-id ${env.BUILD_ID} --branch ${env.BRANCH_NAME} --prefix ${testSpec.dirPath} --s3bucket ci-fail-logs --report-lambda-function report-citogo --build-url ${env.BUILD_URL} --no-compile --test-binary ./${testBinary} --timeout 150s ${testSpec.citogo_extra || ''}"
sh "citogo --flakes 3 --fails 3 --build-id ${env.BUILD_ID} --branch ${env.BRANCH_NAME} --prefix ${testSpec.dirPath} --s3bucket ci-fail-logs --report-lambda-function report-citogo --build-url ${env.BUILD_URL} --no-compile --test-binary ./${testBinary} --timeout 150s ${testSpec.citogo_extra ? testSpec.citogo_extra : ''}"
}
}
}
Expand Down
Loading

0 comments on commit 87690c0

Please sign in to comment.