forked from broadinstitute/gatk
-
Notifications
You must be signed in to change notification settings - Fork 0
/
.travis.yml
189 lines (185 loc) · 8.36 KB
/
.travis.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
language: java
sudo: required
dist: trusty
group: travis_lts
git:
depth: 9999999
lfs_skip_smudge: true
jdk:
- openjdk8
env:
matrix:
- TEST_TYPE=cloud UPLOAD=true
- TEST_TYPE=integration TEST_DOCKER=true TEST_VERBOSITY=minimal
- TEST_TYPE=unit TEST_DOCKER=true TEST_VERBOSITY=minimal
- TEST_TYPE=variantcalling TEST_DOCKER=true TEST_VERBOSITY=minimal
- TEST_TYPE=python TEST_DOCKER=true TEST_VERBOSITY=minimal
- RUN_CNV_GERMLINE_WDL=true
- RUN_CNV_SOMATIC_WDL=true
- RUN_M2_WDL=true
global:
#gradle needs this
- TERM=dumb
#limit gradle jvm memory and disable daemon
- GRADLE_OPTS="-Xmx2048m -Dorg.gradle.daemon=false"
#google cloud stuff
- CLOUDSDK_CORE_DISABLE_PROMPTS=1
- GCLOUD_HOME=$HOME/gcloud/google-cloud-sdk/bin
- CLOUDSDK_PYTHON_SITEPACKAGES=1
- GOOGLE_APPLICATION_CREDENTIALS=$TRAVIS_BUILD_DIR/servicekey.json
#hellbender specific variables
- HELLBENDER_TEST_INPUTS=gs://hellbender/test/resources/
- HELLBENDER_TEST_STAGING=gs://hellbender-test-logs/staging/
- HELLBENDER_TEST_LOGS=/hellbender-test-logs/build_reports/
- HELLBENDER_TEST_PROJECT=broad-dsde-dev
- HELLBENDER_JSON_SERVICE_ACCOUNT_KEY=servicekey.json
#for uploading artifacts
- ARTIFACTORY_USERNAME=gatkci
#artifactory password
- secure: "E0LWXgX3aWSE/DWHXXDx4vrAq4uX6vKg402wToaZ5otbHQ/UP0H7/FA5jomavAXoC46oMVHZcEltZ5OVhuJ0NW8yYxUCecJ1D/YvVQmnfFABcV/qLM+k4e2rYQOKVw/pejB2gG8XdTA+XE2WyTeENbmIkputS8f1ndKWCmZxuuk="
# For cromwell jar download
- CROMWELL_JAR=$HOME/cromwell-30.2.jar
matrix:
fast_finish: true
include:
- jdk: oraclejdk8
env: TEST_TYPE=integration TEST_VERBOSITY=minimal
before_cache:
- rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
- rm -fr $HOME/.gradle/caches/*/plugin-resolution/
cache:
directories:
- $HOME/.gradle/caches/
- $HOME/.gradle/wrapper/
- $HOME/.m2
- $HOME/gcloud/
- $HOME/site-library
before_install:
- REPORT_PATH=${TRAVIS_BRANCH}_${TRAVIS_JOB_NUMBER};
- if [[ $TRAVIS_SECURE_ENV_VARS == true && $TRAVIS_EVENT_TYPE != cron ]]; then
echo "Test report will be written to https://storage.googleapis.com$HELLBENDER_TEST_LOGS$REPORT_PATH/tests/test/index.html";
fi
#setup google cloud and github authentication
- if [ $TRAVIS_SECURE_ENV_VARS == true ]; then
export BOTO_CONFIG=/dev/null;
openssl aes-256-cbc -K $encrypted_29f3b7c4d8c3_key -iv $encrypted_29f3b7c4d8c3_iv -in resources_for_CI/servicekey.json.enc -out servicekey.json -d;
scripts/install_gcloud.sh;
printf 'y\n' | $GCLOUD_HOME/gcloud components update;
if [[ $TEST_TYPE == cloud ]]; then
printf 'y\n' | $GCLOUD_HOME/gcloud components install beta;
fi;
$GCLOUD_HOME/gcloud config set project broad-dsde-dev;
$GCLOUD_HOME/gcloud auth activate-service-account --key-file servicekey.json;
$GCLOUD_HOME/gcloud --version
openssl aes-256-cbc -K $encrypted_4823d58debd3_key -iv $encrypted_4823d58debd3_iv -in resources_for_CI/github_deploy_key.enc -out ~/.ssh/id_rsa -d;
chmod 600 ~/.ssh/id_rsa;
echo "Testing github authentication:";
ssh -T [email protected];
echo "Done testing github authentication";
fi
#install R
- if [[ $TEST_DOCKER != true ]]; then
sudo mkdir -p /usr/local/lib/R/;
sudo mkdir -p site-library;
sudo ln -sFv ~/site-library /usr/local/lib/R/site-library;
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9;
sudo add-apt-repository "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/";
sudo apt-get update;
sudo apt-get install -y --force-yes r-base-dev=3.1.3-1trusty;
sudo apt-get install -y --force-yes r-base-core=3.1.3-1trusty;
sudo Rscript scripts/docker/gatkbase/install_R_packages.R;
fi
# Download Cromwell jar -- if you change the version, please change the CROMWELL_JAR env variable above, too.
- if [[ $RUN_CNV_GERMLINE_WDL == true || $RUN_CNV_SOMATIC_WDL == true || $RUN_M2_WDL == true ]]; then
wget -O $CROMWELL_JAR https://github.com/broadinstitute/cromwell/releases/download/30.2/cromwell-30.2.jar;
fi
# Download git lfs files
- git lfs version
- git lfs install
- git lfs fetch
- git lfs checkout
install:
- if [[ $TRAVIS_SECURE_ENV_VARS == false && $TEST_TYPE == cloud ]]; then
echo "Can't run cloud tests without keys so don't bother building";
elif [[ $TEST_DOCKER == true ]]; then
echo "Skip the install because we're doing a docker build";
else
./gradlew assemble;
./gradlew installDist;
if [[ $RUN_CNV_GERMLINE_WDL == true || $RUN_CNV_SOMATIC_WDL == true || $RUN_M2_WDL == true ]]; then
echo "building a shadow jar for the wdl";
./gradlew shadowJar;
elif [[ $TEST_TYPE == cloud ]]; then
echo "building a spark jar for the dataproc tests";
./gradlew sparkjar;
fi;
fi
script:
# run a basic sanity check to be sure that gatk doesn't explode
# run tests
- if [[ $TRAVIS_EVENT_TYPE == cron ]]; then
echo "Not running any tests for nightly builds";
elif [[ $TRAVIS_SECURE_ENV_VARS == false && $TEST_TYPE == cloud ]]; then
echo "Can't run cloud tests without keys so don't run tests";
elif [[ $RUN_CNV_GERMLINE_WDL == true ]]; then
echo "Running CNV germline workflows";
bash scripts/cnv_cromwell_tests/germline/run_cnv_germline_workflows.sh;
elif [[ $RUN_CNV_SOMATIC_WDL == true ]]; then
echo "Running CNV somatic workflows";
bash scripts/cnv_cromwell_tests/somatic/run_cnv_somatic_workflows.sh;
elif [[ $RUN_M2_WDL == true ]]; then
echo "Deleting some unused files before running M2 WDL...";
rm -Rf src/test/resources/large/VQSR;
echo "Running M2 WDL";
sudo bash scripts/m2_cromwell_tests/run_m2_wdl.sh;
elif [[ $TEST_DOCKER == true ]]; then
echo "Building docker image and running appropriate tests..." ;
if [ ${TRAVIS_PULL_REQUEST} != false ]; then
sudo bash build_docker.sh -e FETCH_HEAD -s -u -t ${TRAVIS_PULL_REQUEST};
DOCKER_TAG=FETCH_HEAD;
else
echo ${TRAVIS_COMMIT};
sudo bash build_docker.sh -e ${TRAVIS_COMMIT} -s -u;
DOCKER_TAG=$TRAVIS_COMMIT;
fi;
sudo docker images;
echo ${TEST_TYPE};
sudo mkdir -p build/reports/;
sudo chmod -R a+w build/reports/;
cp scripts/docker/dockertest.gradle .;
sudo docker run -v $(pwd):/gatkCloneMountPoint:cached -v $(pwd)/testJars:/jars:cached --rm -e "TEST_VERBOSITY=minimal" -e "TEST_TYPE=${TEST_TYPE}" -t broadinstitute/gatk:${DOCKER_TAG} bash --init-file /gatk/gatkenv.rc /root/run_unit_tests.sh;
TEST_EXIT_VALUE=$?;
sudo mkdir build/reports/tests/test && sudo cp -rp build/reports/tests/testOnPackagedReleaseJar/* build/reports/tests/test && sudo rm -r build/reports/tests/testOnPackagedReleaseJar;
$( exit ${TEST_EXIT_VALUE} );
else
./gatk PrintReads -I src/test/resources/NA12878.chr17_69k_70k.dictFix.bam -O output.bam;
travis_wait 50 ./gradlew jacocoTestReport;
fi
after_success:
- bash <(curl -s https://codecov.io/bash)
- if [[ $TRAVIS_BRANCH == master && $UPLOAD == true ]]; then ./gradlew uploadArchives; fi
# if this is actually a commit to master and not a pull request build into master, then publish master-snapshot
- if [[ $TRAVIS_BRANCH == master && $TRAVIS_PULL_REQUEST == false && $UPLOAD == true ]]; then
git tag master;
./gradlew uploadArchives;
fi
# This creates and uploads the gatk zip file to the nightly build bucket, only keeping the 10 newest entries
# This also pokes the Dockerhub web api which is securely stored in the travis API with the variable "DOCKERHUB_URL"
- if [[ $TRAVIS_BRANCH == master && $TRAVIS_EVENT_TYPE == cron && $UPLOAD == true ]]; then
$GCLOUD_HOME/gcloud components -q update gsutil;
gsutil ls -l gs://gatk-nightly-builds | grep gatk | sort -r -k 2 | grep -o '\S\+$' | tail -n +11 | xargs -I {} gsutil rm {};
./gradlew bundle;
ZIP_FILE="$(ls build/ | grep .zip)";
echo "Uploading zip to gs://gatk-nightly-builds/";
$GCLOUD_HOME/gsutil -m cp build/$ZIP_FILE gs://gatk-nightly-builds/"$(date +%Y-%m-%d)"-$ZIP_FILE;
echo "Triggering an automatic build on dockerhub";
curl -H "Content-Type:application/json" --data '{"build":true}' -X POST ${DOCKERHUB_URL};
fi
after_failure:
- dmesg | tail -100
after_script:
- if [[ $TRAVIS_SECURE_ENV_VARS == true && $TRAVIS_EVENT_TYPE != cron ]]; then
$GCLOUD_HOME/gcloud components -q update gsutil;
$GCLOUD_HOME/gsutil -m cp -z html -z js -z xml -z css -r build/reports/tests gs:/$HELLBENDER_TEST_LOGS$REPORT_PATH/;
echo "See the test report at https://storage.googleapis.com$HELLBENDER_TEST_LOGS$REPORT_PATH/tests/test/index.html";
fi