forked from openedx/edx-platform
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathall-tests.sh
executable file
·203 lines (174 loc) · 6.82 KB
/
all-tests.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
#!/usr/bin/env bash
set -e
###############################################################################
#
# edx-all-tests.sh
#
# Execute all tests for edx-platform.
#
# This script can be called from a Jenkins
# multiconfiguration job that defines these environment
# variables:
#
# `TEST_SUITE` defines which kind of test to run.
# Possible values are:
#
# - "quality": Run the quality (pep8/pylint) checks
# - "unit": Run the JavaScript and Python unit tests
# (also tests building the Sphinx documentation,
# because we couldn't think of a better place to put it)
# - "lms-acceptance": Run the acceptance (Selenium) tests for the LMS
# - "cms-acceptance": Run the acceptance (Selenium) tests for Studio
# - "bok-choy": Run acceptance tests that use the bok-choy framework
#
# `SHARD` is a number (1, 2, or 3) indicating which subset of the tests
# to build. Currently, "lms-acceptance" and "bok-choy" each have two
# shards (1 and 2), "cms-acceptance" has three shards (1, 2, and 3),
# and all the other test suites have one shard.
#
# For the "bok-choy", the tests are put into shard groups using the nose
# 'attr' decorator (e.g. "@attr('shard_1')"). Currently, anything with
# the 'shard_1' attribute will run in the first shard. All other bok-choy
# tests will run in shard 2.
#
# For the lettuce acceptance tests, ("lms-" and "cms-acceptance") they
# are decorated with "@shard_{}" (e.g. @shard_1 for the first shard).
# The lettuce tests must have a shard specified to be run in jenkins,
# as there is no shard that runs unspecified tests.
#
#
# Jenkins configuration:
#
# - The edx-platform git repository is checked out by the Jenkins git plugin.
#
# - Jenkins logs in as user "jenkins"
#
# - The Jenkins file system root is "/home/jenkins"
#
# - An init script creates a virtualenv at "/home/jenkins/edx-venv"
# with some requirements pre-installed (such as scipy)
#
# Jenkins worker setup:
# See the edx/configuration repo for Jenkins worker provisioning scripts.
# The provisioning scripts install requirements that this script depends on!
#
###############################################################################
# Violations thresholds for failing the build
PYLINT_THRESHOLD=4600
PEP8_THRESHOLD=0
source $HOME/jenkins_env
# Clean up previous builds
git clean -qxfd
# Clear the mongo database
# Note that this prevents us from running jobs in parallel on a single worker.
mongo --quiet --eval 'db.getMongo().getDBNames().forEach(function(i){db.getSiblingDB(i).dropDatabase()})'
# Ensure we have fetched origin/master
# Some of the reporting tools compare the checked out branch to origin/master;
# depending on how the GitHub plugin refspec is configured, this may
# not already be fetched.
git fetch origin master:refs/remotes/origin/master
# Reset the jenkins worker's ruby environment back to
# the state it was in when the instance was spun up.
if [ -e $HOME/edx-rbenv_clean.tar.gz ]; then
rm -rf $HOME/.rbenv
tar -C $HOME -xf $HOME/edx-rbenv_clean.tar.gz
fi
# Bootstrap Ruby requirements so we can run the tests
bundle install
# Ensure the Ruby environment contains no stray gems
bundle clean --force
# Reset the jenkins worker's virtualenv back to the
# state it was in when the instance was spun up.
if [ -e $HOME/edx-venv_clean.tar.gz ]; then
rm -rf $HOME/edx-venv
tar -C $HOME -xf $HOME/edx-venv_clean.tar.gz
fi
# Activate the Python virtualenv
source $HOME/edx-venv/bin/activate
# If the environment variable 'SHARD' is not set, default to 'all'.
# This could happen if you are trying to use this script from
# jenkins and do not define 'SHARD' in your multi-config project.
# Note that you will still need to pass a value for 'TEST_SUITE'
# or else no tests will be executed.
SHARD=${SHARD:="all"}
case "$TEST_SUITE" in
"quality")
paver run_pep8 -l $PEP8_THRESHOLD > pep8.log || { cat pep8.log; EXIT=1; }
paver run_pylint -l $PYLINT_THRESHOLD > pylint.log || { cat pylint.log; EXIT=1; }
# Run quality task. Pass in the 'fail-under' percentage to diff-quality
paver run_quality -p 100
# Need to create an empty test result so the post-build
# action doesn't fail the build.
mkdir -p reports
cat > reports/quality.xml <<END
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="quality" tests="1" errors="0" failures="0" skip="0">
<testcase classname="quality" name="quality" time="0.604"></testcase>
</testsuite>
END
exit $EXIT
;;
"unit")
paver test
paver coverage
;;
"lms-acceptance")
case "$SHARD" in
"all")
paver test_acceptance -s lms --extra_args="-v 3"
;;
*)
paver test_acceptance -s lms --extra_args="-v 3 --tag shard_${SHARD}"
;;
esac
;;
"cms-acceptance")
case "$SHARD" in
"all")
paver test_acceptance -s cms --extra_args="-v 3"
;;
*)
paver test_acceptance -s cms --extra_args="-v 3 --tag shard_${SHARD}"
;;
esac
;;
"bok-choy")
case "$SHARD" in
"all")
paver test_bokchoy
paver bokchoy_coverage
;;
"1")
paver test_bokchoy --extra_args="-a shard_1"
paver bokchoy_coverage
;;
"2")
paver test_bokchoy --extra_args="-a 'shard_2'"
paver bokchoy_coverage
;;
"3")
paver test_bokchoy --extra_args="-a shard_1=False,shard_2=False"
paver bokchoy_coverage
;;
# Default case because if we later define another bok-choy shard on Jenkins
# (e.g. Shard 4) in the multi-config project and expand this file
# with an additional case condition, old branches without that commit
# would not execute any tests on the worker assigned to that shard
# and thus their build would fail.
# This way they will just report 1 test executed and passed.
*)
# Need to create an empty test result so the post-build
# action doesn't fail the build.
# May be unnecessary if we changed the "Skip if there are no test files"
# option to True in the jenkins job definitions.
mkdir -p reports
cat > reports/bok_choy/xunit.xml <<END
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="nosetests" tests="1" errors="0" failures="0" skip="0">
<testcase classname="acceptance.tests" name="shard_placeholder" time="0.001"></testcase>
</testsuite>
END
;;
esac
;;
esac