Skip to content

Commit 04a850a

Browse files
sdolencsdolenc
authored
continuous integration for oxa-tools (microsoft#283)
* removing sudo * return nonzero if failed * use stepdo fork with systemd reload fix * force correct configuration fork * use edx fork * simplify * don't use fork * skip-tags=edxapp-sandbox * refactor git remote add * use msft fork for sanbox.sh * containers can't update hosts file so we a) pass the docker param and b) allow failure for that play * try source instead of bash * bash instead of source * use other hash for cherry pick * fix hosts cherry-pick for all ficus deployments * add relevant github hyperlink for cherry picks * support ansible-bootstrap retries (due to network timeouts) * fix conditional (for hosts fix) * updating hosts file hash fix * cherry pick some ansible try catch fixes * allow ansible-bootsrap.sh retry for ficus too * resolving merge conflict * minor fixes * improve cherry picks and grep * better console messaging for retry-command * configuration for tests * more aggressive error detection * missing quotation * reduce apt console output * reduce console output * reduce console messaging * temporarily verbose * temporarily more verbose * simplify conditional * long job * try to access circle env values * branch and repo optimizations * use circle env details for repo * simplify config before using anchors and aliases * two jobs * fix yaml formatting * try fixing yaml conf * fix yaml conf * fix yml conf * fix yaml conf * fix yml conf * fix yaml conf * fix yml conf * three tests * fix workflows * expand to all six cases * extend keep alive script duration * simplify grep conditionals * use default oxa-tools branch * sort jobs from longest to shortest * remove extraneous set bash directives * setup reasonable branch filters and scheduled triggers * fix yaml (microsoft#282) * yaml fix * yml fix * descriptive names for test jobs
1 parent be1b024 commit 04a850a

File tree

7 files changed

+335
-39
lines changed

7 files changed

+335
-39
lines changed

‎.circleci/config.yml

+123
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
# Python CircleCI 2.0 configuration file
2+
#
3+
# Check https://circleci.com/docs/2.0/language-python/ for more details
4+
#
5+
version: 2
6+
jobs:
7+
FsDevFic:
8+
environment:
9+
- ONEBOX_PARAMS: "--branch Dev --Role Fullstack"
10+
<<: &sharedSetup
11+
# VM allows for "priveleged" containers.
12+
machine: true
13+
working_directory: ~/repo
14+
steps:
15+
- checkout
16+
- run: |
17+
docker pull solita/ubuntu-systemd:16.04
18+
docker run --rm --privileged -v /:/host solita/ubuntu-systemd:16.04 setup
19+
# Prepare cgroup on the host at /tmp/cgroup to avoid error: "failed to connect to bus: no such file or directory"
20+
sudo mkdir -p /tmp/cgroup/systemd && sudo mount -t cgroup systemd /tmp/cgroup/systemd -o ro,noexec,nosuid,nodev,none,name=systemd
21+
- run: |
22+
# Host entries from edx-configuration/playbooks/roles/local_dev/tasks/main.yml
23+
# "priveleged" to support THP changes in edx-configuration/playbooks/roles/mongo_3_2/tasks/main.yml
24+
docker run -d --name $(echo "$ONEBOX_PARAMS" | tr -d "-" | tr -d " ") \
25+
--add-host "localhost preview.localhost":127.0.0.1 \
26+
--privileged \
27+
--security-opt seccomp=unconfined \
28+
--tmpfs /run \
29+
--tmpfs /run/lock \
30+
-v /sys/fs/cgroup:/sys/fs/cgroup:ro \
31+
-t solita/ubuntu-systemd:16.04
32+
- run:
33+
name: run tests
34+
command: source ./tests/deploy-onebox.sh
35+
DsDevFic:
36+
environment:
37+
- ONEBOX_PARAMS: "--branch Dev --Role Devstack"
38+
<<: *sharedSetup
39+
FsGink:
40+
environment:
41+
- ONEBOX_PARAMS: "--branch Ginkgo --Role Fullstack"
42+
<<: *sharedSetup
43+
DsGink:
44+
environment:
45+
- ONEBOX_PARAMS: "--branch Ginkgo --Role Devstack"
46+
<<: *sharedSetup
47+
FsEdxMaster:
48+
environment:
49+
- ONEBOX_PARAMS: "--branch Edx_master --Role Fullstack"
50+
<<: *sharedSetup
51+
DsEdxMaster:
52+
environment:
53+
- ONEBOX_PARAMS: "--branch Edx_master --Role Devstack"
54+
<<: *sharedSetup
55+
FsReleaseFic:
56+
environment:
57+
- ONEBOX_PARAMS: "--branch Release --Role Fullstack"
58+
<<: *sharedSetup
59+
DsReleaseFic:
60+
environment:
61+
- ONEBOX_PARAMS: "--branch Release --Role Devstack"
62+
<<: *sharedSetup
63+
FsMasterFic:
64+
environment:
65+
- ONEBOX_PARAMS: "--branch Master --Role Fullstack"
66+
<<: *sharedSetup
67+
DsMasterFic:
68+
environment:
69+
- ONEBOX_PARAMS: "--branch Master --Role Devstack"
70+
<<: *sharedSetup
71+
72+
# Jobs are sorted from longest duration to shortest (approximately)
73+
# This generally means the shortest duration overall.
74+
workflows:
75+
version: 2
76+
all:
77+
jobs:
78+
- FsGink:
79+
<<: &releaseFilter
80+
filters:
81+
branches:
82+
only:
83+
- oxa/release.fic
84+
- DsEdxMaster:
85+
<<: *releaseFilter
86+
- DsGink:
87+
<<: *releaseFilter
88+
- DsDevFic:
89+
<<: &devFilter
90+
filters:
91+
branches:
92+
only:
93+
- oxa/dev.fic
94+
- DsReleaseFic:
95+
<<: *releaseFilter
96+
- DsMasterFic:
97+
<<: &masterFilter
98+
filters:
99+
branches:
100+
only:
101+
- oxa/master.fic
102+
- FsDevFic:
103+
<<: *devFilter
104+
- FsReleaseFic:
105+
<<: *releaseFilter
106+
- FsMasterFic:
107+
<<: *masterFilter
108+
scheduled:
109+
triggers:
110+
- schedule:
111+
cron: "7 */25 * * *"
112+
filters:
113+
branches:
114+
only:
115+
- oxa/dev.fic
116+
jobs:
117+
# FsEdxMaster takes a little over two hours which is the timeout on circleci
118+
# If we want to re-nenable it then we'll need to do some "perf tuning"
119+
- FsGink
120+
- DsEdxMaster
121+
- DsGink
122+
- DsDevFic
123+
- FsDevFic

‎onebox.sh

+33-12
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ set -ae
99
# static strings
1010
readonly MSFT="microsoft"
1111
readonly EDX="edx"
12+
readonly CONF="configuration"
13+
readonly E_CONF="${EDX}-${CONF}"
1214
readonly USE_MSFT="useMsftRepo"
1315
readonly TAGS="tags/"
1416
readonly FICUS="${TAGS}open-release/ficus.1"
@@ -259,6 +261,20 @@ get_branch()
259261
fi
260262
}
261263

264+
get_current_org()
265+
{
266+
organization=$MSFT
267+
268+
if git status > /dev/null ; then
269+
remoteUrl="$(git config --get remote.origin.url)"
270+
if echo $remoteUrl | grep -i "http" > /dev/null ; then
271+
organization=$(echo $remoteUrl | tr / "\n" | head -4 | tail -1)
272+
fi
273+
fi
274+
275+
echo $organization
276+
}
277+
262278
get_current_branch()
263279
{
264280
prefix='* '
@@ -310,10 +326,10 @@ get_conf_project_name()
310326
{
311327
case "$BRANCH_VERSIONS" in
312328
edx_f|edx_g|edx_master)
313-
echo "configuration"
329+
echo "$CONF"
314330
;;
315331
*)
316-
echo "edx-configuration"
332+
echo "$E_CONF"
317333
;;
318334
esac
319335
}
@@ -344,7 +360,7 @@ wget_wrapper()
344360

345361
install-with-oxa()
346362
{
347-
bootstrap=`wget_wrapper "scripts/bootstrap.sh" "${MSFT}" "oxa-tools" "$(get_current_branch)"`
363+
bootstrap=`wget_wrapper "scripts/bootstrap.sh" "$(get_current_org)" "oxa-tools" "$(get_current_branch)"`
348364

349365
bash $bootstrap \
350366
--role \
@@ -355,6 +371,8 @@ install-with-oxa()
355371
"dev" \
356372
--msft-oauth \
357373
$MSFT_AUTH \
374+
--oxatools-public-github-accountname \
375+
`get_current_org` \
358376
--oxatools-public-github-projectbranch \
359377
`get_current_branch` \
360378
--edxconfiguration-public-github-accountname \
@@ -410,24 +428,28 @@ install-with-edx-native()
410428
# 1. Set the OPENEDX_RELEASE variable:
411429
OPENEDX_RELEASE=${EDX_BRANCH#$TAGS}
412430

431+
# Enable retry
432+
local utilities=`wget_wrapper "templates/stamp/utilities.sh" "$(get_current_org)" "oxa-tools" "$(get_current_branch)"`
433+
source $utilities
434+
413435
# 2. Bootstrap the Ansible installation:
414-
local ans_bootstrap=`wget_wrapper "util/install/ansible-bootstrap.sh" "${EDX}" "$(get_conf_project_name)" "$OPENEDX_RELEASE"`
415-
sudo bash $ans_bootstrap
436+
local ans_bootstrap=`wget_wrapper "util/install/ansible-bootstrap.sh" "${MSFT}" "$E_CONF" "ginkgo1tweaks"`
437+
set +e
438+
retry-command "bash $ans_bootstrap" 3 "$ans_bootstrap"
439+
exit_on_error "Execution of edX ansible bootstrap failed"
440+
set -e
416441

417442
# 3. (Optional) If this is a new installation, randomize the passwords:
418443
# todo: reconcile this w/ -d and /oxa/oxa.yml
419444
local gen_pass=`wget_wrapper "util/install/generate-passwords.sh" "${EDX}" "$(get_conf_project_name)" "$OPENEDX_RELEASE"`
420445
bash $gen_pass
421446

422-
# 3b Enable retry
423-
local utilities=`wget_wrapper "templates/stamp/utilities.sh" "${MSFT}" "oxa-tools" "$(get_current_branch)"`
424-
source $utilities
425-
426447
# 4. Install Open edX:
427-
local sandbox=`wget_wrapper "util/install/sandbox.sh" "${EDX}" "$(get_conf_project_name)" "$OPENEDX_RELEASE"`
448+
local sandbox=`wget_wrapper "util/install/sandbox.sh" "${MSFT}" "$E_CONF" "ginkgo1tweaks"`
428449
devstack_preconditions $sandbox
429450
set +e
430-
retry-command "bash $sandbox" 8 "$sandbox" "fixPackages"
451+
retry-command "bash $sandbox --skip-tags=edxapp-sandbox" 8 "$sandbox" "fixPackages"
452+
exit_on_error "Execution of edX sandbox playbook failed"
431453
set -e
432454

433455
# get status of edx services
@@ -448,7 +470,6 @@ test_args
448470

449471
set_dynamic_vars
450472

451-
set -x
452473
# We currently use sandbox.sh for ginkgo+. Therefore, it doesn't have our customizations.
453474
# - (fullstack) This is because vagrant-fullstack.yml was removed in March 2017 and
454475
# - (devstack) Something about our customizations result in an "elastic search" error

‎playbooks/roles/edxapp/files/enable_msft_oauth.sh

+8-12
Original file line numberDiff line numberDiff line change
@@ -25,21 +25,17 @@ fix_platform()
2525

2626
log "cherry-pick change to support MSA"
2727

28-
count=`grep -i "live" lms/envs/aws.py | wc -l`
29-
if (( "$count" == 0 )) ; then
28+
if ! ( grep -i "live" lms/envs/aws.py ) ; then
3029
log "Ensure remote has commit"
31-
count=`git remote | grep "msft_plat" | wc -l`
32-
if (( "$count" == 0 )) ; then
33-
git remote add msft_plat https://github.com/microsoft/edx-platform.git
34-
fi
35-
git fetch msft_plat > /dev/null 2>&1
30+
add_remote msft_plat https://github.com/microsoft/edx-platform.git
3631

37-
# Ficus Fix
38-
hash=86a53b8353ac76fbb761d4fa465bc0f363e264b3
32+
# Ficus fix. Apply
33+
# https://github.com/Microsoft/edx-platform/pull/158
34+
hash=030848e7d7c79dccd080df77f02894426c3232fd
3935

40-
# Ginkgo fix
41-
count=`grep -i "social_core" lms/envs/aws.py | wc -l`
42-
if (( "$count" > 0 )) ; then
36+
# Ginkgo and later fix. Applying part of
37+
# https://github.com/Microsoft/edx-platform/compare/ginkgo1...Microsoft:ginkgo1tweaks
38+
if grep -i "social_core" lms/envs/aws.py ; then
4339
hash=dd939e404c9f762b71eabb67f3340c14ba5ba9c3
4440
fi
4541

‎scripts/bootstrap.sh

+35-14
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22
# Copyright (c) Microsoft Corporation. All Rights Reserved.
33
# Licensed under the MIT license. See LICENSE file on the project webpage for details.
44

5-
set -x
6-
75
# argument defaults
86
EDX_ROLE=""
97
DEPLOYMENT_ENV="dev"
@@ -270,7 +268,6 @@ setup_overrides_file()
270268
required_value()
271269
{
272270
if [[ -z $2 ]] ; then
273-
set +x
274271
echo -e "\033[1;36m"
275272
echo -e "\n\n Use onebox.sh for fullstack or devstack instead of invoking bootstrap.sh directly.\n\n"
276273
echo -e '\033[0m'
@@ -299,20 +296,43 @@ verify_state()
299296

300297
fix_jdk()
301298
{
302-
count=`grep -i "8u65" playbooks/roles/oraclejdk/defaults/main.yml | wc -l`
303-
if [[ "$count" -gt "0" ]] ; then
299+
# Apply https://github.com/edx/configuration/pull/3881
300+
if grep -i "8u65" playbooks/roles/oraclejdk/defaults/main.yml ; then
304301
cherry_pick_wrapper 0ca865c9b0da42bed83459389ae35e2551860472 "$EDXAPP_SU_EMAIL"
305302
fi
306303
}
307304

308305
fix_npm_python()
309306
{
310-
count=`grep -i "node_modules" playbooks/roles/edxapp/tasks/deploy.yml | wc -l`
311-
if (( "$count" == 0 )) ; then
307+
# Apply https://github.com/edx/configuration/pull/4101
308+
if ! ( grep -i "node_modules" playbooks/roles/edxapp/tasks/deploy.yml ) ; then
312309
cherry_pick_wrapper 075d69e6c7c5330732ec75346d02df32d087aa92 "$EDXAPP_SU_EMAIL"
313310
fi
314311
}
315312

313+
fix_hosts_file()
314+
{
315+
# Apply https://github.com/Microsoft/edx-configuration/pull/90
316+
add_remote msft_conf "https://github.com/microsoft/edx-configuration.git"
317+
if grep "127.0.0.1 localhost" playbooks/roles/local_dev/tasks/main.yml ; then
318+
cherry_pick_wrapper f3d59dd09dbbd8b60c9049292c3c814f4de715c5 "$EDXAPP_SU_EMAIL"
319+
fi
320+
}
321+
322+
ansible_try_catch()
323+
{
324+
add_remote msft_conf "https://github.com/microsoft/edx-configuration.git"
325+
326+
# Apply https://github.com/Microsoft/edx-configuration/pull/91
327+
cherry_pick_wrapper a6304eaaefc24d2c3c59d57606c059cdd75b1dd4 "$EDXAPP_SU_EMAIL"
328+
329+
# Apply https://github.com/Microsoft/edx-configuration/pull/92
330+
count=$(grep -c -i "name: install python requirements" playbooks/roles/edxapp/tasks/deploy.yml)
331+
if [[ "$count" -lt "2" ]] ; then
332+
cherry_pick_wrapper 5ed320bea2174c37d28b9cc6fe14fa90d83220dd "$EDXAPP_SU_EMAIL"
333+
fi
334+
}
335+
316336
# We should use the existing oxa-tools enlistment if one exists. This
317337
# a) saves us a git clone AND
318338
# b) preserves our current branch/changes
@@ -324,7 +344,7 @@ link_oxa_tools_repo()
324344
# Is the git repo oxa-tools?
325345
pushd ..
326346
actualRemote=`git config --get remote.origin.url | grep -o 'github.com.*' | sed 's/\.git//g'`
327-
count=`echo $OXA_TOOLS_REPO | grep -i $actualRemote | wc -l`
347+
count=`echo $OXA_TOOLS_REPO | grep -i -c $actualRemote`
328348
if [[ "$count" -gt "0" ]] ; then
329349
# Is the repo already at the desired path?
330350
pushd ..
@@ -385,10 +405,15 @@ setup()
385405

386406
# run edx bootstrap and install requirements
387407
cd $CONFIGURATION_PATH
408+
409+
# Cherry pick fixes
388410
fix_jdk
389411
fix_npm_python
412+
fix_hosts_file
413+
ansible_try_catch
414+
390415
ANSIBLE_BOOTSTRAP_SCRIPT=util/install/ansible-bootstrap.sh
391-
bash $ANSIBLE_BOOTSTRAP_SCRIPT
416+
retry-command "bash $ANSIBLE_BOOTSTRAP_SCRIPT" 3 "$ANSIBLE_BOOTSTRAP_SCRIPT"
392417
exit_on_error "Failed executing $ANSIBLE_BOOTSTRAP_SCRIPT"
393418

394419
pip install -r requirements.txt
@@ -598,11 +623,7 @@ setup_overrides_file
598623
# setup crumbs for tracking purposes
599624
TARGET_FILE=/var/log/bootstrap-$EDX_ROLE.log
600625

601-
if [ "$CRON_MODE" == "1" ];
602-
then
603-
# turn off the debug messages since we have proper logging by now
604-
# set +x
605-
626+
if [[ "$CRON_MODE" == "1" ]] ; then
606627
echo "Cron execution for ${EDX_ROLE} on ${HOSTNAME} detected."
607628

608629
# check if we need to run the setup

‎templates/stamp/utilities.sh

+13-1
Original file line numberDiff line numberDiff line change
@@ -336,7 +336,7 @@ retry-command()
336336
local tasksOfPrev=
337337
local alreadyUpgraded=
338338
for (( a=1; a<=$retry_count; a++ )) ; do
339-
message="$optionalDescription attempt number: $a"
339+
message="$optionalDescription attempt number: $a of $retry_count"
340340

341341
# Some failures can be resolved by fixing packages.
342342
if [[ -n "$fix_packages" ]] ; then
@@ -638,6 +638,18 @@ sync_repo()
638638
popd
639639
}
640640

641+
add_remote()
642+
{
643+
local remoteName=$1
644+
local remoteUrl="$2"
645+
646+
if ! ( git remote | grep "$remoteName" ) ; then
647+
git remote add $remoteName $remoteUrl
648+
fi
649+
650+
git fetch $remoteName > /dev/null 2>&1
651+
}
652+
641653
cherry_pick_wrapper()
642654
{
643655
local hash=$1

0 commit comments

Comments
 (0)