Skip to content

Commit 508f985

Browse files
yf225yf225
authored
Parallel build for tutorial, and move build to CircleCI (#366)
1 parent e05cfa4 commit 508f985

19 files changed

+546
-60
lines changed

‎.circleci/config.yml

+223
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
install_official_git_client: &install_official_git_client
2+
name: Install Official Git Client
3+
no_output_timeout: "1h"
4+
command: |
5+
set -e
6+
sudo apt-get -qq update
7+
sudo apt-get -qq install openssh-client git
8+
9+
pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
10+
machine:
11+
image: default
12+
steps:
13+
- checkout
14+
- run:
15+
name: Set Up CI Environment
16+
no_output_timeout: "1h"
17+
command: |
18+
set -e
19+
20+
curl -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
21+
echo "deb https://nvidia.github.io/libnvidia-container/ubuntu14.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
22+
echo "deb https://nvidia.github.io/nvidia-container-runtime/ubuntu14.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
23+
echo "deb https://nvidia.github.io/nvidia-docker/ubuntu14.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
24+
25+
sudo apt-get -qq update
26+
sudo apt-get -qq remove linux-image-generic linux-headers-generic linux-generic
27+
sudo apt-get -qq install \
28+
linux-headers-$(uname -r) \
29+
linux-image-generic \
30+
moreutils \
31+
nvidia-docker2 \
32+
expect-dev
33+
34+
sudo pkill -SIGHUP dockerd
35+
36+
sudo pip -q install awscli==1.16.35
37+
38+
if [ -n "${CUDA_VERSION}" ]; then
39+
wget 'https://s3.amazonaws.com/ossci-linux/nvidia_driver/NVIDIA-Linux-x86_64-396.26.run'
40+
sudo /bin/bash ./NVIDIA-Linux-x86_64-396.26.run -s --no-drm
41+
nvidia-smi
42+
fi
43+
44+
# This IAM user only allows read-write access to ECR
45+
export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_ECR_READ_ONLY}
46+
export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_ECR_READ_ONLY}
47+
eval $(aws ecr get-login --region us-east-1 --no-include-email)
48+
- run:
49+
name: Build
50+
no_output_timeout: "20h"
51+
command: |
52+
set -e
53+
54+
# Get latest PyTorch docker image version
55+
curl -O https://raw.githubusercontent.com/pytorch/pytorch/master/.circleci/config.yml
56+
while read line; do
57+
if [[ "$line" == *PyTorchDockerVersion* ]]; then
58+
export pyTorchDockerImageTag=${line:23} # len("# PyTorchDockerVersion:") == 23
59+
echo "PyTorchDockerImageTag: "${pyTorchDockerImageTag}
60+
break
61+
fi
62+
done < config.yml
63+
64+
cat >/home/circleci/project/ci_build_script.sh <<EOL
65+
# =================== The following code will be executed inside Docker container ===================
66+
set -ex
67+
68+
.jenkins/build.sh
69+
# =================== The above code will be executed inside Docker container ===================
70+
EOL
71+
chmod +x /home/circleci/project/ci_build_script.sh
72+
73+
export DOCKER_IMAGE=${DOCKER_IMAGE}:${pyTorchDockerImageTag}
74+
echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
75+
docker pull ${DOCKER_IMAGE} >/dev/null
76+
if [ -n "${CUDA_VERSION}" ]; then
77+
export id=$(docker run --runtime=nvidia -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
78+
else
79+
export id=$(docker run -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
80+
fi
81+
82+
echo "declare -x JOB_BASE_NAME=${CIRCLE_JOB}" > /home/circleci/project/env
83+
echo "declare -x COMMIT_ID=${CIRCLE_SHA1}" >> /home/circleci/project/env
84+
echo "declare -x COMMIT_SOURCE=${CIRCLE_BRANCH}" >> /home/circleci/project/env
85+
if [[ "$COMMIT_SOURCE" == master ]]; then
86+
echo "declare -x AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_PYTORCH_TUTORIAL_BUILD_MASTER_S3_BUCKET}" >> /home/circleci/project/env
87+
echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_PYTORCH_TUTORIAL_BUILD_MASTER_S3_BUCKET}" >> /home/circleci/project/env
88+
echo "declare -x GITHUB_PYTORCHBOT_USERNAME=${GITHUB_PYTORCHBOT_USERNAME}" >> /home/circleci/project/env
89+
echo "declare -x GITHUB_PYTORCHBOT_TOKEN=${GITHUB_PYTORCHBOT_TOKEN}" >> /home/circleci/project/env
90+
else
91+
echo "declare -x AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_PYTORCH_TUTORIAL_BUILD_PR_S3_BUCKET}" >> /home/circleci/project/env
92+
echo "declare -x AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_PYTORCH_TUTORIAL_BUILD_PR_S3_BUCKET}" >> /home/circleci/project/env
93+
fi
94+
95+
docker cp /home/circleci/project/. "$id:/var/lib/jenkins/workspace"
96+
97+
export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_build_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
98+
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
99+
100+
pytorch_tutorial_build_worker_defaults: &pytorch_tutorial_build_worker_defaults
101+
environment:
102+
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda8-cudnn6-py3"
103+
CUDA_VERSION: "8"
104+
resource_class: gpu.medium
105+
<<: *pytorch_tutorial_build_defaults
106+
107+
pytorch_tutorial_build_manager_defaults: &pytorch_tutorial_build_manager_defaults
108+
environment:
109+
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda8-cudnn6-py3"
110+
resource_class: medium
111+
<<: *pytorch_tutorial_build_defaults
112+
113+
version: 2
114+
jobs:
115+
pytorch_tutorial_build_worker_0:
116+
<<: *pytorch_tutorial_build_worker_defaults
117+
118+
pytorch_tutorial_build_worker_1:
119+
<<: *pytorch_tutorial_build_worker_defaults
120+
121+
pytorch_tutorial_build_worker_2:
122+
<<: *pytorch_tutorial_build_worker_defaults
123+
124+
pytorch_tutorial_build_worker_3:
125+
<<: *pytorch_tutorial_build_worker_defaults
126+
127+
pytorch_tutorial_build_worker_4:
128+
<<: *pytorch_tutorial_build_worker_defaults
129+
130+
pytorch_tutorial_build_worker_5:
131+
<<: *pytorch_tutorial_build_worker_defaults
132+
133+
pytorch_tutorial_build_worker_6:
134+
<<: *pytorch_tutorial_build_worker_defaults
135+
136+
pytorch_tutorial_build_worker_7:
137+
<<: *pytorch_tutorial_build_worker_defaults
138+
139+
pytorch_tutorial_build_worker_8:
140+
<<: *pytorch_tutorial_build_worker_defaults
141+
142+
pytorch_tutorial_build_worker_9:
143+
<<: *pytorch_tutorial_build_worker_defaults
144+
145+
pytorch_tutorial_build_worker_10:
146+
<<: *pytorch_tutorial_build_worker_defaults
147+
148+
pytorch_tutorial_build_worker_11:
149+
<<: *pytorch_tutorial_build_worker_defaults
150+
151+
pytorch_tutorial_build_worker_12:
152+
<<: *pytorch_tutorial_build_worker_defaults
153+
154+
pytorch_tutorial_build_worker_13:
155+
<<: *pytorch_tutorial_build_worker_defaults
156+
157+
pytorch_tutorial_build_worker_14:
158+
<<: *pytorch_tutorial_build_worker_defaults
159+
160+
pytorch_tutorial_build_worker_15:
161+
<<: *pytorch_tutorial_build_worker_defaults
162+
163+
pytorch_tutorial_build_worker_16:
164+
<<: *pytorch_tutorial_build_worker_defaults
165+
166+
pytorch_tutorial_build_worker_17:
167+
<<: *pytorch_tutorial_build_worker_defaults
168+
169+
pytorch_tutorial_build_worker_18:
170+
<<: *pytorch_tutorial_build_worker_defaults
171+
172+
pytorch_tutorial_build_worker_19:
173+
<<: *pytorch_tutorial_build_worker_defaults
174+
175+
pytorch_tutorial_build_manager:
176+
<<: *pytorch_tutorial_build_manager_defaults
177+
178+
workflows:
179+
version: 2
180+
build:
181+
jobs:
182+
- pytorch_tutorial_build_worker_0:
183+
context: org-member
184+
- pytorch_tutorial_build_worker_1:
185+
context: org-member
186+
- pytorch_tutorial_build_worker_2:
187+
context: org-member
188+
- pytorch_tutorial_build_worker_3:
189+
context: org-member
190+
- pytorch_tutorial_build_worker_4:
191+
context: org-member
192+
- pytorch_tutorial_build_worker_5:
193+
context: org-member
194+
- pytorch_tutorial_build_worker_6:
195+
context: org-member
196+
- pytorch_tutorial_build_worker_7:
197+
context: org-member
198+
- pytorch_tutorial_build_worker_8:
199+
context: org-member
200+
- pytorch_tutorial_build_worker_9:
201+
context: org-member
202+
- pytorch_tutorial_build_worker_10:
203+
context: org-member
204+
- pytorch_tutorial_build_worker_11:
205+
context: org-member
206+
- pytorch_tutorial_build_worker_12:
207+
context: org-member
208+
- pytorch_tutorial_build_worker_13:
209+
context: org-member
210+
- pytorch_tutorial_build_worker_14:
211+
context: org-member
212+
- pytorch_tutorial_build_worker_15:
213+
context: org-member
214+
- pytorch_tutorial_build_worker_16:
215+
context: org-member
216+
- pytorch_tutorial_build_worker_17:
217+
context: org-member
218+
- pytorch_tutorial_build_worker_18:
219+
context: org-member
220+
- pytorch_tutorial_build_worker_19:
221+
context: org-member
222+
- pytorch_tutorial_build_manager:
223+
context: org-member

0 commit comments

Comments
 (0)