Skip to content

Commit 92216c0

Browse files
denispalabdenispalab
authored
Create buildkite-pipeline-in-disk.sh
1 parent 0e5a58b commit 92216c0

File tree

1 file changed

+365
-0
lines changed

1 file changed

+365
-0
lines changed

‎ci/buildkite-pipeline-in-disk.sh

+365
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,365 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Builds a buildkite pipeline based on the environment variables
4+
#
5+
6+
set -e
7+
cd "$(dirname "$0")"/..
8+
9+
output_file=${1:-/dev/stderr}
10+
11+
if [[ -n $CI_PULL_REQUEST ]]; then
12+
IFS=':' read -ra affected_files <<< "$(buildkite-agent meta-data get affected_files)"
13+
if [[ ${#affected_files[*]} -eq 0 ]]; then
14+
echo "Unable to determine the files affected by this PR"
15+
exit 1
16+
fi
17+
else
18+
affected_files=()
19+
fi
20+
21+
annotate() {
22+
if [[ -n $BUILDKITE ]]; then
23+
buildkite-agent annotate "$@"
24+
fi
25+
}
26+
27+
# Checks if a CI pull request affects one or more path patterns. Each
28+
# pattern argument is checked in series. If one of them found to be affected,
29+
# return immediately as such.
30+
#
31+
# Bash regular expressions are permitted in the pattern:
32+
# affects .rs$ -- any file or directory ending in .rs
33+
# affects .rs -- also matches foo.rs.bar
34+
# affects ^snap/ -- anything under the snap/ subdirectory
35+
# affects snap/ -- also matches foo/snap/
36+
# Any pattern starting with the ! character will be negated:
37+
# affects !^docs/ -- anything *not* under the docs/ subdirectory
38+
#
39+
affects() {
40+
if [[ -z $CI_PULL_REQUEST ]]; then
41+
# affected_files metadata is not currently available for non-PR builds so assume
42+
# the worse (affected)
43+
return 0
44+
fi
45+
# Assume everyting needs to be tested when any Dockerfile changes
46+
for pattern in ^ci/docker-rust/Dockerfile ^ci/docker-rust-nightly/Dockerfile "$@"; do
47+
if [[ ${pattern:0:1} = "!" ]]; then
48+
for file in "${affected_files[@]}"; do
49+
if [[ ! $file =~ ${pattern:1} ]]; then
50+
return 0 # affected
51+
fi
52+
done
53+
else
54+
for file in "${affected_files[@]}"; do
55+
if [[ $file =~ $pattern ]]; then
56+
return 0 # affected
57+
fi
58+
done
59+
fi
60+
done
61+
62+
return 1 # not affected
63+
}
64+
65+
66+
# Checks if a CI pull request affects anything other than the provided path patterns
67+
#
68+
# Syntax is the same as `affects()` except that the negation prefix is not
69+
# supported
70+
#
71+
affects_other_than() {
72+
if [[ -z $CI_PULL_REQUEST ]]; then
73+
# affected_files metadata is not currently available for non-PR builds so assume
74+
# the worse (affected)
75+
return 0
76+
fi
77+
78+
for file in "${affected_files[@]}"; do
79+
declare matched=false
80+
for pattern in "$@"; do
81+
if [[ $file =~ $pattern ]]; then
82+
matched=true
83+
fi
84+
done
85+
if ! $matched; then
86+
return 0 # affected
87+
fi
88+
done
89+
90+
return 1 # not affected
91+
}
92+
93+
94+
start_pipeline() {
95+
echo "# $*" > "$output_file"
96+
echo "steps:" >> "$output_file"
97+
}
98+
99+
command_step() {
100+
cat >> "$output_file" <<EOF
101+
- name: "$1"
102+
command: "$2"
103+
timeout_in_minutes: $3
104+
artifact_paths: "log-*.txt"
105+
EOF
106+
}
107+
108+
109+
trigger_secondary_step() {
110+
cat >> "$output_file" <<"EOF"
111+
- trigger: "solana-secondary"
112+
branches: "!pull/*"
113+
async: true
114+
build:
115+
message: "${BUILDKITE_MESSAGE}"
116+
commit: "${BUILDKITE_COMMIT}"
117+
branch: "${BUILDKITE_BRANCH}"
118+
env:
119+
TRIGGERED_BUILDKITE_TAG: "${BUILDKITE_TAG}"
120+
EOF
121+
}
122+
123+
wait_step() {
124+
echo " - wait" >> "$output_file"
125+
}
126+
127+
all_test_steps() {
128+
command_step checks ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-checks.sh" 20
129+
wait_step
130+
131+
# Coverage...
132+
if affects \
133+
.rs$ \
134+
Cargo.lock$ \
135+
Cargo.toml$ \
136+
^ci/rust-version.sh \
137+
^ci/test-coverage.sh \
138+
^scripts/coverage.sh \
139+
; then
140+
command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 40
141+
wait_step
142+
else
143+
annotate --style info --context test-coverage \
144+
"Coverage skipped as no .rs files were modified"
145+
fi
146+
# Coverage in disk...
147+
if affects \
148+
.rs$ \
149+
Cargo.lock$ \
150+
Cargo.toml$ \
151+
^ci/rust-version.sh \
152+
^ci/test-coverage.sh \
153+
^scripts/coverage-in-disk.sh \
154+
; then
155+
command_step coverage-in-disk ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 40
156+
wait_step
157+
else
158+
annotate --style info --context test-coverage \
159+
"Coverage skipped as no .rs files were modified"
160+
fi
161+
# Full test suite
162+
command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 60
163+
wait_step
164+
165+
# BPF test suite
166+
if affects \
167+
.rs$ \
168+
Cargo.lock$ \
169+
Cargo.toml$ \
170+
^ci/rust-version.sh \
171+
^ci/test-stable-bpf.sh \
172+
^ci/test-stable.sh \
173+
^ci/test-local-cluster.sh \
174+
^core/build.rs \
175+
^fetch-perf-libs.sh \
176+
^programs/ \
177+
^sdk/ \
178+
; then
179+
cat >> "$output_file" <<"EOF"
180+
- command: "ci/test-stable-bpf.sh"
181+
name: "stable-bpf"
182+
timeout_in_minutes: 20
183+
artifact_paths: "bpf-dumps.tar.bz2"
184+
agents:
185+
- "queue=default"
186+
EOF
187+
else
188+
annotate --style info \
189+
"Stable-BPF skipped as no relevant files were modified"
190+
fi
191+
192+
# Perf test suite
193+
if affects \
194+
.rs$ \
195+
Cargo.lock$ \
196+
Cargo.toml$ \
197+
^ci/rust-version.sh \
198+
^ci/test-stable-perf.sh \
199+
^ci/test-stable.sh \
200+
^ci/test-local-cluster.sh \
201+
^core/build.rs \
202+
^fetch-perf-libs.sh \
203+
^programs/ \
204+
^sdk/ \
205+
; then
206+
cat >> "$output_file" <<"EOF"
207+
- command: "ci/test-stable-perf.sh"
208+
name: "stable-perf"
209+
timeout_in_minutes: 20
210+
artifact_paths: "log-*.txt"
211+
agents:
212+
- "queue=cuda"
213+
EOF
214+
else
215+
annotate --style info \
216+
"Stable-perf skipped as no relevant files were modified"
217+
fi
218+
219+
# Downstream backwards compatibility
220+
if affects \
221+
.rs$ \
222+
Cargo.lock$ \
223+
Cargo.toml$ \
224+
^ci/rust-version.sh \
225+
^ci/test-stable-perf.sh \
226+
^ci/test-stable.sh \
227+
^ci/test-local-cluster.sh \
228+
^core/build.rs \
229+
^fetch-perf-libs.sh \
230+
^programs/ \
231+
^sdk/ \
232+
^scripts/build-downstream-projects.sh \
233+
; then
234+
cat >> "$output_file" <<"EOF"
235+
- command: "scripts/build-downstream-projects.sh"
236+
name: "downstream-projects"
237+
timeout_in_minutes: 30
238+
EOF
239+
else
240+
annotate --style info \
241+
"downstream-projects skipped as no relevant files were modified"
242+
fi
243+
244+
# Downstream Anchor projects backwards compatibility
245+
if affects \
246+
.rs$ \
247+
Cargo.lock$ \
248+
Cargo.toml$ \
249+
^ci/rust-version.sh \
250+
^ci/test-stable-perf.sh \
251+
^ci/test-stable.sh \
252+
^ci/test-local-cluster.sh \
253+
^core/build.rs \
254+
^fetch-perf-libs.sh \
255+
^programs/ \
256+
^sdk/ \
257+
^scripts/build-downstream-anchor-projects.sh \
258+
; then
259+
cat >> "$output_file" <<"EOF"
260+
- command: "scripts/build-downstream-anchor-projects.sh"
261+
name: "downstream-anchor-projects"
262+
timeout_in_minutes: 10
263+
EOF
264+
else
265+
annotate --style info \
266+
"downstream-anchor-projects skipped as no relevant files were modified"
267+
fi
268+
269+
# Wasm support
270+
if affects \
271+
^ci/test-wasm.sh \
272+
^ci/test-stable.sh \
273+
^sdk/ \
274+
; then
275+
command_step wasm ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-wasm.sh" 20
276+
else
277+
annotate --style info \
278+
"wasm skipped as no relevant files were modified"
279+
fi
280+
281+
# Benches...
282+
if affects \
283+
.rs$ \
284+
Cargo.lock$ \
285+
Cargo.toml$ \
286+
^ci/rust-version.sh \
287+
^ci/test-coverage.sh \
288+
^ci/test-bench.sh \
289+
; then
290+
command_step bench "ci/test-bench.sh" 30
291+
else
292+
annotate --style info --context test-bench \
293+
"Bench skipped as no .rs files were modified"
294+
fi
295+
296+
command_step "local-cluster" \
297+
". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-local-cluster.sh" \
298+
40
299+
300+
command_step "local-cluster-flakey" \
301+
". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-local-cluster-flakey.sh" \
302+
10
303+
304+
command_step "local-cluster-slow" \
305+
". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-local-cluster-slow.sh" \
306+
30
307+
}
308+
309+
pull_or_push_steps() {
310+
command_step sanity "ci/test-sanity.sh" 5
311+
wait_step
312+
313+
# Check for any .sh file changes
314+
if affects .sh$; then
315+
command_step shellcheck "ci/shellcheck.sh" 5
316+
wait_step
317+
fi
318+
319+
# Run the full test suite by default, skipping only if modifications are local
320+
# to some particular areas of the tree
321+
if affects_other_than ^.buildkite ^.mergify .md$ ^docs/ ^web3.js/ ^explorer/ ^.gitbook; then
322+
all_test_steps
323+
fi
324+
325+
# web3.js, explorer and docs changes run on Travis or Github actions...
326+
}
327+
328+
329+
if [[ -n $BUILDKITE_TAG ]]; then
330+
start_pipeline "Tag pipeline for $BUILDKITE_TAG"
331+
332+
annotate --style info --context release-tag \
333+
"https://github.com/solana-labs/solana/releases/$BUILDKITE_TAG"
334+
335+
# Jump directly to the secondary build to publish release artifacts quickly
336+
trigger_secondary_step
337+
exit 0
338+
fi
339+
340+
341+
if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
342+
echo "+++ Affected files in this PR"
343+
for file in "${affected_files[@]}"; do
344+
echo "- $file"
345+
done
346+
347+
start_pipeline "Pull request pipeline for $BUILDKITE_BRANCH"
348+
349+
# Add helpful link back to the corresponding Github Pull Request
350+
annotate --style info --context pr-backlink \
351+
"Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
352+
353+
if [[ $GITHUB_USER = "dependabot[bot]" ]]; then
354+
command_step dependabot "ci/dependabot-pr.sh" 5
355+
wait_step
356+
fi
357+
pull_or_push_steps
358+
exit 0
359+
fi
360+
361+
start_pipeline "Push pipeline for ${BUILDKITE_BRANCH:-?unknown branch?}"
362+
pull_or_push_steps
363+
wait_step
364+
trigger_secondary_step
365+
exit 0

0 commit comments

Comments
 (0)