forked from aptos-labs/aptos-core
-
Notifications
You must be signed in to change notification settings - Fork 0
/
config.yml
304 lines (298 loc) · 10.7 KB
/
config.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
version: 2.1
orbs:
aws-cli: circleci/[email protected]
aws-ecr: circleci/[email protected]
kubernetes: circleci/[email protected]
jobs:
build-benchmarks:
machine:
image: ubuntu-2004:current
resource_class: xlarge
steps:
- dev-setup
- run: cargo x bench --no-run
crypto:
docker:
- image: cimg/base:stable
resource_class: medium
steps:
- dev-setup
- run: cargo nextest --nextest-profile ci --package aptos-crypto --features='u32' --no-default-features
- run: cargo nextest --nextest-profile ci --package aptos-crypto --features='u64' --no-default-features
lint:
docker:
- image: cimg/base:2020.01
resource_class: medium
steps:
- dev-setup
- run: sudo apt-get install shellcheck --assume-yes --no-install-recommends
- run: shellcheck scripts/dev_setup.sh
- run: shellcheck scripts/dockerhub_prune.sh
- run: shellcheck docker/build_push.sh
- run: shellcheck docker/docker_republish.sh
- run: shellcheck scripts/weekly-dep-report.sh
- run: cargo x lint
- run: cargo xclippy --workspace --all-targets
- run: cargo fmt
- run: cargo xfmt --check
e2e-test:
machine:
image: ubuntu-2004:current
resource_class: 2xlarge
steps:
- dev-setup
- run: RUST_BACKTRACE=full cargo nextest --nextest-profile ci --partition hash:1/1 --test-threads 5 --package smoke-test
unit-test:
machine:
image: ubuntu-2004:current
resource_class: 2xlarge
steps:
- dev-setup
- run: cargo xtest --doc --unit --changed-since "origin/main"
- run: cargo nextest --nextest-profile ci --partition hash:1/1 --unit --exclude backup-cli --changed-since "origin/main"
docker-build-push:
machine:
image: ubuntu-2004:current
resource_class: medium
parameters:
addl_tag:
description: Additional image tag
type: string
default: latest
steps:
- checkout
- aws-setup
- run: echo "export IMAGE_TAG=dev_$(git rev-parse --short=8 HEAD)" >> $BASH_ENV
- run:
name: Build or skip
shell: /bin/bash
command: |
MANIFEST=$(aws ecr batch-get-image --repository-name aptos/validator --image-ids imageTag=$IMAGE_TAG --query 'images[].imageManifest' --output text)
echo $MANIFEST
if [ -z "$MANIFEST" ]; then
echo "Image tag $IMAGE_TAG not present. Starting build all..."
./docker/build-aws.sh --build-all --version $(git rev-parse --short=8 HEAD) --addl_tags "<<parameters.addl_tag>>"
else
echo "Image tag $IMAGE_TAG already present. Skipping build..."
echo "Continue retagging to <<parameters.addl_tag>>"
imgs=( validator forge init validator_tcb tools faucet )
ret=0
for img in "${imgs[@]}"
do
MANIFEST=$(aws ecr batch-get-image --repository-name aptos/${img} --image-ids imageTag=$IMAGE_TAG --query 'images[].imageManifest' --output text)
put_img_out=$(aws ecr put-image --repository-name aptos/${img} --image-tag main --image-manifest "$MANIFEST" 2>&1)
ret=$?
# ok if image tag exists and cannot overwrite
echo $put_img_out | grep 'ImageAlreadyExistsException' && ret=0
done
exit $ret
fi
ecr-dockerhub-mirror:
machine:
image: ubuntu-2004:current
resource_class: medium
parameters:
addl_tag:
description: Additional image tag
type: string
default: main
steps:
- checkout
- aws-setup
- aws-ecr-setup
- run: echo "export IMAGE_TAG=dev_$(git rev-parse --short=8 HEAD)" >> $BASH_ENV
- run:
name: Get latest built main image
shell: /bin/bash
command: |
imgs=( validator forge init validator_tcb tools faucet )
ret=0
for img in "${imgs[@]}"
do
docker pull "${AWS_ECR_ACCOUNT_URL}/aptos/${img}:${IMAGE_TAG}" || ret=$?
done
exit $ret
- run:
name: Tag image
shell: /bin/bash
command: |
imgs=( validator forge init validator_tcb tools faucet )
org=aptoslab
ret=0
for img in "${imgs[@]}"
do
docker tag "${AWS_ECR_ACCOUNT_URL}/aptos/${img}:${IMAGE_TAG}" "${org}/${img}:${IMAGE_TAG}"
docker tag "${AWS_ECR_ACCOUNT_URL}/aptos/${img}:${IMAGE_TAG}" "${org}/${img}:<<parameters.addl_tag>>" || ret=$?
done
exit $ret
- dockerhub-setup
- run:
name: Push image to Dockerhub
shell: /bin/bash
command: |
# imgs=( validator forge init validator_tcb tools faucet )
imgs=( validator forge init validator_tcb tools faucet )
org=aptoslab
ret=0
for img in "${imgs[@]}"
do
docker push "${org}/${img}:${IMAGE_TAG}"
docker push "${org}/${img}:<<parameters.addl_tag>>" || ret=$?
done
exit $ret
forge-k8s:
docker:
- image: cimg/base:stable
resource_class: medium
steps:
- checkout
- aws-setup
- deploy-setup
- run: echo "export IMAGE_TAG=dev_$(git rev-parse --short=8 HEAD)" >> $BASH_ENV
# since we're running with `--build-all`, assume that if it passes, we have all images required for Forge
- run: aws ecr describe-images --repository-name="aptos/validator" --image-ids=imageTag=$IMAGE_TAG
# XXX: make forge non-blocking for now
- run:
name: Run Forge
shell: /bin/bash
command: |
FGI_REPRO_CMD="./scripts/fgi/run --tag $IMAGE_TAG --suite land_blocking --report forge_report.json"
eval $FGI_REPRO_CMD
FGI_EXIT_CODE=$?
echo "export FGI_REPRO_CMD=$FGI_REPRO_CMD" >> $BASH_ENV
echo "export FGI_EXIT_CODE=$FGI_EXIT_CODE" >> $BASH_ENV
- run:
name: Post Forge test results
shell: /bin/bash
command: |
commit_message=$(git log -1 --pretty=%B)
PR_NUMBER=$(echo "${commit_message}" | grep 'Closes: #' | tail -1 | sed 's/Closes: #//')
echo "Repro: $FGI_REPRO_CMD"
cat \<<EOF > forge_comment.txt
Forge run: ${CIRCLE_BUILD_URL}
Forge Test Result: \`$(cat forge_report.json | jq -r .text)\`
EOF
# replace all newlines
FORGE_COMMENT=$(awk '{printf "%s\\n", $0}' forge_comment.txt)
curl -s -H "Authorization: token ${FORGE_GH_TOKEN}" \
-X POST -d "{\"body\": \"${FORGE_COMMENT}\"}" \
"https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/issues/${PR_NUMBER}/comments"
exit 0
sdk-typescript-test:
docker:
- image: circleci/node:16.13.1-browsers
steps:
- checkout
- run: cd ./ecosystem/typescript/sdk && yarn install
- run: cd ./ecosystem/typescript/sdk && yarn lint
- run: cd ./ecosystem/typescript/sdk && yarn test
- run: cd ./ecosystem/typescript/sdk && yarn build
workflows:
### on bors action ###
# Build the PR binaries and run various tests
# Build the Docker images and run Forge tests
build-test-deploy:
when:
condition:
or:
- equal: [ auto, << pipeline.git.branch >> ]
- equal: [ canary, << pipeline.git.branch >> ]
jobs:
# - build-benchmarks
- crypto
- e2e-test
- lint
- unit-test
- sdk-typescript-test
- docker-build-push:
context: aws-dev
- forge-k8s:
context: aws-dev
requires:
- docker-build-push
### on devnet branch update ###
# Ensure the latest is built on the "devnet" branch, and mirror from ECR to Dockerhub
devnet-branch-cut:
when:
equal: [ devnet, << pipeline.git.branch >> ]
jobs:
- docker-build-push:
context: aws-dev
addl_tag: devnet
- ecr-dockerhub-mirror:
context:
- aws-dev
- docker-aptoslabsbots
addl_tag: devnet
requires:
- docker-build-push
### on continuous_push scheduled pipeline ###
# Build the latest on "main" branch
continuous-push:
when:
and:
- equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
- equal: [ "continuous_push", << pipeline.schedule.name >> ]
jobs:
- docker-build-push:
context: aws-dev
addl_tag: main
### on nightly scheduled pipeline ###
# Ensure the latest on "main" branch is built, and mirror from ECR to Dockerhub
nightly:
when:
and:
- equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
- equal: [ "nightly", << pipeline.schedule.name >> ]
jobs:
- docker-build-push:
context: aws-dev
addl_tag: main
- ecr-dockerhub-mirror:
context:
- aws-dev
- docker-aptoslabsbots
addl_tag: main
requires:
- docker-build-push
commands:
dev-setup:
steps:
- checkout
- run: sudo apt-get update
- run: sudo apt-get install build-essential ca-certificates clang curl git libssl-dev pkg-config --no-install-recommends --assume-yes
- run: curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable
- run: cat $HOME/.cargo/env >> $BASH_ENV
deploy-setup:
steps:
- kubernetes/install-kubectl
- run:
name: Install Helm
# https://helm.sh/docs/intro/install/#from-apt-debianubuntu
command: |
curl https://baltocdn.com/helm/signing.asc | sudo apt-key add -
sudo apt-get install apt-transport-https --yes
echo "deb https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list
sudo apt-get update
sudo apt-get install helm
### Sets up the permissions required for accessing AWS resources
aws-setup:
steps:
- aws-cli/install
# uses envs AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION
- aws-cli/setup
### Sets up the permissions for using AWS ECR from Docker
aws-ecr-setup:
steps:
- run:
name: Compose AWS Env Variables
command: |
echo 'export AWS_ECR_ACCOUNT_URL="${AWS_ECR_ACCOUNT_NUM}.dkr.ecr.${AWS_REGION}.amazonaws.com"' >> $BASH_ENV
- aws-ecr/ecr-login
### Sets up the permissions for using Dockerhub
dockerhub-setup:
steps:
- run:
name: Docker login
command: |
echo $DOCKERHUB_PASSWORD | docker login -u $DOCKERHUB_USERNAME --password-stdin