-
Notifications
You must be signed in to change notification settings - Fork 15
/
Makefile
242 lines (187 loc) · 9.36 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
# cleans your local docker instance of containers, images & volumes. This will remove everything, including items not related to this kaskada project
docker/clean:
docker container prune -f
docker image prune -af
docker volume prune -f
docker system prune -af
ent/clean:
find wren/ent/ -type f ! -name 'generate.go' ! -name '.gitignore' ! -regex '.*/schema/.*' -delete
ent/generate: ent/clean
cd wren && go generate ./ent
# starts a postgres container, runs the create migration tool, then stops the postgres container
ent/create-migrations:
docker run --rm --name migrate_postgres --detach -p 5433:5432 --env POSTGRES_PASSWORD=mpostgres123 postgres:14.3-alpine
sleep 5
cd wren && go run -mod=mod db/main.go
docker stop migrate_postgres
# fixes the migration checksum file (atlas.sum) after any manual changes are applied to the migration files
ent/fix-checksum:
docker run --rm -v "$(shell pwd)/wren/db:/db" arigaio/atlas:0.8.0 migrate hash --dir file://db/migrations
# starts a postgres container, applies all the current migrations, runs the schema inspect tool, then stops the postgres container
ent/update-schema:
docker run --rm --name migrate_postgres --detach -p 5433:5432 --env POSTGRES_PASSWORD=mpostgres123 postgres:14.3-alpine
sleep 5
docker run --rm -v "$(shell pwd)/wren/db/migrations:/migrations" --network host migrate/migrate -path=/migrations/ -database postgres://postgres:[email protected]:5433/postgres?sslmode=disable up
docker run --rm --network host arigaio/atlas:0.8.0 schema inspect -u "postgres://postgres:[email protected]:5433/postgres?sslmode=disable" > wren/db/schema.hcl
docker stop migrate_postgres
# removes all previously generated protobuf code from the repo
proto/clean:
@ find gen/proto/go -type f ! -name '*.pb.ent.go' ! -name '.gitignore' ! -name 'go.*' -delete
@ echo 'checks = ["inherit", "-ST1012"]' > gen/proto/go/staticcheck.conf
# use this varible with `$(call buf_docker_fn,<path>,<additional_docker_flags>)`
# where `<path>` is the desired path from the root of the repo for the buf call
# and `<additional_docker_flags>` are optional additional docker flags to use
buf_docker_fn = docker run --rm --network host --workdir /workspace/proto ${1} \
--volume "$(shell pwd):/workspace" \
--volume "${HOME}/.cache:/root/.cache" \
bufbuild/buf:1.17.0
proto/fmt:
@ $(call buf_docker_fn) format -w
proto/lint:
@ $(call buf_docker_fn) lint
# generates the protobuf libraries
.PHONY: proto/kaskada
proto/kaskada:
@ $(call buf_docker_fn) generate
# regenerates all the protobuf libraries and docs outputted by wren
proto/generate: proto/clean proto/kaskada
@ echo done!
test/int/docker-up:
docker compose -f ./tests/integration/docker-compose.yml up --build --remove-orphans --force-recreate
test/int/docker-up-s3:
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.s3.yml up --build --remove-orphans --force-recreate
test/int/docker-up-dependencies-only:
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.s3.yml up --build --remove-orphans --force-recreate minio pulsar
test/int/docker-up-postgres:
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.postgres.yml up --build --remove-orphans --force-recreate
test/int/docker-up-postgres-s3:
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.postgres.yml -f ./tests/integration/docker-compose.s3.yml up --build --remove-orphans --force-recreate
test/int/run-api-docker:
cd tests/integration/api && ENV=local-docker go run github.com/onsi/ginkgo/v2/ginkgo -vv ./...
test/int/run-api-s3-docker:
cd tests/integration/api && ENV=local-docker OBJECT_STORE_TYPE=s3 OBJECT_STORE_PATH=/data go run github.com/onsi/ginkgo/v2/ginkgo -vv ./...
test/int/run-api-postgres-docker:
cd tests/integration/api && ENV=local-docker DB_DIALECT="postgres" go run github.com/onsi/ginkgo/v2/ginkgo -vv ./...
test/int/run-api-postgres-s3-docker:
cd tests/integration/api && ENV=local-docker DB_DIALECT="postgres" OBJECT_STORE_TYPE=s3 OBJECT_STORE_PATH=/data go run github.com/onsi/ginkgo/v2/ginkgo -vv ./...
test/int/run-api:
cd tests/integration/api && ENV=local-local go run github.com/onsi/ginkgo/v2/ginkgo -vv ./...
test/int/run-api-s3:
cd tests/integration/api && ENV=local-local OBJECT_STORE_TYPE=s3 OBJECT_STORE_PATH=/data go run github.com/onsi/ginkgo/v2/ginkgo -vv ./...
####
## CI related targets
####
ci/integration/tests/docker-compose-up:
export DOCKER_BUILDKIT=1
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.ci.yml up --build --detach
ci/integration/tests/docker-compose-logs:
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.ci.yml logs -t
ci/integration/tests/docker-compose-logs-kaskada-only:
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.ci.yml logs -t kaskada
ci/integration/tests/docker-compose-down:
export DOCKER_BUILDKIT=1
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.ci.yml down
ci/integration/tests/run/api: test/int/run-api-docker
####
## S3 CI Integration Tests
####
ci/integration/tests/docker-compose-up-s3:
export DOCKER_BUILDKIT=1
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.ci.yml -f ./tests/integration/docker-compose.s3.yml up --build --detach
ci/integration/tests/docker-compose-down-s3:
export DOCKER_BUILDKIT=1
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.ci.yml -f ./tests/integration/docker-compose.s3.yml down
ci/integration/tests/docker-compose-s3-logs-kaskada-only:
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.ci.yml -f ./tests/integration/docker-compose.s3.yml logs -t kaskada
ci/integration/tests/docker-compose-s3-logs:
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.ci.yml -f ./tests/integration/docker-compose.s3.yml logs -t
ci/integration/tests/run/api-s3: test/int/run-api-s3-docker
####
## Postgres CI Integration Tests
####
ci/integration/tests/docker-compose-up-postgres:
export DOCKER_BUILDKIT=1
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.ci.yml -f ./tests/integration/docker-compose.postgres.yml up --build --detach
ci/integration/tests/docker-compose-down-postgres:
export DOCKER_BUILDKIT=1
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.ci.yml -f ./tests/integration/docker-compose.postgres.yml down
ci/integration/tests/run/api-postgres: test/int/run-api-postgres
####
## Postgres + S3 CI Integration Tests
####
ci/integration/tests/docker-compose-up-postgres-s3:
export DOCKER_BUILDKIT=1
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.ci.yml -f ./tests/integration/docker-compose.postgres.yml -f ./tests/integration/docker-compose.s3.yml up --build --detach
ci/integration/tests/docker-compose-down-postgres-s3:
export DOCKER_BUILDKIT=1
docker compose -f ./tests/integration/docker-compose.yml -f ./tests/integration/docker-compose.ci.yml -f ./tests/integration/docker-compose.postgres.yml -f ./tests/integration/docker-compose.s3.yml down
ci/integration/tests/run/api-postgres-s3: test/int/run-api-postgres-s3
wren/build:
cp NOTICE wren/
cd wren && go build -o wren main.go
wren/generate-mocks:
docker run -v $(shell pwd)/wren:/src/wren -v$(shell pwd)/gen:/src/gen -w /src/wren vektra/mockery:v2.28.2
wren/lint:
go install honnef.co/go/tools/cmd/staticcheck@latest
staticcheck ./...
wren/run:
cp NOTICE wren/
cd wren && \
DB_IN_MEMORY=false \
DB_PATH=$(shell pwd)/tests/integration/data/kaskada.db \
DEBUG=true \
OBJECT_STORE_PATH=$(shell pwd)/tests/integration/data \
go run main.go
wren/run-s3:
cp NOTICE wren/
cd wren && \
AWS_ACCESS_KEY_ID=kaskada \
AWS_SECRET_ACCESS_KEY=kaskada123 \
AWS_REGION=us-west-2 \
DB_IN_MEMORY=false \
DB_PATH=$(shell pwd)/tests/integration/data/kaskada.db \
DEBUG=true \
OBJECT_STORE_TYPE=s3 \
OBJECT_STORE_BUCKET=integration \
OBJECT_STORE_PATH=$(shell pwd)/tests/integration/data \
OBJECT_STORE_DISABLE_SSL=true \
OBJECT_STORE_ENDPOINT=http://127.0.0.1:9000 \
OBJECT_STORE_FORCE_PATH_STYLE=true \
go run main.go
wren/test:
cp NOTICE wren/
cd wren && go test ./...
.PHONY: sparrow/run sparrow/run-release
sparrow/run:
cargo run -p sparrow-main serve
sparrow/run-s3:
AWS_ENDPOINT=http://127.0.0.1:9000 \
AWS_ALLOW_HTTP=true \
AWS_ACCESS_KEY_ID=kaskada \
AWS_SECRET_ACCESS_KEY=kaskada123 \
AWS_REGION=us-west-2 \
cargo run -p sparrow-main serve
sparrow/run-release:
cargo run --release -p sparrow-main serve
docker/debug: wren/build
cargo build -p sparrow-main
mkdir -p release
cp main release/wren
cp target/debug/sparrow-main release
DOCKER_BUILDKIT=1 docker build -f Dockerfile.release .
rm -fr release
docker/release: wren/build
cargo build --release -p sparrow-main
mkdir -p release
cp main release/wren
cp target/debug/sparrow-main release
DOCKER_BUILDKIT=1 docker build -f Dockerfile.release .
rm -fr release
python/setup:
cd clients/python && poetry install
python/test:
cd clients/python && poetry run poe test
python/build:
cd clients/python && poetry build
python/install: python/build
pip install clients/python/dist/*.whl;