-
Notifications
You must be signed in to change notification settings - Fork 138
/
Makefile
449 lines (364 loc) · 15.1 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
.DEFAULT_GOAL := app
# ---------------------------------------------------------------------------
# Config
ROOT_DIR:=.
UID=${shell id -u}
GID=${shell id -g}
LIB_SOURCE_DIR="${ROOT_DIR}/lib"
LIB_DEBUG_DIR="${ROOT_DIR}/build/Debug"
LIB_RELEASE_DIR="${ROOT_DIR}/build/Release"
LIB_RELWITHDEBINFO_DIR="${ROOT_DIR}/build/RelWithDebInfo"
LIB_XRAY_DIR="${ROOT_DIR}/build/Xray"
DUCKDB_WASM_DIR="${ROOT_DIR}/packages/duckdb/src/wasm"
TARGET=eh
DUCKDB_HASH=${shell cd submodules/duckdb && git reflog -n 1 | head -c 10}
CACHE_DIRS=${ROOT_DIR}/.ccache/ ${ROOT_DIR}/.emscripten_cache/
DOCKER_EXEC_ENVIRONMENT=docker compose run duckdb-wasm-ci
CORES=$(shell grep -c ^processor /proc/cpuinfo 2>/dev/null || sysctl -n hw.ncpu)
GTEST_FILTER=*
JS_FILTER=
EXTENSION_CACHE_DIR="${ROOT_DIR}/.ccache/extension"
JSON_EXTENSION_CACHE_FILE="${EXTENSION_CACHE_DIR}/json"
cpp_lib: lib_tests
# ---------------------------------------------------------------------------
# Formatting
# Format all source files
.PHONY: format
format:
${ROOT_DIR}/scripts/format.sh
.PHONY: check_format
check_format:
${ROOT_DIR}/scripts/format.sh check
# ---------------------------------------------------------------------------
# Building
.PHONY: set_environment
set_environment:
command -v emcc &> /dev/null && EXEC_ENVIRONMENT="" && echo '\033[1m=== Using native mode ===\033[0m' && echo 'Emscripten from' && which emcc || (EXEC_ENVIRONMENT=echo ${DOCKER_EXEC_ENVIRONMENT} && echo '\033[1m === Using docker environment === \033[0m')
build/data:
${ROOT_DIR}/scripts/generate_uni.sh
${ROOT_DIR}/scripts/generate_tpch_tbl.sh 0.01
${ROOT_DIR}/scripts/generate_tpch_tbl.sh 0.1
${ROOT_DIR}/scripts/generate_tpch_tbl.sh 0.25
${ROOT_DIR}/scripts/generate_tpch_tbl.sh 0.5
${ROOT_DIR}/scripts/generate_tpch_arrow.sh 0.01
${ROOT_DIR}/scripts/generate_tpch_arrow.sh 0.1
${ROOT_DIR}/scripts/generate_tpch_arrow.sh 0.25
${ROOT_DIR}/scripts/generate_tpch_arrow.sh 0.5
${ROOT_DIR}/scripts/generate_tpch_sqlite.sh 0.01
${ROOT_DIR}/scripts/generate_tpch_sqlite.sh 0.1
${ROOT_DIR}/scripts/generate_tpch_sqlite.sh 0.25
${ROOT_DIR}/scripts/generate_tpch_sqlite.sh 0.5
${ROOT_DIR}/scripts/generate_tpch_duckdb.sh 0.01
${ROOT_DIR}/scripts/generate_tpch_duckdb.sh 0.1
${ROOT_DIR}/scripts/generate_tpch_duckdb.sh 0.25
${ROOT_DIR}/scripts/generate_tpch_duckdb.sh 0.5
touch build/data
# Compile the core in debug mode
.PHONY: lib
lib: submodules
mkdir -p ${LIB_DEBUG_DIR}
cmake -S ${LIB_SOURCE_DIR} -B ${LIB_DEBUG_DIR} \
-DCMAKE_BUILD_TYPE=Debug \
-DCMAKE_EXPORT_COMPILE_COMMANDS=1
make -C${LIB_DEBUG_DIR} -j${CORES}
# Compile the core in release mode with debug symbols
.PHONY: lib_relwithdebinfo
lib_relwithdebinfo: submodules
mkdir -p ${LIB_RELWITHDEBINFO_DIR}
cmake -S ${LIB_SOURCE_DIR} -B ${LIB_RELWITHDEBINFO_DIR} \
-DCMAKE_BUILD_TYPE=RelWithDebInfo
make -C${LIB_RELWITHDEBINFO_DIR} -j${CORES}
DUCKDB_SOURCES := $(wildcard ${ROOT_DIR}/submodules/duckdb/src/*)
# Compile the core in release mode with debug symbols
# XXX Compiling gtest with a recent clang fails at the moment, recheck xray on linux when fixed
.PHONY: lib_xray
lib_xray: submodules
mkdir -p ${LIB_XRAY_DIR}
cmake -S ${LIB_SOURCE_DIR} -B ${LIB_XRAY_DIR} \
-DWITH_XRAY=1 \
-DCMAKE_BUILD_TYPE=RelWithDebInfo
make -C${LIB_XRAY_DIR} -j${CORES}
# Compile the core in release mode
.PHONY: lib_release
lib_release: submodules
mkdir -p ${LIB_RELEASE_DIR}
cmake -S ${LIB_SOURCE_DIR} -B ${LIB_RELEASE_DIR} \
-DCMAKE_BUILD_TYPE=Release
make -C${LIB_RELEASE_DIR} -j${CORES}
# Instrument execution traces with xray
.PHONY: xray_tester
xray_tester: lib_xray
rm ${LIB_XRAY_DIR}/xray-log.tester.*
cd ${LIB_XRAY_DIR} && XRAY_OPTIONS="patch_premain=true xray_mode=xray-basic" ${LIB_XRAY_DIR}/tester --source_dir ${LIB_SOURCE_DIR} --gtest_filter=${GTEST_FILTER}
llvm-xray-12 convert -symbolize ${LIB_XRAY_DIR}/xray-log.tester.* -instr_map ${LIB_XRAY_DIR}/tester -output-format=trace_event | gzip -9 > ${LIB_XRAY_DIR}/xray-log.tester.gz
# Perf the library with linux perf
.PHONY: lib_perf
perf_tester: lib_relwithdebinfo
perf record --call-graph dwarf ${LIB_RELWITHDEBINFO_DIR}/tester --source_dir ${LIB_SOURCE_DIR} --gtest_filter=${GTEST_FILTER}
hotspot ./perf.data
# Test the core library
.PHONY: lib_tests
lib_tests: lib
${LIB_DEBUG_DIR}/tester --source_dir ${LIB_SOURCE_DIR} --gtest_filter=${GTEST_FILTER}
# Debug the core library
.PHONY: lib_tests_lldb
lib_tests_lldb: lib
lldb ${LIB_DEBUG_DIR}/tester -- --source_dir ${LIB_SOURCE_DIR} --gtest_filter=${GTEST_FILTER}
# Debug the core library
.PHONY: lib_tests_gdb
lib_tests_gdb: lib
gdb --args ${LIB_DEBUG_DIR}/tester --source_dir ${LIB_SOURCE_DIR} --gtest_filter=${GTEST_FILTER}
# Test the core library
.PHONY: lib_tests_rel
lib_tests_rel: lib_relwithdebinfo
${LIB_RELWITHDEBINFO_DIR}/tester --source_dir ${LIB_SOURCE_DIR} --gtest_filter=${GTEST_FILTER}
# Test the core library
.PHONY: lib_tests_rel_lldb
lib_tests_rel_lldb: lib_relwithdebinfo
lldb ${LIB_RELWITHDEBINFO_DIR}/tester -- --source_dir ${LIB_SOURCE_DIR} --gtest_filter=${GTEST_FILTER}
# Debug the library
.PHONY: lib_debug
lib_debug: lib
lldb --args ${LIB_DEBUG_DIR}/tester ${LIB_SOURCE_DIR}
bench_build: yarn_install
yarn workspace @duckdb/benchmarks build
touch build/bench_build
.PHONY: bench_tpch_aq
bench_tpch_aq: build/data bench_build yarn_install
yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.01
yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.1
yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.5
# Run all benchmarks for the paper
.PHONY: bench_tpch
bench_tpch_paper: build/data bench_build yarn_install
yarn workspace @duckdb/benchmarks build
yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.01
yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.1
yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.5
yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.01
yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.1
yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.5
yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.01
yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.1
yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.5
yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.01
yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.1
yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.5
.PHONY: bench_all
bench_all: bench_build bench_internal bench_system_sort_int bench_system_sum_csv bench_system_sort_int bench_system_regex bench_system_join_2 bench_system_join_3 build/data
yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.1
yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.25
yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.5
.PHONY: bench_internal
bench_internal: bench_build
yarn workspace @duckdb/benchmarks bench:internal
.PHONY: bench_system_sort_int
bench_system_sort_int: bench_build
yarn workspace @duckdb/benchmarks bench:system:sort:int
.PHONY: bench_system_sum_csv
bench_system_sum_csv: bench_build
yarn workspace @duckdb/benchmarks bench:system:sum:csv
.PHONY: bench_system_sum_int
bench_system_sum_int: bench_build
yarn workspace @duckdb/benchmarks bench:system:sum:int
.PHONY: bench_system_regex
bench_system_regex: bench_build
yarn workspace @duckdb/benchmarks bench:system:regex
.PHONY: bench_system_join_2
bench_system_join_2: bench_build
yarn workspace @duckdb/benchmarks bench:system:join:2
.PHONY: bench_system_join_3
bench_system_join_3: bench_build
yarn workspace @duckdb/benchmarks bench:system:join:3
.PHONY: bench_system_tpch
bench_system_tpch_duckdb: bench_build build/data
yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.1
.PHONY: wasm_caches
wasm_caches: $(DUCKDB_SOURCES)
mkdir -p ${ROOT_DIR}/.ccache ${ROOT_DIR}/.emscripten_cache
chown -R $(id -u):$(id -g) ${ROOT_DIR}/.ccache ${ROOT_DIR}/.emscripten_cache
rm -rf ${EXTENSION_CACHE_DIR}
mkdir -p ${EXTENSION_CACHE_DIR}
chown -R $(id -u):$(id -g) ${EXTENSION_CACHE_DIR}
mkdir -p ${CACHE_DIRS}
ifeq (${DUCKDB_JSON}, 1)
touch ${JSON_EXTENSION_CACHE_FILE}
endif
wrapped_wasm_caches:
${EXEC_ENVIRONMENT} make wasm_caches
mkdir -p build
touch build/wrapped_wasm_caches
check_duckdb: $(DUCKDB_SOURCES)
(cd ${ROOT_DIR}/build/dev/mvp && make clean) || true
(cd ${ROOT_DIR}/build/dev/eh && make clean) || true
(cd ${ROOT_DIR}/build/dev/coi && make clean) || true
(cd ${ROOT_DIR}/build/relsize/mvp && make clean) || true
(cd ${ROOT_DIR}/build/relsize/eh && make clean) || true
(cd ${ROOT_DIR}/build/relsize/coi && make clean) || true
(cd ${ROOT_DIR}/build/relperf/mvp && make clean) || true
(cd ${ROOT_DIR}/build/relperf/eh && make clean) || true
(cd ${ROOT_DIR}/build/relperf/coi && make clean) || true
(cd ${ROOT_DIR}/build/debug/mvp && make clean) || true
(cd ${ROOT_DIR}/build/debug/eh && make clean) || true
(cd ${ROOT_DIR}/build/debug/coi && make clean) || true
touch check_duckdb
wasm_setup: set_environment check_duckdb wrapped_wasm_caches
touch wasm_setup
.PHONY: wasm_dev
wasm_dev: wasm_setup
${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev mvp
${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev eh
${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev coi
.PHONY: wasm_relperf
wasm_relperf: wasm_setup
${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf mvp
${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf eh
${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf coi
.PHONY: wasm_relsize
wasm_relsize: wasm_setup
${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize mvp
${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize eh
${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize coi
.PHONY: wasm_debug
wasm_debug: wasm_setup
${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug mvp
${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug eh
${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug coi
wasm: wasm_relperf
.PHONY: wasm_star
wasm_star: wasm_relsize wasm_relperf wasm_dev wasm_debug
# Build the duckdb library in debug mode
.PHONY: js_debug
js_debug: build/bootstrap yarn_install
yarn workspace @duckdb/duckdb-wasm build:debug
# Build the duckdb library in release mode
.PHONY: js_release
js_release: yarn_install
yarn workspace @duckdb/duckdb-wasm build:release
# Build the duckdb docs
.PHONY: docs
docs: yarn_install
yarn workspace @duckdb/duckdb-wasm docs
# Run the duckdb javascript tests
.PHONY: js_tests
js_tests: js_debug build/data
ifeq (${DUCKDB_WASM_LOADABLE_EXTENSIONS}, 1)
# FIXME: build also for node and restore those tests
yarn workspace @duckdb/duckdb-wasm test:chrome || echo "--- Parquet tests expected to fail!"
else
yarn workspace @duckdb/duckdb-wasm test
endif
.PHONY: js_tests_release
js_tests_release: js_release
ifeq (${DUCKDB_WASM_LOADABLE_EXTENSIONS}, 1)
# FIXME: build also for node and restore those tests
yarn workspace @duckdb/duckdb-wasm test:chrome || echo "--- Parquet tests expected to fail!"
else
yarn workspace @duckdb/duckdb-wasm test
endif
# Run the duckdb javascript tests in browser
.PHONY: js_tests_browser
js_tests_browser: js_debug build/data
yarn workspace @duckdb/duckdb-wasm test:chrome
# Run the duckdb javascript tests in browser
.PHONY: js_tests_browser_debug
js_tests_browser_debug: js_debug build/data
yarn workspace @duckdb/duckdb-wasm test:browser:debug
# Run the duckdb javascript tests on nodejs
.PHONY: js_tests_node
js_tests_node: js_debug build/data
yarn workspace @duckdb/duckdb-wasm test:node --filter=${JS_FILTER}
.PHONY: js_tests_node_debug
js_tests_node_debug: js_debug build/data
yarn workspace @duckdb/duckdb-wasm test:node:debug --filter=${JS_FILTER}
wasmpack: yarn_install
yarn workspace @duckdb/duckdb-wasm-shell install:wasmpack
.PHONY: shell
shell: build/bootstrap js_debug
yarn workspace @duckdb/duckdb-wasm-shell build:debug
.PHONY: shell_release
shell_release: js_release
yarn workspace @duckdb/duckdb-wasm-shell build:release
.PHONY: app_start
app_start: yarn_install
yarn workspace @duckdb/duckdb-wasm-app start
.PHONY: app_start_corp
app_start_corp:
yarn workspace @duckdb/duckdb-wasm-app start:corp
.PHONY: app
app: wasm wasmpack shell docs js_tests_release
yarn workspace @duckdb/duckdb-wasm-app build:release
build_loadable:
DUCKDB_PLATFORM=wasm_${TARGET} DUCKDB_WASM_LOADABLE_EXTENSIONS=1 GEN=ninja ./scripts/wasm_build_lib.sh relsize ${TARGET}
build_loadable_unsigned: build_loadable
# need to propagate the unsigned flag
serve_loadable_base: wasmpack shell docs
yarn workspace @duckdb/duckdb-wasm-app build:release
cp -r build/extension_repository packages/duckdb-wasm-app/build/release/.
.PHONY: serve_local
serve_local: build_loadable_unsigned serve_loadable_base
npx http-server packages/duckdb-wasm-app/build/release -o "#queries=v0,SET-custom_extension_repository%3D'http%3A%2F%2F127.0.0.1%3A8080%2Fextension_repository'~" -a 127.0.0.1 -p 8080
.PHONY: serve
serve: build_loadable serve_loadable_base
npx http-server packages/duckdb-wasm-app/build/release -o
.PHONY: app_server
app_server:
python3 -m http.server 9003 --bind 127.0.0.1 --directory ./packages/duckdb-wasm-app/build/release/
# C++ formatting
.PHONY: clang_format
clang_format:
python3 ./scripts/run_clang_format.py \
--exclude ./lib/build \
--exclude ./lib/third_party \
-r ./lib/
# JS formatting
.PHONY: eslint
eslint:
yarn workspace @duckdb/duckdb-wasm run lint
yarn workspace @duckdb/duckdb-wasm-shell run lint
yarn workspace @duckdb/benchmarks run lint
# Install all yarn packages
.PHONY: yarn_install
yarn_install:
yarn
yarn install
.PHONY: examples
examples: yarn_install
yarn workspace @duckdb/duckdb-wasm-examples-bare-node test
yarn workspace @duckdb/duckdb-wasm-examples-bare-browser build
yarn workspace @duckdb/duckdb-wasm-examples-esbuild-node build
yarn workspace @duckdb/duckdb-wasm-examples-esbuild-node test
yarn workspace @duckdb/duckdb-wasm-examples-esbuild-browser build
# ---------------------------------------------------------------------------
# Environment
# Generate the compile commands for the language server
.PHONY: compile_commands
compile_commands:
mkdir -p ${LIB_DEBUG_DIR}
cmake -S ${LIB_SOURCE_DIR} -B ${LIB_DEBUG_DIR} \
-DCMAKE_BUILD_TYPE=Debug \
-DCMAKE_EXPORT_COMPILE_COMMANDS=1
ln -sf ${LIB_DEBUG_DIR}/compile_commands.json ${LIB_SOURCE_DIR}/compile_commands.json
# Clean the repository
.PHONY: clean
clean:
rm -rf build
rm -rf target
cd packages/duckdb-wasm-shell && rm -rf node_modules
rm -rf packages/duckdb-wasm-app/build
rm -rf submodules/duckdb/build
rm -rf packages/duckdb-wasm/dist
build/docker_ci_image:
command -v emcc &> /dev/null || docker compose build
touch build/docker_ci_image
patch_duckdb:
(find patches/duckdb/* -type f -name '*.patch' -print0 | xargs -0 cat | patch -p1 --forward -d submodules/duckdb) || true
(find patches/arrow/* -type f -name '*.patch' -print0 | xargs -0 cat | patch -p1 --forward -d submodules/arrow) || true
apply_patches: patch_duckdb
submodules:
git submodule update --init --recursive
touch submodules
# Build infrastructure and packages required for development
build/bootstrap: submodules yarn_install
mkdir -p build
touch build/bootstrap