[CI] reintroduce docker stage for wasm tests (apache#5565)

* [DOCKER] Introduce ci-wasm

* Add Jenkinsfile

* Rename prepare to prepwasm so it won't run by default

.gitignore

@@ -6,7 +6,7 @@ __pycache__/
 # C extensions
 *.so
 *.ll
-
+.npm
 # Distribution / packaging
 .Python
 env/

Jenkinsfile

@@ -46,6 +46,7 @@
 ci_lint = "tvmai/ci-lint:v0.60"
 ci_gpu = "tvmai/ci-gpu:v0.63"
 ci_cpu = "tvmai/ci-cpu:v0.62"
+ci_wasm = "tvmai/ci-wasm:v0.60"
 ci_i386 = "tvmai/ci-i386:v0.52"
 
 // tvm libraries
@@ -169,6 +170,18 @@ stage('Build') {
         }
       }
     }
+    },
+  'BUILD: WASM': {
+    node('CPU') {
+      ws(per_exec_ws("tvm/build-wasm")) {
+        init_git()
+        sh "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh"
+        make(ci_wasm, 'build', '-j2')
+        timeout(time: max_time, unit: 'MINUTES') {
+          sh "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh"
+        }
+      }
+    }
   },
   'BUILD : i386': {
     node('CPU') {

docker/Dockerfile.ci_emscripten → docker/Dockerfile.ci_wasm

@@ -14,26 +14,36 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-# For CPU
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 RUN apt-get update --fix-missing
 
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
 RUN bash /install/ubuntu_install_core.sh
 
-COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh
-
-COPY install/ubuntu_install_emscripten.sh /install/ubuntu_install_emscripten.sh
-RUN bash /install/ubuntu_install_emscripten.sh
+COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
+RUN bash /install/ubuntu1804_install_python.sh
 
 COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh
 RUN bash /install/ubuntu_install_python_package.sh
 
-RUN chmod a+rwx -R /emsdk-portable
-RUN cp -r /emsdk-portable  /emsdk-portable-backup
-RUN mv /emsdk-portable  /emsdk-portable-x
-RUN mv /emsdk-portable-backup /emsdk-portable
-RUN cp /root/.emscripten /emsdk-portable/
+COPY install/ubuntu1804_install_llvm.sh /install/ubuntu1804_install_llvm.sh
+RUN bash /install/ubuntu1804_install_llvm.sh
+
+COPY install/ubuntu_install_java.sh /install/ubuntu_install_java.sh
+RUN bash /install/ubuntu_install_java.sh
+
+COPY install/ubuntu_install_antlr.sh /install/ubuntu_install_antlr.sh
+RUN bash /install/ubuntu_install_antlr.sh
+
+COPY install/ubuntu_install_nodejs.sh /install/ubuntu_install_nodejs.sh
+RUN bash /install/ubuntu_install_nodejs.sh
+
+COPY install/ubuntu_install_emscripten.sh /install/ubuntu_install_emscripten.sh
+RUN bash /install/ubuntu_install_emscripten.sh
+
+ENV EMSDK=/emsdk
+ENV PATH=${PATH}:${EMSDK}:${EMSDK}/upstream/emscripten
+ENV EMSCRIPTEN=${EMSDK}/upstream/emscripten
+ENV BINARYEN=${EMSDK}/upstream
+ENV LLVM=${EMSDK}/upstream/bin

docker/bash.sh

@@ -88,7 +88,7 @@ ${DOCKER_BINARY} run --rm --pid=host\
     -e "CI_BUILD_UID=$(id -u)" \
     -e "CI_BUILD_GROUP=$(id -g -n)" \
     -e "CI_BUILD_GID=$(id -g)" \
-    -e "PYTHONPATH=python:topi/python"\
+    -e "PYTHONPATH=/workspace/python:/workspace/topi/python"\
     -e "CI_PYTEST_ADD_OPTIONS=$CI_PYTEST_ADD_OPTIONS" \
     ${CUDA_ENV}\
     ${CI_DOCKER_EXTRA_PARAMS[@]} \

docker/install/ubuntu_install_emscripten.sh

@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,20 +20,8 @@ set -e
 set -u
 set -o pipefail
 
-alias make="make -j4"
-
-# Get latest cmake
-wget -q https://cmake.org/files/v3.8/cmake-3.8.2-Linux-x86_64.tar.gz
-tar xf cmake-3.8.2-Linux-x86_64.tar.gz
-export PATH=/cmake-3.8.2-Linux-x86_64/bin/:${PATH}
-
-wget -q https://s3.amazonaws.com/mozilla-games/emscripten/releases/emsdk-portable.tar.gz
-tar xf emsdk-portable.tar.gz
-cd emsdk-portable
-./emsdk update
+cd /
+git clone https://github.com/emscripten-core/emsdk.git
+cd emsdk
 ./emsdk install latest
 ./emsdk activate latest
-# Clone and pull latest sdk
-./emsdk install clang-incoming-64bit
-./emsdk activate clang-incoming-64bit
-cd ..

docker/install/ubuntu_install_nodejs.sh

@@ -25,8 +25,6 @@ apt-get install -y curl
 
 # The node install script fetched and executed here will update the
 # apt source list, hence the second apt-get update is necessary.
-curl -s -S -L https://deb.nodesource.com/setup_8.x | bash -
+curl -s -S -L https://deb.nodesource.com/setup_14.x | bash -
 apt-get update
 apt-get install -y nodejs
-
-npm install eslint jsdoc ws

tests/scripts/task_web_test.sh → tests/scripts/task_config_build_wasm.sh

@@ -19,25 +19,20 @@
 set -e
 set -u
 
-export PYTHONPATH=python
-
-cp /emsdk-portable/.emscripten ~/.emscripten
-source /emsdk-portable/emsdk_env.sh
-
-export EM_CONFIG=${HOME}/.emscripten
-export EM_CACHE=${HOME}/.emscripten_cache
-
-echo "Build TVM Web runtime..."
-make web
-
-echo "Prepare test libraries..."
-python tests/web/prepare_test_libs.py
-
-echo "Start testing..."
-
-for test in tests/web/test_*.js; do
-    echo node $test
-    node $test
-done
-
-echo "All tests finishes..."
+mkdir -p build
+cd build
+cp ../cmake/config.cmake .
+
+echo set\(USE_SORT ON\) >> config.cmake
+echo set\(USE_MICRO ON\) >> config.cmake
+echo set\(USE_MICRO_STANDALONE_RUNTIME ON\) >> config.cmake
+echo set\(USE_GRAPH_RUNTIME_DEBUG ON\) >> config.cmake
+echo set\(USE_VM_PROFILER ON\) >> config.cmake
+echo set\(USE_EXAMPLE_EXT_RUNTIME ON\) >> config.cmake
+echo set\(USE_LLVM llvm-config-10\) >> config.cmake
+echo set\(USE_ANTLR ON\) >> config.cmake
+echo set\(CMAKE_CXX_COMPILER g++\) >> config.cmake
+echo set\(CMAKE_CXX_FLAGS -Werror\) >> config.cmake
+echo set\(HIDE_PRIVATE_SYMBOLS ON\) >> config.cmake
+echo set\(USE_VTA_TSIM ON\) >> config.cmake
+echo set\(USE_VTA_FSIM ON\) >> config.cmake

tests/scripts/task_web_build.sh → tests/scripts/task_web_wasm.sh

@@ -15,6 +15,17 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-cp /emsdk-portable/.emscripten ~/.emscripten
-source /emsdk-portable/emsdk_env.sh
-make -j4
+
+set -e
+set -u
+
+export PYTHONPATH=`pwd`/python
+
+cd web
+npm install
+npm run lint
+npm run prepwasm
+npm run bundle
+npm run test
+npm run typedoc
+cd ..

web/package.json

@@ -4,7 +4,7 @@
   "license": "Apache-2.0",
   "version": "0.7.0",
   "scripts": {
-    "prepare": "make && python3 tests/python/prepare_test_libs.py",
+    "prepwasm": "make && python3 tests/python/prepare_test_libs.py",
     "build": "tsc -b && make rmtypedep",
     "lint": "eslint -c .eslintrc.json .",
     "typedoc": "typedoc .",