Add release test runnning full asan python test (ray-project#8836)

.bazelrc

@@ -62,7 +62,8 @@ build:tsan --copt -g
 build:tsan --copt -fno-omit-frame-pointer
 build:tsan --linkopt -fsanitize=thread
 
-# Memory sanitizer configuration:
+# Memory sanitizer configuration
+# If you change these make sure you also change ci/asan_tests/bazelrc
 build:asan --strip=never
 build:asan --copt -g
 build:asan --copt -fsanitize=address

ci/asan_tests/Makefile

@@ -0,0 +1,28 @@
+SHELL=/bin/bash
+
+echo:
+	@echo "Available commands: setup, run, recompile"
+
+setup:
+	echo "Setting up the environment"
+	pip install -r ray-project/requirements.txt
+	pip install -U pytest
+
+	echo "Installing cython example"
+	cd "${HOME}"/ray/doc/examples/cython; python setup.py install --user
+
+	echo "Settting up the shell"
+	cp bazelrc "${HOME}"/.bazelrc # Setup cache
+	echo "LD_PRELOAD=/usr/lib/gcc/x86_64-linux-gnu/7/libasan.so" >> "${HOME}"/.bashrc
+	echo "ASAN_OPTIONS=detect_leaks=0" >> "${HOME}"/.bashrc
+
+	echo "Compiling ray"
+	cd "${HOME}"/ray; git fetch; git pull origin master
+	cd "${HOME}"/ray/python; pip install -e . --verbose || true
+
+run:
+	./run_asan_tests.sh
+
+recompile:
+	cd "${HOME}"/ray; git fetch; git checkout "${git_sha}"
+	cd "${HOME}"/ray/python; pip install -e . --verbose || true

ci/asan_tests/bazelrc

@@ -0,0 +1,30 @@
+# This bazelrc specifies configuration for the release asan test.
+# It exists because we want everything to be "default" configuration
+# instead of "--config=ci" or "--config=asan".
+
+# Cache
+build --remote_cache=https://storage.googleapis.com/ray-bazel-cache
+build --disk_cache=/tmp/bazel-cache
+build --repository_cache=/tmp/bazel-cache
+build --remote_upload_local_results=false
+
+# Asan
+build --strip=never
+build --copt -g
+build --copt -fsanitize=address
+build --copt -fno-sanitize=vptr
+build --copt -DADDRESS_SANITIZER
+build --copt -fno-omit-frame-pointer
+build --linkopt -fsanitize=address
+build --linkopt -fno-sanitize=vptr
+test --action_env=ASAN_OPTIONS=detect_leaks=0
+test --action_env=LD_PRELOAD=/usr/lib/gcc/x86_64-linux-gnu/7/libasan.so
+
+# Travis test config
+test --flaky_test_attempts=3
+test --nocache_test_results
+test --show_timestamps
+test --spawn_strategy=local
+test --test_output=errors
+test --test_verbose_timeout_warnings
+test --jobs 1

ci/asan_tests/ray-project/cluster.yaml

@@ -0,0 +1,49 @@
+# This file is generated by `ray project create`.
+
+# A unique identifier for the head node and workers of this cluster.
+cluster_name: asan_python_tests
+
+# The maximum number of workers nodes to launch in addition to the head
+# node. This takes precedence over min_workers. min_workers defaults to 0.
+max_workers: 0
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-2
+    availability_zone: us-west-2a
+
+head_node:
+    InstanceType: m5.4xlarge
+    ImageId: ami-0d1cd67c26f5fca19  # Default Ubuntu 18.04 AMI.
+
+    # Set primary volume to 50 GiB
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 50
+
+# How Ray will authenticate with newly launched nodes.
+auth:
+    ssh_user: ubuntu
+
+setup_commands:
+    # Install basics.
+    - sudo apt-get update -q
+    - sudo apt-get install -y -q build-essential curl unzip
+    # Install Anaconda.
+    - wget -q https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh
+    - bash Anaconda3-5.0.1-Linux-x86_64.sh -b -p $HOME/anaconda3 || true
+    - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
+    - pip install -U pip || true
+    # Clone Ray.
+    - cd $HOME; rm -rf $HOME/ray; git clone https://github.com/ray-project/ray || true
+    # Install Node.js in order to build the dashboard.
+    - curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash
+    - sudo apt-get install -y nodejs || true
+    - cd ray/python/ray/dashboard/client; npm ci; npm run build
+    # Set up Ray dependencies.
+    - which bazel || bash -c "cd $HOME/ray; ci/travis/install-bazel.sh"
+
+# Command to start ray on the head node. You don't need to change this.
+head_start_ray_commands: []

ci/asan_tests/ray-project/project.yaml

@@ -0,0 +1,38 @@
+# This file is generated by `ray project create`.
+
+name: asan_python_tests
+
+# description: A short description of the project.
+# The URL of the repo this project is part of.
+# repo: ...
+
+cluster:
+  config: ray-project/cluster.yaml
+
+# environment:
+  # dockerfile: The dockerfile to be built and ran the commands with.
+  # dockerimage: The docker image to be used to run the project in, e.g. ubuntu:18.04.
+  # requirements: ray-project/requirements.txt
+
+commands:
+  - name: Setup
+    command: make setup
+  - name: Run
+    command: make run
+  - name: Recompile
+    command: |
+      git_sha={{git_sha}} make recompile
+    params:
+    - name: git_sha # Ray version string.
+      default: "master"
+
+# Pathnames for files and directories that should be saved
+# in a snapshot but that should not be synced with a# session. Pathnames can be relative to the project
+# directory or absolute. Generally, this should be files
+# that were created by an active session, such as
+# application checkpoints and logs.
+output_files: [
+  # For example, uncomment this to save the logs from the
+  # last ray job.
+  # "/tmp/ray/session_latest",
+]

ci/asan_tests/ray-project/requirements.txt

@@ -0,0 +1,37 @@
+aiohttp
+blist
+boto3
+cython==0.29.0
+dataclasses
+dm-tree
+feather-format
+flask
+grpcio
+gym
+kubernetes
+lxml
+networkx
+numba
+opencv-python-headless
+openpyxl
+pandas==0.24.2
+Pillow
+prometheus_client
+py-spy
+pygments
+pytest
+pytest-asyncio
+pytest-rerunfailures
+pytest-sugar
+pytest-timeout
+pyyaml
+requests
+scikit-learn==0.22.2
+scipy==1.4.1
+tabulate
+tensorflow==2.0.1
+torch
+torchvision
+uvicorn
+werkzeug
+xlrd

ci/asan_tests/run_asan_tests.sh

@@ -0,0 +1,15 @@
+set -e
+set -x
+
+export LD_PRELOAD=/usr/lib/gcc/x86_64-linux-gnu/7/libasan.so
+export ASAN_OPTIONS=detect_leaks=0
+
+cd $HOME/ray
+# async plasma test
+python -m pytest -v --durations=5 --timeout=300 python/ray/experimental/test/async_test.py
+
+# Ray tests
+bazel test --test_tag_filters=-jenkins_only python/ray/serve/...
+bazel test --test_tag_filters=-jenkins_only python/ray/dashboard/...
+bazel test --test_tag_filters=-jenkins_only python/ray/tests/...
+bazel test --test_tag_filters=-jenkins_only python/ray/tune/...