Ultra Merge (huawei-noah#430)

* migrating to smarts

* initial changes to SAC and ultra registry

* added info_adapter and removed src

* adding environment score

* moved env_score to info

* removed unnecessary variable

* applied change to ppo

* updating dqn

* changing the action space to discrete

* updating ddpg

* updating policy params

* adding old README

* adding docker for ultra

* removing worker.py and run_agent.py

* updating docker file

* Update Dockerfile

* Update run_container.sh

* Update Dockerfile

* Fix registry import (huawei-noah#385)

* ULTRA Fix `test-formatting` (huawei-noah#386)

* Fix registry import

* `make format`

* Update Dockerfile

* Removed proxy details. (huawei-noah#389)

* Fixed ULTRA README.md values and directories that were old. (huawei-noah#398)

* renaming UltraAgentSpec to BaselineAgentSpec for clarity

* overriding step module for ultra

* cleanup highway changes

* cleanup highway changes

* fixing the format

* removing tests for updates in future

* applied comments to cleanup

* removing docker folder

* removing proxy from Makefile and cleanup Dockerfile

* README docker step update

* README update remove badges

* update docker steo

* adding headers

* fixing the format

Co-authored-by: Kimia Hassanzadeh <[email protected]>
Co-authored-by: adai <[email protected]>
Co-authored-by: christianjans <[email protected]>

cli/cli.py

@@ -22,6 +22,7 @@
 from .envision import envision_cli
 from .studio import scenario_cli
 from .zoo import zoo_cli
+from .ultra import ultra_cli
 
 
 @click.group()
@@ -32,6 +33,7 @@ def scl():
 scl.add_command(envision_cli)
 scl.add_command(scenario_cli)
 scl.add_command(zoo_cli)
+scl.add_command(ultra_cli)
 
 
 if __name__ == "__main__":

cli/ultra.py

@@ -0,0 +1,79 @@
+# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+import os
+import sys
+import glob
+import shutil
+import subprocess
+from pathlib import Path
+
+import click
+from rich import print
+
+
+@click.group(name="ultra")
+def ultra_cli():
+    pass
+
+
+@ultra_cli.command(name="build", help="Build a policy")
+@click.argument("policy", type=click.Path(exists=True), metavar="<policy>")
+def build_policy(policy):
+    def clean():
+        subprocess.check_call([sys.executable, "setup.py", "clean", "--all"])
+
+    def build():
+        cwd = Path(os.getcwd())
+        subprocess.check_call([sys.executable, "setup.py", "bdist_wheel"])
+        results = sorted(glob.glob("./dist/*.whl"), key=os.path.getmtime, reverse=True)
+        assert len(results) > 0, f"No policy package was built at path={cwd}"
+
+        wheel = Path(results[0])
+        dst_path = cwd / wheel.name
+        shutil.move(wheel.resolve(), cwd / wheel.name)
+        return dst_path
+
+    os.chdir(policy)
+    clean()
+    wheel_path = build()
+    clean()
+    print(
+        f"""
+Policy built successfully and is available at,
+
+\t[bold]{wheel_path}[/bold]
+
+You can now add it to the policy ultra if you want to make it available to scenarios.
+"""
+    )
+
+
+@ultra_cli.command(name="worker", help="Start the agent worker")
+@click.argument("auth_key", type=str, default=None)
+@click.argument("port", default=7432, type=int)
+def worker(auth_key, port):
+    from smarts.zoo.worker import listen
+
+    auth_key = auth_key if auth_key else ""
+    listen(port, auth_key)
+
+
+ultra_cli.add_command(build_policy)
+ultra_cli.add_command(worker)

gen_header.sh

@@ -4,15 +4,15 @@ files=""
 
 if [[ $# -eq 0 ]]; then
     # No specific file specified, collate all files excluding auto generated files of *_pb2.py and *_pb2_grpc.py
-    python_files="$(find ./cli ./smarts ./benchmark ./envision -name '*.py' ! -name '*_pb2.py' ! -name '*_pb2_grpc.py')"
+    python_files="$(find ./cli ./smarts ./benchmark ./envision ./ultra -name '*.py' ! -name '*_pb2.py' ! -name '*_pb2_grpc.py')"
     js_files="$(find ./envision/web/src -name '*.js')"
-    files="$python_files $js_files"  
+    files="$python_files $js_files"
 else
     # use files specified in the args
     for file in "$@"; do
         files="$files $file"
-    done 
-fi 
+    done
+fi
 
 # Check and add license notice
 for file in $files; do

ultra/Dockerfile

@@ -0,0 +1,139 @@
+FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+#------------------------------------------------------------------------------
+# Certificate Setup
+#------------------------------------------------------------------------------
+#RUN apt-get update && apt-get install -y openssh-server
+RUN apt-get install ca-certificates
+RUN update-ca-certificates
+RUN apt-get install apt-transport-https
+
+#------------------------------------------------------------------------------
+# Set timezone
+#------------------------------------------------------------------------------
+# Prevent tzdata from trying to be interactive
+# ENV TZ=Europe/Minsk
+# RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+RUN echo "Acquire::Check-Valid-Until \"false\";\nAcquire::Check-Date \"false\";" | \
+          cat > /etc/apt/apt.conf.d/10no--check-valid-until
+
+#------------------------------------------------------------------------------
+# Nvidia Drivers Installation
+#------------------------------------------------------------------------------
+#ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display
+#RUN apt-get update && apt-get install --no-install-recommends -y mesa-utils \
+#    && rm -rf /var/lib/apt/lists/*
+
+#------------------------------------------------------------------------------
+# Install dependencies
+#------------------------------------------------------------------------------
+# http://bugs.python.org/issue19846
+# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
+ENV LANG C.UTF-8
+
+RUN echo "Installing dependencies"
+RUN apt-get update --fix-missing && \
+    apt-get install -y \
+        software-properties-common && \
+    add-apt-repository -y ppa:deadsnakes/ppa && \
+    add-apt-repository -y ppa:sumo/stable && \
+    apt-get update && \
+    apt-get install -y \
+        wget \
+        python3.7 \
+        python3.7-dev \
+        python3.7-venv \
+        sumo \
+        sumo-tools \
+        sumo-doc \
+        libspatialindex-dev \
+        libsm6 \
+        libxext6 \
+        libxrender-dev
+
+# Install pip dependencies
+RUN wget https://bootstrap.pypa.io/get-pip.py -O get-pip.py && \
+    python3.7 get-pip.py && \
+    pip install --upgrade pip
+
+#------------------------------------------------------------------------------
+# Display
+#------------------------------------------------------------------------------
+RUN echo "Installing XDummy"
+ENV DISPLAY :1
+
+RUN apt-get install -y \
+    xserver-xorg-video-dummy \
+    x11-apps
+
+# VOLUME /tmp/.X11-unix
+RUN wget -O /etc/X11/xorg.conf http://xpra.org/xorg.conf
+
+#------------------------------------------------------------------------------
+# Simulator
+#------------------------------------------------------------------------------
+RUN echo "Setup SMARTS Dependencies"
+ENV SUMO_HOME /usr/share/sumo
+
+#------------------------------------------------------------------------------
+# Fix Git
+#------------------------------------------------------------------------------
+RUN cp /etc/apt/sources.list /etc/apt/sources.list~ && \
+    sed -i 's/# deb-src/deb-src/' /etc/apt/sources.list && \
+    apt-get update
+
+RUN apt-get install -y \
+    git
+
+RUN apt-get install build-essential fakeroot dpkg-dev -y && \
+    apt-get build-dep git -y && \
+    apt-get install libcurl4-openssl-dev -y  && \
+    cd ~ && \
+    mkdir source-git && \
+    cd source-git/ && \
+    apt-get source git && \
+    cd git-2.*.*/ && \
+    sed -i 's/libcurl4-gnutls-dev/libcurl4-openssl-dev/' ./debian/control && \
+    sed -i '/TEST\s*=\s*test/d' ./debian/rules && \
+    dpkg-buildpackage -rfakeroot -b -uc -us && \
+    dpkg -i ../git_*ubuntu*.deb
+
+#------------------------------------------------------------------------------
+# Clean-up
+#------------------------------------------------------------------------------
+RUN echo "Cleaning-up"
+RUN apt-get autoremove && \
+    rm -rf /var/lib/apt/lists/*
+
+#------------------------------------------------------------------------------
+# Copy src files
+#------------------------------------------------------------------------------
+# Install SMARTS
+RUN mkdir -p /SMARTS
+COPY ./setup.py /SMARTS/
+
+# Add https://github.com/krallin/tini
+ARG TINI_VERSION=v0.18.0
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
+RUN chmod +x /tini
+#COPY docker/entrypoint.sh /usr/local/bin/entrypoint.sh
+#RUN chmod +x /usr/local/bin/entrypoint.sh
+
+
+#------------------------------------------------------------------------------
+# Entrypoint
+#------------------------------------------------------------------------------
+# For Envision
+EXPOSE 8081
+
+# TODO: Find a better place to put this (e.g. through systemd). As it stands now ctrl-c
+#       could close x-server. Even though it's "running in the background".
+RUN echo "/usr/bin/Xorg " \
+    "-noreset +extension GLX +extension RANDR +extension RENDER" \
+    "-logfile ./xdummy.log -config /etc/X11/xorg.conf -novtswitch $DISPLAY &" >> ~/.bashrc
+
+CMD ["/bin/bash"]
+# Once in container can then run,
+#   python3 examples/competition

ultra/MANIFEST.in

@@ -0,0 +1 @@
+include sac/checkpoint/checkpoint

ultra/Makefile

@@ -0,0 +1,10 @@
+# Name of image
+IMAGE = ultra
+# Docker TAG
+TAG = gpu
+build:
+	docker build \
+		-t $(IMAGE):$(TAG) \
+		--network=host \
+		.
+.PHONY: build

ultra/README.md

@@ -0,0 +1,89 @@
+# ULTRA
+
+Unprotected Left Turn using Reinforcement-learning Agents
+---
+Ultra provides a gym-based environment using SMARTS for tackling intersection navigation and more specifically unprotected left turn.
+
+Here is the summary of key features:
+ - creating customized scenarios with different levels of difficulty
+ - defining/analyzing traffic designs including low-mid-high densities
+ - defining/analyzing social-vehicle behaviors
+ - configurable train/test parameters
+ - train/test custom RL algorithms against benchmark results
+
+
+ignore this step if you already have smarts environment
+### Setup
+  ```sh
+  python3.7 -m venv .smarts
+  # 1-activate virtual environment to install all dependencies
+  source .smarts/bin/activate
+  # 2-install black for formatting (if you wish to contribute)
+  pip install black
+  # 3-install dependencies
+  pip install -e .[train]
+  ```
+
+### Generate scenarios
+
+Supported tasks are listed under `ultra/scenarios/`.
+
+- For first time generating the maps:
+  ```sh
+  $ scl scenario build-all ultra/scenarios/pool
+  ```
+  > Maps only need to be compiled once and you don't need to run them again unless they are modifed
+- Generate scenarios:
+  ``` sh
+  # task 1 generate 1000 sscenarios (800 train, 200 test) and supports 2 levels of difficulties for more info refer to our documentaion
+  $ python ultra/scenarios/interface.py generate --task 1 --level easy
+  ```
+  > After running the command above, train and test scenarios are added under the task directory
+
+### Run envision in the background
+if you are running the train and evalaution in headless mode, ignore this step
+```sh
+$ ./ultra/env/envision_base.sh
+```
+envision runs as a background process, you can view the visualization on `localhost:8081/`
+
+### Train and Evalutaion
+We currently support PPO/SAC/TD3/DQN policies
+
+- Train
+  Training runs the policy with default configuration:
+  `
+  {train episodes: 1000000, maxsteps: 1200, timestep: 0.1 sec, evaluation rate: 10000, evaluation episodes: 200}
+  `
+  - For each experiment a folder with timestamp is added automatically under `logs/` and it saves tensorboad log, models and pkls
+  - For every 10000 observation, evaluation is called automatically and policy is saved under `logs/your-expierment-name/models/`
+  ```sh
+  $ python ultra/train.py --task 1 --level easy
+  # other arguments example: --episodes 20000 --eval-rate 500 --eval-episodes 200 --timestep 1 --headless
+  ```
+### Run Evaluation Separately
+After training your agent, your models should be saved under `logs/your-experiment-name/models/` and you can re-run the evaluation:
+  ```sh
+  $ python ultra/evaluate.py --task 1 --level easy --models logs/your-expierment-name/models
+  # other arguments --episodes 20000 --timestep 1 --headless
+  ```
+
+### Docker
+- Build a docker image and run container
+  ```sh
+  $ cd path/to/repository/SMARTS/ultra
+  $ docker build -t <container name> --network=host . # or run make
+  $ docker run \
+         -it \
+         --volume=/tmp/.X11-unix:/tmp/.X11-unix:rw \
+         --privileged \
+         --env="XAUTHORITY=/tmp/.docker.xauth" \
+         --env="QT_X11_NO_MITSHM=1" \
+         --volume=/usr/lib/nvidia-384:/usr/lib/nvidia-384 \
+         --volume=/usr/lib32/nvidia-384:/usr/lib32/nvidia-384 \
+         --runtime=nvidia \
+         --device /dev/dri \
+         --volume=$DIR:/SMARTS \ #  fill $DIR with the path to SMARTS to mount
+         --name=ultra \
+         ultra:gpu
+  ```

ultra/__init__.py

@@ -0,0 +1,21 @@
+# MIT License
+#
+# Copyright (C) 2021. Huawei Technologies Co., Ltd. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.