Packaging and PyPI distributing (facebookresearch#1192)

.circleci/config.yml

@@ -157,7 +157,7 @@ jobs:
                 conda install -y pytorch==1.12.1=py3.9_cuda11.3_cudnn8.3.2_0 torchvision==0.13.1=py39_cu113 cudatoolkit=11.3 -c pytorch -c nvidia
               fi
               touch ~/miniconda/pytorch_installed
-              python -c 'import torch; print("Has cuda ? ",torch.cuda.is_available()); print("torch version : ",torch.__version__);'
+              python -c 'import torch; print("Has cuda?", torch.cuda.is_available()); print("torch version:", torch.__version__);'
       - restore_cache:
           keys:
             - v1-habitat-sim-{{ checksum "./hsim_sha" }}
@@ -314,15 +314,84 @@ jobs:
       - run:
           name: Ensure non-editable mode works
           command: |
+              cd habitat-lab
               export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
-              . activate habitat; cd habitat-lab
-              python -m pip install habitat-lab/
-              python -c "import habitat"
-              python -m pip install habitat-baselines/
-              python -c "import habitat_baselines"
+              conda create -y -n non-editable-install python=3.9
+              . activate non-editable-install
+              conda install -y -c conda-forge -c aihabitat-nightly habitat-sim
+              pip install habitat-lab/
+              python -c 'import habitat; print("habitat version:", habitat.__version__)'
+              pip install habitat-baselines/
+              python -c 'import habitat_baselines; print("habitat_baselines version:", habitat_baselines.__version__)'
+
+      - run:
+          name: Ensure build and intall work
+          command: |
+              cd habitat-lab
+              export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
+
+              conda create -y -n build-env python=3.9
+              . activate build-env
+              pip install --upgrade build
+              python -m build -s -w -C--global-option=egg_info -C--global-option=--tag-date habitat-lab/
+              python -m build -s -w -C--global-option=egg_info -C--global-option=--tag-date habitat-baselines/
+
+              conda create -y -n bdist-install python=3.9
+              . activate bdist-install
+              conda install -y -c conda-forge -c aihabitat-nightly habitat-sim
+              conda create -n sdist-install --clone bdist-install
+
+              # install from built distribution:
+              . activate bdist-install
+              pip install habitat-lab/dist/habitat_lab*.whl
+              python -c 'import habitat; print("habitat version:", habitat.__version__)'
+              pip install habitat-baselines/dist/habitat_baselines*.whl
+              python -c 'import habitat_baselines; print("habitat_baselines version:", habitat_baselines.__version__)'
+
+              # install from source distribution:
+              . activate sdist-install
+              pip install habitat-lab/dist/habitat-lab*.tar.gz
+              python -c 'import habitat; print("habitat version:", habitat.__version__)'
+              pip install habitat-baselines/dist/habitat-baselines*.tar.gz
+              python -c 'import habitat_baselines; print("habitat_baselines version:", habitat_baselines.__version__)'
+
       - store_artifacts:
           path: habitat-lab/data/profile  # This is the benchmark profile
-
+  pypi_deploy:
+    <<: *gpu
+    parameters:
+      username_env_var:
+        description: "Twine username environment variable name"
+        type: env_var_name
+        default: TESTPYPI_USERNAME
+      password_env_var:
+        description: "Twine password environment variable name"
+        type: env_var_name
+        default: TESTPYPI_PASSWORD
+      repository:
+        description: "Twine repository name (possible options: testpypi or pypi)"
+        type: string
+        default: "testpypi"
+    steps:
+      - checkout:
+          path: ./habitat-lab
+      - run:
+          name: Deploy Habitat-Lab and Habitat-Baselines to PyPI
+          command: |
+              cd habitat-lab
+              export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
+              . activate build-env
+              pip install --upgrade twine
+              twine upload \\
+                --username ${<< parameters.username_env_var >>} \\
+                --password ${<< parameters.password_env_var >>} \\
+                --repository ${<< parameters.repository >>} \\
+                habitat-lab/dist/*
+              twine upload \\
+                --username ${<< parameters.username_env_var >>} \\
+                --password ${<< parameters.password_env_var >>} \\
+                --repository ${<< parameters.repository >>} \\
+                habitat-baselines/dist/*
 
 workflows:
   version: 2
@@ -331,3 +400,40 @@ workflows:
       - pre-commit
       - python_lint
       - install_and_test_ubuntu
+  testpypi_nightly:
+    triggers:
+      - schedule:
+          cron: "0 7 * * *"
+          filters:
+            branches:
+              only: main
+    jobs:
+      - pre-commit
+      - python_lint
+      - install_and_test_ubuntu
+      - pypi_deploy:
+          requires:
+            - pre-commit
+            - python_lint
+            - install_and_test_ubuntu
+  version_pipy_release:
+    jobs:
+      - pre-commit:
+          filters: &version_filter
+            tags:
+              only: /^v[0-9]+(\.[0-9]+)*.*/ # v0.1.5-rc1
+            branches:
+              ignore: /.*/
+      - python_lint:
+          filters: *version_filter
+      - install_and_test_ubuntu:
+          filters: *version_filter
+      - pypi_deploy:
+          filters: *version_filter
+          username_env_var: PYPI_USERNAME
+          password_env_var: PYPI_PASSWORD
+          repository: "pypi"
+          requires:
+            - pre-commit
+            - python_lint
+            - install_and_test_ubuntu

.pre-commit-config.yaml

@@ -18,7 +18,6 @@ repos:
     -   id: check-merge-conflict
     -   id: check-toml
     -   id: check-yaml
-        exclude: "habitat-baselines/habitat_baselines/slambased/data/"
     -   id: mixed-line-ending
         args: ['--fix=lf']
 
@@ -107,7 +106,6 @@ repos:
     rev: v0.8.0.4
     hooks:
     -   id: shellcheck
-        exclude: ^habitat-baselines/habitat_baselines/slambased/
 
 -   repo: https://github.com/AleksaC/circleci-cli-py
     rev: v0.1.17183

docs/pages/habitat2.rst

@@ -28,7 +28,7 @@ Habitat includes an implementation of DD-PPO. As an example, start training a pi
 
 .. code:: sh
 
-    python -u habitat_baselines/run.py \
+    python -u -m habitat_baselines.run \
         --config-name=rearrange/rl_skill.yaml
 
 This trains the Pick skill by default. To train the other skills, specify: :code:`benchmark/rearrange=skill_name` where :code:`skill_name` can be :code:`close_cab`, :code:`close_fridge`, :code:`open_fridge`, :code:`pick`, :code:`place`, or :code:`nav_to_obj`. See `here <https://github.com/facebookresearch/habitat-lab/tree/main/habitat-baselines/habitat_baselines#baselines>`__  for more information on how to run with Habitat Baselines.
@@ -41,7 +41,7 @@ Here we will detail how to run the Task-Planning with Skills trained via reinfor
 
 .. code:: sh
 
-    python -u habitat_baselines/run.py \
+    python -u -m habitat_baselines.run \
         --config-name=rearrange/rl_skill.yaml \
         checkpoint_folder=./place_checkpoints/ \
         benchmark/rearrange=place
@@ -54,7 +54,7 @@ Here we will detail how to run the Task-Planning with Skills trained via reinfor
 
 .. code:: sh
 
-    python -u habitat_baselines/run.py \
+    python -u -m habitat_baselines.run \
         --config-name=rearrange/tp_srl.yaml \
         habitat_baselines.evaluate=True \
         benchmark/rearrange=tidy_house

habitat-baselines/MANIFEST.in

@@ -1 +1,2 @@
 include habitat_baselines/py.typed
+recursive-include habitat_baselines requirements.txt

habitat-baselines/habitat_baselines/README.md → habitat-baselines/README.md

@@ -25,22 +25,22 @@ For training on sample data please follow steps in the repository README. You sh
 
 **train**:
 ```bash
-python -u habitat_baselines/run.py \
+python -u -m habitat_baselines.run \
   --config-name=pointnav/ppo_pointnav_example.yaml
 ```
 
 You can reduce training time by changing the trainer from the default implement to [VER](rl/ver/README.md) by
 setting `trainer_name` to `"ver"` in either the config or via the command line.
 
 ```bash
-python -u habitat_baselines/run.py \
+python -u -m habitat_baselines.run \
   --config-name=pointnav/ppo_pointnav_example.yaml \
   habitat_baselines.trainer_name=ver
 ```
 
 **test**:
 ```bash
-python -u habitat_baselines/run.py \
+python -u -m habitat_baselines.run \
   --config-name=pointnav/ppo_pointnav_example.yaml \
   habitat_baselines.evaluate=True
 ```
@@ -60,13 +60,13 @@ To run the following examples, you need the [ReplicaCAD dataset](https://github.
 To train a high-level policy, while using pre-learned low-level skills (SRL baseline from [Habitat2.0](https://arxiv.org/abs/2106.14405)), you can run:
 
 ```bash
-python -u habitat-baselines/habitat_baselines/run.py \
+python -u -m habitat_baselines.run \
   --config-name=rearrange/rl_hierarchical.yaml
 ```
 To run a rearrangement episode with oracle low-level skills and a fixed task planner, run:
 
 ```bash
-python -u habitat-baselines/habitat_baselines/run.py \
+python -u -m habitat_baselines.run \
   --config-name=rearrange/rl_hierarchical.yaml \
   habitat_baselines.evaluate=True \
   habitat_baselines/rl/policy=hl_fixed \

habitat-baselines/habitat_baselines/il/README.md

@@ -39,7 +39,7 @@ Configuration for training the VQA (answering) model can be found in `habitat_ba
 ### Train:
 
 ```
-python -u habitat_baselines/run.py \
+python -u -m habitat_baselines.run \
   --config-name=eqa/il_eqa_cnn_pretrain.yaml
 ```
 
@@ -48,7 +48,7 @@ Training checkpoints are by default stored in `data/eqa/eqa_cnn_pretrain/checkpo
 ### Eval:
 
 ```
-python -u habitat_baselines/run.py \
+python -u -m habitat_baselines.run \
   --config-name=eqa/il_eqa_cnn_pretrain.yaml \
   habitat_baselines.evaluate=True
 ```
@@ -84,7 +84,7 @@ The VQA trainer picks the EQA CNN pre-trained encoder checkpoint by default from
 ### Train:
 
 ```
-python -u habitat_baselines/run.py \
+python -u -m habitat_baselines.run \
   --config-name=eqa/il_vqa.yaml
 ```
 
@@ -99,7 +99,7 @@ After downloading the pre-trained model, add its path to the config file's `eval
 ### Eval:
 
 ```
-python -u habitat_baselines/run.py \
+python -u -m habitat_baselines.run \
   --config-name=/eqa/il_vqa.yaml \
   habitat_baselines.evaluate=True
 ```
@@ -128,7 +128,7 @@ The trainer also picks the EQA CNN pre-trained encoder checkpoint by default fro
 ### Train:
 
 ```
-python -u habitat_baselines/run.py \
+python -u -m habitat_baselines.run \
   --config-name=eqa/il_pacman_nav.yaml
 ```
 
@@ -138,7 +138,7 @@ Training checkpoints are by default stored in `data/eqa/nav/checkpoints`.
 ### Eval:
 
 ```
-python -u habitat_baselines/run.py \
+python -u -m habitat_baselines.run \
   --config-name=eqa/il_pacman_nav.yaml \
   habitat_baselines.evaluate=True
 ```

habitat-baselines/habitat_baselines/rl/ver/README.md

@@ -27,7 +27,7 @@ There are two components that serve auxiliary functions:
 To use VER, simply change the `trainer_name` to `"ver"` in either the config or via the command line:
 
 ```bash
-python -u habitat_baselines/run.py \
+python -u -m habitat_baselines.run \
   --config-name=pointnav/ppo_pointnav_example.yaml \
   habitat_baselines.trainer_name=ver
 ```

habitat-baselines/habitat_baselines/run.py

@@ -70,8 +70,8 @@ def execute_exp(config: "DictConfig", run_type: str) -> None:
             "--exp-config is now --config-name and is a config path inside habitat-baselines/habitat_baselines/config/. \n"
             "--run-type train is replaced with habitat_baselines.evaluate=False (default) and --run-type eval is replaced with habitat_baselines.evaluate=True.\n"
             "instead of calling:\n\n"
-            "python habitat-baselines/habitat_baselines/run.py --exp-config habitat-baselines/habitat_baselines/config/<path-to-config> --run-type train/eval\n\n"
+            "python -u -m habitat_baselines.run --exp-config habitat-baselines/habitat_baselines/config/<path-to-config> --run-type train/eval\n\n"
             "You now need to do:\n\n"
-            "python habitat-baselines/habitat_baselines/run.py --config-name=<path-to-config> habitat_baselines.evaluate=False/True\n"
+            "python -u -m habitat_baselines.run --config-name=<path-to-config> habitat_baselines.evaluate=False/True\n"
         )
     main()

habitat-baselines/habitat_baselines/version.py

@@ -4,4 +4,4 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-VERSION = "0.2.3"
+VERSION = "0.2.4.dev"

habitat-baselines/setup.py

@@ -45,7 +45,7 @@ def get_package_version():
         version=get_package_version(),
         include_package_data=True,
         description="Habitat-Baselines: Embodied AI baselines.",
-        long_description=read("habitat_baselines/README.md", encoding="utf8"),
+        long_description=read("README.md", encoding="utf8"),
         long_description_content_type="text/markdown",
         author="Meta AI Research",
         license="MIT License",

habitat-lab/MANIFEST.in

@@ -1,3 +1,4 @@
 graft habitat/config
 graft habitat/utils/visualizations/assets
 include habitat/py.typed
+include requirements.txt

habitat-lab/habitat/config/README.md

@@ -329,15 +329,15 @@ class SimulatorConfig(HabitatBaseConfig):
 ```
 - [Parameter sweeping and multirun](https://hydra.cc/docs/tutorials/basic/running_your_app/multi-run/). For example, launching 3 experiments with three different learning rates:
 ```bash
-python -u habitat-baselines/habitat_baselines/run.py --config-name=config.yaml  \
+python -u -m habitat_baselines.run --config-name=config.yaml  \
 –-multirun habitat_baselines.rl.ppo.lr 2.5e-4,2.5e-5,2.5e-6
 ```
 - Seamless [SLURM](https://slurm.schedmd.com/documentation.html) integration through
   [Submitit Launcher](https://hydra.cc/docs/plugins/submitit_launcher/).
   To enable the feature Submitit plugin should be installed: `pip install hydra-submitit-launcher --upgrade`
   and `submitit_slurm` launcher specified in the command line `hydra/launcher=submitit_slurm`:
 ```bash
-python -u habitat-baselines/habitat_baselines/run.py --config-name=config.yaml  \
+python -u -m habitat_baselines.run --config-name=config.yaml  \
 hydra/launcher=submitit_slurm --multirun
 ```
 - Making the config key required by setting its value to `MISSING`. For example, we require the user to explicitly
@@ -367,14 +367,14 @@ config = habitat.get_config("benchmark/nav/pointnav/pointnav_gibson.yaml")
 #### via command line
 Override config values:
 ```bash
-python -u habitat_baselines/run.py --config-name=pointnav/ddppo_pointnav.yaml \
+python -u -m habitat_baselines.run --config-name=pointnav/ddppo_pointnav.yaml \
 habitat.environment.max_episode_steps=250 \
 habitat_baselines.total_num_steps=100
 ```
 
 Override the Config Group Option value:
 ```bash
-python -u habitat_baselines/run.py --config-name=pointnav/ddppo_pointnav.yaml \
+python -u -m habitat_baselines.run --config-name=pointnav/ddppo_pointnav.yaml \
 benchmark/nav/pointnav=pointnav_hm3d  # overriding benchmark config to be pointnav_hm3d
 ```
 

habitat-lab/habitat/utils/profiling_wrapper.py

@@ -9,7 +9,7 @@
 Example of capturing an Nsight Systems profile with Habitat-lab:
 export HABITAT_PROFILING=1
 export NSYS_NVTX_PROFILER_REGISTER_ONLY=0  # required when using capture range
-path/to/nvidia/nsight-systems/bin/nsys profile --sample=none --trace=nvtx --trace-fork-before-exec=true --capture-range=nvtx -p "habitat_capture_range" --stop-on-range-end=true --output=my_profile --export=sqlite python habitat_baselines/run.py --config-name=pointnav/ppo_pointnav.yaml profiling.capture_start_step 200 profiling.num_steps_to_capture 100
+path/to/nvidia/nsight-systems/bin/nsys profile --sample=none --trace=nvtx --trace-fork-before-exec=true --capture-range=nvtx -p "habitat_capture_range" --stop-on-range-end=true --output=my_profile --export=sqlite python -m habitat_baselines.run --config-name=pointnav/ppo_pointnav.yaml profiling.capture_start_step=200 profiling.num_steps_to_capture=100
 # look for my_profile.qdrep in working directory
 """
 

habitat-lab/habitat/version.py

@@ -4,4 +4,4 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-VERSION = "0.2.3"
+VERSION = "0.2.4.dev"