Huggingface data migration (facebookresearch#1450)

* huggingface migration support for CI * specify --no-prune for test data download for CI compat. * remove coda scene download on CI, seems unused
llach · Jul 17, 2023 · 79ef715 · 79ef715
1 parent fd4148b
commit 79ef715
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 31 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -203,38 +203,12 @@ jobs:
       - run:
           name: Download test data
           command: |
+              sudo apt install git-lfs
               export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
               . activate habitat
-              if [ ! -f ./habitat-sim/data/scene_datasets/habitat-test-scenes/van-gogh-room.glb ]
-              then
-                cd habitat-sim
-                wget http://dl.fbaipublicfiles.com/habitat/habitat-test-scenes.zip
-                unzip habitat-test-scenes.zip
-                rm habitat-test-scenes.zip
-                cd -
-              fi
-              if [ ! -f ./habitat-sim/data/robots/franka_panda/panda_arm.urdf ]
-              then
-                python -m habitat_sim.utils.datasets_download --uids franka_panda --data-path habitat-sim/data/
-              fi
-              if [ ! -f ./habitat-sim/data/robots/hab_spot_arm/urdf/hab_spot_arm.urdf ]
-              then
-                python -m habitat_sim.utils.datasets_download --uids hab_spot_arm --data-path habitat-sim/data/ --replace
-              fi
-              if [ ! -f ./habitat-sim/data/robots/hab_stretch/urdf/hab_stretch.urdf ]
-              then
-                python -m habitat_sim.utils.datasets_download --uids hab_stretch --data-path habitat-sim/data/
-              fi
-      - run:
-          name: Download coda scene
-          command: |
-            if [ ! -f ./habitat-sim/data/scene_datasets/coda/coda.glb ]
-            then
-              cd habitat-sim
-              wget --no-check-certificate 'https://docs.google.com/uc?export=download&id=1Pc-J6pZzXEd8RSeLM94t3iwO8q_RQ853' -O coda.zip
-              unzip coda.zip -d data/scene_datasets
-              rm coda.zip
-            fi
+              git lfs install
+              conda install -y gitpython
+              python -m habitat_sim.utils.datasets_download --uids ci_test_assets franka_panda hab_spot_arm --data-path habitat-sim/data/ --no-replace --no-prune
       - run:
           name: Run sim benchmark
           command: |

diff --git a/test/test_baseline_trainers.py b/test/test_baseline_trainers.py
@@ -49,7 +49,9 @@
 @pytest.fixture(scope="module", autouse=True)
 def download_data():
     # Download the needed datasets
-    data_downloader.main(["--uids", "rearrange_task_assets", "--no-replace"])
+    data_downloader.main(
+        ["--uids", "rearrange_task_assets", "--no-replace", "--no-prune"]
+    )
 
 
 @pytest.mark.skipif(