Merge pull request mila-iqia#18 from mila-udem/dima-handcrafted-tests

Bootstrap the work on regression tests

.circleci/config.yml

@@ -23,6 +23,12 @@ jobs:
         - run: python3 -m scripts.eval_bot --level PutNextLocal --num_runs 100 | grep "100.0%"
         - run: python3 -m scripts.eval_bot --level SynthLoc --num_runs 100 | grep "100.0%"
 
+        # Check that bot works on a few hand-crafted tests
+        - run: python3 -m scripts.evaluate --model=BOT  --env=BabyAI-TestGoToBlocked-v0 --episodes=1 | grep 'S 1.00'
+        # Uncomment me when #17 is merged
+        # - run: python3 -m scripts.evaluate --model=BOT  --env=BabyAI-TestPutNextToBlocked-v0 --episodes=1 | grep 'S 1.00'
+
+
         # Quickly test the generation of bot demos
         - run: python3 -m scripts.make_agent_demos --env BabyAI-GoToRedBallGrey-v0 --episodes 100 --valid-episodes 32
 

babyai/levels/__init__.py

@@ -2,5 +2,6 @@
 
 from . import iclr19_levels
 from . import bonus_levels
+from . import test_levels
 
 from .levelgen import test, level_dict

babyai/levels/test_levels.py

@@ -0,0 +1,67 @@
+"""
+Regression tests.
+"""
+
+import numpy as np
+
+import gym
+from .verifier import *
+from .levelgen import *
+from gym_minigrid.minigrid import *
+
+
+class Level_TestGoToBlocked(RoomGridLevel):
+    """
+    Go to a yellow ball that is blocked with a lot of red balls.
+    """
+
+    def __init__(self, room_size=8, seed=None):
+        super().__init__(
+            num_rows=1,
+            num_cols=1,
+            room_size=9,
+            seed=seed
+        )
+
+    def gen_mission(self):
+        self.place_agent()
+        self.start_pos = np.array([3, 3])
+        self.start_dir = 0
+        obj = Ball('yellow')
+        self.grid.set(1, 1, obj)
+        for i in (1, 2, 3):
+            for j in (1, 2, 3):
+                if (i, j) not in [(1 ,1), (3, 3)]:
+                    self.grid.set(i, j, Ball('red'))
+        self.instrs = GoToInstr(ObjDesc(obj.type, obj.color))
+
+
+
+class Level_TestPutNextToBlocked(RoomGridLevel):
+    """
+    Pick up a yellow ball and put it next to a blocked blue ball.
+    """
+
+    def __init__(self, room_size=8, seed=None):
+        super().__init__(
+            num_rows=1,
+            num_cols=1,
+            room_size=9,
+            seed=seed
+        )
+
+    def gen_mission(self):
+        self.place_agent()
+        self.start_pos = np.array([3, 3])
+        self.start_dir = 0
+        obj1 = Ball('yellow')
+        obj2 = Ball('blue')
+        self.place_obj(obj1, (4, 4), (1, 1))
+        self.place_obj(obj2, (1, 1), (1, 1))
+        self.grid.set(1, 2, Ball('red'))
+        self.grid.set(2, 1, Ball('red'))
+        self.instrs = PutNextInstr(ObjDesc(obj1.type, obj1.color),
+                                   ObjDesc(obj2.type, obj2.color))
+
+
+register_levels(__name__, globals())

babyai/utils/__init__.py

@@ -2,7 +2,7 @@
 import random
 import numpy
 import torch
-from babyai.utils.agent import load_agent, ModelAgent
+from babyai.utils.agent import load_agent, ModelAgent, DemoAgent, BotAgent
 from babyai.utils.demos import (
     load_demos, save_demos, synthesize_demos, get_demos_path)
 from babyai.utils.format import ObssPreprocessor, IntObssPreprocessor, get_vocab_path

scripts/evaluate.py

@@ -48,13 +48,13 @@ def main(args, seed, episodes):
         episodes = len(agent.demos)
 
     # Evaluate
-    if isinstance(agent, utils.ModelAgent):
-        if not args.contiguous_episodes:
-            logs = batch_evaluate(agent, args.env, seed, episodes)
-        else:
-            logs = evaluate(agent, env, episodes, False)
-    else:
+    if isinstance(agent, utils.DemoAgent):
         logs = evaluate_demo_agent(agent, episodes)
+    elif isinstance(agent, utils.BotAgent) or args.contiguous_episodes:
+        logs = evaluate(agent, env, episodes, False)
+    else:
+        logs = batch_evaluate(agent, args.env, seed, episodes)
+
 
     return logs