Skip to content

Commit

Permalink
non-learning changes
Browse files Browse the repository at this point in the history
  • Loading branch information
sraychau committed Mar 6, 2023
1 parent af9a737 commit 2a12269
Show file tree
Hide file tree
Showing 2 changed files with 123 additions and 11 deletions.
15 changes: 9 additions & 6 deletions baselines/config/nonlearning.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
BASE_TASK_CONFIG_PATH: multion/config/multion_3on.yaml
BASE_TASK_CONFIG_PATH: multion/config/multion_3on_one_scene.yaml
LOG_FILE: non_learning.log
LOG_INTERVAL: 1
VIDEO_OPTION: ["disk"] # options: "disk", "tensorboard"
VIDEO_DIR: experiments/non_learning/testing/3on/videos
RESULTS_DIR: experiments/non_learning/testing/3on
LOG_INTERVAL: 100
VIDEO_OPTION: [] # options: "disk", "tensorboard"
VIDEO_DIR: experiments/non_learning/mon2.0/3on/oracle/train/videos
RESULTS_DIR: experiments/non_learning/mon2.0/3on/oracle/train/00014-nYYcLpSzihC

EVAL:
SPLIT: train
# any num greater than the actual episode count evals every episode
EPISODE_COUNT: 20
EPISODE_COUNT: 1
EVAL_NONLEARNING: True
NONLEARNING:
# OracleAgent or RandomAgent or HandcraftedAgent
AGENT: OracleAgent

RL:
NEW_REWARD_STRUCTURE: True

119 changes: 114 additions & 5 deletions baselines/nonlearning_agents.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,22 @@
import math
from collections import defaultdict

import torch
import numpy as np
from habitat import logger
from habitat.config.default import Config
from habitat.core.agent import Agent
from habitat.sims.habitat_simulator.actions import HabitatSimActions
from habitat.tasks.nav.shortest_path_follower import ShortestPathFollower
from habitat.utils.visualizations.utils import observations_to_image, append_text_to_image
from habitat.utils.visualizations.utils import append_text_to_image
from baselines.common.viz_utils import (
observations_to_image
)
from baselines.common.utils import (
draw_projection,
rotate_tensor,
to_grid
)
from tqdm import tqdm

from habitat_baselines.utils.common import generate_video
Expand Down Expand Up @@ -41,7 +50,8 @@ def evaluate_agent(config: Config) -> None:
"OracleAgent",
"RandomAgent",
"HandcraftedAgent",
], "EVAL.NONLEARNING.AGENT must be either OracleAgent or RandomAgent or HandcraftedAgent."
"EpisodesValidator",
], "EVAL.NONLEARNING.AGENT must be either OracleAgent or RandomAgent or HandcraftedAgent or EpisodesValidator."

if config.EVAL.NONLEARNING.AGENT == "OracleAgent":
agent = OracleAgent(config.TASK_CONFIG, env)
Expand All @@ -55,6 +65,46 @@ def evaluate_agent(config: Config) -> None:
if config.EVAL.EPISODE_COUNT > 0
else len(env.episodes))

# Validate episodes
if config.EVAL.NONLEARNING.AGENT == "EpisodesValidator":
faulty_episodes = {}
faulty_scenes = []
total_num_faulty_epi = 0

validator = EpisodesValidator(env)
# for i in tqdm(range(num_episodes)):
# obs = env.reset()
# episode = env.current_episode
for episode in env.episodes:
if not validator.goals_reachable(episode):
total_num_faulty_epi += 1
scene_name = os.path.basename(episode.scene_id)

_epi = []
if scene_name in faulty_episodes and "episodes" in faulty_episodes[scene_name]:
_epi = faulty_episodes[scene_name]["episodes"]

_epi.append(episode.episode_id)
count_epi = len(_epi)
faulty_episodes[scene_name] = {}
faulty_episodes[scene_name]["num_episodes"] = count_epi
faulty_episodes[scene_name]["episodes"] = _epi

faulty_scenes.append(scene_name)

with open(config.RESULTS_DIR + f"/faulty_episodes_{split}.json", "w") as f:
json.dump(faulty_episodes, f, indent=4)

faulty_scenes = list(set(faulty_scenes))
with open(config.RESULTS_DIR + f"/faulty_scenes_{split}.json", "w") as f:
json.dump({"faulty_scenes": faulty_scenes}, f, indent=4)

logger.info(f"{len(env.episodes)} episodes validated. {total_num_faulty_epi} episodes found faulty.")
logger.info(f"{len(faulty_scenes)} scenes found faulty.")

return

error_episodes = {}
for i in tqdm(range(num_episodes)):
obs = env.reset()
agent.reset()
Expand All @@ -64,8 +114,34 @@ def evaluate_agent(config: Config) -> None:
while not done:
action = agent.act(obs)
obs, reward, done, info = env.step(action)
logger.info("----------Start")
logger.info(f"Reward={str(reward)}")
_infos = extract_scalars_from_info(info, METRICS_BLACKLIST)
for k,v in _infos.items():
logger.info(f"{k}={str(v)}")
logger.info("----------End")
if len(config.VIDEO_OPTION) > 0:
frame = observations_to_image(obs, info=info)
# Projection code
grid_x, grid_y = to_grid(
-110.0,
110.0,
275,
obs['gps']
)
projection1 = draw_projection(obs['rgb'], obs['depth'] * 10, 13, 275, -110.0, 110.0)
projection1 = projection1.squeeze(0).squeeze(0).permute(1, 2, 0)

projection1 = rotate_tensor(projection1.permute(2, 0, 1).unsqueeze(0), torch.tensor(-(obs["compass"])).unsqueeze(0))
projection1 = projection1.squeeze(0).permute(1, 2, 0)

# s = map_config.egocentric_map_size
# temp = torch.max(test_global_map_visualization[i][grid_x - math.floor(s/2):grid_x + math.ceil(s/2),grid_y - math.floor(s/2):grid_y + math.ceil(s/2),:], projection1)
# test_global_map_visualization[i][grid_x - math.floor(s/2):grid_x + math.ceil(s/2),grid_y - math.floor(s/2):grid_y + math.ceil(s/2),:] = temp
# global_map1 = rotate_tensor(test_global_map_visualization[i][grid_x - math.floor(51/2):grid_x + math.ceil(51/2),grid_y - math.floor(51/2):grid_y + math.ceil(51/2),:].permute(2, 1, 0).unsqueeze(0), torch.tensor(-(observations[i]["compass"])).unsqueeze(0)).squeeze(0).permute(1, 2, 0).numpy()
# egocentric_map = torch.sum(test_global_map[i, grid_x - math.floor(51/2):grid_x+math.ceil(51/2), grid_y - math.floor(51/2):grid_y + math.ceil(51/2),:], dim=2)

frame = observations_to_image(obs, projection1.data.numpy(), None, None, info, [action])
#frame = observations_to_image(obs, info=info, action=[action])
txt_to_show = ('Action: '+ str(action) +
'; Dist_to_multi_goal:' + str(round(info['distance_to_multi_goal'],2)) +
'; Dist_to_curr_goal:' + str(round(info['distance_to_currgoal'],2)) +
Expand All @@ -85,6 +161,15 @@ def evaluate_agent(config: Config) -> None:
)
rgb_frames.append(frame)

# save out error episodes
if float(info['success']) < 1.0:
scene_name = os.path.basename(env.current_episode.scene_id)
if scene_name not in error_episodes:
error_episodes[scene_name] = {}

error_episodes[scene_name][env.current_episode.episode_id] = {}
error_episodes[scene_name][env.current_episode.episode_id]["metrics"] = info

if len(config.VIDEO_OPTION) > 0:
generate_video(
video_option=config.VIDEO_OPTION,
Expand All @@ -105,13 +190,17 @@ def evaluate_agent(config: Config) -> None:
stats[m] += v

stats = {k: v / num_episodes for k, v in stats.items()}
stats["num_episodes"] = num_episodes

logger.info(f"Averaged benchmark for {config.EVAL.NONLEARNING.AGENT}:")
for stat_key in stats.keys():
logger.info("{}: {:.3f}".format(stat_key, stats[stat_key]))

with open(f"stats_{config.EVAL.NONLEARNING.AGENT}_{split}.json", "w") as f:
with open(config.RESULTS_DIR + f"/stats_{config.EVAL.NONLEARNING.AGENT}_{split}.json", "w") as f:
json.dump(stats, f, indent=4)

with open(os.path.join(config.RESULTS_DIR, f"stats_{config.EVAL.NONLEARNING.AGENT}_{split}_error_episodes.json"), "w") as f:
json.dump(error_episodes, f, indent=4)

class RandomAgent(Agent):
r"""Selects an action at each time step by sampling from the oracle action
Expand Down Expand Up @@ -187,4 +276,24 @@ def act(self, observations):
best_action = np.random.choice(self.actions)
self.num_tries -= 1

return best_action
return best_action

class EpisodesValidator():
def __init__(self, env):
self.env = env._env
self.sim = self.env.sim

def goals_reachable(self, episode):
distance_to_subgoal = 0
prev_position = episode.start_position

for goal in episode.goals:
distance_to_subgoal += self.sim.geodesic_distance(
prev_position, goal.position
)
prev_position = goal.position

if (distance_to_subgoal==float("inf")):
return False

return True

0 comments on commit 2a12269

Please sign in to comment.