From 8fe4bd3480e0c1fa72e5ad500243b02c0eaeef23 Mon Sep 17 00:00:00 2001 From: tomasvr Date: Mon, 1 May 2023 19:05:09 +0800 Subject: [PATCH] Stage is now include in name of model directory. --- README.md | 54 +++++++++--------- .../{ddpg_0 => ddpg_0_stage9}/_figure.png | Bin .../_hyperparams_20220808-030634.txt | 0 .../_test_stage1_eps8000_20230501-183843.txt | 3 + .../_test_stage4_eps8000_20221214-143249.txt | 0 .../_test_stage8_eps8000_20221214-134026.txt | 0 .../_test_stage8_eps8000_20221214-134411.txt | 0 .../_test_stage8_eps8000_20221214-134609.txt | 0 .../_test_stage8_eps8000_20221214-135143.txt | 0 .../_test_stage9_eps8000_20221214-135936.txt | 0 .../_test_stage9_eps8000_20221214-140423.txt | 0 .../_train_stage9_20220808-030634.txt | 0 .../actor_stage9_episode8000.pt | Bin .../critic_stage9_episode8000.pt | Bin .../stage9_agent.pkl | Bin .../stage9_episode8000.pkl | Bin .../target_actor_stage9_episode8000.pt | Bin .../target_critic_stage9_episode8000.pt | Bin .../{td3_0 => td3_0_stage9}/_figure.png | Bin .../_hyperparams_20221003-130835.txt | 0 .../_test_stage4_eps7400_20221214-153620.txt | 0 .../_test_stage8_eps7400_20221214-135334.txt | 0 .../_test_stage8_eps7400_20221214-135415.txt | 0 .../_test_stage8_eps7400_20221214-135517.txt | 0 .../_test_stage8_eps7400_20221214-135530.txt | 0 .../_train_stage9_20221003-130835.txt | 0 .../actor_stage9_episode7400.pt | Bin .../critic_stage9_episode7100.pt | Bin .../critic_stage9_episode7400.pt | Bin .../{td3_0 => td3_0_stage9}/stage9_agent.pkl | Bin .../stage9_episode7400.pkl | Bin .../target_actor_stage9_episode7400.pt | Bin .../target_critic_stage9_episode7400.pt | Bin .../turtlebot3_drl/common/settings.py | 4 +- .../turtlebot3_drl/common/storagemanager.py | 10 ++-- .../turtlebot3_drl/common/utilities.py | 2 +- .../turtlebot3_drl/drl_agent/drl_agent.py | 24 ++++---- 37 files changed, 51 insertions(+), 46 deletions(-) rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/_figure.png (100%) rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/_hyperparams_20220808-030634.txt (100%) create mode 100644 src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage1_eps8000_20230501-183843.txt rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/_test_stage4_eps8000_20221214-143249.txt (100%) rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/_test_stage8_eps8000_20221214-134026.txt (100%) rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/_test_stage8_eps8000_20221214-134411.txt (100%) rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/_test_stage8_eps8000_20221214-134609.txt (100%) rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/_test_stage8_eps8000_20221214-135143.txt (100%) rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/_test_stage9_eps8000_20221214-135936.txt (100%) rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/_test_stage9_eps8000_20221214-140423.txt (100%) rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/_train_stage9_20220808-030634.txt (100%) rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/actor_stage9_episode8000.pt (100%) rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/critic_stage9_episode8000.pt (100%) rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/stage9_agent.pkl (100%) rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/stage9_episode8000.pkl (100%) rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/target_actor_stage9_episode8000.pt (100%) rename src/turtlebot3_drl/model/examples/{ddpg_0 => ddpg_0_stage9}/target_critic_stage9_episode8000.pt (100%) rename src/turtlebot3_drl/model/examples/{td3_0 => td3_0_stage9}/_figure.png (100%) rename src/turtlebot3_drl/model/examples/{td3_0 => td3_0_stage9}/_hyperparams_20221003-130835.txt (100%) rename src/turtlebot3_drl/model/examples/{td3_0 => td3_0_stage9}/_test_stage4_eps7400_20221214-153620.txt (100%) rename src/turtlebot3_drl/model/examples/{td3_0 => td3_0_stage9}/_test_stage8_eps7400_20221214-135334.txt (100%) rename src/turtlebot3_drl/model/examples/{td3_0 => td3_0_stage9}/_test_stage8_eps7400_20221214-135415.txt (100%) rename src/turtlebot3_drl/model/examples/{td3_0 => td3_0_stage9}/_test_stage8_eps7400_20221214-135517.txt (100%) rename src/turtlebot3_drl/model/examples/{td3_0 => td3_0_stage9}/_test_stage8_eps7400_20221214-135530.txt (100%) rename src/turtlebot3_drl/model/examples/{td3_0 => td3_0_stage9}/_train_stage9_20221003-130835.txt (100%) rename src/turtlebot3_drl/model/examples/{td3_0 => td3_0_stage9}/actor_stage9_episode7400.pt (100%) rename src/turtlebot3_drl/model/examples/{td3_0 => td3_0_stage9}/critic_stage9_episode7100.pt (100%) rename src/turtlebot3_drl/model/examples/{td3_0 => td3_0_stage9}/critic_stage9_episode7400.pt (100%) rename src/turtlebot3_drl/model/examples/{td3_0 => td3_0_stage9}/stage9_agent.pkl (100%) rename src/turtlebot3_drl/model/examples/{td3_0 => td3_0_stage9}/stage9_episode7400.pkl (100%) rename src/turtlebot3_drl/model/examples/{td3_0 => td3_0_stage9}/target_actor_stage9_episode7400.pt (100%) rename src/turtlebot3_drl/model/examples/{td3_0 => td3_0_stage9}/target_critic_stage9_episode7400.pt (100%) diff --git a/README.md b/README.md index a49bb47..fc14746 100644 --- a/README.md +++ b/README.md @@ -196,18 +196,18 @@ You should see the gazebo GUI come up with the robot model loaded and two moving In a second terminal run ``` -ros2 run turtlebot3_drl drl_gazebo +ros2 run turtlebot3_drl gazebo_goals ``` In a third terminal run ``` -ros2 run turtlebot3_drl drl_environment +ros2 run turtlebot3_drl environment ``` And lastly, in the fourth terminal run the ddpg agent For DDPG: ``` -ros2 run turtlebot3_drl drl_agent ddpg 1 +ros2 run turtlebot3_drl train_agent ddpg ``` The first argument indicates whether we are testing or training (0 = testing, 1 = training) @@ -218,12 +218,12 @@ The first argument indicates whether we are testing or training (0 = testing, 1 for TD3: ``` -ros2 run turtlebot3_drl drl_agent td3 1 +ros2 run turtlebot3_drl train_agent td3 ``` for DQN: ``` -ros2 run turtlebot3_drl drl_agent dqn 1 +ros2 run turtlebot3_drl train_agent dqn ``` Your robot should now be moving and training progress is being printed to the terminals! @@ -239,21 +239,16 @@ The current state of the agent (weights, parameters, replay buffer and graphs) w In order to load a model for testing (e.g. ddpg_0 at episode 500) the following command should be used: ``` -ros2 run turtlebot3_drl drl_agent ddpg 0 "ddpg_0" 500 +ros2 run turtlebot3_drl test_agent ddpg "ddpg_0" 500 ``` In order to load a model to continue training (e.g. ddpg_0 at episode 500) the following command should be used: ``` -ros2 run turtlebot3_drl drl_agent ddpg 1 "ddpg_0" 500 +ros2 run turtlebot3_drl train_agent ddpg "ddpg_0" 500 ``` -**Note:** If you are loading a model on a different stage than it was trained on (e.g. for transfer learning or testing generalizabilty) you have to add a 4th argument specifying the current stage. For example, model ddpg_0 which was trained on stage 4 can be evaluated in stage 3 using the following command -``` -ros2 run turtlebot3_drl drl_agent ddpg 0 "ddpg_0" 500 3 -``` - -(the original training stage is specified in training logfile (e.g _train_**stage2**_*.txt) +**Note:** You can also test (or continue training) a model on a different stage than where it was originally trained on. ### Loading one of the included example models @@ -266,28 +261,26 @@ ros2 launch turtlebot3_gazebo turtlebot3_drl_stage9.launch.py Terminal 2: ``` -ros2 run turtlebot3_drl drl_gazebo +ros2 run turtlebot3_drl gazebo_goals ``` Terminal 3: ``` -ros2 run turtlebot3_drl drl_environment +ros2 run turtlebot3_drl environment ``` Terminal 4: For DDPG: ``` -ros2 run turtlebot3_drl drl_agent ddpg 0 'examples/ddpg_0' 8000 +ros2 run turtlebot3_drl test_agent ddpg 'examples/ddpg_0' 8000 ``` Or, for TD3 ``` -ros2 run turtlebot3_drl drl_agent td3 0 'examples/td3_0' 7400 +ros2 run turtlebot3_drl test_agent td3 'examples/td3_0' 7400 ``` -The pretrained model should then start to navigate successfully. - -Note: Do not include 'examples/' in the command when running models trained on your own machine. +You should then see the example model navigate successfully towards the goal ### Switching environments @@ -356,15 +349,22 @@ The visual should mainly be used during evaluation as it can slow down training ## Command Specification -**drl_agent:** +**train_agent:** + +```ros2 run turtlebot3_drl train_agent [algorithm=dqn/ddpg/td3] [loadmodel=\path\to\model] [loadepisode=episode] ``` + +* `algorithm`: algorithm to run, one of either: `dqn`, `ddpg`, `td3` +* `modelpath`: path to the model to be loaded to continue training +* `loadepisode`: is the episode to load from `modelpath` + +**test_agent:** + +```ros2 run turtlebot3_drl test_agent [algorithm=dqn/ddpg/td3] [loadmodel=\path\to\model] [loadepisode=episode] ``` -```ros2 run turtlebot3_drl drl_agent [algorithm=dqn/ddpg/td3] [mode=0/1] [loadmodel=\path\to\model] [loadepisode=episode] [trainingstage=stage]``` +* `algorithm`: algorithm to run, one of either: `dqn`, `ddpg`, `td3` +* `modelpath`: path to model to be loaded for testing +* `loadepisode`: is the episode to load from `modelpath` -`algorithm` can be either: `dqn`, `ddpg`, `td3` -`mode` is either: `0` (training) or `1` (evaluating) -`modelpath` is the path to the model to load -`loadepisode` is the episode to load from `modelpath` -`trainingstage` is the original training stage of `modelpath` (if different from current stage) ## Physical Robot diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/_figure.png b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/_figure.png similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/_figure.png rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/_figure.png diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/_hyperparams_20220808-030634.txt b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/_hyperparams_20220808-030634.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/_hyperparams_20220808-030634.txt rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/_hyperparams_20220808-030634.txt diff --git a/src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage1_eps8000_20230501-183843.txt b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage1_eps8000_20230501-183843.txt new file mode 100644 index 0000000..3cae945 --- /dev/null +++ b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage1_eps8000_20230501-183843.txt @@ -0,0 +1,3 @@ +episode, outcome, step, episode_duration, distance, s/cw/co/t +1, 1, 1974, 1.432716391998838, 5.1544623374938965, 1/0/0/0/0 +2, 1, 11785, 8.245651064997219, 1.9160521030426025, 2/0/0/0/0 diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/_test_stage4_eps8000_20221214-143249.txt b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage4_eps8000_20221214-143249.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/_test_stage4_eps8000_20221214-143249.txt rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage4_eps8000_20221214-143249.txt diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/_test_stage8_eps8000_20221214-134026.txt b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage8_eps8000_20221214-134026.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/_test_stage8_eps8000_20221214-134026.txt rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage8_eps8000_20221214-134026.txt diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/_test_stage8_eps8000_20221214-134411.txt b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage8_eps8000_20221214-134411.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/_test_stage8_eps8000_20221214-134411.txt rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage8_eps8000_20221214-134411.txt diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/_test_stage8_eps8000_20221214-134609.txt b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage8_eps8000_20221214-134609.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/_test_stage8_eps8000_20221214-134609.txt rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage8_eps8000_20221214-134609.txt diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/_test_stage8_eps8000_20221214-135143.txt b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage8_eps8000_20221214-135143.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/_test_stage8_eps8000_20221214-135143.txt rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage8_eps8000_20221214-135143.txt diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/_test_stage9_eps8000_20221214-135936.txt b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage9_eps8000_20221214-135936.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/_test_stage9_eps8000_20221214-135936.txt rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage9_eps8000_20221214-135936.txt diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/_test_stage9_eps8000_20221214-140423.txt b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage9_eps8000_20221214-140423.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/_test_stage9_eps8000_20221214-140423.txt rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/_test_stage9_eps8000_20221214-140423.txt diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/_train_stage9_20220808-030634.txt b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/_train_stage9_20220808-030634.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/_train_stage9_20220808-030634.txt rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/_train_stage9_20220808-030634.txt diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/actor_stage9_episode8000.pt b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/actor_stage9_episode8000.pt similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/actor_stage9_episode8000.pt rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/actor_stage9_episode8000.pt diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/critic_stage9_episode8000.pt b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/critic_stage9_episode8000.pt similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/critic_stage9_episode8000.pt rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/critic_stage9_episode8000.pt diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/stage9_agent.pkl b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/stage9_agent.pkl similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/stage9_agent.pkl rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/stage9_agent.pkl diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/stage9_episode8000.pkl b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/stage9_episode8000.pkl similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/stage9_episode8000.pkl rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/stage9_episode8000.pkl diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/target_actor_stage9_episode8000.pt b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/target_actor_stage9_episode8000.pt similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/target_actor_stage9_episode8000.pt rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/target_actor_stage9_episode8000.pt diff --git a/src/turtlebot3_drl/model/examples/ddpg_0/target_critic_stage9_episode8000.pt b/src/turtlebot3_drl/model/examples/ddpg_0_stage9/target_critic_stage9_episode8000.pt similarity index 100% rename from src/turtlebot3_drl/model/examples/ddpg_0/target_critic_stage9_episode8000.pt rename to src/turtlebot3_drl/model/examples/ddpg_0_stage9/target_critic_stage9_episode8000.pt diff --git a/src/turtlebot3_drl/model/examples/td3_0/_figure.png b/src/turtlebot3_drl/model/examples/td3_0_stage9/_figure.png similarity index 100% rename from src/turtlebot3_drl/model/examples/td3_0/_figure.png rename to src/turtlebot3_drl/model/examples/td3_0_stage9/_figure.png diff --git a/src/turtlebot3_drl/model/examples/td3_0/_hyperparams_20221003-130835.txt b/src/turtlebot3_drl/model/examples/td3_0_stage9/_hyperparams_20221003-130835.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/td3_0/_hyperparams_20221003-130835.txt rename to src/turtlebot3_drl/model/examples/td3_0_stage9/_hyperparams_20221003-130835.txt diff --git a/src/turtlebot3_drl/model/examples/td3_0/_test_stage4_eps7400_20221214-153620.txt b/src/turtlebot3_drl/model/examples/td3_0_stage9/_test_stage4_eps7400_20221214-153620.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/td3_0/_test_stage4_eps7400_20221214-153620.txt rename to src/turtlebot3_drl/model/examples/td3_0_stage9/_test_stage4_eps7400_20221214-153620.txt diff --git a/src/turtlebot3_drl/model/examples/td3_0/_test_stage8_eps7400_20221214-135334.txt b/src/turtlebot3_drl/model/examples/td3_0_stage9/_test_stage8_eps7400_20221214-135334.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/td3_0/_test_stage8_eps7400_20221214-135334.txt rename to src/turtlebot3_drl/model/examples/td3_0_stage9/_test_stage8_eps7400_20221214-135334.txt diff --git a/src/turtlebot3_drl/model/examples/td3_0/_test_stage8_eps7400_20221214-135415.txt b/src/turtlebot3_drl/model/examples/td3_0_stage9/_test_stage8_eps7400_20221214-135415.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/td3_0/_test_stage8_eps7400_20221214-135415.txt rename to src/turtlebot3_drl/model/examples/td3_0_stage9/_test_stage8_eps7400_20221214-135415.txt diff --git a/src/turtlebot3_drl/model/examples/td3_0/_test_stage8_eps7400_20221214-135517.txt b/src/turtlebot3_drl/model/examples/td3_0_stage9/_test_stage8_eps7400_20221214-135517.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/td3_0/_test_stage8_eps7400_20221214-135517.txt rename to src/turtlebot3_drl/model/examples/td3_0_stage9/_test_stage8_eps7400_20221214-135517.txt diff --git a/src/turtlebot3_drl/model/examples/td3_0/_test_stage8_eps7400_20221214-135530.txt b/src/turtlebot3_drl/model/examples/td3_0_stage9/_test_stage8_eps7400_20221214-135530.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/td3_0/_test_stage8_eps7400_20221214-135530.txt rename to src/turtlebot3_drl/model/examples/td3_0_stage9/_test_stage8_eps7400_20221214-135530.txt diff --git a/src/turtlebot3_drl/model/examples/td3_0/_train_stage9_20221003-130835.txt b/src/turtlebot3_drl/model/examples/td3_0_stage9/_train_stage9_20221003-130835.txt similarity index 100% rename from src/turtlebot3_drl/model/examples/td3_0/_train_stage9_20221003-130835.txt rename to src/turtlebot3_drl/model/examples/td3_0_stage9/_train_stage9_20221003-130835.txt diff --git a/src/turtlebot3_drl/model/examples/td3_0/actor_stage9_episode7400.pt b/src/turtlebot3_drl/model/examples/td3_0_stage9/actor_stage9_episode7400.pt similarity index 100% rename from src/turtlebot3_drl/model/examples/td3_0/actor_stage9_episode7400.pt rename to src/turtlebot3_drl/model/examples/td3_0_stage9/actor_stage9_episode7400.pt diff --git a/src/turtlebot3_drl/model/examples/td3_0/critic_stage9_episode7100.pt b/src/turtlebot3_drl/model/examples/td3_0_stage9/critic_stage9_episode7100.pt similarity index 100% rename from src/turtlebot3_drl/model/examples/td3_0/critic_stage9_episode7100.pt rename to src/turtlebot3_drl/model/examples/td3_0_stage9/critic_stage9_episode7100.pt diff --git a/src/turtlebot3_drl/model/examples/td3_0/critic_stage9_episode7400.pt b/src/turtlebot3_drl/model/examples/td3_0_stage9/critic_stage9_episode7400.pt similarity index 100% rename from src/turtlebot3_drl/model/examples/td3_0/critic_stage9_episode7400.pt rename to src/turtlebot3_drl/model/examples/td3_0_stage9/critic_stage9_episode7400.pt diff --git a/src/turtlebot3_drl/model/examples/td3_0/stage9_agent.pkl b/src/turtlebot3_drl/model/examples/td3_0_stage9/stage9_agent.pkl similarity index 100% rename from src/turtlebot3_drl/model/examples/td3_0/stage9_agent.pkl rename to src/turtlebot3_drl/model/examples/td3_0_stage9/stage9_agent.pkl diff --git a/src/turtlebot3_drl/model/examples/td3_0/stage9_episode7400.pkl b/src/turtlebot3_drl/model/examples/td3_0_stage9/stage9_episode7400.pkl similarity index 100% rename from src/turtlebot3_drl/model/examples/td3_0/stage9_episode7400.pkl rename to src/turtlebot3_drl/model/examples/td3_0_stage9/stage9_episode7400.pkl diff --git a/src/turtlebot3_drl/model/examples/td3_0/target_actor_stage9_episode7400.pt b/src/turtlebot3_drl/model/examples/td3_0_stage9/target_actor_stage9_episode7400.pt similarity index 100% rename from src/turtlebot3_drl/model/examples/td3_0/target_actor_stage9_episode7400.pt rename to src/turtlebot3_drl/model/examples/td3_0_stage9/target_actor_stage9_episode7400.pt diff --git a/src/turtlebot3_drl/model/examples/td3_0/target_critic_stage9_episode7400.pt b/src/turtlebot3_drl/model/examples/td3_0_stage9/target_critic_stage9_episode7400.pt similarity index 100% rename from src/turtlebot3_drl/model/examples/td3_0/target_critic_stage9_episode7400.pt rename to src/turtlebot3_drl/model/examples/td3_0_stage9/target_critic_stage9_episode7400.pt diff --git a/src/turtlebot3_drl/turtlebot3_drl/common/settings.py b/src/turtlebot3_drl/turtlebot3_drl/common/settings.py index 941db56..c6b91d9 100644 --- a/src/turtlebot3_drl/turtlebot3_drl/common/settings.py +++ b/src/turtlebot3_drl/turtlebot3_drl/common/settings.py @@ -3,7 +3,7 @@ ENABLE_STACKING = False ENABLE_VISUAL = False # Meant to be used only during evaluation/testing phase ENABLE_TRUE_RANDOM_GOALS = False # If false, goals are taken randomly from a list of known valid goal positions -MODEL_STORE_INTERVAL = 100 # Store the model weights every N episodes +MODEL_STORE_INTERVAL = 3 # Store the model weights every N episodes # DRL parameters ACTION_SIZE = 2 # Not used for DQN, see DQN_ACTION_SIZE @@ -15,7 +15,7 @@ LEARNING_RATE = 0.003 TAU = 0.003 -OBSERVE_STEPS = 25000 # At training start random actions are taken for N steps for better exploration +OBSERVE_STEPS = 0 # At training start random actions are taken for N steps for better exploration STEP_TIME = 0.01 # Delay between steps, can be set to 0 EPSILON_DECAY = 0.9995 # Epsilon decay per step EPSILON_MINIMUM = 0.05 diff --git a/src/turtlebot3_drl/turtlebot3_drl/common/storagemanager.py b/src/turtlebot3_drl/turtlebot3_drl/common/storagemanager.py index 9a2fed5..6a4df5e 100644 --- a/src/turtlebot3_drl/turtlebot3_drl/common/storagemanager.py +++ b/src/turtlebot3_drl/turtlebot3_drl/common/storagemanager.py @@ -6,7 +6,7 @@ import torch class StorageManager: - def __init__(self, name, stage, load_session, load_episode, device): + def __init__(self, name, load_session, load_episode, device, stage): if load_session and name not in load_session: print(f"ERROR: wrong combination of command and model! make sure command is: {name}_agent") while True: @@ -15,18 +15,18 @@ def __init__(self, name, stage, load_session, load_episode, device): if 'examples' in load_session: self.machine_dir = (os.getenv('DRLNAV_BASE_PATH') + '/src/turtlebot3_drl/model/') self.name = name - self.stage = stage + self.stage = load_session[-1] if load_session else stage self.session = load_session self.load_episode = load_episode self.session_dir = os.path.join(self.machine_dir, self.session) self.map_location = device - def new_session_dir(self): + def new_session_dir(self, stage): i = 0 - session_dir = os.path.join(self.machine_dir, f"{self.name}_{i}") + session_dir = os.path.join(self.machine_dir, f"{self.name}_{i}_stage{stage}") while(os.path.exists(session_dir)): i += 1 - session_dir = os.path.join(self.machine_dir, f"{self.name}_{i}") + session_dir = os.path.join(self.machine_dir, f"{self.name}_{i}_stage{stage}") self.session = f"{self.name}_{i}" print(f"making new model dir: {self.session}") os.makedirs(session_dir) diff --git a/src/turtlebot3_drl/turtlebot3_drl/common/utilities.py b/src/turtlebot3_drl/turtlebot3_drl/common/utilities.py index bd095a9..0f0cabd 100644 --- a/src/turtlebot3_drl/turtlebot3_drl/common/utilities.py +++ b/src/turtlebot3_drl/turtlebot3_drl/common/utilities.py @@ -12,7 +12,7 @@ import xml.etree.ElementTree as ET with open('/tmp/drlnav_current_stage.txt', 'r') as f: - test_stage = int(f.read()) + stage = int(f.read()) def check_gpu(): print("gpu torch available: ", torch.cuda.is_available()) diff --git a/src/turtlebot3_drl/turtlebot3_drl/drl_agent/drl_agent.py b/src/turtlebot3_drl/turtlebot3_drl/drl_agent/drl_agent.py index bec85df..722c0ad 100644 --- a/src/turtlebot3_drl/turtlebot3_drl/drl_agent/drl_agent.py +++ b/src/turtlebot3_drl/turtlebot3_drl/drl_agent/drl_agent.py @@ -43,19 +43,17 @@ from ..common.replaybuffer import ReplayBuffer class DrlAgent(Node): - def __init__(self, training, algorithm, load_session="", load_episode=0, train_stage=util.test_stage): + def __init__(self, training, algorithm, load_session="", load_episode=0): super().__init__(algorithm + '_agent') self.algorithm = algorithm self.training = int(training) self.load_session = load_session self.episode = int(load_episode) - self.train_stage = train_stage if (not self.training and not self.load_session): - quit("ERROR no test agent specified") + quit("Invalid command: Testing but no model to load specified (example format: ros2 run turtlebot3_drl test_agent ddpg ddpg_0_stage4 1)") self.device = util.check_gpu() - self.sim_speed = util.get_simulation_speed(self.train_stage) - print(f"{'training' if (self.training) else 'testing' } on stage: {util.test_stage}") - + self.sim_speed = util.get_simulation_speed(util.stage) + print(f"{'training' if (self.training) else 'testing' } on stage: {util.stage}") self.total_steps = 0 self.observe_steps = OBSERVE_STEPS @@ -66,7 +64,7 @@ def __init__(self, training, algorithm, load_session="", load_episode=0, train_s elif self.algorithm == 'td3': self.model = TD3(self.device, self.sim_speed) else: - quit(f"invalid algorithm specified: {self.algorithm}, chose one of: ddpg, td3, td3conv") + quit(f"invalid algorithm specified: {self.algorithm}, choose one of: dqn, ddpg, td3") self.replay_buffer = ReplayBuffer(self.model.buffer_size) self.graph = Graph() @@ -75,7 +73,7 @@ def __init__(self, training, algorithm, load_session="", load_episode=0, train_s # Model loading # # ===================================================================== # - self.sm = StorageManager(self.algorithm, self.train_stage, self.load_session, self.episode, self.device) + self.sm = StorageManager(self.algorithm, self.load_session, self.episode, self.device, util.stage) if self.load_session: del self.model @@ -83,16 +81,16 @@ def __init__(self, training, algorithm, load_session="", load_episode=0, train_s self.model.device = self.device self.sm.load_weights(self.model.networks) if self.training: - self.replay_buffer.buffer = self.sm.load_replay_buffer(self.model.buffer_size, os.path.join(self.load_session, 'stage'+str(self.train_stage)+'_latest_buffer.pkl')) + self.replay_buffer.buffer = self.sm.load_replay_buffer(self.model.buffer_size, os.path.join(self.load_session, 'stage'+str(self.sm.stage)+'_latest_buffer.pkl')) self.total_steps = self.graph.set_graphdata(self.sm.load_graphdata(), self.episode) print(f"global steps: {self.total_steps}") print(f"loaded model {self.load_session} (eps {self.episode}): {self.model.get_model_parameters()}") else: - self.sm.new_session_dir(util.test_stage) + self.sm.new_session_dir(util.stage) self.sm.store_model(self.model) self.graph.session_dir = self.sm.session_dir - self.logger = Logger(self.training, self.sm.machine_dir, self.sm.session_dir, self.sm.session, self.model.get_model_parameters(), self.model.get_model_configuration(), str(util.test_stage), self.algorithm, self.episode) + self.logger = Logger(self.training, self.sm.machine_dir, self.sm.session_dir, self.sm.session, self.model.get_model_parameters(), self.model.get_model_configuration(), str(util.stage), self.algorithm, self.episode) if ENABLE_VISUAL: self.visual = DrlVisual(self.model.state_size, self.model.hidden_size) self.model.attach_visual(self.visual) @@ -202,5 +200,9 @@ def main_test(args=sys.argv[1:]): args = ['0'] + args main(args) +def main_real(args=sys.argv[1:]): + args = ['0'] + args + main(args) + if __name__ == '__main__': main() \ No newline at end of file