test_dqn_mlp_explain_ft06.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "from rl.agent import DQNAgent\n",
    "from gymjsp.jsspenv import HeuristicJsspEnv\n",
    "from ortools_scheduler import ORtools_scheduler\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "from copy import deepcopy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instances = [\"ft06\", \"la01\", \"la06\", \"la11\", \"la21\", \"la31\", \"la36\", \"orb01\", \"swv01\", \"swv06\", \"swv11\", \"yn1\"]\n",
    "# instances = [\"swv06\"]\n",
    "num_episodes = 1000\n",
    "memory_size = 100000\n",
    "batch_size = 64\n",
    "target_update = 100\n",
    "noisy = False\n",
    "plotting_inteval = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 随机环境"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_rate = 0.5\n",
    "cv = 0.2\n",
    "n = 10 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "instance = \"ft06\"\n",
    "policy_file = f\"policies/dqn_mlp/{instance}_num_episodes={num_episodes}_memory_size={memory_size}_target_update={target_update}_noisy={noisy}.pth\"\n",
    "env = HeuristicJsspEnv(instance)\n",
    "agent = DQNAgent(env, memory_size, batch_size, target_update, noisy=noisy)\n",
    "agent.load_dqn(policy_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent action, makespan = 65, score = -2.100097125097125\n",
      "[4, 4, 4, 3, 0, 0, 4, 0, 0]\n"
     ]
    }
   ],
   "source": [
    "state = env.reset()\n",
    "done = False\n",
    "agent_actions = []\n",
    "score = 0\n",
    "while not done:\n",
    "    action = agent.select_action(state, determine=True)\n",
    "    agent_actions.append(int(action))\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "    state = next_state\n",
    "    score += reward\n",
    "makespan = info[\"makespan\"]\n",
    "print(f\"Agent action, makespan = {makespan}, score = {score}\")\n",
    "print(agent_actions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent action = 6, makespan = 59, score = -2.4800505050505044\n"
     ]
    }
   ],
   "source": [
    "state = env.reset()\n",
    "done = False\n",
    "agent_actions = []\n",
    "score = 0\n",
    "while not done:\n",
    "    action = 6\n",
    "    agent_actions.append(int(action))\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "    state = next_state\n",
    "    score += reward\n",
    "makespan = info[\"makespan\"]\n",
    "print(f\"Agent action = 6, makespan = {makespan}, score = {score}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent action, makespan = 65, score = -26.916666666666654\n",
      "[4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 6, 6, 6, 7, 5, 5, 0, 5, 5, 5, 5, 1, 1, 1, 1, 1, 1, 1, 5, 5, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 2, 2, 2, 7, 7, 7, 7, 2, 2, 2, 2, 2, 7, 5, 0]\n"
     ]
    }
   ],
   "source": [
    "env = HeuristicJsspEnv(\"ft06\", schedule_cycle=1)\n",
    "state = env.reset()\n",
    "done = False\n",
    "agent_actions = []\n",
    "score = 0\n",
    "while not done:\n",
    "    action = agent.select_action(state, determine=True)\n",
    "    agent_actions.append(int(action))\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "    state = next_state\n",
    "    score += reward\n",
    "makespan = info[\"makespan\"]\n",
    "print(f\"Agent action, makespan = {makespan}, score = {score}\")\n",
    "print(agent_actions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "instance = \"ft06\"\n",
    "policy_file = f\"policies/dqn_mlp/{instance}_num_episodes=500_memory_size={memory_size}_target_update={target_update}_noisy={noisy}_cycle=1.pth\"\n",
    "env = HeuristicJsspEnv(instance, schedule_cycle=1)\n",
    "agent = DQNAgent(env, memory_size, batch_size, target_update, noisy=noisy)\n",
    "agent.load_dqn(policy_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent action, makespan = 59, score = -19.833333333333332\n"
     ]
    }
   ],
   "source": [
    "state = env.reset()\n",
    "done = False\n",
    "agent_actions = []\n",
    "score = 0\n",
    "while not done:\n",
    "    action = agent.select_action(state, determine=True)\n",
    "    agent_actions.append(int(action))\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "    state = next_state\n",
    "    score += reward\n",
    "makespan = info[\"makespan\"]\n",
    "print(f\"Agent action, makespan = {makespan}, score = {score}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 对比随机动作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent action, makespan = 2077\n"
     ]
    }
   ],
   "source": [
    "state = env.reset()\n",
    "done = False\n",
    "agent_actions = []\n",
    "while not done:\n",
    "    action = agent.select_action(state, determine=True)\n",
    "    agent_actions.append(int(action))\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "    state = next_state\n",
    "makespan = info[\"makespan\"]\n",
    "print(f\"Agent action, makespan = {makespan}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random action, makespan = 2375.10\n"
     ]
    }
   ],
   "source": [
    "makespans = []\n",
    "for _ in range(10):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    while not done:\n",
    "        action = np.random.randint(0, 8)\n",
    "        next_state, reward, done, info = env.step(action)\n",
    "        state = next_state\n",
    "\n",
    "    makespan = info[\"makespan\"]\n",
    "    makespans.append(makespan)\n",
    "print(f\"Random action, makespan = {np.mean(makespans):.2f}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 对比 heuristic rule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent action=6, makespan = 2287\n"
     ]
    }
   ],
   "source": [
    "state = env.reset()\n",
    "done = False\n",
    "agent_actions = []\n",
    "while not done:\n",
    "    action = 6\n",
    "    agent_actions.append(int(action))\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "    state = next_state\n",
    "makespan = info[\"makespan\"]\n",
    "print(f\"Agent action=6, makespan = {makespan}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### agent train on swv06, test on swv07-swv09"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On instance swv07, makespan = 1914\n",
      "On instance swv08, makespan = 2366\n",
      "On instance swv09, makespan = 2074\n"
     ]
    }
   ],
   "source": [
    "test_instances = [f\"swv0{x}\" for x in range(7,10)]\n",
    "for instance in test_instances:\n",
    "    test_env = HeuristicJsspEnv(instance)\n",
    "    state = test_env.reset()\n",
    "    done = False\n",
    "    while not done:\n",
    "        action = agent.select_action(state, determine=True)\n",
    "        next_state, reward, done, info = test_env.step(action)\n",
    "        state = next_state\n",
    "    makespan = info[\"makespan\"]\n",
    "    print(f\"On instance {instance}, makespan = {makespan}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On instance swv07, makespan = 2173\n",
      "On instance swv08, makespan = 2518\n",
      "On instance swv09, makespan = 2472\n"
     ]
    }
   ],
   "source": [
    "test_instances = [f\"swv0{x}\" for x in range(7,10)]\n",
    "for instance in test_instances:\n",
    "    test_env = HeuristicJsspEnv(instance)\n",
    "    state = test_env.reset()\n",
    "    done = False\n",
    "    while not done:\n",
    "        action = np.random.randint(0, 8)\n",
    "        next_state, reward, done, info = test_env.step(action)\n",
    "        state = next_state\n",
    "    makespan = info[\"makespan\"]\n",
    "    print(f\"On instance {instance}, makespan = {makespan}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 哪些动作做的多"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent choose action 3 for 49 times\n",
      "Agent choose action 6 for 154 times\n",
      "Agent choose action 7 for 38 times\n",
      "Agent choose action 5 for 19 times\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "for i in Counter(agent_actions).keys():\n",
    "    print(f\"Agent choose action {i} for {Counter(agent_actions)[i]} times\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "action | rule  \n",
    "3        shortest processing time  \n",
    "6        most operations remaining  \n",
    "7        least operations remaining  \n",
    "5        shortest processing time remained  "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 哪些state重要"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = agent._get_dqn()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 0.0000,  0.0000, -1.0000,  ...,  0.0000,  0.0000,  0.7333],\n",
      "       device='cuda:0', requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "state = env.reset()\n",
    "state = torch.FloatTensor(state).to(agent.device)\n",
    "state.requires_grad_(True)\n",
    "print(state)\n",
    "model.eval()\n",
    "Q = model(state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "external_grad = torch.tensor(np.ones(8)).to(agent.device)\n",
    "Q.backward(gradient=external_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-0.0558,  0.0433, -0.0338,  ..., -0.6092, -1.2825, -0.1516],\n",
       "       device='cuda:0')"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_grad = state.grad.detach().cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15.880271"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state_grad.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([   7,   62,   97,  247,  307,  347,  397,  427,  477,  527,  537,\n",
       "         587,  677,  687,  737,  867,  977, 1297, 1447, 1507, 1517, 1577,\n",
       "        1597, 1607, 1907, 1917, 2012, 2057, 2077, 2097, 2187, 2357, 2376,\n",
       "        2467, 2537, 2547, 2677, 2707, 2817, 2846, 2847, 2976], dtype=int64),)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.where(state_grad>3)              # 阈值为 3"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "显然agent考虑的主要是 7 结尾的特征，即工件的剩余加工时间"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent consider state 7 for 100 times\n",
      "Agent consider state 2 for 78 times\n",
      "Agent consider state 8 for 43 times\n",
      "Agent consider state 6 for 64 times\n"
     ]
    }
   ],
   "source": [
    "a = np.where(state_grad>1)[0]           # 阈值为 1\n",
    "a = [x%10 for x in a]\n",
    "for i in Counter(a).keys():\n",
    "    print(f\"Agent consider state {i} for {Counter(a)[i]} times\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2 - type  \n",
    "6 - waiting_time  \n",
    "7 - remain_time  \n",
    "8 - doable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([12.657236, 15.880271, 10.319552, 14.00865 , 11.86575 ],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state_grad[np.where(state_grad>10)]   # 阈值为 10"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 第一个动作，有多重要"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Agent first action is 0, makespan = 2049\n",
      "Agent first action is 1, makespan = 2052\n",
      "Agent first action is 2, makespan = 2085\n",
      "Agent first action is 3, makespan = 2077\n",
      "Agent first action is 4, makespan = 1951\n",
      "Agent first action is 5, makespan = 2241\n",
      "Agent first action is 6, makespan = 2049\n",
      "Agent first action is 7, makespan = 2049\n"
     ]
    }
   ],
   "source": [
    "for first_action in range(8):\n",
    "    state = env.reset()\n",
    "    next_state, reward, done, info = env.step(first_action)\n",
    "    state = next_state\n",
    "    while not done:\n",
    "        action = agent.select_action(state, determine=True)\n",
    "        next_state, reward, done, info = env.step(action)\n",
    "        state = next_state\n",
    "    makespan = info[\"makespan\"]\n",
    "    print(f\"Agent first action is {first_action}, makespan = {makespan}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "实际agent选择action=3，可以看出不是最好。agent应该学到了根据这些工件的剩余加工时间来调度，"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### enhanced policy by greedy search\n",
    "greedy can be substituted by MCTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_greedy_action(env_start):\n",
    "    \"\"\"Given env_start at some status, simulate the process with first action [0,1,2,3,4,5,6,7]\n",
    "    and return the action with the lowest makespan.\n",
    "    \"\"\"\n",
    "    makespans = []\n",
    "    for a in range(8):\n",
    "        env = deepcopy(env_start)\n",
    "        next_state, reward, done, info = env.step(a)\n",
    "        state = next_state\n",
    "        while not done:\n",
    "            action = agent.select_action(state, determine=True)\n",
    "            next_state, reward, done, info = env.step(action)\n",
    "            state = next_state\n",
    "        makespans.append(info[\"makespan\"])\n",
    "    return np.argmin(makespans), makespans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2049, 2052, 2085, 2077, 1951, 2241, 2049, 2049]\n",
      "[1981, 1984, 1984, 1951, 1981, 1984, 1951, 1981]\n",
      "[2009, 1951, 2009, 1951, 1951, 2009, 1951, 2125]\n",
      "[1951, 1951, 1951, 1951, 1951, 1951, 1951, 1951]\n",
      "[1951, 1951, 1951, 1951, 1951, 1951, 1951, 1951]\n",
      "[1993, 1996, 1951, 1993, 1951, 2013, 1951, 1996]\n",
      "[1951, 1951, 1951, 1951, 1951, 1951, 1951, 1951]\n",
      "[1951, 1951, 1951, 1951, 1951, 1951, 1951, 1951]\n"
     ]
    }
   ],
   "source": [
    "step_size =30\n",
    "\n",
    "done = False\n",
    "state = env.reset()\n",
    "while not done:\n",
    "    a, makespans = get_greedy_action(env)\n",
    "    next_state, reward, done, info = env.step(a)\n",
    "    state = next_state\n",
    "    print(makespans)\n",
    "    steps = 0\n",
    "    while steps < step_size:\n",
    "        action = agent.select_action(state, determine=True)\n",
    "        next_state, reward, done, info = env.step(action)\n",
    "        state = next_state\n",
    "        steps += 1\n",
    "        if done:\n",
    "            break"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "只有 first action 能优化makespan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2049, 2052, 2085, 2077, 1951, 2241, 2049, 2049]\n",
      "[2082, 2142, 2109, 1951, 2058, 2045, 2000, 2082]\n",
      "[1951, 2026, 2026, 1951, 1951, 2026, 1951, 1951]\n",
      "[1951, 2025, 2025, 1951, 2025, 1951, 2025, 1951]\n",
      "[1951, 2045, 2045, 1951, 2045, 1981, 2045, 1981]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_8940\\1018890013.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mwhile\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m     \u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmakespans\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_greedy_action\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0menv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m     \u001b[0mnext_state\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minfo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m     \u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnext_state\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_8940\\3722718511.py\u001b[0m in \u001b[0;36mget_greedy_action\u001b[1;34m(env_start)\u001b[0m\n\u001b[0;32m     10\u001b[0m         \u001b[1;32mwhile\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m             \u001b[0maction\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0magent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mselect_action\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdetermine\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m             \u001b[0mnext_state\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minfo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maction\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m             \u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnext_state\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m         \u001b[0mmakespans\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minfo\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"makespan\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32md:\\桌面文件夹\\桌面\\毕设\\github\\DRL-for-Job-Shop-Scheduling\\gymjsp\\jsspenv.py\u001b[0m in \u001b[0;36mstep\u001b[1;34m(self, action)\u001b[0m\n\u001b[0;32m   1127\u001b[0m         \u001b[0mg\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdone\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mobserve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1128\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1129\u001b[1;33m         \u001b[0mobservation\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mg2s\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1130\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1131\u001b[0m         info = {\n",
      "\u001b[1;32md:\\桌面文件夹\\桌面\\毕设\\github\\DRL-for-Job-Shop-Scheduling\\gymjsp\\jsspenv.py\u001b[0m in \u001b[0;36mg2s\u001b[1;34m(self, g, state_len)\u001b[0m\n\u001b[0;32m    570\u001b[0m             \u001b[0mjob_id\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstep_id\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnodes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnodes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    571\u001b[0m             \u001b[0mt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprior_processing_time_matrix\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mjob_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mstep_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 572\u001b[1;33m             \u001b[0mchanged_t\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprocessing_time_matrix\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mjob_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mstep_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    573\u001b[0m             \u001b[0msum_t\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprior_processing_time_matrix\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mjob_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    574\u001b[0m             \u001b[0mstate\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mn\u001b[0m \u001b[1;33m*\u001b[0m \u001b[1;36m10\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnodes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m/\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_jobs\u001b[0m  \u001b[1;31m# job id\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "step_size =10\n",
    "\n",
    "done = False\n",
    "state = env.reset()\n",
    "while not done:\n",
    "    a, makespans = get_greedy_action(env)\n",
    "    next_state, reward, done, info = env.step(a)\n",
    "    state = next_state\n",
    "    print(makespans)\n",
    "    steps = 0\n",
    "    while steps < step_size:\n",
    "        action = agent.select_action(state, determine=True)\n",
    "        next_state, reward, done, info = env.step(action)\n",
    "        state = next_state\n",
    "        steps += 1\n",
    "        if done:\n",
    "            break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "dce9ce0b8994960befdaf1abc06919148bccb19973be1d090d69590bd56698c8"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}