signate_stu_v16.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "signate_stu_v16.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNYIbtp5ES5mOYNfiT5zDbW",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "820514a4048c4df4800e4242b51ebf03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9895209405ff4e85af994cb04c8db191",
              "IPY_MODEL_75013f10c0f64c5abd892ac862789c98",
              "IPY_MODEL_4873eccbaef94232a4af2debf3824db7"
            ],
            "layout": "IPY_MODEL_01442dabfdb142729ea09e4436470d43"
          }
        },
        "9895209405ff4e85af994cb04c8db191": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c1c202998e554a7788cec5bd76d7b502",
            "placeholder": "​",
            "style": "IPY_MODEL_cba2b895dc9440c3958b9476882bf29c",
            "value": "Downloading: 100%"
          }
        },
        "75013f10c0f64c5abd892ac862789c98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3bf9558647e14326ba4568777d25081e",
            "max": 433,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f5a3e85859ed43459e3c4fb4f551d939",
            "value": 433
          }
        },
        "4873eccbaef94232a4af2debf3824db7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_09ea2a6998404fbc8d4040b63fc6d523",
            "placeholder": "​",
            "style": "IPY_MODEL_7820ede015ac462e948a8315c6e12eee",
            "value": " 433/433 [00:00&lt;00:00, 13.3kB/s]"
          }
        },
        "01442dabfdb142729ea09e4436470d43": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c1c202998e554a7788cec5bd76d7b502": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cba2b895dc9440c3958b9476882bf29c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3bf9558647e14326ba4568777d25081e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f5a3e85859ed43459e3c4fb4f551d939": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "09ea2a6998404fbc8d4040b63fc6d523": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7820ede015ac462e948a8315c6e12eee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "21558f936ef946df8f7c1761251eb225": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_783737417daf4df4bfece22cdecfb2b7",
              "IPY_MODEL_edc42424840742a7bea1bef465ecb686",
              "IPY_MODEL_c37521c2614c4f3e895e3a155169494a"
            ],
            "layout": "IPY_MODEL_f1c396100a3341d98c4fb1d6c84a6bc1"
          }
        },
        "783737417daf4df4bfece22cdecfb2b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0a0dd6ac19364208acb4fa0da70e2730",
            "placeholder": "​",
            "style": "IPY_MODEL_1d312370b6114a3d99d1dec7a11dcbaf",
            "value": "Downloading: 100%"
          }
        },
        "edc42424840742a7bea1bef465ecb686": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4f2b0ce6a131466dad1b969f1e3a1f0b",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bd789f44e5f741a3b5fcc034cac1823f",
            "value": 231508
          }
        },
        "c37521c2614c4f3e895e3a155169494a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_87cb83d17dff4e2ba05ae6c0e802a918",
            "placeholder": "​",
            "style": "IPY_MODEL_7c1bf27168194994b068492ff4c8d7a9",
            "value": " 232k/232k [00:00&lt;00:00, 2.36MB/s]"
          }
        },
        "f1c396100a3341d98c4fb1d6c84a6bc1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a0dd6ac19364208acb4fa0da70e2730": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1d312370b6114a3d99d1dec7a11dcbaf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4f2b0ce6a131466dad1b969f1e3a1f0b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bd789f44e5f741a3b5fcc034cac1823f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "87cb83d17dff4e2ba05ae6c0e802a918": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7c1bf27168194994b068492ff4c8d7a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/t-nakatani/signate_stu22/blob/exp/signate_stu_v16.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "だらだら乱数を固定しないまま来てしまったので固定  \n",
        "高速化のためにデータも限定"
      ],
      "metadata": {
        "id": "GDti4y2d6yb_"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "ofvp5QVbEagJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f6d504a7-d840-43c6-fe5d-920d9efabc88"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import os\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "ROOT = \"/content/drive/MyDrive/Colab Notebooks/signate/\"\n",
        "\n",
        "#学習用データと評価用データの読み込み\n",
        "train = pd.read_csv(os.path.join(ROOT, \"train_cleaned.csv\"))\n",
        "test = pd.read_csv(os.path.join(ROOT, \"test_cleaned.csv\"))\n",
        "\n",
        "jobdic = dict(zip([1, 2, 3, 4], ['DataScientist', 'ML Engineer', 'Software Engineer', 'Consultant']))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train = train[:100]\n",
        "test = test[:100]"
      ],
      "metadata": {
        "id": "_K_OY71-iCKI"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q transformers==3"
      ],
      "metadata": {
        "id": "YisF3KcxVqbT"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "_Oa9Vjvy9R_T"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import transformers\n",
        "import torch\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "# from transformers import RobertaTokenizer, RobertaModel\n",
        "# from transformers import BertModel, AutoTokenizer \n",
        "\n",
        "from transformers import AutoTokenizer, AutoModel, AutoConfig\n",
        "# from transformers import BertTokenizer, BertModel\n",
        "from torch import optim\n",
        "from torch import cuda\n",
        "import time\n",
        "from matplotlib import pyplot as plt\n",
        "from tqdm import tqdm"
      ],
      "metadata": {
        "id": "IrNv3cPZOec-"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# seeds\n",
        "SEED = 2022\n",
        "def seed_everything(seed):\n",
        "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "    np.random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    if torch.cuda.is_available():\n",
        "        torch.cuda.manual_seed(seed)\n",
        "        torch.cuda.manual_seed_all(seed)\n",
        "\n",
        "seed_everything(SEED)\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    torch.backends.cudnn.benchmark = False\n",
        "    current_device = torch.cuda.current_device()\n",
        "    print(\"Device:\", torch.cuda.get_device_name(current_device))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qnz-PWYD9S92",
        "outputId": "4cf2d7b4-5985-4fa6-927c-108eafaca4aa"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Device: Tesla P100-PCIE-16GB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Datasetの定義\n",
        "class CreateDataset(Dataset):\n",
        "  def __init__(self, X, y, tokenizer, max_len):\n",
        "    self.X = X\n",
        "    self.y = y\n",
        "    self.tokenizer = tokenizer\n",
        "    self.max_len = max_len\n",
        "\n",
        "  def __len__(self):  # len(Dataset)で返す値を指定\n",
        "    return len(self.X)\n",
        "\n",
        "  def __getitem__(self, index):  # Dataset[index]で返す値を指定\n",
        "    text = self.X[index]\n",
        "    inputs = self.tokenizer.encode_plus(\n",
        "      text,\n",
        "      add_special_tokens=True,\n",
        "      max_length=self.max_len,\n",
        "      pad_to_max_length=True,\n",
        "      truncation=True\n",
        "    )\n",
        "    ids = inputs['input_ids']\n",
        "    mask = inputs['attention_mask']\n",
        "    labels = self.y[index]\n",
        "\n",
        "    return {\n",
        "      'ids': torch.LongTensor(ids),\n",
        "      'mask': torch.LongTensor(mask),\n",
        "      'labels': torch.Tensor(labels)\n",
        "    }"
      ],
      "metadata": {
        "id": "bTbjxDE60aFs"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "1. from_pretrainedの文字列でconfig.from_pretrainedを呼び出し\n",
        "2. config.from_pretrainedの結果をconfigにセット"
      ],
      "metadata": {
        "id": "yYsvP_-hf-pQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pandas._config import config\n",
        "# BERT分類モデルの定義\n",
        "class BERTClass(torch.nn.Module):\n",
        "  def __init__(self, drop_rate, otuput_size):\n",
        "    super().__init__()\n",
        "    # self.config = AutoConfig.from_pretrained(\"allenai/scibert_scivocab_uncased\", num_labels=4)\n",
        "    self.config = AutoConfig.from_pretrained(\"bert-base-uncased\")\n",
        "    self.bert = AutoModel.from_config(config=self.config)\n",
        "    self.drop = torch.nn.Dropout(drop_rate)\n",
        "    self.fc = torch.nn.Linear(768, otuput_size)  # BERTの出力に合わせて768次元を指定\n",
        "\n",
        "  def forward(self, ids, mask):\n",
        "    _, out = self.bert(ids, attention_mask=mask)\n",
        "    out = self.fc(self.drop(out))\n",
        "    # out = self.bert(ids, attention_mask=mask)\n",
        "    # out = self.fc(self.drop(out[:, 0, :]))\n",
        "    return out"
      ],
      "metadata": {
        "id": "M38JBNnv7QA4"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def calculate_loss_and_accuracy(model, criterion, loader, device, epoch, max_epoch):\n",
        "  \"\"\" 損失・正解率を計算\"\"\"\n",
        "  model.eval()\n",
        "  loss = 0.0\n",
        "  total = 0\n",
        "  correct = 0\n",
        "  with torch.no_grad():\n",
        "    with tqdm(loader) as pbar:\n",
        "      pbar.set_description('valid')\n",
        "      for i, data in enumerate(pbar):\n",
        "        ids = data['ids'].to(device)\n",
        "        mask = data['mask'].to(device)\n",
        "        labels = data['labels'].to(device)\n",
        "\n",
        "\n",
        "        outputs = model(ids, mask)\n",
        "        loss += criterion(outputs, labels).item()\n",
        "\n",
        "        pred = torch.argmax(outputs, dim=-1).cpu().numpy() # バッチサイズの長さの予測ラベル配列\n",
        "        labels = torch.argmax(labels, dim=-1).cpu().numpy()  # バッチサイズの長さの正解ラベル配列\n",
        "        total += len(labels)\n",
        "        correct += (pred == labels).sum().item()\n",
        "        pbar.set_postfix(Loss=loss/(i+1), Accuracy=correct/total)\n",
        "\n",
        "  return loss / len(loader), correct / total"
      ],
      "metadata": {
        "id": "KFQSOOn7SbPp"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def train_val(train_loader, val_loader, model_path):\n",
        "  device = 'cuda' if cuda.is_available() else 'cpu'\n",
        "  model = BERTClass(DROP_RATE, OUTPUT_SIZE).to(device)\n",
        "  criterion = torch.nn.CrossEntropyLoss()\n",
        "  optimizer = torch.optim.AdamW(params=model.parameters(), lr=LEARNING_RATE)\n",
        "  \n",
        "  log_valid = []\n",
        "  best_acc = 0\n",
        "  \n",
        "\n",
        "  # train\n",
        "  for epoch in range(NUM_EPOCHS):\n",
        "    print('\\nepoch:', epoch+1)\n",
        "    loss_train = correct = total = 0\n",
        "    model.train()\n",
        "    with tqdm(train_loader) as pbar:\n",
        "      pbar.set_description('train')\n",
        "      for i, data in enumerate(pbar):\n",
        "        ids = data['ids'].to(device)\n",
        "        mask = data['mask'].to(device)\n",
        "        labels = data['labels'].to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        outputs = model(ids, mask)\n",
        "        loss = criterion(outputs, labels)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        \n",
        "        loss_train += loss.item()\n",
        "        pred = torch.argmax(outputs, dim=-1).cpu().numpy()\n",
        "        labels = torch.argmax(labels, dim=-1).cpu().numpy()\n",
        "        total += len(labels)\n",
        "        correct += (pred == labels).sum().item()\n",
        "        pbar.set_postfix(Loss=loss_train/(i+1), Accuracy=correct/total)\n",
        "\n",
        "    loss_valid, acc_valid = calculate_loss_and_accuracy(model, criterion, val_loader, device, epoch, NUM_EPOCHS)\n",
        "    log_valid.append([loss_valid, acc_valid])\n",
        "    if best_acc < acc_valid:\n",
        "      best_acc = acc_valid\n",
        "      \n",
        "      torch.save(model, os.path.join(ROOT, model_path))\n",
        "      print('==== model saved ====')\n",
        "\n",
        "  return log_valid"
      ],
      "metadata": {
        "id": "T9Hy83GPWD1C"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "DROP_RATE = 0.1\n",
        "OUTPUT_SIZE = 4\n",
        "BATCH_SIZE = 16\n",
        "NUM_EPOCHS = 10\n",
        "LEARNING_RATE = 2e-5\n",
        "N_CV = 5\n",
        "VERSION = 'v16'"
      ],
      "metadata": {
        "id": "XfF5KLQJWJCG"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import sklearn.metrics\n",
        "def train_val_predict(\n",
        "        df_train,       # 学習用のデータ\n",
        "        text_column,    # 対象のカラム名\n",
        "        target_column,  # 目的変数のカラム名\n",
        "        df_valid=None,  # 検証用データ\n",
        "        df_val_test=None,       # 予測用データ\n",
        "        model_file_prefix=\"\",  # 保存時のファイル名識別子\n",
        "        epochs=NUM_EPOCHS,\n",
        "        batch_size=BATCH_SIZE,\n",
        "    ):\n",
        "  \n",
        "    # model\n",
        "    model_path = \"{}_{}.pth\".format(model_file_prefix, VERSION)\n",
        "    model = BERTClass(DROP_RATE, OUTPUT_SIZE)\n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    optimizer = torch.optim.AdamW(params=model.parameters(), lr=LEARNING_RATE)\n",
        "    device = 'cuda' if cuda.is_available() else 'cpu'\n",
        "\n",
        "    # dataset\n",
        "    df_val_test[target_column].fillna(0)\n",
        "\n",
        "    max_len = 128\n",
        "    tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n",
        "    dataset_train = CreateDataset(df_train[text_column].values, pd.get_dummies(df_train[target_column]).values, tokenizer, max_len)\n",
        "    dataset_val = CreateDataset(df_valid[text_column].values, pd.get_dummies(df_valid[target_column]).values, tokenizer, max_len)\n",
        "    dataset_test = CreateDataset(df_val_test[text_column].values, pd.get_dummies(df_val_test[target_column]).values, tokenizer, max_len)\n",
        "\n",
        "    dataloader_train = DataLoader(dataset_train, batch_size=BATCH_SIZE, shuffle=True)\n",
        "    dataloader_valid = DataLoader(dataset_val, batch_size=BATCH_SIZE, shuffle=True)\n",
        "    dataloader_test = DataLoader(dataset_test, batch_size=BATCH_SIZE, shuffle=False)\n",
        "\n",
        "    # train\n",
        "    print('\\n================  start train  ================')\n",
        "    metric = train_val(dataloader_train, dataloader_valid, model_path)\n",
        "    print('================  end train  ================\\n')\n",
        "\n",
        "    # predict\n",
        "    model = torch.load(os.path.join(ROOT, model_path))\n",
        "    model.eval()\n",
        "    pred_y_list = []\n",
        "    emb_list = []\n",
        "    with torch.no_grad():\n",
        "      with tqdm(dataloader_test) as pbar:\n",
        "        pbar.set_description('test')\n",
        "        for data in pbar:\n",
        "          ids = data['ids'].to(device)\n",
        "          mask = data['mask'].to(device)\n",
        "          labels = data['labels'].to(device)\n",
        "\n",
        "          output = model.forward(ids, mask)\n",
        "          pred = torch.argmax(output, dim=-1).cpu().numpy()\n",
        "\n",
        "          pred_y_list.extend(pred)\n",
        "          emb_list.extend(output.cpu().numpy())\n",
        "    return metric, pred_y_list, emb_list"
      ],
      "metadata": {
        "id": "La2mPphur18q"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import sklearn.model_selection\n",
        "\n",
        "text_column, target_column, n_splits = ('description', 'jobflag', N_CV)\n",
        "\n",
        "df = pd.concat([train, test], ignore_index=True, sort=False)\n",
        "df_train = df[df[target_column].notnull()]\n",
        "df_test = df[df[target_column].isnull()]\n",
        "\n",
        "df_train_idx = df_train.index\n",
        "\n",
        "# store result\n",
        "df_pred = pd.DataFrame(df.index, columns=[\"index\"]).set_index(\"index\")\n",
        "df_emb = pd.DataFrame(df.index, columns=[\"index\"]).set_index(\"index\")\n",
        "df_emb_pred = None\n",
        "metric_list = []\n",
        "all_emb = []\n",
        "\n",
        "# cross validation\n",
        "kf = sklearn.model_selection.StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1234)\n",
        "for i, (train_idx, test_idx) in enumerate(kf.split(df_train, df_train[target_column])):\n",
        "  df_train_sub = df_train.iloc[train_idx]\n",
        "  df_test_sub = df_train.iloc[test_idx]\n",
        "\n",
        "  df_val_test = pd.concat([df_test_sub, df_test], ignore_index=True, sort=False)\n",
        "\n",
        "  model_file_prefix = \"cv_{}\".format(i)\n",
        "\n",
        "  # train\n",
        "  metric, pred_y_list, emb_list = train_val_predict(\n",
        "      df_train=df_train_sub, \n",
        "      text_column=text_column,\n",
        "      target_column=target_column, \n",
        "      df_valid=df_test_sub,\n",
        "      df_val_test=df_val_test,\n",
        "      model_file_prefix=model_file_prefix,\n",
        "  )\n",
        "  metric_list.append(metric)\n",
        "  all_emb.append(emb_list)\n",
        "  # break"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "820514a4048c4df4800e4242b51ebf03",
            "9895209405ff4e85af994cb04c8db191",
            "75013f10c0f64c5abd892ac862789c98",
            "4873eccbaef94232a4af2debf3824db7",
            "01442dabfdb142729ea09e4436470d43",
            "c1c202998e554a7788cec5bd76d7b502",
            "cba2b895dc9440c3958b9476882bf29c",
            "3bf9558647e14326ba4568777d25081e",
            "f5a3e85859ed43459e3c4fb4f551d939",
            "09ea2a6998404fbc8d4040b63fc6d523",
            "7820ede015ac462e948a8315c6e12eee",
            "21558f936ef946df8f7c1761251eb225",
            "783737417daf4df4bfece22cdecfb2b7",
            "edc42424840742a7bea1bef465ecb686",
            "c37521c2614c4f3e895e3a155169494a",
            "f1c396100a3341d98c4fb1d6c84a6bc1",
            "0a0dd6ac19364208acb4fa0da70e2730",
            "1d312370b6114a3d99d1dec7a11dcbaf",
            "4f2b0ce6a131466dad1b969f1e3a1f0b",
            "bd789f44e5f741a3b5fcc034cac1823f",
            "87cb83d17dff4e2ba05ae6c0e802a918",
            "7c1bf27168194994b068492ff4c8d7a9"
          ]
        },
        "id": "QB2vixNEo7_B",
        "outputId": "7cb04714-6740-4af3-a52a-a7b2dde942c4"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/433 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "820514a4048c4df4800e4242b51ebf03"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "21558f936ef946df8f7c1761251eb225"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "================  start train  ================\n",
            "\n",
            "epoch: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:04<00:00,  1.07it/s, Accuracy=0.25, Loss=1.49]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 10.07it/s, Accuracy=0.4, Loss=1.35]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.10it/s, Accuracy=0.375, Loss=1.38]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.40it/s, Accuracy=0.4, Loss=1.28]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.33it/s, Accuracy=0.338, Loss=1.27]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.76it/s, Accuracy=0.3, Loss=1.14]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.27it/s, Accuracy=0.375, Loss=1.32]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.09it/s, Accuracy=0.4, Loss=1.16]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.32it/s, Accuracy=0.4, Loss=1.29]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.58it/s, Accuracy=0.55, Loss=1.21]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.29it/s, Accuracy=0.388, Loss=1.27]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.78it/s, Accuracy=0.4, Loss=1.21]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.27it/s, Accuracy=0.412, Loss=1.24]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.40it/s, Accuracy=0.4, Loss=1.13]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.27it/s, Accuracy=0.4, Loss=1.24]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.22it/s, Accuracy=0.4, Loss=1.15]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.25it/s, Accuracy=0.412, Loss=1.22]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 17.00it/s, Accuracy=0.35, Loss=1.15]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.26it/s, Accuracy=0.35, Loss=1.26]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 13.83it/s, Accuracy=0.4, Loss=1.14]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "================  end train  ================\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "test: 100%|██████████| 8/8 [00:00<00:00, 11.12it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "================  start train  ================\n",
            "\n",
            "epoch: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.22it/s, Accuracy=0.325, Loss=1.48]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 15.25it/s, Accuracy=0.25, Loss=1.51]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.30it/s, Accuracy=0.287, Loss=1.38]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 15.85it/s, Accuracy=0.4, Loss=1.23]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.31it/s, Accuracy=0.4, Loss=1.27]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.22it/s, Accuracy=0.4, Loss=1.27]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.32it/s, Accuracy=0.4, Loss=1.26]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.16it/s, Accuracy=0.4, Loss=1.27]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.27it/s, Accuracy=0.412, Loss=1.25]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 15.77it/s, Accuracy=0.35, Loss=1.23]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.32it/s, Accuracy=0.438, Loss=1.26]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 15.44it/s, Accuracy=0.4, Loss=1.32]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.20it/s, Accuracy=0.438, Loss=1.25]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 13.90it/s, Accuracy=0.4, Loss=1.14]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.14it/s, Accuracy=0.412, Loss=1.22]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 13.34it/s, Accuracy=0.4, Loss=1.23]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.13it/s, Accuracy=0.412, Loss=1.21]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 13.45it/s, Accuracy=0.4, Loss=1.2]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.09it/s, Accuracy=0.537, Loss=1.17]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.51it/s, Accuracy=0.35, Loss=1.11]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "================  end train  ================\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "test: 100%|██████████| 8/8 [00:00<00:00, 12.87it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "================  start train  ================\n",
            "\n",
            "epoch: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.32it/s, Accuracy=0.338, Loss=1.37]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.11it/s, Accuracy=0.4, Loss=1.23]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.26it/s, Accuracy=0.4, Loss=1.26]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 15.78it/s, Accuracy=0.35, Loss=1.2]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.26it/s, Accuracy=0.3, Loss=1.29]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 15.41it/s, Accuracy=0.4, Loss=1.23]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.31it/s, Accuracy=0.412, Loss=1.27]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.24it/s, Accuracy=0.4, Loss=1.28]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.27it/s, Accuracy=0.362, Loss=1.26]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 17.01it/s, Accuracy=0.3, Loss=1.16]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.13it/s, Accuracy=0.388, Loss=1.23]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 13.37it/s, Accuracy=0.4, Loss=1.21]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  3.97it/s, Accuracy=0.45, Loss=1.17]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 13.98it/s, Accuracy=0.35, Loss=1.2]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.00it/s, Accuracy=0.487, Loss=1.11]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 13.44it/s, Accuracy=0.4, Loss=1.17]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.24it/s, Accuracy=0.525, Loss=1.07]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 14.84it/s, Accuracy=0.35, Loss=1.95]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.29it/s, Accuracy=0.6, Loss=0.962]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 15.27it/s, Accuracy=0.35, Loss=1.45]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "================  end train  ================\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "test: 100%|██████████| 8/8 [00:00<00:00, 12.78it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "================  start train  ================\n",
            "\n",
            "epoch: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.30it/s, Accuracy=0.275, Loss=1.39]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 15.51it/s, Accuracy=0.4, Loss=1.28]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.29it/s, Accuracy=0.388, Loss=1.27]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.43it/s, Accuracy=0.4, Loss=1.22]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.29it/s, Accuracy=0.438, Loss=1.22]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.66it/s, Accuracy=0.5, Loss=1.31]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.24it/s, Accuracy=0.388, Loss=1.24]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.62it/s, Accuracy=0.4, Loss=1.19]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.30it/s, Accuracy=0.463, Loss=1.23]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.81it/s, Accuracy=0.4, Loss=1.42]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.31it/s, Accuracy=0.35, Loss=1.24]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.07it/s, Accuracy=0.3, Loss=1.19]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.29it/s, Accuracy=0.412, Loss=1.25]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.56it/s, Accuracy=0.5, Loss=1.42]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.29it/s, Accuracy=0.438, Loss=1.22]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.36it/s, Accuracy=0.55, Loss=1.12]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.30it/s, Accuracy=0.5, Loss=1.13]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.06it/s, Accuracy=0.5, Loss=1.03]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.33it/s, Accuracy=0.537, Loss=1.06]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.30it/s, Accuracy=0.5, Loss=1.07]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "================  end train  ================\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "test: 100%|██████████| 8/8 [00:00<00:00, 12.96it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "================  start train  ================\n",
            "\n",
            "epoch: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.01it/s, Accuracy=0.35, Loss=1.55]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 14.24it/s, Accuracy=0.4, Loss=1.33]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.26it/s, Accuracy=0.275, Loss=1.37]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.27it/s, Accuracy=0.25, Loss=1.44]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.25it/s, Accuracy=0.25, Loss=1.33]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 17.09it/s, Accuracy=0.4, Loss=1.38]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.26it/s, Accuracy=0.4, Loss=1.26]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 17.23it/s, Accuracy=0.4, Loss=1.39]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.25it/s, Accuracy=0.438, Loss=1.25]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.58it/s, Accuracy=0.4, Loss=1.4]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.29it/s, Accuracy=0.425, Loss=1.24]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 17.28it/s, Accuracy=0.4, Loss=1.26]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.23it/s, Accuracy=0.388, Loss=1.26]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 16.72it/s, Accuracy=0.4, Loss=1.5]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  4.18it/s, Accuracy=0.4, Loss=1.25]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 14.23it/s, Accuracy=0.4, Loss=1.17]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  3.99it/s, Accuracy=0.438, Loss=1.22]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00,  7.30it/s, Accuracy=0.4, Loss=1.51]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 5/5 [00:01<00:00,  2.86it/s, Accuracy=0.375, Loss=1.22]\n",
            "valid: 100%|██████████| 2/2 [00:00<00:00, 14.16it/s, Accuracy=0.4, Loss=1.31]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "================  end train  ================\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "test: 100%|██████████| 8/8 [00:00<00:00, 12.90it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "all_emb_ = np.array([np.array(emb)[-len(df_test):] for emb in all_emb])"
      ],
      "metadata": {
        "id": "eK-UI_G3wzIG"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mean_df = pd.DataFrame([[np.mean(arr_val) for arr_val in arr_vec] for arr_vec in np.moveaxis(all_emb_, 0, 2)])\n",
        "preds = [np.argmax(arr)+1 for arr in mean_df.to_numpy()]\n",
        "df_submit = pd.DataFrame([df_test.id.to_list(), preds]).T\n",
        "df_submit.to_csv(os.path.join(ROOT, f'{VERSION}_esemble.csv'), index=None, header=None)"
      ],
      "metadata": {
        "id": "zHARK-Z-lvL8"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "acc_list = []\n",
        "for item in metric_list:\n",
        "  acc_list.append((np.max([acc for loss, acc in item])))\n",
        "print(np.mean(acc_list))"
      ],
      "metadata": {
        "id": "JVEj-oYop9as",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ab22f29a-705a-4483-aeda-180a220569bd"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.4600000000000001\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for label, arr in zip(df_train[target_column][:10], pd.get_dummies(df_train[target_column]).values[:10]):\n",
        "  print(label, arr)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PY_0LCy8Q2xR",
        "outputId": "146916b8-7f9c-4762-ff31-2c344612646f"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "3.0 [0 0 1 0]\n",
            "3.0 [0 0 1 0]\n",
            "4.0 [0 0 0 1]\n",
            "3.0 [0 0 1 0]\n",
            "4.0 [0 0 0 1]\n",
            "3.0 [0 0 1 0]\n",
            "4.0 [0 0 0 1]\n",
            "4.0 [0 0 0 1]\n",
            "1.0 [1 0 0 0]\n",
            "1.0 [1 0 0 0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "check_ml = pd.read_csv(os.path.join(ROOT, f'v14_esemble.csv'), header=None) "
      ],
      "metadata": {
        "id": "Mu2pV8PvRNdh"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "check_ml[1] = 2"
      ],
      "metadata": {
        "id": "enL2oPLt8A1V"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "check_ml.to_csv(os.path.join(ROOT, 'check_ml.csv'), index=None, header=None)"
      ],
      "metadata": {
        "id": "0hmF4Cy58VNg"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "eiC4Ia0o8YOk"
      },
      "execution_count": 19,
      "outputs": []
    }
  ]
}