BERT_Korean_Tokenizer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "5b0843766d3f4ac785c7dba85254d605": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6356e0609f9f49d5996ef7f4f77fbd2d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ca07bf481be7473ab1b22babaa76c3da",
              "IPY_MODEL_ab60a81459a84ae19fa922aa4ce27e8a"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "6356e0609f9f49d5996ef7f4f77fbd2d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "ca07bf481be7473ab1b22babaa76c3da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4a4ff12bb4604faf8c1cd79156713854",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 995526,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 995526,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_88f28f34de9e41cc948c7aebb4035589"
          },
          "model_module_version": "1.5.0"
        },
        "ab60a81459a84ae19fa922aa4ce27e8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6cb6badcbbd34359be9dad2c8af93098",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 996k/996k [00:00&lt;00:00, 1.67MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_51eafe68808a4ffbac05605381c2d5a3"
          },
          "model_module_version": "1.5.0"
        },
        "4a4ff12bb4604faf8c1cd79156713854": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "88f28f34de9e41cc948c7aebb4035589": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "6cb6badcbbd34359be9dad2c8af93098": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "51eafe68808a4ffbac05605381c2d5a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "93234e1bd6444d819b130d83402d2d7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ad295e710380441588473f810a9210d7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e6e0a2274e7f420f91dc97a143315da0",
              "IPY_MODEL_8d879e2bbea04536aaa1ee5d356bb7c3"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "ad295e710380441588473f810a9210d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "e6e0a2274e7f420f91dc97a143315da0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a0dbf235f20c497186d319b5b1558dd9",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 29,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 29,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ed834b7f997141479ab90216655e230a"
          },
          "model_module_version": "1.5.0"
        },
        "8d879e2bbea04536aaa1ee5d356bb7c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_32c340873ce247e88df66c73309eecdc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 29.0/29.0 [00:00&lt;00:00, 50.2B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2ef88e8c35374ca69203a64d209745ea"
          },
          "model_module_version": "1.5.0"
        },
        "a0dbf235f20c497186d319b5b1558dd9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "ed834b7f997141479ab90216655e230a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "32c340873ce247e88df66c73309eecdc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "2ef88e8c35374ca69203a64d209745ea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "e4dcb08aab3748b18a10d0f5daaf3554": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f1ed2fc28a3e499fa784d5aa1777a77b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_598f160635264f138769ae94a127455c",
              "IPY_MODEL_693887ccba30416586e2085b7e36118b"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "f1ed2fc28a3e499fa784d5aa1777a77b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "598f160635264f138769ae94a127455c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b61fb58de9be4c84b7767bf69e32c5d9",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1961828,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1961828,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1ac7cea5aaba45af9eddeaaee02e1e5a"
          },
          "model_module_version": "1.5.0"
        },
        "693887ccba30416586e2085b7e36118b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_cc353722b52045efb9009ef79c7d56b7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.96M/1.96M [00:00&lt;00:00, 6.43MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9169c98d7ee5423ba5b6eb3c4dbbeda7"
          },
          "model_module_version": "1.5.0"
        },
        "b61fb58de9be4c84b7767bf69e32c5d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "1ac7cea5aaba45af9eddeaaee02e1e5a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "cc353722b52045efb9009ef79c7d56b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "9169c98d7ee5423ba5b6eb3c4dbbeda7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dscoool/datastructure/blob/main/BERT_Korean_tokenizer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hctUswBCe6OI"
      },
      "source": [
        "# 한국어 개체명 인식 개요\n",
        "이번 강의에서는 버트를 활용해서 한국어 개체명 인식을 다뤄보고자 합니다.\n",
        "![Imgur](https://i.imgur.com/PDdTLjy.png)\n",
        "데이터는 https://github.com/naver/nlp-challenge/ 에서 받아왔습니다.\n",
        "\n",
        "개체명 추출 리더보드에서 제공되는 코퍼스는 문장에 나타난 개체명을 14개 분류 카테고리로 주석 작업이 되어있습니다.\n",
        "\n",
        "![Imgur](https://i.imgur.com/it4uTE3.png)\n",
        "\n",
        "문장을 입력하면 다음과 같은 형식으로 개체명이 분류됩니다.  \n",
        "\n",
        "![Imgur](https://i.imgur.com/RpuZf6R.png)\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o9NQjPIe3vE2"
      },
      "source": [
        "# 목차\n",
        "이번 실습은 <b>1) 네이버 개체명 인식 데이터 불러오기 및 전처리 2) BERT 인풋 만들기 3) 버트를 활용한 개체명 인식 모델 만들기 4) 훈련 및 성능 검증 5) 실제 데이터로 실습하기</b>로 구성되어 있습니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hMBwJT0r38tj"
      },
      "source": [
        "# BERT를 활용하여 한국어 개체명 인식기 만들기"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Amc52mb94Jzq"
      },
      "source": [
        "## 개체명 인식 데이터 불러오기 및 전처리"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z3rnyfkihRVq"
      },
      "source": [
        "개체명 인식을 위한 데이터를 다운 받습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Xv5IzTBeecA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "af8096fe-f6ef-4fab-e71b-28c9486c7d74"
      },
      "source": [
        "!wget https://github.com/naver/nlp-challenge/raw/master/missions/ner/data/train/train_data"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-04-08 04:48:45--  https://github.com/naver/nlp-challenge/raw/master/missions/ner/data/train/train_data\n",
            "Resolving github.com (github.com)... 140.82.112.3\n",
            "Connecting to github.com (github.com)|140.82.112.3|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://raw.githubusercontent.com/naver/nlp-challenge/master/missions/ner/data/train/train_data [following]\n",
            "--2021-04-08 04:48:46--  https://raw.githubusercontent.com/naver/nlp-challenge/master/missions/ner/data/train/train_data\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 16945023 (16M) [text/plain]\n",
            "Saving to: ‘train_data’\n",
            "\n",
            "train_data          100%[===================>]  16.16M  47.9MB/s    in 0.3s    \n",
            "\n",
            "2021-04-08 04:48:47 (47.9 MB/s) - ‘train_data’ saved [16945023/16945023]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3MeOwf3FhdgE"
      },
      "source": [
        "분석에 필요한 모듈들을 임포트 합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d0dhGSE3ehh-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "be054168-4ef4-4782-c00f-95b5a427a0cb"
      },
      "source": [
        "!pip install transformers\n",
        "!pip install sentencepiece\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from transformers import *\n",
        "import json\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "import os\n",
        "import re\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/81/91/61d69d58a1af1bd81d9ca9d62c90a6de3ab80d77f27c5df65d9a2c1f5626/transformers-4.5.0-py3-none-any.whl (2.1MB)\n",
            "\u001b[K     |████████████████████████████████| 2.2MB 6.7MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 27.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/08/cd/342e584ee544d044fb573ae697404ce22ede086c9e87ce5960772084cad0/sacremoses-0.0.44.tar.gz (862kB)\n",
            "\u001b[K     |████████████████████████████████| 870kB 20.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.44-cp37-none-any.whl size=886084 sha256=713752039c6d34585842187f0f00738f9b484717309582b63f15bcff2129ad2c\n",
            "  Stored in directory: /root/.cache/pip/wheels/3e/fb/c0/13ab4d63d537658f448366744654323077c4d90069b6512f3c\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.44 tokenizers-0.10.2 transformers-4.5.0\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f5/99/e0808cb947ba10f575839c43e8fafc9cc44e4a7a2c8f79c60db48220a577/sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 6.7MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.95\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1SBHyIvJhieB"
      },
      "source": [
        "train 데이터를 불러 오겠습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "45hQv0Bienkt",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 415
        },
        "outputId": "e4614d58-3e74-4f22-81b4-1461e4c599ed"
      },
      "source": [
        "train = pd.read_csv(\"train_data\", names=['src', 'tar'], sep=\"\\t\")\n",
        "train = train.reset_index()\n",
        "train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>src</th>\n",
              "      <th>tar</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>비토리오</td>\n",
              "      <td>PER_B</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>양일</td>\n",
              "      <td>DAT_B</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>만에</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>영사관</td>\n",
              "      <td>ORG_B</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>감호</td>\n",
              "      <td>CVL_B</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>769060</th>\n",
              "      <td>2</td>\n",
              "      <td>어째</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>769061</th>\n",
              "      <td>3</td>\n",
              "      <td>뭔가</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>769062</th>\n",
              "      <td>4</td>\n",
              "      <td>수상쩍은</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>769063</th>\n",
              "      <td>5</td>\n",
              "      <td>좌담</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>769064</th>\n",
              "      <td>6</td>\n",
              "      <td>．</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>769065 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        index   src    tar\n",
              "0           1  비토리오  PER_B\n",
              "1           2    양일  DAT_B\n",
              "2           3    만에      -\n",
              "3           4   영사관  ORG_B\n",
              "4           5    감호  CVL_B\n",
              "...       ...   ...    ...\n",
              "769060      2    어째      -\n",
              "769061      3    뭔가      -\n",
              "769062      4  수상쩍은      -\n",
              "769063      5    좌담      -\n",
              "769064      6     ．      -\n",
              "\n",
              "[769065 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eC2QfGRbhmJJ"
      },
      "source": [
        "train 데이터에 마침표가 이상한 것들이 많아서 확실하게 .으로 수정해 주겠습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ms0gXmslimQc"
      },
      "source": [
        "train['src'] = train['src'].str.replace(\"．\", \".\", regex=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X2zgWiWAhiXY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 415
        },
        "outputId": "9faf36e1-57ea-4508-ca75-38c0b27e285d"
      },
      "source": [
        "train.loc[train['src']=='.']"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>src</th>\n",
              "      <th>tar</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>6</td>\n",
              "      <td>.</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>15</td>\n",
              "      <td>.</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>10</td>\n",
              "      <td>.</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>77</th>\n",
              "      <td>24</td>\n",
              "      <td>.</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>19</td>\n",
              "      <td>.</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>769013</th>\n",
              "      <td>11</td>\n",
              "      <td>.</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>769036</th>\n",
              "      <td>23</td>\n",
              "      <td>.</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>769053</th>\n",
              "      <td>17</td>\n",
              "      <td>.</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>769058</th>\n",
              "      <td>5</td>\n",
              "      <td>.</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>769064</th>\n",
              "      <td>6</td>\n",
              "      <td>.</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>57254 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        index src tar\n",
              "15          6   .   -\n",
              "30         15   .   -\n",
              "40         10   .   -\n",
              "77         24   .   -\n",
              "96         19   .   -\n",
              "...       ...  ..  ..\n",
              "769013     11   .   -\n",
              "769036     23   .   -\n",
              "769053     17   .   -\n",
              "769058      5   .   -\n",
              "769064      6   .   -\n",
              "\n",
              "[57254 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IJHg2kIUht7L"
      },
      "source": [
        "데이터를 전처리 해주겠습니다.  \n",
        "한글, 영어, 소문자, 대문자, . 이외의 단어들을 모두 제거하겠습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XnsxeMY9Z7vv"
      },
      "source": [
        "train['src'] = train['src'].astype(str)\n",
        "train['tar'] = train['tar'].astype(str)\n",
        "\n",
        "train['src'] = train['src'].str.replace(r'[^ㄱ-ㅣ가-힣0-9a-zA-Z.]+', \"\", regex=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QekBoou4h1HK"
      },
      "source": [
        "데이터를 리스트 형식으로 변환합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gj8CBxAtGVpT"
      },
      "source": [
        "data = [list(x) for x in train[['index', 'src', 'tar']].to_numpy()]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q9Tup6qziOMA"
      },
      "source": [
        "데이터를 잘 보면 (인덱스, 단어, 개체) 로 이루어 진 것을 알 수 있습니다.  \n",
        "인덱스가 1,2,3,4,5.. 이렇게 이어지다가 다시 1,2,3,4, 로 바뀌는데 숫자가 바뀌기 전까지가 한 문장을 의미합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ETMBXgxPiH63",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "25d284ca-fb77-4a17-9fd6-66e02958f50a"
      },
      "source": [
        "print(data[:20])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1, '비토리오', 'PER_B'], [2, '양일', 'DAT_B'], [3, '만에', '-'], [4, '영사관', 'ORG_B'], [5, '감호', 'CVL_B'], [6, '용퇴', '-'], [7, '항룡', '-'], [8, '압력설', '-'], [9, '의심만', '-'], [10, '가율', '-'], [1, '이', '-'], [2, '음경동맥의', '-'], [3, '직경이', '-'], [4, '8', 'NUM_B'], [5, '19mm입니다', 'NUM_B'], [6, '.', '-'], [1, '9세이브로', 'NUM_B'], [2, '구완', '-'], [3, '30위인', 'NUM_B'], [4, 'LG', 'ORG_B']]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "59Lv_fQKh7Lj"
      },
      "source": [
        "라벨들을 추출하고, 딕셔너리 형식으로 저장하도록 하겠습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jQm5tzwSQVH_"
      },
      "source": [
        "label = train['tar'].unique().tolist()\n",
        "label_dict = {word:i for i, word in enumerate(label)}\n",
        "label_dict.update({\"[PAD]\":len(label_dict)})\n",
        "index_to_ner = {i:j for j, i in label_dict.items()}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CCFeeGVoRos7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d71fb100-fad5-4706-bb14-4fe6343c1a2d"
      },
      "source": [
        "print(label_dict)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'PER_B': 0, 'DAT_B': 1, '-': 2, 'ORG_B': 3, 'CVL_B': 4, 'NUM_B': 5, 'LOC_B': 6, 'EVT_B': 7, 'TRM_B': 8, 'TRM_I': 9, 'EVT_I': 10, 'PER_I': 11, 'CVL_I': 12, 'NUM_I': 13, 'TIM_B': 14, 'TIM_I': 15, 'ORG_I': 16, 'DAT_I': 17, 'ANM_B': 18, 'MAT_B': 19, 'MAT_I': 20, 'AFW_B': 21, 'FLD_B': 22, 'LOC_I': 23, 'AFW_I': 24, 'PLT_B': 25, 'FLD_I': 26, 'ANM_I': 27, 'PLT_I': 28, '[PAD]': 29}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kClinJNx2gkb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "62511e6f-a109-4551-dfaf-106d86d1947f"
      },
      "source": [
        "print(index_to_ner)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{0: 'PER_B', 1: 'DAT_B', 2: '-', 3: 'ORG_B', 4: 'CVL_B', 5: 'NUM_B', 6: 'LOC_B', 7: 'EVT_B', 8: 'TRM_B', 9: 'TRM_I', 10: 'EVT_I', 11: 'PER_I', 12: 'CVL_I', 13: 'NUM_I', 14: 'TIM_B', 15: 'TIM_I', 16: 'ORG_I', 17: 'DAT_I', 18: 'ANM_B', 19: 'MAT_B', 20: 'MAT_I', 21: 'AFW_B', 22: 'FLD_B', 23: 'LOC_I', 24: 'AFW_I', 25: 'PLT_B', 26: 'FLD_I', 27: 'ANM_I', 28: 'PLT_I', 29: '[PAD]'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MqwmtXbBihQM"
      },
      "source": [
        "데이터를 문장들과 개체들로 분리합니다.  \n",
        "tups[0], tups[1],... 에 각각의 문장에 해당하는 단어와 개체 번호가 저장이 되게 됩니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aBX31tk0HCIP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e0c1fcf0-dcc2-4f37-c699-b4323dfdabad"
      },
      "source": [
        "tups = []\n",
        "temp_tup = []\n",
        "temp_tup.append(data[0][1:])\n",
        "sentences = []\n",
        "targets = []\n",
        "for i, j, k in data:\n",
        "\n",
        "  if i != 1:\n",
        "    temp_tup.append([j,label_dict[k]])\n",
        "  if i == 1:\n",
        "    if len(temp_tup) != 0:\n",
        "      tups.append(temp_tup)\n",
        "      temp_tup = []\n",
        "      temp_tup.append([j,label_dict[k]])\n",
        "\n",
        "tups.pop(0)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['비토리오', 'PER_B']]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "weH4ntmzM0If",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "713d2dab-ae5e-4d24-e6c6-3967547677d6"
      },
      "source": [
        "print(tups[0], tups[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[['비토리오', 0], ['양일', 1], ['만에', 2], ['영사관', 3], ['감호', 4], ['용퇴', 2], ['항룡', 2], ['압력설', 2], ['의심만', 2], ['가율', 2]] [['이', 2], ['음경동맥의', 2], ['직경이', 2], ['8', 5], ['19mm입니다', 5], ['.', 2]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FUWClNZni1Pi"
      },
      "source": [
        "tups를 보면 [(단어, 개체), (단어, 개체), (단어, 개체)]의 형식으로 저장이 되어 있는데, 이거를 (단어, 단어, 단어, 단어), (개체, 개체, 개체, 개체) 형식으로 변환하도록 하겠습니다.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xXcYNq0DTKk7"
      },
      "source": [
        "sentences = []\n",
        "targets = []\n",
        "for tup in tups:\n",
        "  sentence = []\n",
        "  target = []\n",
        "  sentence.append(\"[CLS]\")\n",
        "  target.append(label_dict['-'])\n",
        "  for i, j in tup:\n",
        "    sentence.append(i)\n",
        "    target.append(j)\n",
        "  sentence.append(\"[SEP]\")\n",
        "  target.append(label_dict['-'])\n",
        "  sentences.append(sentence)\n",
        "  targets.append(target)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QMmSZyygUatW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b82ea976-7b70-42bd-9d66-840acb67659e"
      },
      "source": [
        "sentences[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['[CLS]',\n",
              " '비토리오',\n",
              " '양일',\n",
              " '만에',\n",
              " '영사관',\n",
              " '감호',\n",
              " '용퇴',\n",
              " '항룡',\n",
              " '압력설',\n",
              " '의심만',\n",
              " '가율',\n",
              " '[SEP]']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qPvbW9H1UeUd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f6dbcedf-b959-461e-9c81-ecb64f072cb4"
      },
      "source": [
        "targets[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[2, 0, 1, 2, 3, 4, 2, 2, 2, 2, 2, 2]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q7TeCA1I4NkT"
      },
      "source": [
        "## 버트 인풋 만들기"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2njeCZEKjGdu"
      },
      "source": [
        "구글의 multilinguial-bert를 활용하도록 하겠습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JfIdFdAVOKfD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 169,
          "referenced_widgets": [
            "5b0843766d3f4ac785c7dba85254d605",
            "6356e0609f9f49d5996ef7f4f77fbd2d",
            "ca07bf481be7473ab1b22babaa76c3da",
            "ab60a81459a84ae19fa922aa4ce27e8a",
            "4a4ff12bb4604faf8c1cd79156713854",
            "88f28f34de9e41cc948c7aebb4035589",
            "6cb6badcbbd34359be9dad2c8af93098",
            "51eafe68808a4ffbac05605381c2d5a3",
            "93234e1bd6444d819b130d83402d2d7b",
            "ad295e710380441588473f810a9210d7",
            "e6e0a2274e7f420f91dc97a143315da0",
            "8d879e2bbea04536aaa1ee5d356bb7c3",
            "a0dbf235f20c497186d319b5b1558dd9",
            "ed834b7f997141479ab90216655e230a",
            "32c340873ce247e88df66c73309eecdc",
            "2ef88e8c35374ca69203a64d209745ea",
            "e4dcb08aab3748b18a10d0f5daaf3554",
            "f1ed2fc28a3e499fa784d5aa1777a77b",
            "598f160635264f138769ae94a127455c",
            "693887ccba30416586e2085b7e36118b",
            "b61fb58de9be4c84b7767bf69e32c5d9",
            "1ac7cea5aaba45af9eddeaaee02e1e5a",
            "cc353722b52045efb9009ef79c7d56b7",
            "9169c98d7ee5423ba5b6eb3c4dbbeda7"
          ]
        },
        "outputId": "b59464b7-6772-401e-c12f-037b0663f1e1"
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5b0843766d3f4ac785c7dba85254d605",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=995526.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "93234e1bd6444d819b130d83402d2d7b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=29.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e4dcb08aab3748b18a10d0f5daaf3554",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1961828.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zyapNcSIjnqM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "69392624-5c83-4bb6-9496-79023fbc8e23"
      },
      "source": [
        "tokenizer.tokenize(\"대한민국 만세.\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['대한민국', '만', '##세', '.']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vpl_V5_HjSxp"
      },
      "source": [
        "여기서부터가 중요한데, 문장을 토크나이징 하고 개체(target)을 토크나이징 한 문장에 맞추도록 하겠습니다.  \n",
        "문장 \"대한민국 만세.\" 는 사실 (대한민국, 개체1), (만세., 개체2) 을 가지고 있는데 토크나이징을 하면 '▁대한민국', '▁만', '세', '.' 로 토크나이징이 됩니다.  \n",
        "여기서 그렇다면 ( ▁대한민국, 개체1) , (▁만, 개체2), (세, 개체2), (., 개체 2) 와 같은 방식으로 각 개체를 부여해주어야 합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W3p-p4W0Cc62"
      },
      "source": [
        "def tokenize_and_preserve_labels(sentence, text_labels):\n",
        "  tokenized_sentence = []\n",
        "  labels = []\n",
        "\n",
        "  for word, label in zip(sentence, text_labels):\n",
        "\n",
        "    tokenized_word = tokenizer.tokenize(word)\n",
        "    n_subwords = len(tokenized_word)\n",
        "\n",
        "    tokenized_sentence.extend(tokenized_word)\n",
        "    labels.extend([label] * n_subwords)\n",
        "\n",
        "  return tokenized_sentence, labels"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0U6gAmqhCvRF"
      },
      "source": [
        "tokenized_texts_and_labels = [\n",
        "                              tokenize_and_preserve_labels(sent, labs)\n",
        "                              for sent, labs in zip(sentences, targets)]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ar9Rd9e-kd3J",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "63bffa12-20d1-4a08-fa71-2217a6c92131"
      },
      "source": [
        "print(tokenized_texts_and_labels[:2])\n",
        "# [(문장, 개체들), (문장, 개체들),...] 형식으로 저장되어 있음."
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[(['[CLS]', '비', '##토', '##리', '##오', '양', '##일', '만', '##에', '영', '##사', '##관', '감', '##호', '용', '##퇴', '항', '##룡', '압', '##력', '##설', '의', '##심', '##만', '가', '##율', '[SEP]'], [2, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 4, 4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]), (['[CLS]', '이', '음', '##경', '##동', '##맥', '##의', '직', '##경', '##이', '8', '19', '##mm', '##입', '##니다', '.', '[SEP]'], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 5, 5, 5, 5, 2, 2])]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ABHN43Oekyq9"
      },
      "source": [
        "(문장, 개체들), (문장, 개체들) 을 [문장, 문장, 문장, ...] , [개체들, 개체들 개체들,,,,]로 분리해주도록 하겠습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RNcXW5YeDI_r"
      },
      "source": [
        "tokenized_texts = [token_label_pair[0] for token_label_pair in tokenized_texts_and_labels]\n",
        "labels = [token_label_pair[1] for token_label_pair in tokenized_texts_and_labels]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dUrWCDs00nzw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c36da420-499e-4888-fdd4-2bcc91f5e1cb"
      },
      "source": [
        "tokenized_texts[1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['[CLS]',\n",
              " '이',\n",
              " '음',\n",
              " '##경',\n",
              " '##동',\n",
              " '##맥',\n",
              " '##의',\n",
              " '직',\n",
              " '##경',\n",
              " '##이',\n",
              " '8',\n",
              " '19',\n",
              " '##mm',\n",
              " '##입',\n",
              " '##니다',\n",
              " '.',\n",
              " '[SEP]']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SusPxxR40r8E",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e93f6502-2fab-4766-c13f-b84a1136f116"
      },
      "source": [
        "labels[1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 5, 5, 5, 5, 5, 2, 2]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0W004Btok8Qb"
      },
      "source": [
        "문장의 길이가 상위 2.5%(88) 인 지점을 기준으로 문장의 길이를 정하도록 하겠습니다.  \n",
        "만약 문장의 길이가 88보다 크면 문장이 잘리게 되고, 길이가 88보다 작다면 패딩이 되어 모든 문장의 길이가 88로 정해지게 됩니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QSy59ymvVLiM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cedfb7b0-2744-436c-c2a4-6fde808b8b38"
      },
      "source": [
        "print(np.quantile(np.array([len(x) for x in tokenized_texts]), 0.975))\n",
        "max_len = 88\n",
        "bs = 32"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "96.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J3dyW2avlPX4"
      },
      "source": [
        "버트에 인풋으로 들어갈 train 데이터를 만들도록 하겠습니다.  \n",
        "버트 인풋으로는   \n",
        "input_ids : 문장이 토크나이즈 된 것이 숫자로 바뀐 것,   \n",
        "attention_masks : 문장이 토크나이즈 된 것 중에서 패딩이 아닌 부분은 1, 패딩인 부분은 0으로 마스킹  \n",
        "[input_ids, attention_masks]가 인풋으로 들어갑니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Iy2Aops4DzYa"
      },
      "source": [
        "input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],\n",
        "                          maxlen=max_len, dtype = \"int\", value=tokenizer.convert_tokens_to_ids(\"[PAD]\"), truncating=\"post\", padding=\"post\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nxn9G_poE-mL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "39c7e610-ced9-429f-a789-6ee5d26a7655"
      },
      "source": [
        "input_ids[1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([   101,   9638,   9634,  31720,  18778, 118915,  10459,   9707,\n",
              "        31720,  10739,    129,  10270,  17525,  58303,  48345,    119,\n",
              "          102,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MKfLbZjolxvo"
      },
      "source": [
        "정답에 해당하는 개체들을 만들어 보겠습니다.  \n",
        "패딩에 해당하는 부분은 label_dict([PAD])(29)가 들어가게 되겠습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "natWaP5FEdDf"
      },
      "source": [
        "tags = pad_sequences([lab for lab in labels], maxlen=max_len, value=label_dict[\"[PAD]\"], padding='post',\\\n",
        "                     dtype='int', truncating='post')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BvlojVT_FBw5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8b56dd87-e0fb-44a6-a8a6-1ae75e4345c7"
      },
      "source": [
        "tags[1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  5,  5,  5,  5,  5,  2,  2,\n",
              "       29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,\n",
              "       29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,\n",
              "       29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,\n",
              "       29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29,\n",
              "       29, 29, 29])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q484H2TAmC9y"
      },
      "source": [
        "어텐션 마스크를 만들어 주겠습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B_AYPsSnFStM"
      },
      "source": [
        "attention_masks = np.array([[int(i != tokenizer.convert_tokens_to_ids(\"[PAD]\")) for i in ii] for ii in input_ids])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U1KW7qkTmG5w"
      },
      "source": [
        "train 데이터에서 10% 만큼을 validation 데이터로 분리해 주겠습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m2Xzq_g9Fcn0"
      },
      "source": [
        "tr_inputs, val_inputs, tr_tags, val_tags = train_test_split(input_ids, tags,\n",
        "                                                            random_state=2018, test_size=0.1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gLkM6hwfFjxf"
      },
      "source": [
        "tr_masks, val_masks, _, _ = train_test_split(attention_masks, input_ids,\n",
        "                                             random_state=2018, test_size=0.1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-MOH3QOJ4ZPc"
      },
      "source": [
        "## 개체명 인식 모델 만들기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jrS1jwfoN99Z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "33e5164d-a57d-4da1-de08-ec6fae886d07"
      },
      "source": [
        "# TPU 작동을 위해 실행\n",
        "resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])\n",
        "tf.config.experimental_connect_to_cluster(resolver)\n",
        "tf.tpu.experimental.initialize_tpu_system(resolver)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:absl:Entering into master device scope: /job:worker/replica:0/task:0/device:CPU:0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.24.98.122:8470\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.24.98.122:8470\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.tpu.topology.Topology at 0x7fad58031fd0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WLPa5rijMvjq"
      },
      "source": [
        "SEQ_LEN = max_len\n",
        "def create_model():\n",
        "  model = TFBertModel.from_pretrained(\"bert-base-multilingual-cased\", from_pt=True, num_labels=len(label_dict), output_attentions = False,\n",
        "    output_hidden_states = False)\n",
        "\n",
        "  token_inputs = tf.keras.layers.Input((SEQ_LEN,), dtype=tf.int32, name='input_word_ids') # 토큰 인풋\n",
        "  mask_inputs = tf.keras.layers.Input((SEQ_LEN,), dtype=tf.int32, name='input_masks') # 마스크 인풋\n",
        "\n",
        "  bert_outputs = model([token_inputs, mask_inputs])\n",
        "  bert_outputs = bert_outputs[0] # shape : (Batch_size, max_len, 30(개체의 총 개수))\n",
        "  nr = tf.keras.layers.Dense(30, activation='softmax')(bert_outputs) # shape : (Batch_size, max_len, 30)\n",
        "\n",
        "  nr_model = tf.keras.Model([token_inputs, mask_inputs], nr)\n",
        "\n",
        "  nr_model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.00002), loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False),\n",
        "      metrics=['sparse_categorical_accuracy'])\n",
        "  nr_model.summary()\n",
        "  return nr_model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9PKDPCca4i8N"
      },
      "source": [
        "## 훈련 및 성능 검증"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CjKJmS5LNAsd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "896c2b0e-a429-4133-a956-3cb8049762c0"
      },
      "source": [
        "strategy = tf.distribute.experimental.TPUStrategy(resolver)\n",
        "# TPU를 활용하기 위해 context로 묶어주기\n",
        "with strategy.scope():\n",
        "  nr_model = create_model()\n",
        "  nr_model.fit([tr_inputs, tr_masks], tr_tags, validation_data=([val_inputs, val_masks], val_tags), epochs=3, shuffle=False, batch_size=bs)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:absl:`tf.distribute.experimental.TPUStrategy` is deprecated, please use  the non experimental symbol `tf.distribute.TPUStrategy` instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n",
            "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFBertModel: ['cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight']\n",
            "- This IS expected if you are initializing TFBertModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFBertModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the weights of TFBertModel were initialized from the PyTorch model.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_2\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_word_ids (InputLayer)     [(None, 88)]         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_masks (InputLayer)        [(None, 88)]         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "tf_bert_model_2 (TFBertModel)   TFBaseModelOutputWit 177853440   input_word_ids[0][0]             \n",
            "                                                                 input_masks[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 88, 30)       23070       tf_bert_model_2[0][0]            \n",
            "==================================================================================================\n",
            "Total params: 177,876,510\n",
            "Trainable params: 177,876,510\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "Epoch 1/3\n",
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_2/bert/pooler/dense/kernel:0', 'tf_bert_model_2/bert/pooler/dense/bias:0'] when minimizing the loss.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_2/bert/pooler/dense/kernel:0', 'tf_bert_model_2/bert/pooler/dense/bias:0'] when minimizing the loss.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_2/bert/pooler/dense/kernel:0', 'tf_bert_model_2/bert/pooler/dense/bias:0'] when minimizing the loss.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_2/bert/pooler/dense/kernel:0', 'tf_bert_model_2/bert/pooler/dense/bias:0'] when minimizing the loss.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "1843/1843 [==============================] - ETA: 0s - loss: 0.3336 - sparse_categorical_accuracy: 0.9091WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "1843/1843 [==============================] - 285s 114ms/step - loss: 0.3336 - sparse_categorical_accuracy: 0.9091 - val_loss: 0.1556 - val_sparse_categorical_accuracy: 0.9525\n",
            "Epoch 2/3\n",
            "1843/1843 [==============================] - 161s 87ms/step - loss: 0.1492 - sparse_categorical_accuracy: 0.9548 - val_loss: 0.1403 - val_sparse_categorical_accuracy: 0.9572\n",
            "Epoch 3/3\n",
            "1843/1843 [==============================] - 161s 87ms/step - loss: 0.1185 - sparse_categorical_accuracy: 0.9634 - val_loss: 0.1372 - val_sparse_categorical_accuracy: 0.9589\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ybu66VKQ4b5H"
      },
      "source": [
        "# 만약 TPU를 사용하지 않고 GPU를 사용한다면\n",
        "#nr_model = create_model()\n",
        "#nr_model.fit([tr_inputs, tr_masks], tr_tags, validation_data=([val_inputs, val_masks], val_tags), epochs=3, shuffle=False, batch_size=bs)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lmsc9SvzUDYp"
      },
      "source": [
        "from sklearn.metrics import precision_score, recall_score, f1_score, classification_report"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x90MqYVZo6By",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "88cd845a-e921-435b-8291-5bd0911fde8a"
      },
      "source": [
        "y_predicted = nr_model.predict([val_inputs, val_masks])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kjRRoCc-XHDV"
      },
      "source": [
        "f_label = [i for i, j in label_dict.items()]\n",
        "val_tags_l = [index_to_ner[x] for x in np.ravel(val_tags).astype(int).tolist()]\n",
        "y_predicted_l = [index_to_ner[x] for x in np.ravel(np.argmax(y_predicted, axis=2)).astype(int).tolist()]\n",
        "f_label.remove(\"[PAD]\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_GnAt491m2hk"
      },
      "source": [
        "각 개체별 f1 score를 측정하도록 하겠습니다.  \n",
        "참고로 micro avg는 전체 정답을 기준으로 f1 score을 측정한 것이며,  \n",
        "macro avg는 각 개체별 f1 score를 가중평균 한 것입니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mMRAtG9HZCRS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7be46f94-a388-40e9-9ddc-935c629ee92f"
      },
      "source": [
        "print(classification_report(val_tags_l, y_predicted_l, labels=f_label))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "       PER_B       0.82      0.86      0.84     11508\n",
            "       DAT_B       0.90      0.84      0.87      4354\n",
            "           -       0.95      0.95      0.95    136806\n",
            "       ORG_B       0.87      0.80      0.83     13511\n",
            "       CVL_B       0.75      0.81      0.78     15837\n",
            "       NUM_B       0.95      0.91      0.93     11161\n",
            "       LOC_B       0.85      0.76      0.80      5912\n",
            "       EVT_B       0.81      0.79      0.80      3887\n",
            "       TRM_B       0.82      0.70      0.76      6052\n",
            "       TRM_I       0.38      0.43      0.40       670\n",
            "       EVT_I       0.73      0.83      0.77      1833\n",
            "       PER_I       0.73      0.69      0.71      1578\n",
            "       CVL_I       0.47      0.40      0.43       869\n",
            "       NUM_I       0.69      0.76      0.72      1538\n",
            "       TIM_B       0.80      0.91      0.85       630\n",
            "       TIM_I       0.90      0.84      0.87       219\n",
            "       ORG_I       0.55      0.73      0.63      1481\n",
            "       DAT_I       0.80      0.90      0.85       937\n",
            "       ANM_B       0.77      0.62      0.69      1385\n",
            "       MAT_B       0.47      0.38      0.42        58\n",
            "       MAT_I       0.00      0.00      0.00         0\n",
            "       AFW_B       0.61      0.61      0.61      1303\n",
            "       FLD_B       0.51      0.57      0.54       455\n",
            "       LOC_I       0.00      0.00      0.00        35\n",
            "       AFW_I       0.52      0.53      0.53       464\n",
            "       PLT_B       0.35      0.09      0.15        65\n",
            "       FLD_I       0.00      0.00      0.00         4\n",
            "       ANM_I       0.00      0.00      0.00         7\n",
            "       PLT_I       0.00      0.00      0.00         0\n",
            "\n",
            "   micro avg       0.89      0.89      0.89    222559\n",
            "   macro avg       0.59      0.58      0.58    222559\n",
            "weighted avg       0.89      0.89      0.89    222559\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "emi8JP4I4lxX"
      },
      "source": [
        "# 실제 데이터로 실습하기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R6Wet-aDstpw"
      },
      "source": [
        "def ner_inference(test_sentence):\n",
        "\n",
        "\n",
        "  tokenized_sentence = np.array([tokenizer.encode(test_sentence, max_length=max_len, truncation=True, padding='max_length')])\n",
        "  tokenized_mask = np.array([[int(x!=1) for x in tokenized_sentence[0].tolist()]])\n",
        "  ans = nr_model.predict([tokenized_sentence, tokenized_mask])\n",
        "  ans = np.argmax(ans, axis=2)\n",
        "\n",
        "  tokens = tokenizer.convert_ids_to_tokens(tokenized_sentence[0])\n",
        "  new_tokens, new_labels = [], []\n",
        "  for token, label_idx in zip(tokens, ans[0]):\n",
        "    if (token.startswith(\"##\")):\n",
        "      new_labels.append(index_to_ner[label_idx])\n",
        "      new_tokens.append(token[2:])\n",
        "    elif (token=='[CLS]'):\n",
        "      pass\n",
        "    elif (token=='[SEP]'):\n",
        "      pass\n",
        "    elif (token=='[PAD]'):\n",
        "      pass\n",
        "    elif (token != '[CLS]' or token != '[SEP]'):\n",
        "      new_tokens.append(token)\n",
        "      new_labels.append(index_to_ner[label_idx])\n",
        "\n",
        "  for token, label in zip(new_tokens, new_labels):\n",
        "      print(\"{}\\t{}\".format(label, token))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mvYrdoG8t8UV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7e64b739-52c4-45c2-9bed-8e13437bf666"
      },
      "source": [
        "ner_inference(\"문재인 대통령은 1953년 1월 24일 경상남도 거제시에서 아버지 문용형과 어머니 강한옥 사이에서 둘째(장남)로 태어났다.\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "PER_B\t문\n",
            "PER_B\t재\n",
            "PER_B\t인\n",
            "CVL_B\t대통령\n",
            "CVL_B\t은\n",
            "DAT_B\t1953\n",
            "DAT_B\t년\n",
            "DAT_I\t1월\n",
            "DAT_I\t24일\n",
            "LOC_B\t경\n",
            "LOC_B\t상\n",
            "LOC_B\t남도\n",
            "LOC_B\t거\n",
            "LOC_B\t제\n",
            "LOC_B\t시\n",
            "LOC_B\t에서\n",
            "CVL_B\t아버지\n",
            "PER_B\t문\n",
            "PER_B\t용\n",
            "PER_B\t형\n",
            "PER_B\t과\n",
            "CVL_B\t어머니\n",
            "PER_B\t강\n",
            "PER_B\t한\n",
            "PER_B\t옥\n",
            "-\t사이에\n",
            "-\t서\n",
            "NUM_B\t둘\n",
            "NUM_B\t째\n",
            "NUM_I\t(\n",
            "CVL_B\t장\n",
            "CVL_B\t남\n",
            "CVL_B\t)\n",
            "CVL_B\t로\n",
            "-\t태어났다\n",
            "-\t.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LP1sNHYuvXKS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5a7c5100-b68d-4d5a-a47b-93051c977a3a"
      },
      "source": [
        "ner_inference(\"9세이브로 구완 30위인 LG 박찬형은 평균자책점이 16.45로 준수한 편이지만 22이닝 동안 피홈런이 31개나 된다.\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "NUM_B\t9\n",
            "NUM_B\t세\n",
            "NUM_B\t이브\n",
            "NUM_B\t로\n",
            "-\t구\n",
            "-\t완\n",
            "NUM_B\t30\n",
            "NUM_B\t위\n",
            "NUM_B\t인\n",
            "ORG_B\tLG\n",
            "PER_B\t박\n",
            "PER_B\t찬\n",
            "PER_B\t형\n",
            "PER_B\t은\n",
            "-\t평\n",
            "-\t균\n",
            "-\t자\n",
            "-\t책\n",
            "-\t점\n",
            "-\t이\n",
            "NUM_B\t16\n",
            "NUM_B\t.\n",
            "NUM_B\t45\n",
            "NUM_B\t로\n",
            "-\t준\n",
            "-\t수\n",
            "-\t한\n",
            "-\t편\n",
            "-\t이지\n",
            "-\t만\n",
            "NUM_B\t22\n",
            "NUM_B\t이\n",
            "NUM_B\t닝\n",
            "-\t동안\n",
            "TRM_B\t피\n",
            "-\t홈\n",
            "NUM_B\t런\n",
            "TRM_B\t이\n",
            "NUM_B\t31\n",
            "NUM_B\t개\n",
            "NUM_B\t나\n",
            "-\t된다\n",
            "-\t.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8xAN7BaD1R3O",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5601c3be-cb6b-44fd-d4b0-d2db36572fe3"
      },
      "source": [
        "ner_inference(\"인공지능의 역사는 20세기 초반에서 더 거슬러 올라가보면 이미 17~18세기부터 태동하고 있었지만 이때는 인공지능 그 자체보다는 뇌와 마음의 관계에 관한 철학적인 논쟁 수준에 머무르고 있었다. 그럴 수 밖에 없는 것이 당시에는 인간의 뇌 말고는 정보처리기계가 존재하지 않았기 때문이다. \")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-\t인\n",
            "TRM_B\t공\n",
            "-\t지\n",
            "-\t능\n",
            "-\t의\n",
            "-\t역\n",
            "-\t사는\n",
            "-\t20\n",
            "NUM_B\t세기\n",
            "-\t초\n",
            "-\t반\n",
            "-\t에서\n",
            "-\t더\n",
            "-\t거\n",
            "-\t슬\n",
            "-\t러\n",
            "-\t올\n",
            "-\t라\n",
            "-\t가\n",
            "-\t보\n",
            "-\t면\n",
            "-\t이미\n",
            "NUM_B\t17\n",
            "NUM_B\t~\n",
            "NUM_B\t18\n",
            "NUM_B\t세기\n",
            "NUM_B\t부터\n",
            "-\t태\n",
            "-\t동\n",
            "-\t하고\n",
            "-\t있\n",
            "-\t었지만\n",
            "-\t이때\n",
            "-\t는\n",
            "-\t인\n",
            "-\t공\n",
            "-\t지\n",
            "-\t능\n",
            "-\t그\n",
            "-\t자\n",
            "-\t체\n",
            "-\t보다\n",
            "-\t는\n",
            "-\t뇌\n",
            "-\t와\n",
            "-\t마\n",
            "-\t음\n",
            "-\t의\n",
            "-\t관\n",
            "-\t계에\n",
            "-\t관한\n",
            "-\t철\n",
            "-\t학적\n",
            "-\t인\n",
            "-\t논\n",
            "-\t쟁\n",
            "-\t수\n",
            "-\t준\n",
            "-\t에\n",
            "-\t머\n",
            "-\t무\n",
            "-\t르고\n",
            "-\t있었다\n",
            "-\t.\n",
            "-\t그\n",
            "-\t럴\n",
            "-\t수\n",
            "-\t밖\n",
            "-\t에\n",
            "-\t없는\n",
            "-\t것이\n",
            "-\t당시\n",
            "-\t에는\n",
            "-\t인\n",
            "-\t간의\n",
            "-\t뇌\n",
            "-\t말\n",
            "-\t고\n",
            "-\t는\n",
            "-\t정\n",
            "-\t보\n",
            "-\t처\n",
            "-\t리\n",
            "-\t기\n",
            "-\t계\n",
            "-\t가\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}