notebooks/08_warm_starting_encoder_decoder.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Leveraging Pre-trained Checkpoints for Encoder-Decoder Models.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "Gw3IZYrfKl4Z",
        "4M2uzGLV9a_O",
        "9pftbMBeJACc",
        "27iHqRfq6rne",
        "ZwQIEhKOrJpl"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "9a4bcf9eaa6f4f60a3c3a9cd0fd88194": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a1be871fd0774aa39230b11e35ebb0a6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5d186df7b2e04aac8624674e90ebec34",
              "IPY_MODEL_43dac6ef55c144d9ba55467df614c229"
            ]
          }
        },
        "a1be871fd0774aa39230b11e35ebb0a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5d186df7b2e04aac8624674e90ebec34": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d84b7e47a3c24ff9b5683840c819cb0f",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 3711,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3711,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8fefa370353c4b3c8d1f94eb4576e582"
          }
        },
        "43dac6ef55c144d9ba55467df614c229": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_82b204b26f164f86812dab500787c875",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 9.38k/? [00:02&lt;00:00, 3.65kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_237322dff7d345899c1db1484a2c044a"
          }
        },
        "d84b7e47a3c24ff9b5683840c819cb0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8fefa370353c4b3c8d1f94eb4576e582": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "82b204b26f164f86812dab500787c875": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "237322dff7d345899c1db1484a2c044a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e05188a37e784a7d8041afd7136c6054": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cef4d9d182074dafaf5ed6f60590e6f8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_45599cc93c4747868dd71f97b457dbbe",
              "IPY_MODEL_540abae6193b4f99825065ce2278cde4"
            ]
          }
        },
        "cef4d9d182074dafaf5ed6f60590e6f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "45599cc93c4747868dd71f97b457dbbe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_dc069cab9cf341b1b40680e526afd13c",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1690,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1690,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1e7cfcf4470d4d14a3a573b488792cf4"
          }
        },
        "540abae6193b4f99825065ce2278cde4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6144b64f4a444d56ab4dd31a449d1cbc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 9.50k/? [00:00&lt;00:00, 112kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f3e30f37b4da4492becdffebe4bfcef1"
          }
        },
        "dc069cab9cf341b1b40680e526afd13c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1e7cfcf4470d4d14a3a573b488792cf4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6144b64f4a444d56ab4dd31a449d1cbc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f3e30f37b4da4492becdffebe4bfcef1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7859a2b3e52345c186ae5b6aa4ba0c36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cb9b100de0e44e53a808590d6d34826e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c9ff75060f20493e8bd15bb15e6dbb4f",
              "IPY_MODEL_d69756f059dd4f9a94c25661a4824249"
            ]
          }
        },
        "cb9b100de0e44e53a808590d6d34826e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c9ff75060f20493e8bd15bb15e6dbb4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_24eeae5c3e8d4df0bb50d7fd9b211ea9",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f2ca62ce391a40debfbb77f1e0dd0c36"
          }
        },
        "d69756f059dd4f9a94c25661a4824249": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_47e3012f5c0345d79f0f19444c6d8803",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 159M/? [00:34&lt;00:00, 4.59MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0ff2285796f84eb9b12fd30c1d5d875a"
          }
        },
        "24eeae5c3e8d4df0bb50d7fd9b211ea9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f2ca62ce391a40debfbb77f1e0dd0c36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "47e3012f5c0345d79f0f19444c6d8803": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0ff2285796f84eb9b12fd30c1d5d875a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6990216adbba46ffac1fdec6b546d407": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8c7cdecd21d54435b883e7a088ef6f5a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_60673f89a61644afb85f4b9862348432",
              "IPY_MODEL_23c9e0da40774a0cbba440c3a9eb3567"
            ]
          }
        },
        "8c7cdecd21d54435b883e7a088ef6f5a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "60673f89a61644afb85f4b9862348432": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f47f42c2da1c4604ad91def09f9eec73",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1d595dca7d3e4269b66493196f955c61"
          }
        },
        "23c9e0da40774a0cbba440c3a9eb3567": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6661ea196016443ca00c49754e110fdc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 376M/? [00:14&lt;00:00, 25.8MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8e9a979f544c4ca985ccd337ccb074b3"
          }
        },
        "f47f42c2da1c4604ad91def09f9eec73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1d595dca7d3e4269b66493196f955c61": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6661ea196016443ca00c49754e110fdc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8e9a979f544c4ca985ccd337ccb074b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "56c9f2696c744b26879300c152685456": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d2cb5034aee6400da27febf14d10b948",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7174b9ae437d4ee4a9e2e849c977ff9a",
              "IPY_MODEL_1bc7d7ed8a1b495b9b14f70c8ee3bfba"
            ]
          }
        },
        "d2cb5034aee6400da27febf14d10b948": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7174b9ae437d4ee4a9e2e849c977ff9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_631c7c56beb94443a8256be8e155abcc",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 626592,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 626592,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a7372225873a4671b007cf53e43a8c1b"
          }
        },
        "1bc7d7ed8a1b495b9b14f70c8ee3bfba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_28b7153fc1f844d69706b9a27f47c06f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2.11M/? [00:03&lt;00:00, 573kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0451f5a139d743c79904ea69262cff2b"
          }
        },
        "631c7c56beb94443a8256be8e155abcc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a7372225873a4671b007cf53e43a8c1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "28b7153fc1f844d69706b9a27f47c06f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0451f5a139d743c79904ea69262cff2b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5e642e1989c14140987ace2780d01c85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c598b1d4de1b46159e0ab6d41f593573",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b3788d1393bc46c4b667d3eac298e2c3",
              "IPY_MODEL_f92c1350fdcd4694a5d7d8a4f97522f1"
            ]
          }
        },
        "c598b1d4de1b46159e0ab6d41f593573": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b3788d1393bc46c4b667d3eac298e2c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_05c67e2a0f554b03a260ada6e8d0bf73",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 13463354,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 13463354,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1a7492549fbc4335925ed4e4c890a10f"
          }
        },
        "f92c1350fdcd4694a5d7d8a4f97522f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4ede6881ab1d46598e895b9e35036163",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 46.4M/? [00:01&lt;00:00, 35.3MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6f15d47ce5a84e74a2e38e5b1adc63d7"
          }
        },
        "05c67e2a0f554b03a260ada6e8d0bf73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1a7492549fbc4335925ed4e4c890a10f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4ede6881ab1d46598e895b9e35036163": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6f15d47ce5a84e74a2e38e5b1adc63d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d63220e55e03451f8ef8d87fb1a3c475": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_07c8d79de4524a128c90662a6769bbd4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_34e25bcf15d8406abfacaaf271d7a2dd",
              "IPY_MODEL_27997ec8b6e94ca2825179ea7f39dce9"
            ]
          }
        },
        "07c8d79de4524a128c90662a6769bbd4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "34e25bcf15d8406abfacaaf271d7a2dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_017cb45b81b145a2a5615c65337d0881",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 725406,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 725406,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_285519bebc544384bc373839ff71ef56"
          }
        },
        "27997ec8b6e94ca2825179ea7f39dce9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_40e5138495d543428ce291927b118be7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2.43M/? [00:03&lt;00:00, 763kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_44e96cb737b94f79b9fe7ec01e9fe20a"
          }
        },
        "017cb45b81b145a2a5615c65337d0881": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "285519bebc544384bc373839ff71ef56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "40e5138495d543428ce291927b118be7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "44e96cb737b94f79b9fe7ec01e9fe20a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4005f402528243aa999facc2bb3d335d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d4f55df76c404287b0277e898203e7a8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5e4c739ff55945e48124c0379f4d9d95",
              "IPY_MODEL_32a3578a1ed54aed9f0a5c58dd18e58c"
            ]
          }
        },
        "d4f55df76c404287b0277e898203e7a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5e4c739ff55945e48124c0379f4d9d95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9b7c6e3be93a4dc0bb26119a585f17fc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6be70c64f32b4c09b188a46be8557207"
          }
        },
        "32a3578a1ed54aed9f0a5c58dd18e58c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3010957ac5b94132b4fbbad52ed835a4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 287113/0 [00:58&lt;00:00, 4979.40 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1adbedddf59b446bac4886968aabb1a0"
          }
        },
        "9b7c6e3be93a4dc0bb26119a585f17fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6be70c64f32b4c09b188a46be8557207": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3010957ac5b94132b4fbbad52ed835a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1adbedddf59b446bac4886968aabb1a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9d25c4a8783d4b27a49371d85cfad10f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5be7b3e47c0640538826beaf4a25a580",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_593988c055184050a3a35b3ca5add45e",
              "IPY_MODEL_471125eb87fd41d890a97c9db3241b4c"
            ]
          }
        },
        "5be7b3e47c0640538826beaf4a25a580": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "593988c055184050a3a35b3ca5add45e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_670469b341234e9bb11eb602e7710d36",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ea72a63d8e4e4e3dbf8637db14c6f19e"
          }
        },
        "471125eb87fd41d890a97c9db3241b4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8282529cfee24d909611b124faeaece4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 13368/0 [00:02&lt;00:00, 4965.57 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_eab9187f72ea4ea5837b91030fe11bb0"
          }
        },
        "670469b341234e9bb11eb602e7710d36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ea72a63d8e4e4e3dbf8637db14c6f19e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8282529cfee24d909611b124faeaece4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "eab9187f72ea4ea5837b91030fe11bb0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e612249e01bf4cd2b432685b749870dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c26e2f8f192a4abdb1d01630c5e69108",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f434739f47bc4314b0f4bda009ebf933",
              "IPY_MODEL_ca2d9092ae564a96b04a1d374dd2a92c"
            ]
          }
        },
        "c26e2f8f192a4abdb1d01630c5e69108": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f434739f47bc4314b0f4bda009ebf933": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1c7d6b86b5c3411b82d043042469347f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0dd7c6df3edb489e97bc5a4b6c5c4472"
          }
        },
        "ca2d9092ae564a96b04a1d374dd2a92c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c9ba3af29928460fa69c3c118dbee14d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 11490/0 [00:02&lt;00:00, 3873.27 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cee91c0dedfa43cab6f16e6876bc06a2"
          }
        },
        "1c7d6b86b5c3411b82d043042469347f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0dd7c6df3edb489e97bc5a4b6c5c4472": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c9ba3af29928460fa69c3c118dbee14d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cee91c0dedfa43cab6f16e6876bc06a2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "13491b7e71824282a09aa3d79b54429b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2c82a3afff3743149bc8b694669caa76",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b5830cea6bb04bcdaf6b37e66e500f79",
              "IPY_MODEL_2e1d4e151c0548049406c9b99a889f95"
            ]
          }
        },
        "2c82a3afff3743149bc8b694669caa76": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b5830cea6bb04bcdaf6b37e66e500f79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_12b63db1af494ccd91f04564198d41d6",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0c9eab6f217b4a4b85dd04d4b5bb9e57"
          }
        },
        "2e1d4e151c0548049406c9b99a889f95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_24ce686bdacc4610bb0d3454081c26ad",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:55&lt;00:00, 4.17kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_678789bbf40742fb978b8bd9f438c22e"
          }
        },
        "12b63db1af494ccd91f04564198d41d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0c9eab6f217b4a4b85dd04d4b5bb9e57": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "24ce686bdacc4610bb0d3454081c26ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "678789bbf40742fb978b8bd9f438c22e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "72655e8e1ccc4319bf37ff482a0e2305": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_271b96e5a35d489c8ff4fe3922ca9cf9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8f13ee0980d64a9f9b897a5c5c7e642e",
              "IPY_MODEL_8f75d80e802947b8a6116c2c7ab20550"
            ]
          }
        },
        "271b96e5a35d489c8ff4fe3922ca9cf9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8f13ee0980d64a9f9b897a5c5c7e642e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0990b53afaf34317a764d719cc361268",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_49b8e76a66d446d8b87a669798e771ea"
          }
        },
        "8f75d80e802947b8a6116c2c7ab20550": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c7b135aae8bc45219e769608ab04965a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:00&lt;00:00, 946kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1d995b48d4094840b385a8dbe3f38745"
          }
        },
        "0990b53afaf34317a764d719cc361268": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "49b8e76a66d446d8b87a669798e771ea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c7b135aae8bc45219e769608ab04965a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1d995b48d4094840b385a8dbe3f38745": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ab623a8d901044789781355e887998b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4f0716d31ce7470ca6f3f30d11023a30",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_13214334499a4be2adabcd0e3a2098ce",
              "IPY_MODEL_559670ccde114edeaee350cb2bf11faf"
            ]
          }
        },
        "4f0716d31ce7470ca6f3f30d11023a30": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "13214334499a4be2adabcd0e3a2098ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b4b4366968b845f7a7fb5e29a3ee6982",
            "_dom_classes": [],
            "description": "#0: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2500,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2500,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7f75959ba2d644b1a7e51d867894a4e0"
          }
        },
        "559670ccde114edeaee350cb2bf11faf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3cadfe17c4354080b3f39c6631c42f16",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2500/2500 [00:36&lt;00:00, 68.26ex/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d3a4cc6d56984d0b9bd540de69524bd7"
          }
        },
        "b4b4366968b845f7a7fb5e29a3ee6982": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7f75959ba2d644b1a7e51d867894a4e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3cadfe17c4354080b3f39c6631c42f16": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d3a4cc6d56984d0b9bd540de69524bd7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9b8fac4e1d164201b2aee090d2808994": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_078263580d3f4bd0b3e82b75d184f66d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3d13e90a757d4442bbba7c15754f6917",
              "IPY_MODEL_aea6dd73e1804da782260d948fcf1242"
            ]
          }
        },
        "078263580d3f4bd0b3e82b75d184f66d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3d13e90a757d4442bbba7c15754f6917": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e52a41b9784e43498621b9a9756906e5",
            "_dom_classes": [],
            "description": "#1: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2500,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2500,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d330fc9742394baf8c0e8106d9f86e4e"
          }
        },
        "aea6dd73e1804da782260d948fcf1242": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_72ec123bd5a340eea68c2a11b5e0cada",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2500/2500 [00:37&lt;00:00, 67.54ex/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c4db0c00f00a4fd3be952c43d8eec786"
          }
        },
        "e52a41b9784e43498621b9a9756906e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d330fc9742394baf8c0e8106d9f86e4e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "72ec123bd5a340eea68c2a11b5e0cada": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c4db0c00f00a4fd3be952c43d8eec786": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3aa4de28d47d4c3fac88271f391351fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5ef6ff2d5a7b48f682f48b41d73850e4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8711623ff5fc4341b727b1d1e9e1005c",
              "IPY_MODEL_ceb56899765a469c96713ff39d600dad"
            ]
          }
        },
        "5ef6ff2d5a7b48f682f48b41d73850e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8711623ff5fc4341b727b1d1e9e1005c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_69aead212e8645a8842bc5bc7a9b35fc",
            "_dom_classes": [],
            "description": "#3: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2500,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2500,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_00abeaaea3734b3f911bf897a0053384"
          }
        },
        "ceb56899765a469c96713ff39d600dad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2c6c618c2b304eb89c5cb796e49da5a7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2500/2500 [00:36&lt;00:00, 67.94ex/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5598c4414a2f4f76818cc1abbbd17d77"
          }
        },
        "69aead212e8645a8842bc5bc7a9b35fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "00abeaaea3734b3f911bf897a0053384": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2c6c618c2b304eb89c5cb796e49da5a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5598c4414a2f4f76818cc1abbbd17d77": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "779b46b3ddb645efafce1abeae8b4f1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f8d765bd954342f3bcfff58551bc6095",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d15bbef314fa4154a0fae86e3ee3205a",
              "IPY_MODEL_86bfcd4e109f41e39d814c362bad93e6"
            ]
          }
        },
        "f8d765bd954342f3bcfff58551bc6095": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d15bbef314fa4154a0fae86e3ee3205a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_aa6c2bfcd4104839aea3de1db223b5f5",
            "_dom_classes": [],
            "description": "#2: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2500,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2500,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_739f97bbb6ec42fbb5e2a9df0e960d8f"
          }
        },
        "86bfcd4e109f41e39d814c362bad93e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_562880f2d6504d29b9d48c815918c7c0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2500/2500 [00:37&lt;00:00, 67.43ex/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_93102375b016495c937bbd50ba5eac8e"
          }
        },
        "aa6c2bfcd4104839aea3de1db223b5f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "739f97bbb6ec42fbb5e2a9df0e960d8f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "562880f2d6504d29b9d48c815918c7c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "93102375b016495c937bbd50ba5eac8e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7bbc19eddb6740e2a2caddb6db63c501": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d638f3862f8c48f8ae537b5494185c9a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8dc7da1ba2d2491ab6e2b77db0eae508",
              "IPY_MODEL_36b28c1b21cf4a2091a8fe2c588f4005"
            ]
          }
        },
        "d638f3862f8c48f8ae537b5494185c9a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8dc7da1ba2d2491ab6e2b77db0eae508": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_02a6c48208c04ad185ea1b80aa5574d1",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d2ef3fbc86b54bb88b12a073c0d7a141"
          }
        },
        "36b28c1b21cf4a2091a8fe2c588f4005": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0f95672d388f4f90a2d95dc84674b489",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:00&lt;00:00,  6.07ba/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4079b4184ebc4140b3d4a8cd07e74551"
          }
        },
        "02a6c48208c04ad185ea1b80aa5574d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d2ef3fbc86b54bb88b12a073c0d7a141": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0f95672d388f4f90a2d95dc84674b489": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4079b4184ebc4140b3d4a8cd07e74551": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "67b48f2dabe0455293ff6ff51dfee2ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9ab45cb1bc4b42ed9682e6ef4c8eac13",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_bbbbe2f83efe42b7b65c404cfb559aa1",
              "IPY_MODEL_2278f2e4c00546ab8a9616745c54f188"
            ]
          }
        },
        "9ab45cb1bc4b42ed9682e6ef4c8eac13": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bbbbe2f83efe42b7b65c404cfb559aa1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_fb5a37123a2a435a83196fda0449bfa7",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 8,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 8,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_604569ab53d641cf8d3a88651c89c791"
          }
        },
        "2278f2e4c00546ab8a9616745c54f188": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c6d3e9577f3145dab6b29fd6d882455d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 8/8 [00:00&lt;00:00, 10.81ba/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0b41e8a1e3614f54b653f471cc309ad8"
          }
        },
        "fb5a37123a2a435a83196fda0449bfa7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "604569ab53d641cf8d3a88651c89c791": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c6d3e9577f3145dab6b29fd6d882455d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0b41e8a1e3614f54b653f471cc309ad8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b0b66fbc1c044bb79bce9642df312005": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6b86ca19dbcb490c84b1e18918ff2655",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_070d62e3fbab456ea931f688339ef2e9",
              "IPY_MODEL_b06505ffcfc8456993310dddc90a3ff3"
            ]
          }
        },
        "6b86ca19dbcb490c84b1e18918ff2655": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "070d62e3fbab456ea931f688339ef2e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_01b075a2aafa4e2aaabe8096fe9848cf",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9323291a5c954f9992fe55789a35eebf"
          }
        },
        "b06505ffcfc8456993310dddc90a3ff3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4e1d64901cf14b688c075abb4a15537b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2/2 [00:15&lt;00:00,  7.82s/ba]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_76e86da8909540dcbaf6bd83a4c6c3ac"
          }
        },
        "01b075a2aafa4e2aaabe8096fe9848cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9323291a5c954f9992fe55789a35eebf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4e1d64901cf14b688c075abb4a15537b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "76e86da8909540dcbaf6bd83a4c6c3ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c459ef7385394e8b9727e52af8ba14e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d0f4cd37d0454eb3b532940c8bd29e6b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fceacd2050714f37898dccc96e3615b6",
              "IPY_MODEL_502370e87e3c4c048768d15633e3edfc"
            ]
          }
        },
        "d0f4cd37d0454eb3b532940c8bd29e6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fceacd2050714f37898dccc96e3615b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4727afce4d0b405e8b2ddc18709a2a25",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3031118e0790487db4d9cdee7c6a60de"
          }
        },
        "502370e87e3c4c048768d15633e3edfc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_49a36c8bebc94064b19631318bf01cf0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:00&lt;00:00, 2.17kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d6ed9aaf2ac94e2da3fb94e807732ea8"
          }
        },
        "4727afce4d0b405e8b2ddc18709a2a25": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3031118e0790487db4d9cdee7c6a60de": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "49a36c8bebc94064b19631318bf01cf0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d6ed9aaf2ac94e2da3fb94e807732ea8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fe2ab4e1e8984425930cdae883be5951": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_601473895b044099bdb5e189bdbcf165",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_872fca6183524208acf8959926d2b3c6",
              "IPY_MODEL_9428736d1d4c46a498c7b79eb816c06a"
            ]
          }
        },
        "601473895b044099bdb5e189bdbcf165": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "872fca6183524208acf8959926d2b3c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_29f3be857e7e4987bc546f1424204841",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_315b2b27d17648bcaa68f0ec0070ad86"
          }
        },
        "9428736d1d4c46a498c7b79eb816c06a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_817a5c1e27864737b7407c38c461f3f8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:14&lt;00:00, 30.2MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5e85f88a9eca4b8f9314ea5ae3d35944"
          }
        },
        "29f3be857e7e4987bc546f1424204841": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "315b2b27d17648bcaa68f0ec0070ad86": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "817a5c1e27864737b7407c38c461f3f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5e85f88a9eca4b8f9314ea5ae3d35944": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "155dc2a8d8ae49bf9ef09d9638b321b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3819c7abd556463e824b384cdd2c08c4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3ef8d61808d848668991493b47924850",
              "IPY_MODEL_3c0a1fd8624a4013aa2c9c72f81a2201"
            ]
          }
        },
        "3819c7abd556463e824b384cdd2c08c4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3ef8d61808d848668991493b47924850": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c3997491d4cb42a9a923ff11293e3cc5",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_eac59ca126ed4d86b9f2188f33651c60"
          }
        },
        "3c0a1fd8624a4013aa2c9c72f81a2201": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_10ead3d44543440790b441baf21a152b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:00&lt;00:00, 2.75kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_349fc462a45c410eadfc04ff1348d86b"
          }
        },
        "c3997491d4cb42a9a923ff11293e3cc5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "eac59ca126ed4d86b9f2188f33651c60": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "10ead3d44543440790b441baf21a152b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "349fc462a45c410eadfc04ff1348d86b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "af3d97966c114cc9a805726d43a44193": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_db5c7f4020044d16b996332e4c4801cf",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_81418013f14c4b969e7e8746d69e5d1a",
              "IPY_MODEL_d206f8cf11524ff99817912536824743"
            ]
          }
        },
        "db5c7f4020044d16b996332e4c4801cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "81418013f14c4b969e7e8746d69e5d1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b70855e9ecd34ae18cb7f38351fc03b7",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 435779157,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 435779157,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8c1c013cc8ce463dbf5eaa6ca377136b"
          }
        },
        "d206f8cf11524ff99817912536824743": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_004282ce3a9449c39e06c32f69ddc28e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 436M/436M [00:10&lt;00:00, 42.4MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_652a52c856a948909158cd2d653405a6"
          }
        },
        "b70855e9ecd34ae18cb7f38351fc03b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8c1c013cc8ce463dbf5eaa6ca377136b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "004282ce3a9449c39e06c32f69ddc28e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "652a52c856a948909158cd2d653405a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2fd461deea6146ca9bd5938207a77420": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f35dfd2f3ea346e9b6e86d59d29129de",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e46962da15fb411e91f9db4ff329156e",
              "IPY_MODEL_32c27fe6fcc049eba015c51fda3ced13"
            ]
          }
        },
        "f35dfd2f3ea346e9b6e86d59d29129de": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e46962da15fb411e91f9db4ff329156e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_865b3b7f5bb7412696ff6441f2b7fd60",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1717,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1717,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_391dd711451a49bcb4234f48add40a12"
          }
        },
        "32c27fe6fcc049eba015c51fda3ced13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dd128062d2594bf9b0ed771a28336699",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 4.20k/? [00:00&lt;00:00, 15.4kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3ad51a6456ab4ab082e016fca466bacb"
          }
        },
        "865b3b7f5bb7412696ff6441f2b7fd60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "391dd711451a49bcb4234f48add40a12": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dd128062d2594bf9b0ed771a28336699": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3ad51a6456ab4ab082e016fca466bacb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "001c9215afbb4a178835b138385289cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_22adeb51790b481d8c62f4c4ca2375fc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8f611825f13b44bd889ccd1ee3950ebb",
              "IPY_MODEL_74281bb4a2be473392d729e4c1a3f251"
            ]
          }
        },
        "22adeb51790b481d8c62f4c4ca2375fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8f611825f13b44bd889ccd1ee3950ebb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_068321dbd7254d0a8c1767d879ec6943",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 3660,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3660,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d62484271f7a4f3080e6db2ebaad9480"
          }
        },
        "74281bb4a2be473392d729e4c1a3f251": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f5f0f0dd629c41aaa41d0df604c583db",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3.66k/3.66k [00:00&lt;00:00, 5.31kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_61e1f2615273479cbab9845024c66ad4"
          }
        },
        "068321dbd7254d0a8c1767d879ec6943": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d62484271f7a4f3080e6db2ebaad9480": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f5f0f0dd629c41aaa41d0df604c583db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "61e1f2615273479cbab9845024c66ad4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1d46e07204cf4a51a1895633073a9563": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ca4a525521d6485580c382967037e99f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fd02a19331f54227a412f526a8df80ac",
              "IPY_MODEL_377e2ee25f824a52ad5a1882071cb463"
            ]
          }
        },
        "ca4a525521d6485580c382967037e99f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fd02a19331f54227a412f526a8df80ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5d0ce87488cc4de8b6514af094dee6a9",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 989691346,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 989691346,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a40f126721864d448a74cb0efd705054"
          }
        },
        "377e2ee25f824a52ad5a1882071cb463": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e6f03cc01f8d4ebcaa521610eb03bfe2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 990M/990M [00:29&lt;00:00, 33.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_69607c45c1fa4d3abae6f710785ac16e"
          }
        },
        "5d0ce87488cc4de8b6514af094dee6a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a40f126721864d448a74cb0efd705054": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e6f03cc01f8d4ebcaa521610eb03bfe2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "69607c45c1fa4d3abae6f710785ac16e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a87d5a96d832455190ae67e63f9242b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_046527ff4e4a4d568b4d8136dbf66d48",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c783495fd2b0477db3347961be75bef3",
              "IPY_MODEL_5890190d211a4709a1430e9f21c10877"
            ]
          }
        },
        "046527ff4e4a4d568b4d8136dbf66d48": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c783495fd2b0477db3347961be75bef3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_75b5bd9fa1014c24bcebf1f17027abeb",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4b8b52bec8d641f2b7bf737507685fe9"
          }
        },
        "5890190d211a4709a1430e9f21c10877": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_af0bcea9cf0949688d33721b02121d80",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:02&lt;00:00, 104kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7d27762447494a8397a2cb703528a6af"
          }
        },
        "75b5bd9fa1014c24bcebf1f17027abeb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4b8b52bec8d641f2b7bf737507685fe9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "af0bcea9cf0949688d33721b02121d80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7d27762447494a8397a2cb703528a6af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0143ea0238c242e09023e8956d1b7c63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_af59c31b4af849b1b6325ed4921987d5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_60da1ffd161744148b15f768bb58f2e1",
              "IPY_MODEL_4f59c2dc0af84b888d2d5b65abc4330a"
            ]
          }
        },
        "af59c31b4af849b1b6325ed4921987d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "60da1ffd161744148b15f768bb58f2e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_665d515b9688490c9bc5334e96a6f06d",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 156,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 156,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_22b8d7cb89a9431a93498b6e55208c94"
          }
        },
        "4f59c2dc0af84b888d2d5b65abc4330a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_30bc39b8ca8b4e5ca6bcc47ca122b45a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 156/156 [00:00&lt;00:00, 199B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_03ecd8bfbae54796b19e0c35d82251bd"
          }
        },
        "665d515b9688490c9bc5334e96a6f06d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "22b8d7cb89a9431a93498b6e55208c94": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "30bc39b8ca8b4e5ca6bcc47ca122b45a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "03ecd8bfbae54796b19e0c35d82251bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f90970b373f1445994b977ef37afc9c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0d51f43f6300485ba70ac633798aa722",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a42c79ec2ca54b288893d49125c2f753",
              "IPY_MODEL_0794b1342edc450588c545f996c21f9b"
            ]
          }
        },
        "0d51f43f6300485ba70ac633798aa722": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a42c79ec2ca54b288893d49125c2f753": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1018157054a74c09b12f687e78c5c37f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 252,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 252,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c1c4eff0ed8e4e7b81ef32f416af67a6"
          }
        },
        "0794b1342edc450588c545f996c21f9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2904dc58a7574dc9be22d59209c350b4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 252/252 [00:00&lt;00:00, 588B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_90f528ea5ba24db89d3d04eb110fd3ad"
          }
        },
        "1018157054a74c09b12f687e78c5c37f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c1c4eff0ed8e4e7b81ef32f416af67a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2904dc58a7574dc9be22d59209c350b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "90f528ea5ba24db89d3d04eb110fd3ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "53297fb5dd8e4ec4b3ee6b611559fd2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_12b5cb483c364956b73eb36e9675ac97",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5f11f11b307142558309e59eed8d9d51",
              "IPY_MODEL_82269c7f62394c5f83d889af910a9747"
            ]
          }
        },
        "12b5cb483c364956b73eb36e9675ac97": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5f11f11b307142558309e59eed8d9d51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_afd72567f8eb4da2a7771c1af707c43c",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 15,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 15,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_88dc3c0cfdd2464fa1660653e4810ab8"
          }
        },
        "82269c7f62394c5f83d889af910a9747": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_848419a18c4e49e0afbc0c72abf60e96",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 15/15 [21:59&lt;00:00, 87.95s/ba]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d015425ac18d43d48f51e491b8bd1c29"
          }
        },
        "afd72567f8eb4da2a7771c1af707c43c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "88dc3c0cfdd2464fa1660653e4810ab8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "848419a18c4e49e0afbc0c72abf60e96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d015425ac18d43d48f51e491b8bd1c29": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/huggingface/blog/blob/add_warm_starting_encoder_decoder/Leveraging_Pre_trained_Checkpoints_for_Encoder_Decoder_Models.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mIBy7uK3Od4B"
      },
      "source": [
        "# **Leveraging Pre-trained Language Model Checkpoints for Encoder-Decoder Models**\n",
        "\n",
        "Transformer-based encoder-decoder models were proposed in [Vaswani et al. (2017)](https://arxiv.org/pdf/1706.03762.pdf) and have recently experienced a surge of interest, *e.g.* [Lewis et al. (2019)](https://arxiv.org/abs/1910.13461), [Raffel et al. (2019)](https://arxiv.org/abs/1910.10683), [Zhang et al. (2020)](https://arxiv.org/abs/1912.08777), [Zaheer et al. (2020)](https://arxiv.org/abs/2007.14062), [Yan et al. (2020)](https://arxiv.org/pdf/2001.04063.pdf).\n",
        "\n",
        "Similar to BERT and GPT2, massive pre-trained encoder-decoder models have shown to significantly boost performance on a variety of *sequence-to-sequence* tasks [Lewis et al. (2019)](https://arxiv.org/abs/1910.13461), [Raffel et al. (2019)](https://arxiv.org/abs/1910.10683). However, due to the enormous computational cost attached to pre-training encoder-decoder models, the development of such models is mainly limited to large companies and institutes.\n",
        "\n",
        "In [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks (2020)](https://arxiv.org/pdf/1907.12461.pdf), Sascha Rothe, Shashi Narayan and Aliaksei Severyn initialize encoder-decoder model with pre-trained *encoder and/or decoder-only* checkpoints (*e.g.* BERT, GPT2) to skip the costly pre-training. The authors show that such *warm-started* encoder-decoder models yield competitive results to large pre-trained encoder-decoder models, such as [*T5*](https://arxiv.org/abs/1910.10683), and [*Pegasus*](https://arxiv.org/abs/1912.08777) on multiple *sequence-to-sequence* tasks at a fraction of the training cost.\n",
        "\n",
        "In this notebook, we will explain in detail how encoder-decoder models can be warm-started, give practical tips based on [Rothe et al. (2020)](https://arxiv.org/pdf/1907.12461.pdf), and finally go over a complete code example showing how to warm-start encoder-decoder models with 🤗Transformers.\n",
        "\n",
        "This notebook is divided into 4 parts:\n",
        "\n",
        "- **Introduction** - *Short summary of pre-trained language models in NLP and the need for warm-starting encoder-decoder models.*\n",
        "- **Warm-starting encoder-decoder models (Theory)** - *Illustrative explanation on how encoder-decoder models are warm-started?* \n",
        "- **Warm-starting encoder-decoder models (Analysis)** - *Summary of [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks (2020)](https://arxiv.org/pdf/1907.12461.pdf) - What model combinations are effective to warm-start encoder-decoder models; How does it differ from task to task?*\n",
        "- **Warm-starting encoder-decoder models with 🤗Transformers (Practice)** - *Complete code example showcasing in-detail how to use the `EncoderDecoderModel` framework to warm-start transformer-based encoder-decoder models.*\n",
        "\n",
        "It is highly recommended (probably even necessary) to have read [this blog post](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/Encoder_Decoder_Model.ipynb) about transformer-based encoder-decoder models.\n",
        "\n",
        "Let's start by giving some back-ground on warm-starting encoder-decoder models.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gw3IZYrfKl4Z"
      },
      "source": [
        "## **Introduction**\n",
        "\n",
        "Recently, pre-trained language models${}^1$ have revolutionized the field of natural language processing (NLP). \n",
        "\n",
        "The first pre-trained language models were based on recurrent neural networks (RNN) as proposed [Dai et al. (2015)](https://arxiv.org/pdf/1511.01432.pdf). *Dai et. al* showed that pre-training an RNN-based model on unlabelled data and subsequently fine-tuning${}^2$ it on a specific task yields better results than training a randomly initialized model directly on such a task.\n",
        "However, it was only in 2018, when pre-trained language models become widely accepted in NLP. [ELMO by Peters et al.](https://arxiv.org/abs/1802.05365) and [ULMFit by Howard et al.](https://arxiv.org/pdf/1801.06146.pdf) were the first pre-trained language model to significantly improve the state-of-the-art on an array of natural language understanding (NLU) tasks. Just a couple of months later, OpenAI and Google published *transformer-based* pre-trained language models, called [GPT by Radford et al.](https://s3-us-west-2.amazonaws.com/openai-assets/research-covers/language-unsupervised/language_understanding_paper.pdf) and [BERT by Devlin et al.](https://arxiv.org/abs/1810.04805) respectively.\n",
        "The improved efficiency of *transformer-based* language models over RNNs allowed GPT2 and BERT to be pre-trained on massive amounts of unlabeled text data. Once pre-trained, BERT and GPT were shown to require very little fine-tuning to shatter state-of-art results on more than a dozen NLU tasks ${}^3$.\n",
        "\n",
        "The capability of pre-trained language models to effectively transfer *task-agnostic* knowledge to *task-specific* knowledge turned out to be a great catalyst for NLU. Whereas engineers and researchers previously had to train a language model from scratch, now publicly available checkpoints of large pre-trained language models can be fine-tuned at a fraction of the cost and time.\n",
        "This can save millions in industry and allows for faster prototyping and better benchmarks in research.\n",
        "\n",
        "Pre-trained language models have established a new level of performance on NLU tasks and more and more research has been built upon leveraging such pre-trained language models for improved NLU systems. However, standalone BERT and GPT models have been less successful for *sequence-to-sequence* tasks, *e.g.* *text-summarization*, *machine translation*, *sentence-rephrasing*, etc. \n",
        "\n",
        "Sequence-to-sequence tasks are defined as a mapping from an input sequence $\\mathbf{X}_{1:n}$ to an output sequence $\\mathbf{Y}_{1:m}$ of *a-priori* unknown output length $m$. Hence, a sequence-to-sequence model should define the conditional probability distribution of the output sequence $\\mathbf{Y}_{1:m}$ conditioned on the input sequence $\\mathbf{X}_{1:n}$:\n",
        "\n",
        "$$ p_{\\theta_{\\text{model}}}(\\mathbf{Y}_{1:m} | \\mathbf{X}_{1:n}). $$\n",
        "\n",
        "Without loss of generality, an input word sequence of $n$ words is hereby represented by the vector sequnece $\\mathbf{X}_{1:n} = \\mathbf{x}_1, \\ldots, \\mathbf{x}_n$ and an output sequence of $m$ words as $\\mathbf{Y}_{1:m} = \\mathbf{y}_1, \\ldots, \\mathbf{y}_m$.\n",
        "\n",
        "Let's see how BERT and GPT2 would be fit to model sequence-to-sequence tasks.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p3CAvlBZ58Fp"
      },
      "source": [
        "### **BERT**\n",
        "\n",
        "BERT is an *encoder-only* model, which maps an input sequence $\\mathbf{X}_{1:n}$ to a *contextualized* encoded sequence $\\mathbf{\\overline{X}}_{1:n}$:\n",
        "\n",
        "$$ f_{\\theta_{\\text{BERT}}}: \\mathbf{X}_{1:n} \\to \\mathbf{\\overline{X}}_{1:n}. $$\n",
        "\n",
        " BERT's contextualized encoded sequence $\\mathbf{\\overline{X}}_{1:n}$ can then further be processed by a classification layer for NLU classification tasks, such as *sentiment analysis*, *natural language inference*, etc. To do so, the classification layer, *i.e.* typically a pooling layer followed by a feed-forward layer, is added as a final layer on top of BERT to map the contextualized encoded sequence $\\mathbf{\\overline{X}}_{1:n}$ to a class $c$:\n",
        "\n",
        " $$\n",
        "  f_{\\theta{\\text{p,c}}}: \\mathbf{\\overline{X}}_{1:n} \\to c.\n",
        " $$ \n",
        "\n",
        "It has been shown that adding a pooling- and classification layer, defined as $\\theta_{\\text{p,c}}$, on top of a pre-trained BERT model $\\theta_{\\text{BERT}}$ and subsequently fine-tuning the complete model $\\{\\theta_{\\text{p,c}}, \\theta_{\\text{BERT}}\\}$ can yield state-of-the-art performances on a variety of NLU tasks, *cf.* to [BERT by Devlin et al.](https://arxiv.org/abs/1810.04805).\n",
        "\n",
        "Let's visualize BERT.\n",
        "\n",
        " ![texte du lien](https://raw.githubusercontent.com/patrickvonplaten/scientific_images/master/bert.png)\n",
        "\n",
        " The BERT model is shown in grey. The model stacks multiple *BERT blocks*, each of which is composed of *bi-directional* self-attention layers (shown in the lower part of the red box) and two feed-forward layers (short in the upper part of the red box). \n",
        " \n",
        " Each BERT block makes use of **bi-directional** self-attention to process an input sequence $\\mathbf{x'}_1, \\ldots, \\mathbf{x'}_n$ (shown in light grey) to a more \"refined\" contextualized output sequence $\\mathbf{x''}_1, \\ldots, \\mathbf{x''}_n$ (shown in slightly darker grey) ${}^4$. The contextualized output sequence of the final BERT block, *i.e.* $\\mathbf{\\overline{X}}_{1:n}$, can then be mapped to a single output class $c$ by adding a *task-specific* classification layer (shown in orange) as explained above.\n",
        "\n",
        "*Encoder-only* models can only map an input sequence to an output sequence of *a priori* known output length. In conclusion, the output dimension does not depend on the input sequence, which makes it disadvantageous and impractical to use encoder-only models for sequence-to-sequence tasks.\n",
        "\n",
        "As for all *encoder-only* models, BERT's architecture corresponds exactly to the architecture of the encoder part of *transformer-based* encoder-decoder models as shown in the \"Encoder\" section in the [Encoder-Decoder notebook](https://colab.research.google.com/drive/19wkOLQIjBBXQ-j3WWTEiud6nGBEw4MdF?usp=sharing).\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W15QvjRh6IQr"
      },
      "source": [
        "### **GPT2**\n",
        "\n",
        "GPT2 is a *decoder-only* model, which makes use of *uni-directional* (*i.e.* \"causal\") self-attention to define a mapping from an input sequence $\\mathbf{Y}_{0: m - 1}$ ${}^1$ to a \"next-word\" logit vector sequence $\\mathbf{L}_{1:m}$:\n",
        "\n",
        "$$ f_{\\theta_{\\text{GPT2}}}: \\mathbf{Y}_{0: m - 1} \\to \\mathbf{L}_{1:m}. $$\n",
        "\n",
        "By processing the logit vectors $\\mathbf{L}_{1:m}$ with the *softmax* operation, the model can define the probability distribution of the word sequence $\\mathbf{Y}_{1:m}$. To be exact, the probability distribution of the word sequence $\\mathbf{Y}_{1:m}$ can be factorized into $m-1$ conditional \"next word\" distributions:\n",
        "\n",
        "$$ p_{\\theta_{\\text{GPT2}}}(\\mathbf{Y}_{1:m}) = \\prod_{i=1}^{m} p_{\\theta_{\\text{GPT2}}}(\\mathbf{y}_i | \\mathbf{Y}_{0:i-1}). $$\n",
        "\n",
        "$p_{\\theta_{\\text{GPT2}}}(\\mathbf{y}_i | \\mathbf{Y}_{0:i-1})$ hereby presents the probability distribution of the next word $\\mathbf{y}_i$ given all previous words $\\mathbf{y}_0, \\ldots, \\mathbf{y}_{i-1}$ ${}^3$ and is defined as the softmax operation applied on the logit vector $\\mathbf{l}_i$. To summarize, the following equations hold true.\n",
        "\n",
        "$$ p_{\\theta_{\\text{gpt2}}}(\\mathbf{y}_i | \\mathbf{Y}_{0:i-1}) = \\textbf{Softmax}(\\mathbf{l}_i) = \\textbf{Softmax}(f_{\\theta_{\\text{GPT2}}}(\\mathbf{Y}_{0: i - 1})).$$\n",
        "\n",
        "For more detail, please refer to the [decoder](https://huggingface.co/blog/encoder-decoder#decoder) section of the encoder-decoder blog post.\n",
        "\n",
        "Let's visualize GPT2 now as well.\n",
        "\n",
        " ![texte du lien](https://raw.githubusercontent.com/patrickvonplaten/scientific_images/master/gpt2.png)\n",
        "\n",
        "Analogous to BERT, GPT2 is composed of a stack of *GPT2 blocks*. In contrast to BERT block, GPT2 block makes use of **uni-directional** self-attention to process some input vectors $\\mathbf{y'}_0, \\ldots, \\mathbf{y'}_{m-1}$ (shown in light blue on the bottom right) to an output vector sequence $\\mathbf{y''}_0, \\ldots, \\mathbf{y''}_{m-1}$ (shown in darker blue on the top right). In addition to the GPT2 block stack, the model also has a linear layer, called *LM Head*, which maps the output vectors of the final GPT2 block to the logit vectors $\\mathbf{l}_1, \\ldots, \\mathbf{l}_m$. As mentioned earlier, a logit vector $\\mathbf{l}_i$ can then be used to sample of new input vector $\\mathbf{y}_i$ ${}^5$.\n",
        "\n",
        "GPT2 is mainly used for *open-domain* text generation. First, an input prompt $\\mathbf{Y}_{0:i-1}$ is fed to the model to yield the conditional distribution  $p_{\\theta_{\\text{gpt2}}}(\\mathbf{y} | \\mathbf{Y}_{0:i-1})$. Then the next word $\\mathbf{y}_i$ is sampled from the distribution (represented by the grey arrows in the graph above) and consequently append to the input. In an auto-regressive fashion the word $\\mathbf{y}_{i+1}$ can then be sampled from $p_{\\theta_{\\text{gpt2}}}(\\mathbf{y} | \\mathbf{Y}_{0:i})$ and so on.\n",
        "\n",
        "GPT2 is therefore well-suited for *language generation*, but less so for *conditional* generation. By setting the input prompt $\\mathbf{Y}_{0: i-1}$ equal to the sequence input $\\mathbf{X}_{1:n}$, GPT2 can very well be used for conditional generation. However, the model architecture has a fundamental drawback compared to the encoder-decoder architecture as explained in [Raffel et al. (2019)](https://arxiv.org/abs/1910.10683) on page 17. In short, uni-directional self-attention forces the model's representation of the sequence input $\\mathbf{X}_{1:n}$ to be unnecessarily limited since $\\mathbf{x}_i$ cannot depend on $\\mathbf{x}_{i+1}, \\forall i \\in \\{1,\\ldots, n\\}$.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FBRkPI-P6Mn2"
      },
      "source": [
        "\n",
        "### **Encoder-Decoder**\n",
        "\n",
        "Because *encoder-only* models require to know the output length *a priori*, they seem unfit for sequence-to-sequence tasks. *Decoder-only* models can function well for sequence-to-sequence tasks, but also have certain architectural limitations as explained above.\n",
        "\n",
        "The current predominant approach to tackle *sequence-to-sequence* tasks are\n",
        "*transformer-based* **encoder-decoder** models - often also called *seq2seq transformer* models. Encoder-decoder models were introduced in [Vaswani et al. (2017)](https://arxiv.org/abs/1706.03762) and since then have been shown to perform better on *sequence-to-sequence* tasks than stand-alone language models (*i.e.* decoder-only models), *e.g.* [Raffel et al. (2020)](https://arxiv.org/pdf/1910.10683.pdf). In essence, an encoder-decoder model is the combination of a *stand-alone* encoder, such as BERT, and a *stand-alone* decoder model, such as GPT2. For more details on the exact architecture of transformer-based encoder-decoder models, please refer to [this blog post](https://huggingface.co/blog/encoder-decoder). \n",
        "\n",
        "Now, we know that freely available checkpoints of large pre-trained *stand-alone* encoder and decoder models, such as *BERT* and *GPT*, can boost performance and reduce training cost for many NLU tasks, We also know that encoder-decoder models are essentially the combination of *stand-alone* encoder and decoder models.\n",
        "This naturally brings up the question of how one can leverage stand-alone model checkpoints for encoder-decoder models and which model combinations are most performant on certain *sequence-to-sequence* tasks.\n",
        " \n",
        "In 2020, Sascha Rothe, Shashi Narayan, and Aliaksei Severyn investigated exactly this question in their paper [**Leveraging Pre-trained Checkpoints for Sequence Generation Tasks**](https://arxiv.org/abs/1907.12461). The paper offers a great analysis of different encoder-decoder model combinations and fine-tuning techniques, which we will study in more detail later.\n",
        "\n",
        "Composing an encoder-decoder model of pre-trained stand-alone model checkpoints is defined as *warm-starting* the encoder-decoder model.\n",
        "The following sections show how warm-starting an encoder-decoder model works in theory, how one can put the theory into practice with 🤗Transformers, and also gives practical tips for better performance.\n",
        "\n",
        "---\n",
        "${}^1$ A *pre-trained language model* is defined as a neural network:\n",
        "- that has been trained on *unlabeled* text data, *i.e.* in a task-agnostic, unsupervised fashion, and\n",
        "- that processes a sequence of input words into a *context-dependent* embedding. *E.g.* the *continuous bag-of-words* and *skip-gram* model from [Mikolov et al. (2013)](https://arxiv.org/abs/1301.3781) is not considered a pre-trained language model because the embeddings are context-agnostic.\n",
        "\n",
        "${}^2$ *Fine-tuning* is defined as the *task-specific* training of a model that has been initialized with the weights of a pre-trained language model.\n",
        "\n",
        "${}^3$  The input vector $\\mathbf{y}_0$ corresponds hereby to the $\\text{BOS}$ embedding vector required to predict the very first output word $\\mathbf{y}_1$.\n",
        "\n",
        "${}^4$ Without loss of generalitiy, we exclude the normalization layers to not clutter the equations and illustrations.\n",
        "\n",
        "${}^5$ For more detail on why uni-directional self-attention is used for \"decoder-only\" models, such as GPT2, and how sampling works exactly, please refer to the [decoder](https://huggingface.co/blog/encoder-decoder#decoder) section of the encoder-decoder blog post."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4M2uzGLV9a_O"
      },
      "source": [
        "## **Warm-starting encoder-decoder models (Theory)**\n",
        "\n",
        "Having read the introduction, we are now familiar with *encoder-only*- and *decoder-only* models. We have noticed that the encoder-decoder model architecture is essentially a composition of a *stand-alone* encoder model and a *stand-alone* decoder model, which led us to the question of how one can *warm-start* encoder-decoder models from *stand-alone* model checkpoints.\n",
        "\n",
        "There are multiple possibilities to warm-start an encoder-decoder model. One can\n",
        "\n",
        "1. initialize both the encoder and decoder part from an *encoder-only* model checkpoint, *e.g.* BERT,\n",
        "2. initialize the encoder part from an *encoder-only* model checkpoint, *e.g.* BERT, and the decoder part from and a *decoder-only* checkpoint, *e.g.* GPT2,\n",
        "3. initialize only the encoder part with an *encoder-only* model checkpoint, or\n",
        "4. initialize only the decoder part with a *decoder-only* model checkpoint.\n",
        "\n",
        "In the following, we will put the focus on possibilities 1. and 2. Possibilities 3. and 4. are trivial after having understood the first two.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aJ7VLfMh6Ssm"
      },
      "source": [
        "### **Recap Encoder-Decoder Model**\n",
        "\n",
        "First, let's do a quick recap of the encoder-decoder architecture.\n",
        "\n",
        " ![texte du lien](https://raw.githubusercontent.com/patrickvonplaten/scientific_images/master/encoder_decoder_reap.png)\n",
        "\n",
        "The encoder (shown in green) is a stack of *encoder blocks*. Each encoder block is composed of a *bi-directional self-attention* layer, and two feed-forward layers ${}^1$. The decoder (shown in orange) is a stack of *decoder blocks*, followed by a dense layer, called *LM Head*. Each decoder block is composed of a *uni-directional self-attention* layer, a *cross-attention* layer, and two feed-forward layers.\n",
        "\n",
        " The encoder maps the input sequence $\\mathbf{X}_{1:n}$ to a contextualized encoded sequence $\\mathbf{\\overline{X}}_{1:n}$ in the exact same way BERT does. \n",
        " The decoder then maps the contextualized encoded sequence $\\mathbf{\\overline{X}}_{1:n}$ and a target sequence $\\mathbf{Y}_{0:m-1}$ to the logit vectors $\\mathbf{L}_{1:m}$. Analogous to GPT2, the logits are then used to define the distribution of the target sequence $\\mathbf{Y}_{1:m}$ conditioned on the input sequence $\\mathbf{X}_{1:n}$ by means of a *softmax* operation.\n",
        "\n",
        "To put it into mathematical terms, first, the conditional distribution is factorized into $m - 1$ conditional distributions of the next word $\\mathbf{y}_i$ by Bayes' rule.\n",
        "\n",
        " $$\n",
        " p_{\\theta_{\\text{enc, dec}}}(\\mathbf{Y}_{1:m} | \\mathbf{X}_{1:n}) = p_{\\theta_{\\text{dec}}}(\\mathbf{Y}_{1:m} | \\mathbf{\\overline{X}}_{1:n}) = \\prod_{i=1}^m p_{\\theta_{\\text{dec}}}(\\mathbf{y}_i | \\mathbf{Y}_{0: i -1}, \\mathbf{\\overline{X}}_{1:n}), \\text{ with }  \\mathbf{\\overline{X}}_{1:n} = f_{\\theta_{\\text{enc}}}(\\mathbf{X}_{1:n}).\n",
        " $$\n",
        "\n",
        "Each \"next-word\" conditional distributions is thereby defined by the *softmax* of the logit vector as follows.\n",
        "\n",
        " $$ p_{\\theta_{\\text{dec}}}(\\mathbf{y}_i | \\mathbf{Y}_{0: i -1}, \\mathbf{\\overline{X}}_{1:n}) = \\textbf{Softmax}(\\mathbf{l}_i). $$\n",
        "\n",
        "For more detail, please refer to the [Encoder-Decoder notebook](https://colab.research.google.com/drive/19wkOLQIjBBXQ-j3WWTEiud6nGBEw4MdF?usp=sharing).\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L-LyGs5Q6X1P"
      },
      "source": [
        "### **Warm-staring Encoder-Decoder with BERT**\n",
        "\n",
        "Let's now illustrate how a pre-trained BERT model can be used to warm-start the encoder-decoder model. BERT's pre-trained weight parameters are used to both initialize the encoder's weight parameters as well as the decoder's weight parameters. \n",
        "To do so, BERT's architecture is compared to the encoder's architecture and all layers of the encoder that also exist in BERT will be initialized with the pre-trained weight parameters of the respective layers. All layers of the encoder that do not exist in BERT will simply have their weight parameters be randomly initialized.\n",
        "\n",
        "Let's visualize.\n",
        "\n",
        "  ![texte du lien](https://raw.githubusercontent.com/patrickvonplaten/scientific_images/master/encoder_decoder/leverage_encoder.png)\n",
        "\n",
        "We can see that the encoder architecture corresponds 1-to-1 to BERT's architecture.\n",
        "The weight parameters of the *bi-directional self-attention layer* and the two *feed-forward layers* of **all** encoder blocks are initialized with the weight parameters of the respective BERT blocks.\n",
        "This is illustrated examplary for the second encoder block (red boxes at bottow) whose weight parameters $\\theta_{\\text{enc}}^{\\text{self-attn}, 2}$ and $\\theta_{\\text{enc}}^{\\text{feed-forward}, 2}$ are set to BERT's weight parameters $\\theta_{\\text{BERT}}^{\\text{feed-forward}, 2}$ and $\\theta_{\\text{BERT}}^{\\text{self-attn}, 2}$, respectively at initialization.\n",
        "\n",
        "Before fine-tuning, the encoder therefore behaves exactly like a pre-trained BERT model. Assuming the input sequence $\\mathbf{x}_1, \\ldots, \\mathbf{x}_n$ (shown in green) passed to the encoder is equal to the input sequence $\\mathbf{x}_1^{\\text{BERT}}, \\ldots, \\mathbf{x}_n^{\\text{BERT}}$ (shown in grey) passed to BERT, this means that the respective output vector sequences $\\mathbf{\\overline{x}}_1, \\ldots, \\mathbf{\\overline{x}}_n$ (shown in darker green) and $\\mathbf{\\overline{x}}_1^{\\text{BERT}}, \\ldots, \\mathbf{\\overline{x}}_n^{\\text{BERT}}$ (shown in darker grey) also have to be equal.\n",
        "\n",
        "\n",
        "Next, let's illustrate how the decoder is warm-started.\n",
        "\n",
        "   ![texte du lien](https://raw.githubusercontent.com/patrickvonplaten/scientific_images/master/encoder_decoder/leverage_decoder.png)\n",
        "\n",
        "The architecture of the decoder is different from BERT's architecture in three ways.\n",
        "\n",
        "1. First, the decoder has to be conditioned on the contextualized encoded sequence $\\mathbf{\\overline{X}}_{1:n}$ by means of cross-attention layers.\n",
        "Consequently, randomly initialized cross-attention layers are added between the self-attention layer and the two feed-forward layers in each BERT block. This is represented exemplary for the second block by $+\\theta_{\\text{dec}}^{\\text{cross-attention, 2}}$ and illustrated by the newly added fully connected graph in red in the lower red box on the right. This necessarily changes the behavior of each modified BERT block so that an input vector, *e.g.* $\\mathbf{y'}_0$ now yields a random output vector $\\mathbf{y''}_0$ (highlighted by the red border around the output vector $\\mathbf{y''}_0$).\n",
        "\n",
        "2. Second, BERT's *bi-directional* self-attention layers have to be changed to *uni-directional* self-attention layers to comply with auto-regressive generation. Because both the bi-directional and the uni-directional self-attention layer are based on the same *key*, *query* and *value* projection weights, the decoder's self-attention layer weights can be initialized with BERT's self-attention layer weights. *E.g.* the query, key and value weight parameters of the decoder's uni-directional self-attention layer are initialized with those of BERT's bi-directional self-attention layer:\n",
        "$$\n",
        "\\theta_{\\text{BERT}}^{\\text{self-attn}, 2} = \\{\\mathbf{W}_{\\text{BERT}, k}^{\\text{self-attn}, 2}, \\mathbf{W}_{\\text{BERT}, v}^{\\text{self-attn}, 2}, \\mathbf{W}_{\\text{BERT}, q}^{\\text{self-attn}, 2} \\} \\to\n",
        "\\theta_{\\text{dec}}^{\\text{self-attn}, 2} = \\{\\mathbf{W}_{\\text{dec}, k}^{\\text{self-attn}, 2}, \\mathbf{W}_{\\text{dec}, v}^{\\text{self-attn}, 2}, \\mathbf{W}_{\\text{dec}, q}^{\\text{self-attn}, 2} \\}.\n",
        "$$\n",
        "However, in *uni-directional* self-attention each token only attends to all previous tokens, so that the decoder's self-attention layers yield different output vectors than BERT's self-attention layers even though they share the same weights. Compare *e.g.*, the decoder's causally connected graph in the right box versus BERT's fully connected graph in the left box.  \n",
        "\n",
        "3. Third, the decoder outputs a sequence of logit vectors $\\mathbf{L}_{1:m}$ in order to define the conditional probability distribution $p_{\\theta_{\\text{dec}}}(\\mathbf{Y}_{1:n} | \\mathbf{\\overline{X}})$. As a result, a *LM Head* layer is added on top of the last decoder block. The weight parameters of the *LM Head* layer usually correspond to the weight parameters of the word embedding $\\mathbf{W}_{\\text{emb}}$ and thus are not randomly initialized. This is illustrated in the top by the initialization $\\theta_{\\text{BERT}}^{\\text{word-emb}} \\to \\theta_{\\text{dec}}^{\\text{lm-head}}$.\n",
        "\n",
        "To conclude, when warm-starting the decoder from a pre-trained BERT model only the cross-attention layer weights are randomly initialized. All other weights including those of the self-attention layer and LM Head are initialized with BERT's pre-trained weight parameters.\n",
        "\n",
        "Having warm-stared the encoder-decoder model, the weights are then fine-tuned on a *sequence-to-sequence* downstream task, such as summarization.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SKaGyb446cmb"
      },
      "source": [
        "### **Warm-staring Encoder-Decoder with BERT and GPT2**\n",
        "\n",
        "Instead of warm-starting both the encoder and decoder with a BERT checkpoint, we can instead leverage the BERT checkpoint for the encoder and a GPT2 checkpoint for the decoder. At first glance, a decoder-only GPT2 checkpoint seems to be better-suited to warm-start the decoder because it has already been trained on causal language modeling and uses *uni-directional* self-attention layers.\n",
        "\n",
        "Let's illustrate how a GPT2 checkpoint can be used to warm-start the decoder.\n",
        "\n",
        "![texte du lien](https://raw.githubusercontent.com/patrickvonplaten/scientific_images/master/encoder_decoder/leverage_decoder_gpt2.png)\n",
        "\n",
        "We can see that decoder is more similar to GPT2 than it is to BERT. The weight parameters of decoder's *LM Head* can directly be initialized with GPT2's *LM Head* weight parameters, *e.g.* $\\theta_{\\text{GPT2}}^{\\text{lm-head}} \\to \\theta_{\\text{dec}}^{\\text{lm-head}}$. \n",
        "In addition, the blocks of the decoder and GPT2 both make use of *uni-directional* self-attention so that the output vectors of the decoder's self-attention layer are equivalent to GPT2's output vectors assuming the input vectors are the same, *e.g.* $\\mathbf{y'}_0^{\\text{GPT2}} = \\mathbf{y'}_0$.\n",
        "In contrast to the BERT-initialized decoder, the GPT2-initialized decoder, therefore, keeps the causal connected graph of the self-attention layer as can be seen in the red boxes on the bottom.\n",
        "\n",
        "Nevertheless, the GPT2-initialized decoder also has to condition the decoder on $\\mathbf{\\overline{X}}_{1:n}$. Analoguos to the BERT-initialized decoder, randomly initialized weight parameters for the cross-attention layer are therefore added to each decoder block. This is illustrated *e.g.* for the second encoder block by $+\\theta_{\\text{dec}}^{\\text{cross-attention, 2}}$.\n",
        "\n",
        "Even though GPT2 resembles the decoder part of an encoder-decoder model more than BERT, a GPT2-initialized decoder will also yield random logit vectors $\\mathbf{L}_{1:m}$ without fine-tuning due to randomly initialized cross-attention layers in every decoder block. It would be interesting to investigate whether a GPT2-initialized decoder yields better results or can be fine-tuned more efficiently.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tDwUkF7O6g_1"
      },
      "source": [
        "### **Encoder-Decoder Weight Sharing**\n",
        "\n",
        "In [Raffel et al. (2020)](https://arxiv.org/pdf/1910.10683.pdf), the authors show that a randomly-initialized encoder-decoder model that shares the encoder's weights with the decoder, and therefore reduces the memory footprint by half, performs only slightly worse than its \"non-shared\" version.\n",
        "Sharing the encoder's weights with the decoder means that all layers of the decoder that are found at the same position in the encoder share the same weight parameters, *i.e.* the same node in the network's computation graph.  \n",
        " *E.g.* the query, key, and value projection matrices of the self-attention layer in the third encoder block, defined as $\\mathbf{W}^{\\text{self-attn}, 3}_{\\text{Enc}, k}$, $\\mathbf{W}^{\\text{self-attn}, 3}_{\\text{Enc}, v}$, $\\mathbf{W}^{\\text{self-attn}, 3}_{\\text{Enc}, q}$ are identical to the respective query, key, and value projections matrices of the self-attention layer in the third decoder block ${}^2$:\n",
        "\n",
        " $$ \\mathbf{W}^{\\text{self-attn}, 3}_{k} = \\mathbf{W}^{\\text{self-attn}, 3}_{\\text{enc}, k} \\equiv \\mathbf{W}^{\\text{self-attn}, 3}_{\\text{dec}, k}, $$\n",
        " $$ \\mathbf{W}^{\\text{self-attn}, 3}_{q} = \\mathbf{W}^{\\text{self-attn}, 3}_{\\text{enc}, q} \\equiv \\mathbf{W}^{\\text{self-attn}, 3}_{\\text{dec}, q}, $$\n",
        "  $$ \\mathbf{W}^{\\text{self-attn}, 3}_{v} = \\mathbf{W}^{\\text{self-attn}, 3}_{\\text{enc}, v} \\equiv \\mathbf{W}^{\\text{self-attn}, 3}_{\\text{dec}, v}, $$\n",
        "\n",
        "As a result, the key projection weights $\\mathbf{W}^{\\text{self-attn}, 3}_{k}, \\mathbf{W}^{\\text{self-attn}, 3}_{v}, \\mathbf{W}^{\\text{self-attn}, 3}_{q}$ are updated twice for each backward propagation pass - once when the gradient is backpropagated through the third decoder block and once when the gradient is backprapageted thourgh the third encoder block.\n",
        "\n",
        "In the same way, we can warm-start an encoder-decoder model by sharing the encoder weights with the decoder.\n",
        "Being able to share the weights between the encoder and decoder requires the decoder architecture (excluding the cross-attention weights) to be identical to the encoder architecture. Therefore, *encoder-decoder weight sharing* is only relevant if the encoder-decoder model is warm-started from a single *encoder-only* pre-trained checkpoint. \n",
        "\n",
        "Great! That was the theory about warm-starting encoder-decoder models. Let's now look at some results.\n",
        "\n",
        "\n",
        "---\n",
        "\n",
        "${}^1$ Without loss of generality, we exclude the normalization layers to not clutter the equations and illustrations.\n",
        "\n",
        "${}^2$ For more detail on how self-attention layers function, please refer to [this section](https://huggingface.co/blog/encoder-decoder#encoder) of the transformer-based encoder-decoder model blog post for the encoder-part (and [this section](https://huggingface.co/blog/encoder-decoder#decoder) for the decoder part respectively)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9pftbMBeJACc"
      },
      "source": [
        "## **Warm-starting encoder-decoder models (Analysis)**\n",
        "\n",
        "In this section, we will summarize the findings on warm-starting encoder-decoder models as presented in [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, and Aliaksei Severyn. The authors compared the performance of warm-started encoder-decoder models to randomly initialized encoder-decoder models on multiple *sequence-to-sequence* tasks, notably *summarization*, *translation*, *sentence splitting*, and *sentence fusion*. \n",
        "\n",
        "To be more precise, the publicly available pre-trained checkpoints of **BERT**, **RoBERTa**, and **GPT2** were leveraged in different variations to warm-start an encoder-decoder model. *E.g.* a BERT-initialised encoder was paired with a BERT-initialized decoder yielding a BERT2BERT model *or* a RoBERTa-initialized encoder was paired with a GPT2-initialized decoder to yield a *RoBERTa2GPT2* model.\n",
        "Additionally, the effect of sharing the encoder and decoder weights (as explained in the previous section) was investigated for RoBERTa, *i.e.* **RoBERTaShare**, and for BERT, *i.e.* **BERTShare**. Randomly or partly randomly initialized encoder-decoder models were used as a baseline, such as a fully randomly initialized encoder-decoder model, coined **Rnd2Rnd** or a BERT-initialized decoder paired with a randomly initialized encoder, defined as **Rnd2BERT**.\n",
        "\n",
        "The following table shows a complete list of all investigated model variants including the number of randomly initialized weights, *i.e.* \"random\", and the number of weights initialized from the respective pre-trained checkpoints, *i.e.* \"leveraged\". All models are based on a 12-layer architecture with 768-dim hidden size embeddings, corresponding to the `bert-base-cased`, `bert-base-uncased`, `roberta-base`, and `gpt2` checkpoints in the 🤗Transformers model hub.\n",
        "\n",
        " Model | random | leveraged | total |\n",
        ":--- | :--- | :--- | :--- |\n",
        "Rnd2Rnd | 221M | 0 | 221M |\n",
        "Rnd2BERT | 112M | 109M | 221M |\n",
        "BERT2Rnd | 112M | 109M | 221M |\n",
        "Rnd2GPT2 | 114M | 125M | 238M |\n",
        "BERT2BERT | 26M | 195M | 221M |\n",
        "BERTShare | 26M | 109M | 135M |\n",
        "RoBERTaShare | 26M | 126M | 152M |\n",
        "BERT2GPT2 | 26M | 234M | 260M |\n",
        "RoBERTa2GPT2 | 26M | 250M | 276M |\n",
        "\n",
        "The model *Rnd2Rnd*, which is based on the BERT2BERT architecture, contains 221M weight parameters - all of which are randomly initialized. The other two \"BERT-based\" baselines *Rnd2BERT* and *BERT2Rnd* have roughly half of their weights, *i.e.* 112M parameters, randomly initialized. The other 109M weight parameters are leveraged from the pre-trained `bert-base-uncased` checkpoint for the encoder- or decoder part respectively. The models *BERT2BERT*, *BERT2GPT2*, and *RoBERTa2GPT2* have all of their encoder weight parameters leveraged (from `bert-base-uncased`, `roberta-base` respectively) and most of the decoder weight parameter weights as well (from `gpt2`, `bert-base-uncased` respectively). 26M decoder weight parameters, which correspond to the 12 cross-attention layers, are thereby randomly initialized. RoBERTa2GPT2 and BERT2GPT2 are compared to the *Rnd2GPT2* baseline. \n",
        "Also, it should be noted that the shared model variants *BERTShare* and *RoBERTaShare* have significantly fewer parameters because all encoder weight parameters are shared with the respective decoder weight parameters.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "27iHqRfq6rne"
      },
      "source": [
        "### **Experiments**\n",
        "\n",
        "The above models were trained and evaluated on four sequence-to-sequence tasks of increasing complexity: sentence-level fusion, sentence-level splitting, translation, and abstractive summarization. The following table shows which datasets were used for each task.\n",
        "\n",
        "Seq2Seq Task | Datasets | Paper | 🤗datasets |\n",
        ":--- | :--- | :--- | :--- |\n",
        "Sentence Fusion | DiscoFuse | [Geva et al. (2019)](https://arxiv.org/abs/1902.10526) | [link](https://huggingface.co/nlp/viewer/?dataset=discofuse&config=discofuse-wikipedia) |\n",
        "Sentence Splitting | WikiSplit | [Botha et al. (2018)](https://arxiv.org/abs/1808.09468) | - |\n",
        "Translation | WMT14 EN => DE| [Bojar et al. (2014)](http://www.aclweb.org/anthology/W/W14/W14-3302) | [link](https://huggingface.co/nlp/viewer/?dataset=wmt14&config=de-en) |\n",
        "         | WMT14 DE => EN | [Bojar et al. (2014)](http://www.aclweb.org/anthology/W/W14/W14-3302) | [link](https://huggingface.co/nlp/viewer/?dataset=wmt14&config=de-en)|\n",
        "Abstractive Summarizaion | CNN/Dailymail | [Hermann et al. (2015)](http://arxiv.org/abs/1704.04368) | [link](https://huggingface.co/nlp/viewer/?dataset=cnn_dailymail&config=3.0.0) |\n",
        "                         | BBC XSum | [Narayan et al. (2018a)](https://arxiv.org/abs/1808.08745) | [link](https://huggingface.co/nlp/viewer/?dataset=xsum) |\n",
        "                         | Gigaword | [Napoles et al. (2012)](http://dx.doi.org/10.18653/v1/D15-1044) | [link](https://huggingface.co/nlp/viewer/?dataset=gigaword) |\n",
        "\n",
        "\n",
        "Depending on the task, a slightly different training regime was used. *E.g.* according to the size of the dataset and the specific task, the number of training steps ranges from 200K to 500K, the batch size is set to either 128 or 256, the input length ranges from 128 to 512 and the output length varies between 32 to 128. It shall be emphasized however that within each task, all models were trained and evaluated using the same hyperparameters to ensure a fair comparison. For more information on the task-specific hyperparameter settings, the reader is advised to see the *Experiments* section in the [paper](https://arxiv.org/pdf/1907.12461.pdf).\n",
        "\n",
        "We will now give a condensed overview of the results for each task.\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g_XGrp_D7Cv2"
      },
      "source": [
        "#### **Sentence Fusion and -Splitting (DiscoFuse, WikiSplit)**\n",
        "\n",
        "**Sentence Fusion** is the task of combining multiple sentences into a single coherent sentence. *E.g.* the two sentences: \n",
        "\n",
        "_As a run-blocker, Zeitler moves relatively well._\n",
        "_Zeitler too often struggles at the point of contact in space._ \n",
        "\n",
        "should be connected with a fitting *linking word*, such as:\n",
        "\n",
        " _As a run-blocker, Zeitler moves relatively well. **However**, **he** too often struggles at the point of contact in space._\n",
        "\n",
        " As can be seen the linking word \"however\" provides a coherent transition from the first sentence to the second one. A model that is capable of generating such a linking word has arguably learned to infer that the two sentences above contrast to each other. \n",
        " \n",
        " The inverse task is called **Sentence splitting** and consists of splitting a single complex sentence into multiple simpler ones that together retain the same meaning. Sentence splitting is considered as an important task in text simplification, *cf.* to [Botha et al. (2018)](https://arxiv.org/pdf/1808.09468.pdf). \n",
        "\n",
        " As an example, the sentence:\n",
        "\n",
        " _Street Rod is the first in a series of two games released for the PC and Commodore 64 in 1989_\n",
        "\n",
        " can be simplified into \n",
        "\n",
        " _Street Rod is the first in a series of two games**. It** was released for the PC and Commodore 64 in 1989_\n",
        "\n",
        "It can be seen that the long sentence tries to convey two important pieces of information. One is that the game was the first of two games being released for the PC, and the second being the year in which it was released.\n",
        "Sentence splitting, therefore, requires the model to understand which part of the sentence should be divided into two sentences, making the task more difficult than sentence fusion.\n",
        "\n",
        "A common metric to evaluate the performance of models on sentence fusion resp. -splitting tasks is *SARI* [(Wu et al. (2016)](https://www.aclweb.org/anthology/Q16-1029/), which is broadly based on the F1-score of label and model output.\n",
        "\n",
        "Let's see how the models perform on sentence fusion and -splitting.\n",
        "\n",
        " Model | 100% DiscoFuse (SARI) | 10% DiscoFuse (SARI) | 100% WikiSplit (SARI) |\n",
        ":--- | :---: | :---: | :---: |\n",
        "Rnd2Rnd | 86.9 | 81.5 | 61.7 |\n",
        "Rnd2BERT | 87.6 | 82.1 | 61.8 |\n",
        "BERT2Rnd | 89.3 | 86.1 | 63.1 |\n",
        "Rnd2GPT2 | 86.5 | 81.4 | 61.3 |\n",
        "BERT2BERT | 89.3 | 86.1 | 63.2 |\n",
        "BERTShare | 89.2 | 86.0 | **63.5** |\n",
        "RoBERTaShare | 89.7 | 86.0 | 63.4 |\n",
        "BERT2GPT2 | 88.4 | 84.1 | 62.4 |\n",
        "RoBERTa2GPT2 | **89.9** | **87.1** | 63.2 |\n",
        "--- | --- | --- | ---|\n",
        "RoBERTaShare (large) | **90.3** | **87.7** | **63.8**\n",
        "\n",
        "The first two columns show the performance of the encoder-decoder models on the DiscoFuse evaluation data. The first column states the results of encoder-decoder models trained on all (100%) of the training data, while the second column shows the results of the models trained only on 10% of the training data. We observe that warm-started models perform significantly better than the randomly initialized baseline models *Rnd2Rnd*, *Rnd2Bert*, and *Rnd2GPT2*. A warm-started *RoBERTa2GPT2* model trained only on 10% of the training data is on par with an *Rnd2Rnd* model trained on 100% of the training data. Interestingly, the *Bert2Rnd* baseline performs equally well as a fully warm-started *Bert2Bert* model, which indicates that warm-starting the encoder-part is more effective than warm-starting the decoder-part. The best results are obtained by *RoBERTa2GPT2*, followed by *RobertaShare*. Sharing encoder and decoder weight parameters does seem to slightly increase the model's performance.\n",
        "\n",
        "On the more difficult sentence splitting task, a similar pattern emerges. Warm-started encoder-decoder models significantly outperform encoder-decoder models whose encoder is randomly initialized and encoder-decoder models with shared weight parameters yield better results than those with uncoupled weight parameters. On sentence splitting the *BertShare* models yields the best performance closely followed by *RobertaShare*.\n",
        "\n",
        "In addition to the 12-layer model variants, the authors also trained and evaluated a 24-layer *RobertaShare (large)* model which outperforms all 12-layer models significantly."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XsNS0xyi6-0l"
      },
      "source": [
        "#### **Machine Translation (WMT14)**\n",
        "\n",
        "Next, the authors evaluated warm-started encoder-decoder models on the probably most common benchmark in machine translation (MT) - the *En* $\\to$ *De* and *De* $\\to$ *En* WMT14 dataset. In this notebook, we present the results on the *newstest2014* eval dataset. Because the benchmark requires the model to understand both an English and a German vocabulary the BERT-initialized encoder-decoder models were warm-started from the multilingual pre-trained checkpoint `bert-base-multilingual-cased`. Because there is no publicly available multilingual RoBERTa checkpoint, RoBERTa-initialized encoder-decoder models were excluded for MT. \n",
        "GPT2-initialized models were initialized from the `gpt2` pre-trained checkpoint as in the previous experiment. The translation results are reported using the BLUE-4 score metric ${}^1$.\n",
        "\n",
        " Model | En $\\to$ De (BLEU-4) | De $\\to$ En (BLEU-4) |\n",
        ":--- | :---: | :---: |\n",
        "Rnd2Rnd | 26.0 | 29.1 | \n",
        "Rnd2BERT | 27.2 | 30.4 |\n",
        "BERT2Rnd | **30.1** | **32.7** |\n",
        "Rnd2GPT2 | 19.6 | 23.2 |\n",
        "BERT2BERT | **30.1** | **32.7** |\n",
        "BERTShare | 29.6 | 32.6 |\n",
        "BERT2GPT2 | 23.2 | 31.4 |\n",
        "--- | --- | --- |\n",
        "BERT2Rnd (large, custom) | **31.7** | **34.2**\n",
        "BERTShare (large, custom) | 30.5 | 33.8 |\n",
        "\n",
        "Again, we observe a significant performance boost by warm-starting the encoder-part, with *BERT2Rnd* and *BERT2BERT* yielding the best results on both the *En* $\\to$ *De* and *De* $\\to$ *En* tasks.  *GPT2* initialized models perform significantly worse even than the *Rnd2Rnd* baseline on *En* $\\to$ *De*. Taking into consideration that the `gpt2` checkpoint was trained only on English text, it is not very surprising that *BERT2GPT2* and *Rnd2GPT2* models have difficulties generating German translations. This hypothesis is supported by the competitive results (*e.g.* 31.4 vs. 32.7) of *BERT2GPT2* on the *De* $\\to$ *En* task for which GPT2's vocabulary fits the English output format. \n",
        "Contrary to the results obtained on sentence fusion and sentence splitting, sharing encoder and decoder weight parameters does not yield a performance boost in MT. Possible reasons for this as stated by the authors include \n",
        " - *the encoder-decoder model capacity is an important factor in MT, and* \n",
        " - *the encoder and decoder have to deal with different grammar and vocabulary*\n",
        "\n",
        "Since the *bert-base-multilingual-cased* checkpoint was trained on more than 100 languages, its vocabulary is probably undesirably large for *En* $\\to$ *De* and *De* $\\to$ *En* MT. Thus, the authors pre-trained a large BERT encoder-only checkpoint on the English and German subset of the Wikipedia dump and subsequently used it to warm-start a *BERT2Rnd* and *BERTShare* encoder-decoder model. Thanks to the improved vocabulary, another significant performance boost is observed, with *BERT2Rnd (large, custom)* significantly outperforming all other models."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "22l31xtp67Ms"
      },
      "source": [
        "#### **Summarization (CNN/Dailymail, BBC XSum, Gigaword)**\n",
        "\n",
        "Finally, the encoder-decoder models were evaluated on the arguably most challenging sequence-to-sequence task - *summarization*. The authors picked three summarization datasets with different characteristics for evaluation: Gigaword (*headline generation*), BBC XSum (*extreme summarization*), and CNN/Dailymayl (*abstractive summarization*).\n",
        "\n",
        "The Gigaword dataset contains sentence-level abstractive summarizations, requiring the model to learn sentence-level understanding, abstraction, and eventually paraphrasing.\n",
        "A typical data sample in Gigaword, such as\n",
        "\n",
        "\"*venezuelan president hugo chavez said thursday he\n",
        "has ordered a probe into a suspected coup plot\n",
        "allegedly involving active and retired military\n",
        "officers .*\",\n",
        "\n",
        "would have a corresponding headline as its label, *e.g.*:\n",
        "\n",
        "\"*chavez orders probe into suspected coup plot*\".\n",
        "\n",
        "The BBC XSum dataset consists of much longer *article-like* text inputs with the labels being mostly single sentence summarizations. This dataset requires the model not only to learn document-level inference but also a high level of abstractive paraphrasing. Some data samples of the BBC XSUM datasets are shown [here](https://huggingface.co/nlp/viewer/?dataset=xsum).\n",
        "\n",
        "For the CNN/Dailmail dataset, documents, which are of similar length than those in the BBC XSum dataset, have to be summarized to bullet-point story highlights. The labels therefore often consist of multiple sentences. Besides document-level understanding, the CNN/Dailymail dataset requires models to be good at copying the most salient information. Some examples can be viewed [here](https://huggingface.co/nlp/viewer/?dataset=cnn_dailymail).\n",
        "\n",
        "The models are evaluated using the [Rouge metric](https://www.aclweb.org/anthology/N03-1020/), whereas the Rouge-2 scores are shown below.\n",
        "\n",
        "Alright, let's take a look at the results.\n",
        "\n",
        " Model | CNN/Dailymail (Rouge-2) | BBC XSum (Rouge-2) | Gigaword (Rouge-2) |\n",
        ":--- | :---: | :---: | :---: |\n",
        "Rnd2Rnd | 14.00 | 10.23 | 18.71 |\n",
        "Rnd2BERT | 15.55 | 11.52 | 18.91 |\n",
        "BERT2Rnd | 17.76 | 15.83 | 19.26 |\n",
        "Rnd2GPT2 | 8.81 | 8.77 | 18.39 |\n",
        "BERT2BERT | 17.84 | 15.24 | 19.68 |\n",
        "BERTShare | 18.10 | 16.12 | **19.81** |\n",
        "RoBERTaShare | **18.95** | **17.50** | 19.70 |\n",
        "BERT2GPT2 | 4.96 | 8.37 | 18.23 |\n",
        "RoBERTa2GPT2 | 14.72 | 5.20 | 19.21 |\n",
        "--- | --- | --- | ---|\n",
        "RoBERTaShare (large) | 18.91 | **18.79** | 19.78 |\n",
        "\n",
        "We observe again that warm-starting the encoder-part gives a significant improvement over models with randomly-initialized encoders, which is especially visible for document-level abstraction tasks, *i.e.* CNN/Dailymail and BBC XSum. This shows that tasks requiring a high level of abstraction benefit more from a pre-trained encoder part than those requiring only sentence-level abstraction. Except for Gigaword GPT2-based encoder-decoder models seem to be unfit for summarization.\n",
        "\n",
        "Furthermore, the shared encoder-decoder models are the best performing models for summarization. *RoBERTaShare* and *BERTShare* are the best performing models on all datasets whereas the margin is especially significant on the BBC XSum dataset on which *RoBERTaShare (large)* outperforms *BERT2BERT* and *BERT2Rnd* by *ca.* 3 Rouge-2 points and *Rnd2Rnd* by more than 8 Rouge-2 points. As brought forward by the authors, \"*this is probably because the BBC summary sentences follow\n",
        "a distribution that is similar to that of the sentences in the document, whereas this is not necessarily the case for the Gigaword headlines and the\n",
        "CNN/DailyMail bullet-point highlights*\". Intuitively this means that in BBC XSum, the input sentences processed by the encoder are very similar in structure to the single sentence summary processed by the decoder, *i.e.* same length, similar choice of words, similar syntax.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lYiF_PXB6zie"
      },
      "source": [
        "### **Conclusion**\n",
        "\n",
        "Alright, let's draw a conclusion and try to derive some practical tips. \n",
        "\n",
        "- We have observed on all tasks that a warm-started encoder-part gives a significant performance boost compared to encoder-decoder models having a randomly initialized encoder. On the other hand, warm-starting the decoder seems to be less important, with *BERT2BERT* being on par with *BERT2Rnd* on most tasks. An intuitive reason would be that since a BERT- or RoBERTa-initialized encoder part has none of its weight parameters randomly initialized, the encoder can fully exploit the acquired knowledge of BERT's or RoBERTa's pre-trained checkpoints, respectively. In contrast, the warm-started decoder always has parts of its weight parameters randomly initialized which possibly makes it much harder to effectively leverage the knowledge acquired by the checkpoint used to initialize the decoder. \n",
        "\n",
        "- Next, we noticed that it is often beneficial to share encoder and decoder weights, especially if the target distribution is similar to the input distribution (*e.g.* BBC XSum). However, for datasets whose target data distribution differs more significantly from the input data distribution and for which model capacity${}^2$ is known to play an important role, *e.g.* WMT14, encoder-decoder weight sharing seems to be disadvantageous.\n",
        "\n",
        "- Finally, we have seen that it is very important that the vocabulary of the pre-trained \"stand-alone\" checkpoints fit the vocabulary required to solve the sequence-to-sequence task. *E.g.* a warm-started BERT2GPT2 encoder-decoder will perform poorly on *En* $\\to$ *De* MT because GPT2 was pre-trained on English whereas the target language is German. The overall poor performance of the *BERT2GPT2*, *Rnd2GPT2*, and *RoBERTa2GPT2* compared to *BERT2BERT*, *BERTShared*, and *RoBERTaShared* suggests that it is more effective to have a shared vocabulary. Also, it shows that initializing the decoder part with a pre-trained GPT2 checkpoint is *not* more effective than initializing it with a pre-trained BERT checkpoint besides GPT2 being more similar to the decoder in its architecture.\n",
        "\n",
        "For each of the above tasks, the most performant models were ported to 🤗Transformers and can be accessed here:\n",
        "\n",
        "- *RoBERTaShared (large)* - *Wikisplit*: [google/roberta2roberta_L-24_wikisplit](https://huggingface.co/google/roberta2roberta_L-24_wikisplit).\n",
        "- *RoBERTaShared (large)* - *Discofuse*: [google/roberta2roberta_L-24_discofuse](https://huggingface.co/google/roberta2roberta_L-24_discofuse).\n",
        "- *BERT2BERT (large)* - *WMT en $\\to$ de*: [google/bert2bert_L-24_wmt_en_de](https://huggingface.co/google/bert2bert_L-24_wmt_en_de).\n",
        "- *BERT2BERT (large)* - *WMT de $\\to$ en*: [google/bert2bert_L-24_wmt_de_en](https://huggingface.co/google/bert2bert_L-24_wmt_de_en).\n",
        "- *RoBERTaShared (large)* - *CNN/Dailymail*: [google/roberta2roberta_L-24_cnn_daily_mail](https://huggingface.co/google/roberta2roberta_L-24_cnn_daily_mail).\n",
        "- *RoBERTaShared (large)* - *BBC XSum*: [google/roberta2roberta_L-24_bbc](https://huggingface.co/google/roberta2roberta_L-24_bbc).\n",
        "- *RoBERTaShared (large)* - *Gigaword*: [google/roberta2roberta_L-24_gigaword](https://huggingface.co/google/roberta2roberta_L-24_gigaword).\n",
        "\n",
        "---\n",
        "\n",
        "${}^1$ To retrieve BLEU-4 scores, a script from the Tensorflow Official Transformer implementation\n",
        "https://github.com/tensorflow/models/tree\n",
        "master/official/nlp/transformer was used. Note\n",
        "that, differently from the tensor2tensor/utils/\n",
        "`get_ende_bleu.sh` used by Vaswani et al. (2017), this\n",
        "script does not split noun compounds, but utf-8\n",
        "quotes were normalized to ascii quotes after having noted that the pre-processed\n",
        "training set contains only ascii quotes.\n",
        "\n",
        "${}^2$ Model capacity is an informal definition of how good the model is at modeling complex patterns. It is also sometimes defined as *the ability of a model to learn from more and more data*. Model capacity is broadly measured by the number of trainable parameters - the more parameters, the higher the model capacity."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H1F58j028eTV"
      },
      "source": [
        "## **Warm-starting encoder-decoder models with 🤗Transformers (Practice)**\n",
        "\n",
        "We have explained the theory of warm-starting encoder-decoder models, analyzed empirical results on multiple datasets, and have derived practical conclusions. Let's now walk through a complete code example showcasing how a **BERT2BERT** model can be warm-started and consequently fine-tuned on the *CNN/Dailymail* summarization task. We will be leveraging the 🤗datasets and 🤗Transformers libraries.\n",
        "\n",
        "In addition, the following list provides a condensed version of this and other notebooks on warm-starting other combinations of encoder-decoder models. \n",
        "\n",
        "- for **BERT2BERT** on *CNN/Dailymail* (a condensed version of this notebook), click [here](https://colab.research.google.com/drive/1Ekd5pUeCX7VOrMx94_czTkwNtLN32Uyu?usp=sharing).\n",
        "- for **RoBERTaShare** on *BBC XSum*, click [here](https://colab.research.google.com/drive/1vHZHXOCFqOXIvdsF8j4WBRaAOAjAroTi?usp=sharing).\n",
        "- for **BERT2Rnd** on *WMT14 En $\\to$ De*, click [here]().\n",
        "- for **RoBERTa2GPT2** on *DiscoFuse*, click [here]().\n",
        "\n",
        "***Note***: This notebook only uses a few training, validation, and test data samples for demonstration purposes. To fine-tune an encoder-decoder model on the full training data, the user should change the training and data preprocessing parameters accordingly as highlighted by the comments.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3FO5ESocXvlK"
      },
      "source": [
        "### **Data Preprocessing**\n",
        "\n",
        "In this section, we show how the data can be pre-processed for training. More importantly, we try to give the reader some insight into the process of deciding how to preprocess the data.\n",
        "\n",
        "We will need datasets and transformers to be installed."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w67vkz3KP9eZ"
      },
      "source": [
        "%%capture\n",
        "!pip install datasets==1.0.2\n",
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "igRJfr-tINdv"
      },
      "source": [
        "Let's start by downloading the *CNN/Dailymail* dataset."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "scAYCFJ-IGZ_",
        "outputId": "64e0036e-1f49-478d-84ac-1499a778a73e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423,
          "referenced_widgets": [
            "9a4bcf9eaa6f4f60a3c3a9cd0fd88194",
            "a1be871fd0774aa39230b11e35ebb0a6",
            "5d186df7b2e04aac8624674e90ebec34",
            "43dac6ef55c144d9ba55467df614c229",
            "d84b7e47a3c24ff9b5683840c819cb0f",
            "8fefa370353c4b3c8d1f94eb4576e582",
            "82b204b26f164f86812dab500787c875",
            "237322dff7d345899c1db1484a2c044a",
            "e05188a37e784a7d8041afd7136c6054",
            "cef4d9d182074dafaf5ed6f60590e6f8",
            "45599cc93c4747868dd71f97b457dbbe",
            "540abae6193b4f99825065ce2278cde4",
            "dc069cab9cf341b1b40680e526afd13c",
            "1e7cfcf4470d4d14a3a573b488792cf4",
            "6144b64f4a444d56ab4dd31a449d1cbc",
            "f3e30f37b4da4492becdffebe4bfcef1",
            "7859a2b3e52345c186ae5b6aa4ba0c36",
            "cb9b100de0e44e53a808590d6d34826e",
            "c9ff75060f20493e8bd15bb15e6dbb4f",
            "d69756f059dd4f9a94c25661a4824249",
            "24eeae5c3e8d4df0bb50d7fd9b211ea9",
            "f2ca62ce391a40debfbb77f1e0dd0c36",
            "47e3012f5c0345d79f0f19444c6d8803",
            "0ff2285796f84eb9b12fd30c1d5d875a",
            "6990216adbba46ffac1fdec6b546d407",
            "8c7cdecd21d54435b883e7a088ef6f5a",
            "60673f89a61644afb85f4b9862348432",
            "23c9e0da40774a0cbba440c3a9eb3567",
            "f47f42c2da1c4604ad91def09f9eec73",
            "1d595dca7d3e4269b66493196f955c61",
            "6661ea196016443ca00c49754e110fdc",
            "8e9a979f544c4ca985ccd337ccb074b3",
            "56c9f2696c744b26879300c152685456",
            "d2cb5034aee6400da27febf14d10b948",
            "7174b9ae437d4ee4a9e2e849c977ff9a",
            "1bc7d7ed8a1b495b9b14f70c8ee3bfba",
            "631c7c56beb94443a8256be8e155abcc",
            "a7372225873a4671b007cf53e43a8c1b",
            "28b7153fc1f844d69706b9a27f47c06f",
            "0451f5a139d743c79904ea69262cff2b",
            "5e642e1989c14140987ace2780d01c85",
            "c598b1d4de1b46159e0ab6d41f593573",
            "b3788d1393bc46c4b667d3eac298e2c3",
            "f92c1350fdcd4694a5d7d8a4f97522f1",
            "05c67e2a0f554b03a260ada6e8d0bf73",
            "1a7492549fbc4335925ed4e4c890a10f",
            "4ede6881ab1d46598e895b9e35036163",
            "6f15d47ce5a84e74a2e38e5b1adc63d7",
            "d63220e55e03451f8ef8d87fb1a3c475",
            "07c8d79de4524a128c90662a6769bbd4",
            "34e25bcf15d8406abfacaaf271d7a2dd",
            "27997ec8b6e94ca2825179ea7f39dce9",
            "017cb45b81b145a2a5615c65337d0881",
            "285519bebc544384bc373839ff71ef56",
            "40e5138495d543428ce291927b118be7",
            "44e96cb737b94f79b9fe7ec01e9fe20a",
            "4005f402528243aa999facc2bb3d335d",
            "d4f55df76c404287b0277e898203e7a8",
            "5e4c739ff55945e48124c0379f4d9d95",
            "32a3578a1ed54aed9f0a5c58dd18e58c",
            "9b7c6e3be93a4dc0bb26119a585f17fc",
            "6be70c64f32b4c09b188a46be8557207",
            "3010957ac5b94132b4fbbad52ed835a4",
            "1adbedddf59b446bac4886968aabb1a0",
            "9d25c4a8783d4b27a49371d85cfad10f",
            "5be7b3e47c0640538826beaf4a25a580",
            "593988c055184050a3a35b3ca5add45e",
            "471125eb87fd41d890a97c9db3241b4c",
            "670469b341234e9bb11eb602e7710d36",
            "ea72a63d8e4e4e3dbf8637db14c6f19e",
            "8282529cfee24d909611b124faeaece4",
            "eab9187f72ea4ea5837b91030fe11bb0",
            "e612249e01bf4cd2b432685b749870dd",
            "c26e2f8f192a4abdb1d01630c5e69108",
            "f434739f47bc4314b0f4bda009ebf933",
            "ca2d9092ae564a96b04a1d374dd2a92c",
            "1c7d6b86b5c3411b82d043042469347f",
            "0dd7c6df3edb489e97bc5a4b6c5c4472",
            "c9ba3af29928460fa69c3c118dbee14d",
            "cee91c0dedfa43cab6f16e6876bc06a2"
          ]
        }
      },
      "source": [
        "import datasets\n",
        "train_data = datasets.load_dataset(\"cnn_dailymail\", \"3.0.0\", split=\"train\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9a4bcf9eaa6f4f60a3c3a9cd0fd88194",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=3711.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e05188a37e784a7d8041afd7136c6054",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1690.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Downloading and preparing dataset cnn_dailymail/3.0.0 (download: 558.32 MiB, generated: 1.28 GiB, post-processed: Unknown size, total: 1.82 GiB) to /root/.cache/huggingface/datasets/cnn_dailymail/3.0.0/3.0.0/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7859a2b3e52345c186ae5b6aa4ba0c36",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Downloading', max=1.0, style=ProgressSt…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6990216adbba46ffac1fdec6b546d407",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Downloading', max=1.0, style=ProgressSt…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "56c9f2696c744b26879300c152685456",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=626592.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5e642e1989c14140987ace2780d01c85",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=13463354.0, style=ProgressStyle(descrip…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d63220e55e03451f8ef8d87fb1a3c475",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=725406.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4005f402528243aa999facc2bb3d335d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9d25c4a8783d4b27a49371d85cfad10f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e612249e01bf4cd2b432685b749870dd",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\rDataset cnn_dailymail downloaded and prepared to /root/.cache/huggingface/datasets/cnn_dailymail/3.0.0/3.0.0/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602. Subsequent calls will reuse this data.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C3lywsK4TXV5"
      },
      "source": [
        "Alright, let's get a first impression of the dataset.\n",
        "Alternatively, the dataset can also be visualized using the awesome [datasets viewer](https://huggingface.co/nlp/viewer/?dataset=cnn_dailymail&config=3.0.0) online.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tQiACfE3Z3yw",
        "outputId": "0dd80140-be43-47f3-cfb0-398fe96c9f1a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "train_data.info.description"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'CNN/DailyMail non-anonymized summarization dataset.\\n\\nThere are two features:\\n  - article: text of news article, used as the document to be summarized\\n  - highlights: joined text of highlights with <s> and </s> around each\\n    highlight, which is the target summary\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yDota3xcZ8EH"
      },
      "source": [
        "Our input is called *article* and our labels are called *highlights*. Let's now print out the first example of the training data to get a feeling for the data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lMbFZz9LTVSf",
        "outputId": "cb981aa4-5669-45b0-e7c7-0ffec82db076",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 709
        }
      },
      "source": [
        "import pandas as pd\n",
        "from IPython.display import display, HTML\n",
        "from datasets import ClassLabel\n",
        "\n",
        "df = pd.DataFrame(train_data[:1])\n",
        "del df[\"id\"]\n",
        "for column, typ in train_data.features.items():\n",
        "      if isinstance(typ, ClassLabel):\n",
        "          df[column] = df[column].transform(lambda i: typ.names[i])\n",
        "display(HTML(df.to_html()))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>article</th>\n",
              "      <th>highlights</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>It's official: U.S. President Barack Obama wants lawmakers to weigh in on whether to use military force in Syria. Obama sent a letter to the heads of the House and Senate on Saturday night, hours after announcing that he believes military action against Syrian targets is the right step to take over the alleged use of chemical weapons. The proposed legislation from Obama asks Congress to approve the use of military force \"to deter, disrupt, prevent and degrade the potential for future uses of chemical weapons or other weapons of mass destruction.\" It's a step that is set to turn an international crisis into a fierce domestic political battle. There are key questions looming over the debate: What did U.N. weapons inspectors find in Syria? What happens if Congress votes no? And how will the Syrian government react? In a televised address from the White House Rose Garden earlier Saturday, the president said he would take his case to Congress, not because he has to -- but because he wants to. \"While I believe I have the authority to carry out this military action without specific congressional authorization, I know that the country will be stronger if we take this course, and our actions will be even more effective,\" he said. \"We should have this debate, because the issues are too big for business as usual.\" Obama said top congressional leaders had agreed to schedule a debate when the body returns to Washington on September 9. The Senate Foreign Relations Committee will hold a hearing over the matter on Tuesday, Sen. Robert Menendez said. Transcript: Read Obama's full remarks . Syrian crisis: Latest developments . U.N. inspectors leave Syria . Obama's remarks came shortly after U.N. inspectors left Syria, carrying evidence that will determine whether chemical weapons were used in an attack early last week in a Damascus suburb. \"The aim of the game here, the mandate, is very clear -- and that is to ascertain whether chemical weapons were used -- and not by whom,\" U.N. spokesman Martin Nesirky told reporters on Saturday. But who used the weapons in the reported toxic gas attack in a Damascus suburb on August 21 has been a key point of global debate over the Syrian crisis. Top U.S. officials have said there's no doubt that the Syrian government was behind it, while Syrian officials have denied responsibility and blamed jihadists fighting with the rebels. British and U.S. intelligence reports say the attack involved chemical weapons, but U.N. officials have stressed the importance of waiting for an official report from inspectors. The inspectors will share their findings with U.N. Secretary-General Ban Ki-moon Ban, who has said he wants to wait until the U.N. team's final report is completed before presenting it to the U.N. Security Council. The Organization for the Prohibition of Chemical Weapons, which nine of the inspectors belong to, said Saturday that it could take up to three weeks to analyze the evidence they collected. \"It needs time to be able to analyze the information and the samples,\" Nesirky said. He noted that Ban has repeatedly said there is no alternative to a political solution to the crisis in Syria, and that \"a military solution is not an option.\" Bergen:  Syria is a problem from hell for the U.S. Obama: 'This menace must be confronted' Obama's senior advisers have debated the next steps to take, and the president's comments Saturday came amid mounting political pressure over the situation in Syria. Some U.S. lawmakers have called for immediate action while others warn of stepping into what could become a quagmire. Some global leaders have expressed support, but the British Parliament's vote against military action earlier this week was a blow to Obama's hopes of getting strong backing from key NATO allies. On Saturday, Obama proposed what he said would be a limited military action against Syrian President Bashar al-Assad. Any military attack would not be open-ended or include U.S. ground forces, he said. Syria's alleged use of chemical weapons earlier this month \"is an assault on human dignity,\" the president said. A failure to respond with force, Obama argued,  \"could lead to escalating use of chemical weapons or their proliferation to terrorist groups who would do our people harm. In a world with many dangers, this menace must be confronted.\" Syria missile strike: What would happen next? Map: U.S. and allied assets around Syria . Obama decision came Friday night . On Friday night, the president made a last-minute decision to consult lawmakers. What will happen if they vote no? It's unclear. A senior administration official told CNN that Obama has the authority to act without Congress -- even if Congress rejects his request for authorization to use force. Obama on Saturday continued to shore up support for a strike on the al-Assad government. He spoke by phone with French President Francois Hollande before his Rose Garden speech. \"The two leaders agreed that the international community must deliver a resolute message to the Assad regime -- and others who would consider using chemical weapons -- that these crimes are unacceptable and those who violate this international norm will be held accountable by the world,\" the White House said. Meanwhile, as uncertainty loomed over how Congress would weigh in, U.S. military officials said they remained at the ready. 5 key assertions: U.S. intelligence report on Syria . Syria: Who wants what after chemical weapons horror . Reactions mixed to Obama's speech . A spokesman for the Syrian National Coalition said that the opposition group was disappointed by Obama's announcement. \"Our fear now is that the lack of action could embolden the regime and they repeat his attacks in a more serious way,\" said spokesman Louay Safi. \"So we are quite concerned.\" Some members of Congress applauded Obama's decision. House Speaker John Boehner, Majority Leader Eric Cantor, Majority Whip Kevin McCarthy and Conference Chair Cathy McMorris Rodgers issued a statement Saturday praising the president. \"Under the Constitution, the responsibility to declare war lies with Congress,\" the Republican lawmakers said. \"We are glad the president is seeking authorization for any military action in Syria in response to serious, substantive questions being raised.\" More than 160 legislators, including 63 of Obama's fellow Democrats, had signed letters calling for either a vote or at least a \"full debate\" before any U.S. action. British Prime Minister David Cameron, whose own attempt to get lawmakers in his country to support military action in Syria failed earlier this week, responded to Obama's speech in a Twitter post Saturday. \"I understand and support Barack Obama's position on Syria,\" Cameron said. An influential lawmaker in Russia -- which has stood by Syria and criticized the United States -- had his own theory. \"The main reason Obama is turning to the Congress:  the military operation did not get enough support either in the world, among allies of the US or in the United States itself,\" Alexei Pushkov, chairman of the international-affairs committee of the Russian State Duma, said in a Twitter post. In the United States, scattered groups of anti-war protesters around the country took to the streets Saturday. \"Like many other Americans...we're just tired of the United States getting involved and invading and bombing other countries,\" said Robin Rosecrans, who was among hundreds at a Los Angeles demonstration. What do Syria's neighbors think? Why Russia, China, Iran stand by Assad . Syria's government unfazed . After Obama's speech, a military and political analyst on Syrian state TV said Obama is \"embarrassed\" that Russia opposes military action against Syria, is \"crying for help\" for someone to come to his rescue and is facing two defeats -- on the political and military levels. Syria's prime minister appeared unfazed by the saber-rattling. \"The Syrian Army's status is on maximum readiness and fingers are on the trigger to confront all challenges,\" Wael Nader al-Halqi said during a meeting with a delegation of Syrian expatriates from Italy, according to a banner on Syria State TV that was broadcast prior to Obama's address. An anchor on Syrian state television said Obama \"appeared to be preparing for an aggression on Syria based on repeated lies.\" A top Syrian diplomat told the state television network that Obama was facing pressure to take military action from Israel, Turkey, some Arabs and right-wing extremists in the United States. \"I think he has done well by doing what Cameron did in terms of taking the issue to Parliament,\" said Bashar Jaafari, Syria's ambassador to the United Nations. Both Obama and Cameron, he said, \"climbed to the top of the tree and don't know how to get down.\" The Syrian government has denied that it used chemical weapons in the August 21 attack, saying that jihadists fighting with the rebels used them in an effort to turn global sentiments against it. British intelligence had put the number of people killed in the attack at more than 350. On Saturday, Obama said \"all told, well over 1,000 people were murdered.\" U.S. Secretary of State John Kerry on Friday cited a death toll of 1,429, more than 400 of them children. No explanation was offered for the discrepancy. Iran: U.S. military action in Syria would spark 'disaster' Opinion: Why strikes in Syria are a bad idea .</td>\n",
              "      <td>Syrian official: Obama climbed to the top of the tree, \"doesn't know how to get down\"\\nObama sends a letter to the heads of the House and Senate .\\nObama to seek congressional approval on military action against Syria .\\nAim is to determine whether CW were used, not by whom, says U.N. spokesman .</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OsC2qcvvV18z"
      },
      "source": [
        "\n",
        "The input data seems to consist of short news articles. Interestingly, the labels appear to be bullet-point-like summaries. At this point, one should probably take a look at a couple of other examples to get a better feeling for the data.\n",
        "\n",
        "One should also notice here that the text is *case-sensitive*. This means that we have to be careful if we want to use *case-insensitive* models.\n",
        "As *CNN/Dailymail* is a summarization dataset, the model will be evaluated using the *ROUGE* metric. \n",
        "Checking the description of *ROUGE* in 🤗datasets, *cf.* [here](https://huggingface.co/metrics/rouge), we can see that the metric is *case-insensitive*, meaning that *upper case* letters will be normalized to *lower case* letters during evaluation. Thus, we can safely leverage *uncased* checkpoints, such as `bert-base-uncased`.\n",
        "\n",
        "Cool! Next, let's get a sense of the length of input data and labels. \n",
        "\n",
        "As models compute length in *token-length*, we will make use of the `bert-base-uncased` tokenizer to compute the article and summary length.\n",
        "\n",
        "First, we load the tokenizer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M0FWEkj8LThp",
        "outputId": "27fe55e1-e674-42d4-93ee-153116d24fb7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 117,
          "referenced_widgets": [
            "13491b7e71824282a09aa3d79b54429b",
            "2c82a3afff3743149bc8b694669caa76",
            "b5830cea6bb04bcdaf6b37e66e500f79",
            "2e1d4e151c0548049406c9b99a889f95",
            "12b63db1af494ccd91f04564198d41d6",
            "0c9eab6f217b4a4b85dd04d4b5bb9e57",
            "24ce686bdacc4610bb0d3454081c26ad",
            "678789bbf40742fb978b8bd9f438c22e",
            "72655e8e1ccc4319bf37ff482a0e2305",
            "271b96e5a35d489c8ff4fe3922ca9cf9",
            "8f13ee0980d64a9f9b897a5c5c7e642e",
            "8f75d80e802947b8a6116c2c7ab20550",
            "0990b53afaf34317a764d719cc361268",
            "49b8e76a66d446d8b87a669798e771ea",
            "c7b135aae8bc45219e769608ab04965a",
            "1d995b48d4094840b385a8dbe3f38745"
          ]
        }
      },
      "source": [
        "from transformers import BertTokenizerFast\n",
        "tokenizer = BertTokenizerFast.from_pretrained(\"bert-base-uncased\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "13491b7e71824282a09aa3d79b54429b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "72655e8e1ccc4319bf37ff482a0e2305",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uUNOm0AALxLA"
      },
      "source": [
        "Next, we make use of `.map()` to compute the length of the article and its summary. Since we know that the maximum length that `bert-base-uncased` can process amounts to 512, we are also interested in the percentage of input samples being longer than the maximum length.\n",
        "Similarly, we compute the percentage of summaries that are longer than 64, and 128 respectively.\n",
        "\n",
        " We can define the `.map()` function as follows."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wakECgirV1ZN"
      },
      "source": [
        "# map article and summary len to dict as well as if sample is longer than 512 tokens\n",
        "def map_to_length(x):\n",
        "  x[\"article_len\"] = len(tokenizer(x[\"article\"]).input_ids)\n",
        "  x[\"article_longer_512\"] = int(x[\"article_len\"] > tokenizer.max_len)\n",
        "  x[\"summary_len\"] = len(tokenizer(x[\"highlights\"]).input_ids)\n",
        "  x[\"summary_longer_64\"] = int(x[\"summary_len\"] > 64)\n",
        "  x[\"summary_longer_128\"] = int(x[\"summary_len\"] > 128)\n",
        "  return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GkzX1bM1N6oz"
      },
      "source": [
        "It should be sufficient to look at the first 10000 samples. We can speed up the mapping by using multiple processes with `num_proc=4`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5yAVH5fkMwLr",
        "outputId": "d04f01c0-fe58-4bfe-f156-830b74457fa1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 291,
          "referenced_widgets": [
            "ab623a8d901044789781355e887998b4",
            "4f0716d31ce7470ca6f3f30d11023a30",
            "13214334499a4be2adabcd0e3a2098ce",
            "559670ccde114edeaee350cb2bf11faf",
            "b4b4366968b845f7a7fb5e29a3ee6982",
            "7f75959ba2d644b1a7e51d867894a4e0",
            "3cadfe17c4354080b3f39c6631c42f16",
            "d3a4cc6d56984d0b9bd540de69524bd7",
            "9b8fac4e1d164201b2aee090d2808994",
            "078263580d3f4bd0b3e82b75d184f66d",
            "3d13e90a757d4442bbba7c15754f6917",
            "aea6dd73e1804da782260d948fcf1242",
            "e52a41b9784e43498621b9a9756906e5",
            "d330fc9742394baf8c0e8106d9f86e4e",
            "72ec123bd5a340eea68c2a11b5e0cada",
            "c4db0c00f00a4fd3be952c43d8eec786",
            "3aa4de28d47d4c3fac88271f391351fd",
            "5ef6ff2d5a7b48f682f48b41d73850e4",
            "8711623ff5fc4341b727b1d1e9e1005c",
            "ceb56899765a469c96713ff39d600dad",
            "69aead212e8645a8842bc5bc7a9b35fc",
            "00abeaaea3734b3f911bf897a0053384",
            "2c6c618c2b304eb89c5cb796e49da5a7",
            "5598c4414a2f4f76818cc1abbbd17d77",
            "779b46b3ddb645efafce1abeae8b4f1f",
            "f8d765bd954342f3bcfff58551bc6095",
            "d15bbef314fa4154a0fae86e3ee3205a",
            "86bfcd4e109f41e39d814c362bad93e6",
            "aa6c2bfcd4104839aea3de1db223b5f5",
            "739f97bbb6ec42fbb5e2a9df0e960d8f",
            "562880f2d6504d29b9d48c815918c7c0",
            "93102375b016495c937bbd50ba5eac8e"
          ]
        }
      },
      "source": [
        "sample_size = 10000\n",
        "data_stats = train_data.select(range(sample_size)).map(map_to_length, num_proc=4)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:1374: FutureWarning: The `max_len` attribute has been deprecated and will be removed in a future version, use `model_max_length` instead.\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "    "
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ab623a8d901044789781355e887998b4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='#0', max=2500.0, style=ProgressStyle(description_width='i…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9b8fac4e1d164201b2aee090d2808994",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='#1', max=2500.0, style=ProgressStyle(description_width='i…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3aa4de28d47d4c3fac88271f391351fd",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='#3', max=2500.0, style=ProgressStyle(description_width='i…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "779b46b3ddb645efafce1abeae8b4f1f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='#2', max=2500.0, style=ProgressStyle(description_width='i…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3eluYonnRL-c"
      },
      "source": [
        "Having computed the length for the first 10000 samples, we should now average them together. For this, we can make use of the `.map()` function with `batched=True` and `batch_size=-1` to have access to all 10000 samples within the `.map()` function."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kALWVxHlOU4A",
        "outputId": "3f3cba41-acb0-41a4-d578-d252f4c2e4dc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85,
          "referenced_widgets": [
            "7bbc19eddb6740e2a2caddb6db63c501",
            "d638f3862f8c48f8ae537b5494185c9a",
            "8dc7da1ba2d2491ab6e2b77db0eae508",
            "36b28c1b21cf4a2091a8fe2c588f4005",
            "02a6c48208c04ad185ea1b80aa5574d1",
            "d2ef3fbc86b54bb88b12a073c0d7a141",
            "0f95672d388f4f90a2d95dc84674b489",
            "4079b4184ebc4140b3d4a8cd07e74551"
          ]
        }
      },
      "source": [
        "def compute_and_print_stats(x):\n",
        "  if len(x[\"article_len\"]) == sample_size:\n",
        "    print(\n",
        "        \"Article Mean: {}, %-Articles > 512:{}, Summary Mean:{}, %-Summary > 64:{}, %-Summary > 128:{}\".format(\n",
        "            sum(x[\"article_len\"]) / sample_size,\n",
        "            sum(x[\"article_longer_512\"]) / sample_size, \n",
        "            sum(x[\"summary_len\"]) / sample_size,\n",
        "            sum(x[\"summary_longer_64\"]) / sample_size,\n",
        "            sum(x[\"summary_longer_128\"]) / sample_size,\n",
        "        )\n",
        "    )\n",
        "\n",
        "output = data_stats.map(\n",
        "  compute_and_print_stats, \n",
        "  batched=True,\n",
        "  batch_size=-1,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7bbc19eddb6740e2a2caddb6db63c501",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Article Mean: 847.6216, %-Articles > 512:0.7355, Summary Mean:57.7742, %-Summary > 64:0.3185, %-Summary > 128:0.0\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e5fW44r9uJo5"
      },
      "source": [
        "We can see that on average an article contains 848 tokens with *ca.* 3/4 of the articles being longer than the model's `max_length` 512. The summary is on average 57 tokens long. Over 30% of our 10000-sample summaries are longer than 64 tokens, but none are longer than 128 tokens.\n",
        "\n",
        "`bert-base-cased` is limited to 512 tokens, which means we would have to cut possibly important information from the article. Because most of the important information is often found at the beginning of articles and because we want to be computationally efficient, we decide to stick to `bert-base-cased` with a `max_length` of 512 in this notebook. This choice is not optimal but has shown to yield [good results](https://arxiv.org/abs/1907.12461) on CNN/Dailymail. Alternatively, one could leverage long-range sequence models, such as [Longformer](https://huggingface.co/allenai/longformer-large-4096) to be used as the encoder.\n",
        "\n",
        "Regarding the summary length, we can see that a length of 128 already includes all of the summary labels. 128 is easily within the limits of `bert-base-cased`, so we decide to limit the generation to 128.\n",
        "\n",
        "Again, we will make use of the `.map()` function - this time to transform each training batch into a batch of model inputs.\n",
        "\n",
        "`\"article\"` and `\"highlights\"` are tokenized and prepared as the Encoder's `\"input_ids\"` and Decoder's `\"decoder_input_ids\"` respectively.\n",
        "\n",
        "`\"labels\"` are shifted automatically to the left for language modeling training.\n",
        "\n",
        "Lastly, it is very important to remember to ignore the loss of the padded labels. In 🤗Transformers this can be done by setting the label to -100. Great, let's write down our mapping function then."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yoN2q0hZUbXN"
      },
      "source": [
        "encoder_max_length=512\n",
        "decoder_max_length=128\n",
        "\n",
        "def process_data_to_model_inputs(batch):\n",
        "  # tokenize the inputs and labels\n",
        "  inputs = tokenizer(batch[\"article\"], padding=\"max_length\", truncation=True, max_length=encoder_max_length)\n",
        "  outputs = tokenizer(batch[\"highlights\"], padding=\"max_length\", truncation=True, max_length=decoder_max_length)\n",
        "\n",
        "  batch[\"input_ids\"] = inputs.input_ids\n",
        "  batch[\"attention_mask\"] = inputs.attention_mask\n",
        "  batch[\"decoder_input_ids\"] = outputs.input_ids\n",
        "  batch[\"decoder_attention_mask\"] = outputs.attention_mask\n",
        "  batch[\"labels\"] = outputs.input_ids.copy()\n",
        "\n",
        "  # because BERT automatically shifts the labels, the labels correspond exactly to `decoder_input_ids`. \n",
        "  # We have to make sure that the PAD token is ignored\n",
        "  batch[\"labels\"] = [[-100 if token == tokenizer.pad_token_id else token for token in labels] for labels in batch[\"labels\"]]\n",
        "\n",
        "  return batch"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oxeMUmjxbqdP"
      },
      "source": [
        "In this notebook, we train and evaluate the model just on a few training examples for demonstration and set the `batch_size` to 4 to prevent out-of-memory issues.  \n",
        "\n",
        "The following line reduces the training data to only the first `32` examples. The cell can be commented out or not run for a full training run. Good results were obtained with a `batch_size` of 16."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rQ2nmUCRbzxY"
      },
      "source": [
        "train_data = train_data.select(range(32))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tYvdujyfro1i"
      },
      "source": [
        "Alright, let's prepare the training data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z396M38-beJE",
        "outputId": "c9251a0b-2e65-4df0-de5c-50eca3204632",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "67b48f2dabe0455293ff6ff51dfee2ed",
            "9ab45cb1bc4b42ed9682e6ef4c8eac13",
            "bbbbe2f83efe42b7b65c404cfb559aa1",
            "2278f2e4c00546ab8a9616745c54f188",
            "fb5a37123a2a435a83196fda0449bfa7",
            "604569ab53d641cf8d3a88651c89c791",
            "c6d3e9577f3145dab6b29fd6d882455d",
            "0b41e8a1e3614f54b653f471cc309ad8"
          ]
        }
      },
      "source": [
        "# batch_size = 16\n",
        "batch_size=4\n",
        "\n",
        "train_data = train_data.map(\n",
        "    process_data_to_model_inputs, \n",
        "    batched=True, \n",
        "    batch_size=batch_size, \n",
        "    remove_columns=[\"article\", \"highlights\", \"id\"]\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "67b48f2dabe0455293ff6ff51dfee2ed",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=8.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YN4Qx8-lr2m7"
      },
      "source": [
        "Taking a look at the processed training dataset we can see that the column names `article`, `highlights`, and `id` have been replaced by the arguments expected by the `EncoderDecoderModel`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GFBL22RxsN1Q",
        "outputId": "65ffbac4-df8f-41ba-8b55-df21b691fd9b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "train_data"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset(features: {'attention_mask': Sequence(feature=Value(dtype='int64', id=None), length=-1, id=None), 'decoder_attention_mask': Sequence(feature=Value(dtype='int64', id=None), length=-1, id=None), 'decoder_input_ids': Sequence(feature=Value(dtype='int64', id=None), length=-1, id=None), 'input_ids': Sequence(feature=Value(dtype='int64', id=None), length=-1, id=None), 'labels': Sequence(feature=Value(dtype='int64', id=None), length=-1, id=None)}, num_rows: 32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lL9HI9E3sPRB"
      },
      "source": [
        "So far, the data was manipulated using Python's `List` format. Let's convert the data to PyTorch Tensors to be trained on GPU."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-vZKuZVvqxlC"
      },
      "source": [
        "train_data.set_format(\n",
        "    type=\"torch\", columns=[\"input_ids\", \"attention_mask\", \"decoder_input_ids\", \"decoder_attention_mask\", \"labels\"],\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EPCjQ5E6Lg89"
      },
      "source": [
        "Awesome, the data processing of the training data is finished. \n",
        "Analogous, we can do the same for the validation data.\n",
        "\n",
        "First, we load 10% of the validation dataset:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lB-bX6qktl1i",
        "outputId": "24db08a6-59c2-4d13-8a42-4e854617a298",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "val_data = datasets.load_dataset(\"cnn_dailymail\", \"3.0.0\", split=\"validation[:10%]\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reusing dataset cnn_dailymail (/root/.cache/huggingface/datasets/cnn_dailymail/3.0.0/3.0.0/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ktlQfG-YtrG2"
      },
      "source": [
        "For demonstration purposes, the validation data is then reduced to just 8 samples,"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c0_fTpFTtxSX"
      },
      "source": [
        "val_data = val_data.select(range(8))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CLXcp_rJtzbt"
      },
      "source": [
        "the mapping function is applied,"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZIYccglk7FYj",
        "outputId": "2d3d0c33-e61a-4f1d-d883-425255aba0c5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "b0b66fbc1c044bb79bce9642df312005",
            "6b86ca19dbcb490c84b1e18918ff2655",
            "070d62e3fbab456ea931f688339ef2e9",
            "b06505ffcfc8456993310dddc90a3ff3",
            "01b075a2aafa4e2aaabe8096fe9848cf",
            "9323291a5c954f9992fe55789a35eebf",
            "4e1d64901cf14b688c075abb4a15537b",
            "76e86da8909540dcbaf6bd83a4c6c3ac"
          ]
        }
      },
      "source": [
        "val_data = val_data.map(\n",
        "    process_data_to_model_inputs, \n",
        "    batched=True, \n",
        "    batch_size=batch_size, \n",
        "    remove_columns=[\"article\", \"highlights\", \"id\"]\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b0b66fbc1c044bb79bce9642df312005",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=2.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ovXI_tnDt5dU"
      },
      "source": [
        "and, finally, the validation data is also converted to PyTorch tensors."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6r2-M5hYt-Vw"
      },
      "source": [
        "val_data.set_format(\n",
        "    type=\"torch\", columns=[\"input_ids\", \"attention_mask\", \"decoder_input_ids\", \"decoder_attention_mask\", \"labels\"],\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1WRG1QRwMUVd"
      },
      "source": [
        "Great! Now we can move to warm-starting the `EncoderDecoderModel`."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aEjb026cNC38"
      },
      "source": [
        "### **Warm-starting the Encoder-Decoder Model**\n",
        "\n",
        "This section explains how an Encoder-Decoder model can be warm-started using the `bert-base-cased` checkpoint.\n",
        "\n",
        "Let's start by importing the `EncoderDecoderModel`. For more detailed information about the `EncoderDecoderModel` class, the reader is advised to take a look at the [documentation](https://huggingface.co/transformers/model_doc/encoderdecoder.html)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kRwMBv0hNMNM"
      },
      "source": [
        "from transformers import EncoderDecoderModel"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SpqSPjsLe1uL"
      },
      "source": [
        "In contrast to other model classes in 🤗Transformers, the `EncoderDecoderModel` class has two methods to load pre-trained weights, namely:\n",
        "\n",
        "1. the \"standard\" `.from_pretrained(...)` method is derived from the general `PretrainedModel.from_pretrained(...)` method and thus corresponds exactly to the the one of other model classes. The function expects a single model identifier, *e.g.* `.from_pretrained(\"google/bert2bert_L-24_wmt_de_en\")` and will load a single `.pt` checkpoint file into the `EncoderDecoderModel` class.\n",
        "\n",
        "2. a special `.from_encoder_decoder_pretrained(...)` method, which can be used to warm-start an encoder-decoder model from two model identifiers - one for the encoder and one for the decoder. The first model identifier is thereby used to load the *encoder*, via `AutoModel.from_pretrained(...)` (see doc [here](https://huggingface.co/transformers/master/model_doc/auto.html?highlight=automodel#automodel)) and the second model identifier is used to load the *decoder* via `AutoModelForCausalLM` (see doc [here](https://huggingface.co/transformers/master/model_doc/auto.html#automodelforcausallm).\n",
        "\n",
        "Alright, let's warm-start our *BERT2BERT* model. As mentioned earlier we will warm-start both the encoder and decoder with the `\"bert-base-cased\"` checkpoint."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tS0UndNoQh8t",
        "outputId": "4031f7f7-c444-4a34-ada2-97879fffd18f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 227,
          "referenced_widgets": [
            "c459ef7385394e8b9727e52af8ba14e8",
            "d0f4cd37d0454eb3b532940c8bd29e6b",
            "fceacd2050714f37898dccc96e3615b6",
            "502370e87e3c4c048768d15633e3edfc",
            "4727afce4d0b405e8b2ddc18709a2a25",
            "3031118e0790487db4d9cdee7c6a60de",
            "49a36c8bebc94064b19631318bf01cf0",
            "d6ed9aaf2ac94e2da3fb94e807732ea8",
            "fe2ab4e1e8984425930cdae883be5951",
            "601473895b044099bdb5e189bdbcf165",
            "872fca6183524208acf8959926d2b3c6",
            "9428736d1d4c46a498c7b79eb816c06a",
            "29f3be857e7e4987bc546f1424204841",
            "315b2b27d17648bcaa68f0ec0070ad86",
            "817a5c1e27864737b7407c38c461f3f8",
            "5e85f88a9eca4b8f9314ea5ae3d35944"
          ]
        }
      },
      "source": [
        "bert2bert = EncoderDecoderModel.from_encoder_decoder_pretrained(\"bert-base-uncased\", \"bert-base-uncased\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c459ef7385394e8b9727e52af8ba14e8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fe2ab4e1e8984425930cdae883be5951",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertLMHeadModel: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertLMHeadModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
            "- This IS NOT expected if you are initializing BertLMHeadModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertLMHeadModel were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['bert.encoder.layer.0.crossattention.self.query.weight', 'bert.encoder.layer.0.crossattention.self.query.bias', 'bert.encoder.layer.0.crossattention.self.key.weight', 'bert.encoder.layer.0.crossattention.self.key.bias', 'bert.encoder.layer.0.crossattention.self.value.weight', 'bert.encoder.layer.0.crossattention.self.value.bias', 'bert.encoder.layer.0.crossattention.output.dense.weight', 'bert.encoder.layer.0.crossattention.output.dense.bias', 'bert.encoder.layer.0.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.0.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.1.crossattention.self.query.weight', 'bert.encoder.layer.1.crossattention.self.query.bias', 'bert.encoder.layer.1.crossattention.self.key.weight', 'bert.encoder.layer.1.crossattention.self.key.bias', 'bert.encoder.layer.1.crossattention.self.value.weight', 'bert.encoder.layer.1.crossattention.self.value.bias', 'bert.encoder.layer.1.crossattention.output.dense.weight', 'bert.encoder.layer.1.crossattention.output.dense.bias', 'bert.encoder.layer.1.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.1.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.2.crossattention.self.query.weight', 'bert.encoder.layer.2.crossattention.self.query.bias', 'bert.encoder.layer.2.crossattention.self.key.weight', 'bert.encoder.layer.2.crossattention.self.key.bias', 'bert.encoder.layer.2.crossattention.self.value.weight', 'bert.encoder.layer.2.crossattention.self.value.bias', 'bert.encoder.layer.2.crossattention.output.dense.weight', 'bert.encoder.layer.2.crossattention.output.dense.bias', 'bert.encoder.layer.2.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.2.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.3.crossattention.self.query.weight', 'bert.encoder.layer.3.crossattention.self.query.bias', 'bert.encoder.layer.3.crossattention.self.key.weight', 'bert.encoder.layer.3.crossattention.self.key.bias', 'bert.encoder.layer.3.crossattention.self.value.weight', 'bert.encoder.layer.3.crossattention.self.value.bias', 'bert.encoder.layer.3.crossattention.output.dense.weight', 'bert.encoder.layer.3.crossattention.output.dense.bias', 'bert.encoder.layer.3.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.3.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.4.crossattention.self.query.weight', 'bert.encoder.layer.4.crossattention.self.query.bias', 'bert.encoder.layer.4.crossattention.self.key.weight', 'bert.encoder.layer.4.crossattention.self.key.bias', 'bert.encoder.layer.4.crossattention.self.value.weight', 'bert.encoder.layer.4.crossattention.self.value.bias', 'bert.encoder.layer.4.crossattention.output.dense.weight', 'bert.encoder.layer.4.crossattention.output.dense.bias', 'bert.encoder.layer.4.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.4.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.5.crossattention.self.query.weight', 'bert.encoder.layer.5.crossattention.self.query.bias', 'bert.encoder.layer.5.crossattention.self.key.weight', 'bert.encoder.layer.5.crossattention.self.key.bias', 'bert.encoder.layer.5.crossattention.self.value.weight', 'bert.encoder.layer.5.crossattention.self.value.bias', 'bert.encoder.layer.5.crossattention.output.dense.weight', 'bert.encoder.layer.5.crossattention.output.dense.bias', 'bert.encoder.layer.5.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.5.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.6.crossattention.self.query.weight', 'bert.encoder.layer.6.crossattention.self.query.bias', 'bert.encoder.layer.6.crossattention.self.key.weight', 'bert.encoder.layer.6.crossattention.self.key.bias', 'bert.encoder.layer.6.crossattention.self.value.weight', 'bert.encoder.layer.6.crossattention.self.value.bias', 'bert.encoder.layer.6.crossattention.output.dense.weight', 'bert.encoder.layer.6.crossattention.output.dense.bias', 'bert.encoder.layer.6.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.6.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.7.crossattention.self.query.weight', 'bert.encoder.layer.7.crossattention.self.query.bias', 'bert.encoder.layer.7.crossattention.self.key.weight', 'bert.encoder.layer.7.crossattention.self.key.bias', 'bert.encoder.layer.7.crossattention.self.value.weight', 'bert.encoder.layer.7.crossattention.self.value.bias', 'bert.encoder.layer.7.crossattention.output.dense.weight', 'bert.encoder.layer.7.crossattention.output.dense.bias', 'bert.encoder.layer.7.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.7.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.8.crossattention.self.query.weight', 'bert.encoder.layer.8.crossattention.self.query.bias', 'bert.encoder.layer.8.crossattention.self.key.weight', 'bert.encoder.layer.8.crossattention.self.key.bias', 'bert.encoder.layer.8.crossattention.self.value.weight', 'bert.encoder.layer.8.crossattention.self.value.bias', 'bert.encoder.layer.8.crossattention.output.dense.weight', 'bert.encoder.layer.8.crossattention.output.dense.bias', 'bert.encoder.layer.8.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.8.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.9.crossattention.self.query.weight', 'bert.encoder.layer.9.crossattention.self.query.bias', 'bert.encoder.layer.9.crossattention.self.key.weight', 'bert.encoder.layer.9.crossattention.self.key.bias', 'bert.encoder.layer.9.crossattention.self.value.weight', 'bert.encoder.layer.9.crossattention.self.value.bias', 'bert.encoder.layer.9.crossattention.output.dense.weight', 'bert.encoder.layer.9.crossattention.output.dense.bias', 'bert.encoder.layer.9.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.9.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.10.crossattention.self.query.weight', 'bert.encoder.layer.10.crossattention.self.query.bias', 'bert.encoder.layer.10.crossattention.self.key.weight', 'bert.encoder.layer.10.crossattention.self.key.bias', 'bert.encoder.layer.10.crossattention.self.value.weight', 'bert.encoder.layer.10.crossattention.self.value.bias', 'bert.encoder.layer.10.crossattention.output.dense.weight', 'bert.encoder.layer.10.crossattention.output.dense.bias', 'bert.encoder.layer.10.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.10.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.11.crossattention.self.query.weight', 'bert.encoder.layer.11.crossattention.self.query.bias', 'bert.encoder.layer.11.crossattention.self.key.weight', 'bert.encoder.layer.11.crossattention.self.key.bias', 'bert.encoder.layer.11.crossattention.self.value.weight', 'bert.encoder.layer.11.crossattention.self.value.bias', 'bert.encoder.layer.11.crossattention.output.dense.weight', 'bert.encoder.layer.11.crossattention.output.dense.bias', 'bert.encoder.layer.11.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.11.crossattention.output.LayerNorm.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hMkM5q7Ehrdk"
      },
      "source": [
        "For once, we should take a good look at the warning here. \n",
        "We can see that two weights corresponding to a `\"cls\"` layer were not used. This should not be a problem because we don't need BERT's CLS layer for *sequence-to-sequence* tasks. \n",
        "Also, we notice that a lot of weights are \"newly\" or randomly initialized. When taking a closer look these weights all correspond to the cross-attention layer, which is exactly what we would expect after having read the theory above.\n",
        "\n",
        "Let's take a closer look at the model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BiCX7C0zxjAp",
        "outputId": "80b233b4-ce08-457c-c2df-a23dac788e48",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "bert2bert"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "EncoderDecoderModel(\n",
              "  (encoder): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (decoder): BertLMHeadModel(\n",
              "    (bert): BertModel(\n",
              "      (embeddings): BertEmbeddings(\n",
              "        (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "        (position_embeddings): Embedding(512, 768)\n",
              "        (token_type_embeddings): Embedding(2, 768)\n",
              "        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (encoder): BertEncoder(\n",
              "        (layer): ModuleList(\n",
              "          (0): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (crossattention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (1): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (crossattention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (2): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (crossattention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (3): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (crossattention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (4): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (crossattention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (5): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (crossattention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (6): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (crossattention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (7): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (crossattention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (8): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (crossattention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (9): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (crossattention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (10): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (crossattention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (11): BertLayer(\n",
              "            (attention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (crossattention): BertAttention(\n",
              "              (self): BertSelfAttention(\n",
              "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (output): BertSelfOutput(\n",
              "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "            (intermediate): BertIntermediate(\n",
              "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            )\n",
              "            (output): BertOutput(\n",
              "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (cls): BertOnlyMLMHead(\n",
              "      (predictions): BertLMPredictionHead(\n",
              "        (transform): BertPredictionHeadTransform(\n",
              "          (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        )\n",
              "        (decoder): Linear(in_features=768, out_features=30522, bias=True)\n",
              "      )\n",
              "    )\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5wRcnvzXxyOB"
      },
      "source": [
        "We see that `bert2bert.encoder` is an instance of `BertModel` and that `bert2bert.decoder` one of `BertLMHeadModel`. \n",
        "However, both instances are now combined into a single `torch.nn.Module` and can thus be saved as a single `.pt` checkpoint file. \n",
        "\n",
        "Let's try it out using the standard `.save_pretrained(...)` method."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1GR3W0pOs7Na"
      },
      "source": [
        "bert2bert.save_pretrained(\"bert2bert\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y53B8J_RwoD3"
      },
      "source": [
        "Similarly, the model can be reloaded using the standard `.from_pretrained(...)` method."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fpZaCNWWwoeg"
      },
      "source": [
        "bert2bert = EncoderDecoderModel.from_pretrained(\"bert2bert\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gA8qu45YzEPz"
      },
      "source": [
        "Awesome. Let's also checkpoint the config."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8lWokpytzKNk",
        "outputId": "6b8cf1e4-ddc2-4b86-99c3-9b313bc32c82",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "bert2bert.config"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "EncoderDecoderConfig {\n",
              "  \"_name_or_path\": \"bert2bert\",\n",
              "  \"architectures\": [\n",
              "    \"EncoderDecoderModel\"\n",
              "  ],\n",
              "  \"decoder\": {\n",
              "    \"_name_or_path\": \"bert-base-uncased\",\n",
              "    \"add_cross_attention\": true,\n",
              "    \"architectures\": [\n",
              "      \"BertForMaskedLM\"\n",
              "    ],\n",
              "    \"attention_probs_dropout_prob\": 0.1,\n",
              "    \"bad_words_ids\": null,\n",
              "    \"bos_token_id\": null,\n",
              "    \"chunk_size_feed_forward\": 0,\n",
              "    \"decoder_start_token_id\": null,\n",
              "    \"do_sample\": false,\n",
              "    \"early_stopping\": false,\n",
              "    \"eos_token_id\": null,\n",
              "    \"finetuning_task\": null,\n",
              "    \"gradient_checkpointing\": false,\n",
              "    \"hidden_act\": \"gelu\",\n",
              "    \"hidden_dropout_prob\": 0.1,\n",
              "    \"hidden_size\": 768,\n",
              "    \"id2label\": {\n",
              "      \"0\": \"LABEL_0\",\n",
              "      \"1\": \"LABEL_1\"\n",
              "    },\n",
              "    \"initializer_range\": 0.02,\n",
              "    \"intermediate_size\": 3072,\n",
              "    \"is_decoder\": true,\n",
              "    \"is_encoder_decoder\": false,\n",
              "    \"label2id\": {\n",
              "      \"LABEL_0\": 0,\n",
              "      \"LABEL_1\": 1\n",
              "    },\n",
              "    \"layer_norm_eps\": 1e-12,\n",
              "    \"length_penalty\": 1.0,\n",
              "    \"max_length\": 20,\n",
              "    \"max_position_embeddings\": 512,\n",
              "    \"min_length\": 0,\n",
              "    \"model_type\": \"bert\",\n",
              "    \"no_repeat_ngram_size\": 0,\n",
              "    \"num_attention_heads\": 12,\n",
              "    \"num_beams\": 1,\n",
              "    \"num_hidden_layers\": 12,\n",
              "    \"num_return_sequences\": 1,\n",
              "    \"output_attentions\": false,\n",
              "    \"output_hidden_states\": false,\n",
              "    \"pad_token_id\": 0,\n",
              "    \"prefix\": null,\n",
              "    \"pruned_heads\": {},\n",
              "    \"repetition_penalty\": 1.0,\n",
              "    \"return_dict\": false,\n",
              "    \"sep_token_id\": null,\n",
              "    \"task_specific_params\": null,\n",
              "    \"temperature\": 1.0,\n",
              "    \"tie_encoder_decoder\": false,\n",
              "    \"tie_word_embeddings\": true,\n",
              "    \"tokenizer_class\": null,\n",
              "    \"top_k\": 50,\n",
              "    \"top_p\": 1.0,\n",
              "    \"torchscript\": false,\n",
              "    \"type_vocab_size\": 2,\n",
              "    \"use_bfloat16\": false,\n",
              "    \"use_cache\": true,\n",
              "    \"vocab_size\": 30522,\n",
              "    \"xla_device\": null\n",
              "  },\n",
              "  \"encoder\": {\n",
              "    \"_name_or_path\": \"bert-base-uncased\",\n",
              "    \"add_cross_attention\": false,\n",
              "    \"architectures\": [\n",
              "      \"BertForMaskedLM\"\n",
              "    ],\n",
              "    \"attention_probs_dropout_prob\": 0.1,\n",
              "    \"bad_words_ids\": null,\n",
              "    \"bos_token_id\": null,\n",
              "    \"chunk_size_feed_forward\": 0,\n",
              "    \"decoder_start_token_id\": null,\n",
              "    \"do_sample\": false,\n",
              "    \"early_stopping\": false,\n",
              "    \"eos_token_id\": null,\n",
              "    \"finetuning_task\": null,\n",
              "    \"gradient_checkpointing\": false,\n",
              "    \"hidden_act\": \"gelu\",\n",
              "    \"hidden_dropout_prob\": 0.1,\n",
              "    \"hidden_size\": 768,\n",
              "    \"id2label\": {\n",
              "      \"0\": \"LABEL_0\",\n",
              "      \"1\": \"LABEL_1\"\n",
              "    },\n",
              "    \"initializer_range\": 0.02,\n",
              "    \"intermediate_size\": 3072,\n",
              "    \"is_decoder\": false,\n",
              "    \"is_encoder_decoder\": false,\n",
              "    \"label2id\": {\n",
              "      \"LABEL_0\": 0,\n",
              "      \"LABEL_1\": 1\n",
              "    },\n",
              "    \"layer_norm_eps\": 1e-12,\n",
              "    \"length_penalty\": 1.0,\n",
              "    \"max_length\": 20,\n",
              "    \"max_position_embeddings\": 512,\n",
              "    \"min_length\": 0,\n",
              "    \"model_type\": \"bert\",\n",
              "    \"no_repeat_ngram_size\": 0,\n",
              "    \"num_attention_heads\": 12,\n",
              "    \"num_beams\": 1,\n",
              "    \"num_hidden_layers\": 12,\n",
              "    \"num_return_sequences\": 1,\n",
              "    \"output_attentions\": false,\n",
              "    \"output_hidden_states\": false,\n",
              "    \"pad_token_id\": 0,\n",
              "    \"prefix\": null,\n",
              "    \"pruned_heads\": {},\n",
              "    \"repetition_penalty\": 1.0,\n",
              "    \"return_dict\": false,\n",
              "    \"sep_token_id\": null,\n",
              "    \"task_specific_params\": null,\n",
              "    \"temperature\": 1.0,\n",
              "    \"tie_encoder_decoder\": false,\n",
              "    \"tie_word_embeddings\": true,\n",
              "    \"tokenizer_class\": null,\n",
              "    \"top_k\": 50,\n",
              "    \"top_p\": 1.0,\n",
              "    \"torchscript\": false,\n",
              "    \"type_vocab_size\": 2,\n",
              "    \"use_bfloat16\": false,\n",
              "    \"use_cache\": true,\n",
              "    \"vocab_size\": 30522,\n",
              "    \"xla_device\": null\n",
              "  },\n",
              "  \"is_encoder_decoder\": true,\n",
              "  \"model_type\": \"encoder_decoder\"\n",
              "}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7k95iWQyzQqG"
      },
      "source": [
        "The config is similarly composed of an encoder config and a decoder config both of which are instances of `BertConfig` in our case.\n",
        "However, the overall config is of type `EncoderDecoderConfig` and is therefore saved as a single `.json` file.\n",
        "\n",
        "In conclusion, one should remember that once an `EncoderDecoderModel` object is instantiated, it provides the same functionality as any other Encoder-Decoder model in 🤗Transformers, *e.g.* [BART](https://huggingface.co/transformers/model_doc/bart.html), [T5](https://huggingface.co/transformers/model_doc/t5.html), [ProphetNet](https://huggingface.co/transformers/model_doc/prophetnet.html), ...\n",
        "The only difference is that an `EncoderDecoderModel` provides the additional `from_encoder_decoder_pretrained(...)` function allowing the model class to be warm-started from any two encoder and decoder checkpoints."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5bPQd9-ZxjmJ"
      },
      "source": [
        "On a side-note, if one would want to create a shared encoder-decoder model, the parameter `tie_encoder_decoder=True` can additionally be passed as follows:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EZokD01chq3x",
        "outputId": "8ed15c1f-841e-421e-8d6e-0acfd293de2b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245,
          "referenced_widgets": [
            "155dc2a8d8ae49bf9ef09d9638b321b5",
            "3819c7abd556463e824b384cdd2c08c4",
            "3ef8d61808d848668991493b47924850",
            "3c0a1fd8624a4013aa2c9c72f81a2201",
            "c3997491d4cb42a9a923ff11293e3cc5",
            "eac59ca126ed4d86b9f2188f33651c60",
            "10ead3d44543440790b441baf21a152b",
            "349fc462a45c410eadfc04ff1348d86b",
            "af3d97966c114cc9a805726d43a44193",
            "db5c7f4020044d16b996332e4c4801cf",
            "81418013f14c4b969e7e8746d69e5d1a",
            "d206f8cf11524ff99817912536824743",
            "b70855e9ecd34ae18cb7f38351fc03b7",
            "8c1c013cc8ce463dbf5eaa6ca377136b",
            "004282ce3a9449c39e06c32f69ddc28e",
            "652a52c856a948909158cd2d653405a6"
          ]
        }
      },
      "source": [
        "shared_bert2bert = EncoderDecoderModel.from_encoder_decoder_pretrained(\"bert-base-cased\", \"bert-base-cased\", tie_encoder_decoder=True)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "155dc2a8d8ae49bf9ef09d9638b321b5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "af3d97966c114cc9a805726d43a44193",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=435779157.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertLMHeadModel: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertLMHeadModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
            "- This IS NOT expected if you are initializing BertLMHeadModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertLMHeadModel were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['bert.encoder.layer.0.crossattention.self.query.weight', 'bert.encoder.layer.0.crossattention.self.query.bias', 'bert.encoder.layer.0.crossattention.self.key.weight', 'bert.encoder.layer.0.crossattention.self.key.bias', 'bert.encoder.layer.0.crossattention.self.value.weight', 'bert.encoder.layer.0.crossattention.self.value.bias', 'bert.encoder.layer.0.crossattention.output.dense.weight', 'bert.encoder.layer.0.crossattention.output.dense.bias', 'bert.encoder.layer.0.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.0.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.1.crossattention.self.query.weight', 'bert.encoder.layer.1.crossattention.self.query.bias', 'bert.encoder.layer.1.crossattention.self.key.weight', 'bert.encoder.layer.1.crossattention.self.key.bias', 'bert.encoder.layer.1.crossattention.self.value.weight', 'bert.encoder.layer.1.crossattention.self.value.bias', 'bert.encoder.layer.1.crossattention.output.dense.weight', 'bert.encoder.layer.1.crossattention.output.dense.bias', 'bert.encoder.layer.1.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.1.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.2.crossattention.self.query.weight', 'bert.encoder.layer.2.crossattention.self.query.bias', 'bert.encoder.layer.2.crossattention.self.key.weight', 'bert.encoder.layer.2.crossattention.self.key.bias', 'bert.encoder.layer.2.crossattention.self.value.weight', 'bert.encoder.layer.2.crossattention.self.value.bias', 'bert.encoder.layer.2.crossattention.output.dense.weight', 'bert.encoder.layer.2.crossattention.output.dense.bias', 'bert.encoder.layer.2.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.2.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.3.crossattention.self.query.weight', 'bert.encoder.layer.3.crossattention.self.query.bias', 'bert.encoder.layer.3.crossattention.self.key.weight', 'bert.encoder.layer.3.crossattention.self.key.bias', 'bert.encoder.layer.3.crossattention.self.value.weight', 'bert.encoder.layer.3.crossattention.self.value.bias', 'bert.encoder.layer.3.crossattention.output.dense.weight', 'bert.encoder.layer.3.crossattention.output.dense.bias', 'bert.encoder.layer.3.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.3.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.4.crossattention.self.query.weight', 'bert.encoder.layer.4.crossattention.self.query.bias', 'bert.encoder.layer.4.crossattention.self.key.weight', 'bert.encoder.layer.4.crossattention.self.key.bias', 'bert.encoder.layer.4.crossattention.self.value.weight', 'bert.encoder.layer.4.crossattention.self.value.bias', 'bert.encoder.layer.4.crossattention.output.dense.weight', 'bert.encoder.layer.4.crossattention.output.dense.bias', 'bert.encoder.layer.4.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.4.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.5.crossattention.self.query.weight', 'bert.encoder.layer.5.crossattention.self.query.bias', 'bert.encoder.layer.5.crossattention.self.key.weight', 'bert.encoder.layer.5.crossattention.self.key.bias', 'bert.encoder.layer.5.crossattention.self.value.weight', 'bert.encoder.layer.5.crossattention.self.value.bias', 'bert.encoder.layer.5.crossattention.output.dense.weight', 'bert.encoder.layer.5.crossattention.output.dense.bias', 'bert.encoder.layer.5.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.5.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.6.crossattention.self.query.weight', 'bert.encoder.layer.6.crossattention.self.query.bias', 'bert.encoder.layer.6.crossattention.self.key.weight', 'bert.encoder.layer.6.crossattention.self.key.bias', 'bert.encoder.layer.6.crossattention.self.value.weight', 'bert.encoder.layer.6.crossattention.self.value.bias', 'bert.encoder.layer.6.crossattention.output.dense.weight', 'bert.encoder.layer.6.crossattention.output.dense.bias', 'bert.encoder.layer.6.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.6.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.7.crossattention.self.query.weight', 'bert.encoder.layer.7.crossattention.self.query.bias', 'bert.encoder.layer.7.crossattention.self.key.weight', 'bert.encoder.layer.7.crossattention.self.key.bias', 'bert.encoder.layer.7.crossattention.self.value.weight', 'bert.encoder.layer.7.crossattention.self.value.bias', 'bert.encoder.layer.7.crossattention.output.dense.weight', 'bert.encoder.layer.7.crossattention.output.dense.bias', 'bert.encoder.layer.7.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.7.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.8.crossattention.self.query.weight', 'bert.encoder.layer.8.crossattention.self.query.bias', 'bert.encoder.layer.8.crossattention.self.key.weight', 'bert.encoder.layer.8.crossattention.self.key.bias', 'bert.encoder.layer.8.crossattention.self.value.weight', 'bert.encoder.layer.8.crossattention.self.value.bias', 'bert.encoder.layer.8.crossattention.output.dense.weight', 'bert.encoder.layer.8.crossattention.output.dense.bias', 'bert.encoder.layer.8.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.8.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.9.crossattention.self.query.weight', 'bert.encoder.layer.9.crossattention.self.query.bias', 'bert.encoder.layer.9.crossattention.self.key.weight', 'bert.encoder.layer.9.crossattention.self.key.bias', 'bert.encoder.layer.9.crossattention.self.value.weight', 'bert.encoder.layer.9.crossattention.self.value.bias', 'bert.encoder.layer.9.crossattention.output.dense.weight', 'bert.encoder.layer.9.crossattention.output.dense.bias', 'bert.encoder.layer.9.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.9.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.10.crossattention.self.query.weight', 'bert.encoder.layer.10.crossattention.self.query.bias', 'bert.encoder.layer.10.crossattention.self.key.weight', 'bert.encoder.layer.10.crossattention.self.key.bias', 'bert.encoder.layer.10.crossattention.self.value.weight', 'bert.encoder.layer.10.crossattention.self.value.bias', 'bert.encoder.layer.10.crossattention.output.dense.weight', 'bert.encoder.layer.10.crossattention.output.dense.bias', 'bert.encoder.layer.10.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.10.crossattention.output.LayerNorm.bias', 'bert.encoder.layer.11.crossattention.self.query.weight', 'bert.encoder.layer.11.crossattention.self.query.bias', 'bert.encoder.layer.11.crossattention.self.key.weight', 'bert.encoder.layer.11.crossattention.self.key.bias', 'bert.encoder.layer.11.crossattention.self.value.weight', 'bert.encoder.layer.11.crossattention.self.value.bias', 'bert.encoder.layer.11.crossattention.output.dense.weight', 'bert.encoder.layer.11.crossattention.output.dense.bias', 'bert.encoder.layer.11.crossattention.output.LayerNorm.weight', 'bert.encoder.layer.11.crossattention.output.LayerNorm.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "The following encoder weights were not tied to the decoder ['bert/pooler']\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-KFgnQ2nuNjw"
      },
      "source": [
        "As a comparison, we can see that the tied model has much fewer parameters as expected."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8UsfN3ayuMbm",
        "outputId": "8d39b6e5-3b69-45bf-9bd4-a21e8e252f2b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(f\"\\n\\nNum Params. Shared: {shared_bert2bert.num_parameters()}, Non-Shared: {bert2bert.num_parameters()}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "Num Params. Shared: 137298244, Non-Shared: 247363386\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1XpiyR9LuiPQ"
      },
      "source": [
        "In this notebook, we will however train a non-shared  *Bert2Bert* model, so we continue with `bert2bert` and not `shared_bert2bert`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "41Lo4Yruudwx"
      },
      "source": [
        "# free memory\n",
        "del shared_bert2bert"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rTet0_tyj0Ly"
      },
      "source": [
        "We have warm-started a `bert2bert` model, but we have not defined all the relevant parameters used for beam search decoding yet.\n",
        "\n",
        "Let's start by setting the special tokens.\n",
        "`bert-base-cased` does not have a `decoder_start_token_id` or `eos_token_id`, so we will use its `cls_token_id` and `sep_token_id` respectively. \n",
        "Also, we should define a `pad_token_id` on the config and make sure the correct `vocab_size` is set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JD2jv3GkyjR-"
      },
      "source": [
        "bert2bert.config.decoder_start_token_id = tokenizer.cls_token_id\n",
        "bert2bert.config.eos_token_id = tokenizer.sep_token_id\n",
        "bert2bert.config.pad_token_id = tokenizer.pad_token_id\n",
        "bert2bert.config.vocab_size = bert2bert.config.encoder.vocab_size"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WOPZ-pDpvSP9"
      },
      "source": [
        "Next, let's define all parameters related to beam search decoding. Since `bart-large-cnn` yields good results on CNN/Dailymail, we will just copy its beam search decoding parameters.\n",
        "\n",
        "For more details on what each of these parameters does, please take a look at [this](https://huggingface.co/blog/how-to-generate) blog post or the [docs](https://huggingface.co/transformers/main_classes/model.html#generative-models)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t004XKklvU6q"
      },
      "source": [
        "bert2bert.config.max_length = 142\n",
        "bert2bert.config.min_length = 56\n",
        "bert2bert.config.no_repeat_ngram_size = 3\n",
        "bert2bert.config.early_stopping = True\n",
        "bert2bert.config.length_penalty = 2.0\n",
        "bert2bert.config.num_beams = 4"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h4GwOGxAtKo5"
      },
      "source": [
        "Alright, let's now start fine-tuning the warm-started *BERT2BERT* model."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u98CLZiTkgzv"
      },
      "source": [
        "### **Fine-Tuning Warm-Started Encoder-Decoder Models**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-gYzA-w96wCt"
      },
      "source": [
        "In this section, we will show how one can make use of the `Seq2SeqTrainer` that can be found under [examples/seq2seq/seq2seq_trainer.py](https://github.com/huggingface/transformers/blob/master/examples/seq2seq/seq2seq_trainer.py) to fine-tune a warm-started encoder-decoder model.\n",
        "\n",
        "Let's first download the `Seq2SeqTrainer` and its training arguments `Seq2SeqTrainingArguments`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pyiwaF0noA5c"
      },
      "source": [
        "%%capture\n",
        "!rm seq2seq_trainer.py\n",
        "!rm seq2seq_training_args.py\n",
        "!wget https://raw.githubusercontent.com/huggingface/transformers/master/examples/seq2seq/seq2seq_trainer.py\n",
        "!wget https://raw.githubusercontent.com/huggingface/transformers/master/examples/seq2seq/seq2seq_training_args.py"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WaUYsq5YKgjb"
      },
      "source": [
        "In addition, we need a couple of python packages to make the `Seq2SeqTrainer` work."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CFf3BB7rKeBc"
      },
      "source": [
        "%%capture\n",
        "!pip install git-python==1.0.3\n",
        "!pip install rouge_score\n",
        "!pip install sacrebleu"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uD0WDdSTLKW3"
      },
      "source": [
        "Alright, let's import the `Seq2SeqTrainer` and the `Seq2SeqTrainingArguments`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jewYMBoKKWwB"
      },
      "source": [
        "from seq2seq_trainer import Seq2SeqTrainer\n",
        "from seq2seq_training_args import Seq2SeqTrainingArguments"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dPUAgo7pxH24"
      },
      "source": [
        "The `Seq2SeqTrainer` extends 🤗Transformer's Trainer for encoder-decoder models.\n",
        "In short, it allows using the `generate(...)` function during evaluation, which is necessary to validate the performance of encoder-decoder models on most *sequence-to-sequence* tasks, such as *summarization*. \n",
        "\n",
        "For more information on the `Trainer`, one should read through [this](https://huggingface.co/transformers/training.html#trainer) short tutorial.\n",
        "\n",
        "Let's begin by configuring the `Seq2SeqTrainingArguments`.\n",
        "\n",
        "The argument `predict_with_generate` should be set to `True`, so that the `Seq2SeqTrainer` runs the `generate(...)` on the validation data and passes the generated output as `predictions` to the `compute_metric(...)` function which we will define later.\n",
        "The additional arguments are derived from `TrainingArguments` and can be read upon [here](https://huggingface.co/transformers/main_classes/trainer.html#trainingarguments).\n",
        "For a complete training run, one should change those arguments as needed. Good default values are commented out below.\n",
        "\n",
        "For more information on the `Seq2SeqTrainer`, the reader is advised to take a look at the [code](https://github.com/huggingface/transformers/blob/master/examples/seq2seq/seq2seq_trainer.py)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LAaTxUpdzshF"
      },
      "source": [
        "training_args = Seq2SeqTrainingArguments(\n",
        "    predict_with_generate=True,\n",
        "    evaluation_strategy=\"steps\",\n",
        "    per_device_train_batch_size=batch_size,\n",
        "    per_device_eval_batch_size=batch_size,\n",
        "    fp16=True, \n",
        "    output_dir=\"./\",\n",
        "    logging_steps=2,\n",
        "    save_steps=10,\n",
        "    eval_steps=4,\n",
        "    # logging_steps=1000,\n",
        "    # save_steps=500,\n",
        "    # eval_steps=7500,\n",
        "    # warmup_steps=2000,\n",
        "    # save_total_limit=3,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z0FWPCFnCgf6"
      },
      "source": [
        "\n",
        "Also, we need to define a function to correctly compute the ROUGE score during validation. Since we activated `predict_with_generate`, the `compute_metrics(...)` function expects `predictions` that were obtained using the `generate(...)` function. \n",
        "Like most summarization tasks, CNN/Dailymail is typically evaluated using the ROUGE score. \n",
        "\n",
        "Let's first load the ROUGE metric using the 🤗datasets library."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "68IHmFYLx09W",
        "outputId": "fc21bc78-50f6-4994-b30e-c7508b24b273",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "2fd461deea6146ca9bd5938207a77420",
            "f35dfd2f3ea346e9b6e86d59d29129de",
            "e46962da15fb411e91f9db4ff329156e",
            "32c27fe6fcc049eba015c51fda3ced13",
            "865b3b7f5bb7412696ff6441f2b7fd60",
            "391dd711451a49bcb4234f48add40a12",
            "dd128062d2594bf9b0ed771a28336699",
            "3ad51a6456ab4ab082e016fca466bacb"
          ]
        }
      },
      "source": [
        "rouge = datasets.load_metric(\"rouge\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2fd461deea6146ca9bd5938207a77420",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1717.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y8UgiZP7N_tB"
      },
      "source": [
        "Next, we will define the `compute_metrics(...)` function. The `rouge` metric computes the score from two lists of strings. Thus we decode both the `predictions` and `labels` - making sure that `-100` is correctly replaced by the `pad_token_id` and remove all special characters by setting `skip_special_tokens=True`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iVKkr2xfNyRh"
      },
      "source": [
        "def compute_metrics(pred):\n",
        "    labels_ids = pred.label_ids\n",
        "    pred_ids = pred.predictions\n",
        "\n",
        "    pred_str = tokenizer.batch_decode(pred_ids, skip_special_tokens=True)\n",
        "    labels_ids[labels_ids == -100] = tokenizer.pad_token_id\n",
        "    label_str = tokenizer.batch_decode(labels_ids, skip_special_tokens=True)\n",
        "\n",
        "    rouge_output = rouge.compute(predictions=pred_str, references=label_str, rouge_types=[\"rouge2\"])[\"rouge2\"].mid\n",
        "\n",
        "    return {\n",
        "        \"rouge2_precision\": round(rouge_output.precision, 4),\n",
        "        \"rouge2_recall\": round(rouge_output.recall, 4),\n",
        "        \"rouge2_fmeasure\": round(rouge_output.fmeasure, 4),\n",
        "    }"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1ik4hZb2yV-b"
      },
      "source": [
        "Great, now we can pass all arguments to the `Seq2SeqTrainer` and start finetuning. Executing the following cell will take *ca.* 10 minutes \n",
        "☕.\n",
        "\n",
        "Finetuning *BERT2BERT* on the complete *CNN/Dailymail* training data takes *ca.*  model takes *ca.* 8h on a single *TITAN RTX* GPU."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cy7i1Q3D1qSI",
        "outputId": "5ca07924-72c1-4703-d678-8f97a2c0b169",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        }
      },
      "source": [
        "# instantiate trainer\n",
        "trainer = Seq2SeqTrainer(\n",
        "    model=bert2bert,\n",
        "    args=training_args,\n",
        "    compute_metrics=compute_metrics,\n",
        "    train_dataset=train_data,\n",
        "    eval_dataset=val_data,\n",
        ")\n",
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/datasets/arrow_dataset.py:835: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  /pytorch/torch/csrc/utils/tensor_numpy.cpp:141.)\n",
            "  return torch.tensor(x, **format_kwargs)\n",
            "/usr/local/lib/python3.6/dist-packages/torch/optim/lr_scheduler.py:123: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
            "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='24' max='24' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [24/24 13:54, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Rouge2 Precision</th>\n",
              "      <th>Rouge2 Recall</th>\n",
              "      <th>Rouge2 Fmeasure</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>9.762589</td>\n",
              "      <td>10.518353</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>7.669090</td>\n",
              "      <td>7.811150</td>\n",
              "      <td>0.002500</td>\n",
              "      <td>0.003900</td>\n",
              "      <td>0.003000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>7.495941</td>\n",
              "      <td>7.724218</td>\n",
              "      <td>0.004200</td>\n",
              "      <td>0.010900</td>\n",
              "      <td>0.006100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>7.291489</td>\n",
              "      <td>7.634698</td>\n",
              "      <td>0.005700</td>\n",
              "      <td>0.010900</td>\n",
              "      <td>0.007500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>7.093002</td>\n",
              "      <td>7.636654</td>\n",
              "      <td>0.003800</td>\n",
              "      <td>0.007800</td>\n",
              "      <td>0.005100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>24</td>\n",
              "      <td>7.212769</td>\n",
              "      <td>7.571666</td>\n",
              "      <td>0.004800</td>\n",
              "      <td>0.010900</td>\n",
              "      <td>0.006600</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=24, training_loss=7.9638017018636065)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YQ7shq14JRJG"
      },
      "source": [
        "Awesome, we should now be fully equipped to finetune a warm-started encoder-decoder model. To check the result of our fine-tuning let's take a look at the saved checkpoints."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t2Yv4OkuJb1B",
        "outputId": "a1bef582-328a-4c8b-b7c0-69122018ebf4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!ls"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "bert2bert      checkpoint-20  runs\t   seq2seq_trainer.py\n",
            "checkpoint-10  __pycache__    sample_data  seq2seq_training_args.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CU1xAmVcJeP6"
      },
      "source": [
        "Finally, we can load the checkpoint as usual via the `EncoderDecoderModel.from_pretrained(...)` method."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J2MvxTwmKVeV"
      },
      "source": [
        "dummy_bert2bert = EncoderDecoderModel.from_pretrained(\"./checkpoint-20\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZwQIEhKOrJpl"
      },
      "source": [
        "### **Evaluation**\n",
        "\n",
        "In a final step, we might want to evaluate the *BERT2BERT* model on the test data.\n",
        "\n",
        "To start, instead of loading the dummy model, let's load a *BERT2BERT* model that was finetuned on the full training dataset. Also, we load its tokenizer, which is just a copy of `bert-base-cased`'s tokenizer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TM2czn1xKxC0",
        "outputId": "6bfef3b6-1acc-4ce3-f575-c354cfc9517c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 267,
          "referenced_widgets": [
            "001c9215afbb4a178835b138385289cd",
            "22adeb51790b481d8c62f4c4ca2375fc",
            "8f611825f13b44bd889ccd1ee3950ebb",
            "74281bb4a2be473392d729e4c1a3f251",
            "068321dbd7254d0a8c1767d879ec6943",
            "d62484271f7a4f3080e6db2ebaad9480",
            "f5f0f0dd629c41aaa41d0df604c583db",
            "61e1f2615273479cbab9845024c66ad4",
            "1d46e07204cf4a51a1895633073a9563",
            "ca4a525521d6485580c382967037e99f",
            "fd02a19331f54227a412f526a8df80ac",
            "377e2ee25f824a52ad5a1882071cb463",
            "5d0ce87488cc4de8b6514af094dee6a9",
            "a40f126721864d448a74cb0efd705054",
            "e6f03cc01f8d4ebcaa521610eb03bfe2",
            "69607c45c1fa4d3abae6f710785ac16e",
            "a87d5a96d832455190ae67e63f9242b4",
            "046527ff4e4a4d568b4d8136dbf66d48",
            "c783495fd2b0477db3347961be75bef3",
            "5890190d211a4709a1430e9f21c10877",
            "75b5bd9fa1014c24bcebf1f17027abeb",
            "4b8b52bec8d641f2b7bf737507685fe9",
            "af0bcea9cf0949688d33721b02121d80",
            "7d27762447494a8397a2cb703528a6af",
            "0143ea0238c242e09023e8956d1b7c63",
            "af59c31b4af849b1b6325ed4921987d5",
            "60da1ffd161744148b15f768bb58f2e1",
            "4f59c2dc0af84b888d2d5b65abc4330a",
            "665d515b9688490c9bc5334e96a6f06d",
            "22b8d7cb89a9431a93498b6e55208c94",
            "30bc39b8ca8b4e5ca6bcc47ca122b45a",
            "03ecd8bfbae54796b19e0c35d82251bd",
            "f90970b373f1445994b977ef37afc9c6",
            "0d51f43f6300485ba70ac633798aa722",
            "a42c79ec2ca54b288893d49125c2f753",
            "0794b1342edc450588c545f996c21f9b",
            "1018157054a74c09b12f687e78c5c37f",
            "c1c4eff0ed8e4e7b81ef32f416af67a6",
            "2904dc58a7574dc9be22d59209c350b4",
            "90f528ea5ba24db89d3d04eb110fd3ad"
          ]
        }
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "\n",
        "bert2bert = EncoderDecoderModel.from_pretrained(\"patrickvonplaten/bert2bert_cnn_daily_mail\").to(\"cuda\")\n",
        "tokenizer = BertTokenizer.from_pretrained(\"patrickvonplaten/bert2bert_cnn_daily_mail\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "001c9215afbb4a178835b138385289cd",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=3660.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1d46e07204cf4a51a1895633073a9563",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=989691346.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a87d5a96d832455190ae67e63f9242b4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0143ea0238c242e09023e8956d1b7c63",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=156.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f90970b373f1445994b977ef37afc9c6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=252.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jOOslzG9K1hM"
      },
      "source": [
        "Next, we load just 2% of *CNN/Dailymail's* test data. For the full evaluation, one should obviously use 100% of the data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oOoSrwWarJAC",
        "outputId": "dfd6be70-a0c9-4758-9c45-690f9e639ada",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "test_data = datasets.load_dataset(\"cnn_dailymail\", \"3.0.0\", split=\"test[:2%]\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reusing dataset cnn_dailymail (/root/.cache/huggingface/datasets/cnn_dailymail/3.0.0/3.0.0/0128610a44e10f25b4af6689441c72af86205282d26399642f7db38fa7535602)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XVD967erLrnM"
      },
      "source": [
        "Now, we can again leverage 🤗dataset's handy `map()` function to generate a summary for each test sample.\n",
        "\n",
        "For each data sample we:\n",
        "\n",
        "- first, tokenize the `\"article\"`,\n",
        "- second, generate the output token ids, and\n",
        "- third, decode the output token ids to obtain our predicted summary."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x274feeXLqrT"
      },
      "source": [
        "def generate_summary(batch):\n",
        "    # cut off at BERT max length 512\n",
        "    inputs = tokenizer(batch[\"article\"], padding=\"max_length\", truncation=True, max_length=512, return_tensors=\"pt\")\n",
        "    input_ids = inputs.input_ids.to(\"cuda\")\n",
        "    attention_mask = inputs.attention_mask.to(\"cuda\")\n",
        "\n",
        "    outputs = bert2bert.generate(input_ids, attention_mask=attention_mask)\n",
        "\n",
        "    output_str = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n",
        "\n",
        "    batch[\"pred_summary\"] = output_str\n",
        "\n",
        "    return batch"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pZecWodQsr0O"
      },
      "source": [
        "Let's run the map function to obtain the *results* dictionary that has the model's predicted summary stored for each sample. Executing the following cell may take *ca.* 10min ☕."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bYmdx-W1NAky",
        "outputId": "241d430f-49f4-4b98-e542-c74c708afa34",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "53297fb5dd8e4ec4b3ee6b611559fd2b",
            "12b5cb483c364956b73eb36e9675ac97",
            "5f11f11b307142558309e59eed8d9d51",
            "82269c7f62394c5f83d889af910a9747",
            "afd72567f8eb4da2a7771c1af707c43c",
            "88dc3c0cfdd2464fa1660653e4810ab8",
            "848419a18c4e49e0afbc0c72abf60e96",
            "d015425ac18d43d48f51e491b8bd1c29"
          ]
        }
      },
      "source": [
        "batch_size = 16  # change to 64 for full evaluation\n",
        "\n",
        "results = test_data.map(generate_summary, batched=True, batch_size=batch_size, remove_columns=[\"article\"])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "53297fb5dd8e4ec4b3ee6b611559fd2b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=15.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0ZhjRk8oNW0J"
      },
      "source": [
        "Finally, we compute the ROUGE score."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yfhyBz-ZsVJe",
        "outputId": "d012af34-250e-48c8-f396-4389e2c96a76",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "rouge.compute(predictions=results[\"pred_summary\"], references=results[\"highlights\"], rouge_types=[\"rouge2\"])[\"rouge2\"].mid"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Score(precision=0.10389454113300968, recall=0.1564771201053348, fmeasure=0.12175271663717585)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7zdm50ZotZqb"
      },
      "source": [
        "That's it. We've shown how to warm-start a *BERT2BERT* model and fine-tune/evaluate it on the CNN/Dailymail dataset.\n",
        "\n",
        "The fully trained *BERT2BERT* model is uploaded to the 🤗model hub under [patrickvonplaten/bert2bert_cnn_daily_mail](https://huggingface.co/patrickvonplaten/bert2bert_cnn_daily_mail). \n",
        "\n",
        "The model achieves a ROUGE-2 score of **18.22** on the full evaluation data, which is even a little better than reported in the paper.\n",
        "\n",
        "For some summarization examples, the reader is advised to use the online inference API of the model, [here](https://huggingface.co/patrickvonplaten/bert2bert_cnn_daily_mail)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LlrCkQGuA4Bw"
      },
      "source": [
        "Thanks a lot to Sascha Rothe, Shashi Narayan, and Aliaksei Severyn from Google Research, and Victor Sanh, Sylvain Gugger, and Thomas Wolf from 🤗Hugging Face for proof-reading and giving very much appreciated feedback."
      ]
    }
  ]
}