Fine_tuning_GPT_2_Español.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Fine-tuning GPT-2 en Español.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e243c0f3792644c68fd750d5b4aea237": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8f0c2825d2bf49ada7efaed9a64844a5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_53a9953d40314e47ae18a8d027cbc2c5",
              "IPY_MODEL_234c28476b2846b881a6ca675d0fdc23",
              "IPY_MODEL_1fe3c29b4a9747db9075e77df98729ba"
            ]
          }
        },
        "8f0c2825d2bf49ada7efaed9a64844a5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "53a9953d40314e47ae18a8d027cbc2c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_adaf5e3e2f1e4ee1a4f698b1334a6d3a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cb1fdca6acda410fa30a27dfa9da93c2"
          }
        },
        "234c28476b2846b881a6ca675d0fdc23": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3d6a6e25fd414415be5cad5668528b96",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ca38a529c24f481b817df7f3441c396c"
          }
        },
        "1fe3c29b4a9747db9075e77df98729ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1a61c779ee6b4d8695a692bbf5e2b82c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 0/? [00:00&lt;?, ? tables/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7b86a0c7d49a4a17ac4bc82d82b0e12b"
          }
        },
        "adaf5e3e2f1e4ee1a4f698b1334a6d3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cb1fdca6acda410fa30a27dfa9da93c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3d6a6e25fd414415be5cad5668528b96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ca38a529c24f481b817df7f3441c396c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": "20px",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a61c779ee6b4d8695a692bbf5e2b82c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7b86a0c7d49a4a17ac4bc82d82b0e12b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3b6969623d7d47d182032547e187c5a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6309e0de39ec4dd4a175bbc43760beb7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f632eea6f5b44fae81592262c04aedb4",
              "IPY_MODEL_a232fea5da2d459281af7917efce2f4c",
              "IPY_MODEL_4f17d23a24c34eb088b0378ff4974ffd"
            ]
          }
        },
        "6309e0de39ec4dd4a175bbc43760beb7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f632eea6f5b44fae81592262c04aedb4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_827a3ba2fa3340e2a9fa83431ae11a1b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fcde75531ec74dcfbfb250fd20ad8151"
          }
        },
        "a232fea5da2d459281af7917efce2f4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_33407f7c0f1a43088fe5f385ac048576",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 840253,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 840253,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_41414473e5c7412b82c2fb91656741dd"
          }
        },
        "4f17d23a24c34eb088b0378ff4974ffd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_49a432f2f312457cad83e7c0e6c43015",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 840k/840k [00:00&lt;00:00, 1.52MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5585509c4eae449fa2e30645d3ec4591"
          }
        },
        "827a3ba2fa3340e2a9fa83431ae11a1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fcde75531ec74dcfbfb250fd20ad8151": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "33407f7c0f1a43088fe5f385ac048576": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "41414473e5c7412b82c2fb91656741dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "49a432f2f312457cad83e7c0e6c43015": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5585509c4eae449fa2e30645d3ec4591": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "eed0101232404cacba582b0178fd7901": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_db8be632c5b64b18aa541683968cf803",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_db2987c730a746d5b5b7c13c9103668a",
              "IPY_MODEL_e59a5e5c5dd54e8da1cdca3640ce2c24",
              "IPY_MODEL_7ccd176a23794736b37bd5776c2aca06"
            ]
          }
        },
        "db8be632c5b64b18aa541683968cf803": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "db2987c730a746d5b5b7c13c9103668a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7a875b9d28f041f4b2fa2411ff983c4f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ce1f548a89da478da6af9f75ea965ed4"
          }
        },
        "e59a5e5c5dd54e8da1cdca3640ce2c24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_dc89967e1c4a43f3a915557e89c838b4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 498530,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 498530,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_719c82095616487b904e0febb724e0ad"
          }
        },
        "7ccd176a23794736b37bd5776c2aca06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9117bc4290eb495db52d1fa870cd582f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 499k/499k [00:00&lt;00:00, 1.39MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1d5a6280ebba42fa933f8f4522de24c0"
          }
        },
        "7a875b9d28f041f4b2fa2411ff983c4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ce1f548a89da478da6af9f75ea965ed4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dc89967e1c4a43f3a915557e89c838b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "719c82095616487b904e0febb724e0ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9117bc4290eb495db52d1fa870cd582f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1d5a6280ebba42fa933f8f4522de24c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bcef9e3b2447466fa0ef50851728d41a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3f2647e850eb4bf2b9ac5c3af893f763",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_69f579dca8284590af10c1ac5662ea6f",
              "IPY_MODEL_a43c34aeaa2848f3a73e6bede3369108",
              "IPY_MODEL_326693a8a77f4705a6e2aefb8b94fb6f"
            ]
          }
        },
        "3f2647e850eb4bf2b9ac5c3af893f763": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "69f579dca8284590af10c1ac5662ea6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8ef937f54e0648a9be8a4f159e5015f8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0619728ea1b64cbfbd1f68781b5d8fdc"
          }
        },
        "a43c34aeaa2848f3a73e6bede3369108": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ff853b181336432f9246f3d08092332b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 262,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 262,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9c0e3ea2c03d451ebf4f89441376938a"
          }
        },
        "326693a8a77f4705a6e2aefb8b94fb6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_521916cd9d4a4ca89a9284283ef0db6a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 262/262 [00:00&lt;00:00, 8.28kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_eb1288534e1f4d71b736999deece5a39"
          }
        },
        "8ef937f54e0648a9be8a4f159e5015f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0619728ea1b64cbfbd1f68781b5d8fdc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ff853b181336432f9246f3d08092332b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9c0e3ea2c03d451ebf4f89441376938a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "521916cd9d4a4ca89a9284283ef0db6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "eb1288534e1f4d71b736999deece5a39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "eeb0b08452694019a801c19bdaba283b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_265e1368bd214bdb9454f555bd676f4b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_44dea4f943a64cd0935847818e1bd390",
              "IPY_MODEL_f997d367354b4153b42c85556e2955a4",
              "IPY_MODEL_452af706e8f843b5acecbdf16665c71e"
            ]
          }
        },
        "265e1368bd214bdb9454f555bd676f4b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "44dea4f943a64cd0935847818e1bd390": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0c000e55ffaf46c59093e5689021c6d1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_22c7e8b8c9be4716aa883d89d80960af"
          }
        },
        "f997d367354b4153b42c85556e2955a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b9668fb49b4d4344a8dda2407fadb3d4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 115,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 115,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4a30d67b29d3400294f61358d6b5b63b"
          }
        },
        "452af706e8f843b5acecbdf16665c71e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a836d1d6070845fb97d0af603aff011b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 115/115 [00:00&lt;00:00, 3.29kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_39116dc131ab4cc89170d79ac40d68ed"
          }
        },
        "0c000e55ffaf46c59093e5689021c6d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "22c7e8b8c9be4716aa883d89d80960af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b9668fb49b4d4344a8dda2407fadb3d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4a30d67b29d3400294f61358d6b5b63b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a836d1d6070845fb97d0af603aff011b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "39116dc131ab4cc89170d79ac40d68ed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "49c34597114d4dad913bd44627345e3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_03467909530a403095c008d17de4d364",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_00dc4491e4a547098b98f27641d1cb50",
              "IPY_MODEL_0356baccbebf495a9a17a28c87abe681",
              "IPY_MODEL_c7f65289cfa24ba9aa0e9a441d5a4a79"
            ]
          }
        },
        "03467909530a403095c008d17de4d364": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "00dc4491e4a547098b98f27641d1cb50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4713b15948ca4fb8b3221091a5bfbdce",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9290772521ae4ba0a6d1b68a67b39e07"
          }
        },
        "0356baccbebf495a9a17a28c87abe681": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e7773eecba314e7bb6494d0e978b205a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 914,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 914,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aef79b311de1407d992ec15e8b1a3161"
          }
        },
        "c7f65289cfa24ba9aa0e9a441d5a4a79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fec7bf69e28946129ac59b586054d1fc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 914/914 [00:00&lt;00:00, 20.8kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6decf01d245744a58ecdbd7b0ae8ba68"
          }
        },
        "4713b15948ca4fb8b3221091a5bfbdce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9290772521ae4ba0a6d1b68a67b39e07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e7773eecba314e7bb6494d0e978b205a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aef79b311de1407d992ec15e8b1a3161": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fec7bf69e28946129ac59b586054d1fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6decf01d245744a58ecdbd7b0ae8ba68": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fc427444d03241748487470cb2d33693": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b012174d4b6d4bfba723c6c07361aeb2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_198f555587324367b047f18b0975f231",
              "IPY_MODEL_84c8604672f3440b9241b4dc47cae279",
              "IPY_MODEL_aa5781bb9d9c4f83ad41950b3c10468a"
            ]
          }
        },
        "b012174d4b6d4bfba723c6c07361aeb2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "198f555587324367b047f18b0975f231": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d9c7b558918d4843b2b78d21a5caa0f7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9e5fa69a27104b17891b2d7a3e89c96c"
          }
        },
        "84c8604672f3440b9241b4dc47cae279": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a46dab8d60dd4565879701e379e91c2d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_88b6475161ad4c9798b5baff0d232429"
          }
        },
        "aa5781bb9d9c4f83ad41950b3c10468a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a214a23777da41588313eaa9bfb002e8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:00&lt;00:00,  2.44ba/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5e3d9022398c4ef398a75e998c405e3e"
          }
        },
        "d9c7b558918d4843b2b78d21a5caa0f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9e5fa69a27104b17891b2d7a3e89c96c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a46dab8d60dd4565879701e379e91c2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "88b6475161ad4c9798b5baff0d232429": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a214a23777da41588313eaa9bfb002e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5e3d9022398c4ef398a75e998c405e3e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5b136f4e60e9491fa55c695e6cfbefb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f966cc741c6e49938255964dd9e9325d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_90e3a560099644169b6cfe92f1ec4615",
              "IPY_MODEL_bac09ab44fe6449993d2a0fad4eb99fa",
              "IPY_MODEL_97bc9b6ae6dd4d278b9f5f64c1c382bc"
            ]
          }
        },
        "f966cc741c6e49938255964dd9e9325d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "90e3a560099644169b6cfe92f1ec4615": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2ae3de5e2c044b88bd7698defd90c402",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_535a9a6be3b2438f87421a88a07c9939"
          }
        },
        "bac09ab44fe6449993d2a0fad4eb99fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c7705508d954476e84bee80c87402305",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_43203d5ffd53490796ba3efdf65e5195"
          }
        },
        "97bc9b6ae6dd4d278b9f5f64c1c382bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bc73dd08bbab416f8680b74144e2ee3b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:00&lt;00:00, 11.31ba/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1d16330089a74da5a84e3bd4defe73ea"
          }
        },
        "2ae3de5e2c044b88bd7698defd90c402": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "535a9a6be3b2438f87421a88a07c9939": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c7705508d954476e84bee80c87402305": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "43203d5ffd53490796ba3efdf65e5195": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bc73dd08bbab416f8680b74144e2ee3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1d16330089a74da5a84e3bd4defe73ea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cdd68dc55e034bca9b3b33fc48b17f2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9e15f650de3747d6b0bb09dbc9837d03",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_116e4d0df6084b6d950963e161983721",
              "IPY_MODEL_8de304bfd8bf4b339108818d03d72178",
              "IPY_MODEL_fecee17e58e24c52aae43dad7761cd31"
            ]
          }
        },
        "9e15f650de3747d6b0bb09dbc9837d03": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "116e4d0df6084b6d950963e161983721": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_14e44a89fa2d453984f1b3082d5a4d93",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_34564151bf7944a4bd5817c5666c2fe0"
          }
        },
        "8de304bfd8bf4b339108818d03d72178": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8b0957916bce44bb9545746d1ad8100e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 261499792,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 261499792,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2c0e51e8ac7e4ce98db042911302b8ee"
          }
        },
        "fecee17e58e24c52aae43dad7761cd31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9258b1acae7b4fc688b9fd8ae7c35552",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 261M/261M [00:05&lt;00:00, 49.6MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ee1ce94145a34bf89585df74efd4349c"
          }
        },
        "14e44a89fa2d453984f1b3082d5a4d93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "34564151bf7944a4bd5817c5666c2fe0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8b0957916bce44bb9545746d1ad8100e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2c0e51e8ac7e4ce98db042911302b8ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9258b1acae7b4fc688b9fd8ae7c35552": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ee1ce94145a34bf89585df74efd4349c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gramagrass/estorrente/blob/main/Fine_tuning_GPT_2_Espa%C3%B1ol.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lhrryhx3r1v3"
      },
      "source": [
        "## Init Env"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_DThoCy6HyQu",
        "outputId": "d5bf25b4-530d-412a-ab75-73a2eac1fe22"
      },
      "source": [
        "# Check that we have a GPU\n",
        "!nvidia-smi"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tue Aug 17 00:13:45 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 470.57.02    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   34C    P8     9W /  70W |      0MiB / 15109MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 945
        },
        "id": "xhoU6bzPH0x1",
        "outputId": "8828d47a-7cac-498d-ad8d-d0f0a3e34ce0"
      },
      "source": [
        "!pip3 install torch==1.7.1+cu110 torchvision==0.8.2+cu110 torchaudio==0.7.2 --upgrade --force-reinstall -f https://download.pytorch.org/whl/torch_stable.html"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Looking in links: https://download.pytorch.org/whl/torch_stable.html\n",
            "Collecting torch==1.7.1+cu110\n",
            "  Downloading https://download.pytorch.org/whl/cu110/torch-1.7.1%2Bcu110-cp37-cp37m-linux_x86_64.whl (1156.8 MB)\n",
            "\u001b[K     |███████████████████████         | 834.1 MB 1.2 MB/s eta 0:04:27tcmalloc: large alloc 1147494400 bytes == 0x55a9fb194000 @  0x7f0694e20615 0x55a9c115b02c 0x55a9c123b17a 0x55a9c115de4d 0x55a9c124fc0d 0x55a9c11d20d8 0x55a9c11ccc35 0x55a9c115f73a 0x55a9c11d1f40 0x55a9c11ccc35 0x55a9c115f73a 0x55a9c11ce93b 0x55a9c1250a56 0x55a9c11cdfb3 0x55a9c1250a56 0x55a9c11cdfb3 0x55a9c1250a56 0x55a9c11cdfb3 0x55a9c115fb99 0x55a9c11a2e79 0x55a9c115e7b2 0x55a9c11d1e65 0x55a9c11ccc35 0x55a9c115f73a 0x55a9c11ce93b 0x55a9c11ccc35 0x55a9c115f73a 0x55a9c11cdb0e 0x55a9c115f65a 0x55a9c11cdd67 0x55a9c11ccc35\n",
            "\u001b[K     |█████████████████████████████▏  | 1055.7 MB 1.2 MB/s eta 0:01:28tcmalloc: large alloc 1434370048 bytes == 0x55aa3f7ea000 @  0x7f0694e20615 0x55a9c115b02c 0x55a9c123b17a 0x55a9c115de4d 0x55a9c124fc0d 0x55a9c11d20d8 0x55a9c11ccc35 0x55a9c115f73a 0x55a9c11d1f40 0x55a9c11ccc35 0x55a9c115f73a 0x55a9c11ce93b 0x55a9c1250a56 0x55a9c11cdfb3 0x55a9c1250a56 0x55a9c11cdfb3 0x55a9c1250a56 0x55a9c11cdfb3 0x55a9c115fb99 0x55a9c11a2e79 0x55a9c115e7b2 0x55a9c11d1e65 0x55a9c11ccc35 0x55a9c115f73a 0x55a9c11ce93b 0x55a9c11ccc35 0x55a9c115f73a 0x55a9c11cdb0e 0x55a9c115f65a 0x55a9c11cdd67 0x55a9c11ccc35\n",
            "\u001b[K     |████████████████████████████████| 1156.7 MB 1.3 MB/s eta 0:00:01tcmalloc: large alloc 1445945344 bytes == 0x55aa94fd6000 @  0x7f0694e20615 0x55a9c115b02c 0x55a9c123b17a 0x55a9c115de4d 0x55a9c124fc0d 0x55a9c11d20d8 0x55a9c11ccc35 0x55a9c115f73a 0x55a9c11cdd67 0x55a9c11ccc35 0x55a9c115f73a 0x55a9c11cdd67 0x55a9c11ccc35 0x55a9c115f73a 0x55a9c11cdd67 0x55a9c11ccc35 0x55a9c115f73a 0x55a9c11cdd67 0x55a9c11ccc35 0x55a9c115f73a 0x55a9c11cdd67 0x55a9c115f65a 0x55a9c11cdd67 0x55a9c11ccc35 0x55a9c115f73a 0x55a9c11ce93b 0x55a9c11ccc35 0x55a9c115f73a 0x55a9c11ce93b 0x55a9c11ccc35 0x55a9c115fdd1\n",
            "\u001b[K     |████████████████████████████████| 1156.8 MB 14 kB/s \n",
            "\u001b[?25hCollecting torchvision==0.8.2+cu110\n",
            "  Downloading https://download.pytorch.org/whl/cu110/torchvision-0.8.2%2Bcu110-cp37-cp37m-linux_x86_64.whl (12.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 12.9 MB 160 kB/s \n",
            "\u001b[?25hCollecting torchaudio==0.7.2\n",
            "  Downloading torchaudio-0.7.2-cp37-cp37m-manylinux1_x86_64.whl (7.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 7.6 MB 7.0 MB/s \n",
            "\u001b[?25hCollecting typing-extensions\n",
            "  Downloading typing_extensions-3.10.0.0-py3-none-any.whl (26 kB)\n",
            "Collecting numpy\n",
            "  Downloading numpy-1.21.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 15.7 MB 175 kB/s \n",
            "\u001b[?25hCollecting pillow>=4.1.1\n",
            "  Downloading Pillow-8.3.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (3.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.0 MB 65.8 MB/s \n",
            "\u001b[?25hInstalling collected packages: typing-extensions, numpy, torch, pillow, torchvision, torchaudio\n",
            "  Attempting uninstall: typing-extensions\n",
            "    Found existing installation: typing-extensions 3.7.4.3\n",
            "    Uninstalling typing-extensions-3.7.4.3:\n",
            "      Successfully uninstalled typing-extensions-3.7.4.3\n",
            "  Attempting uninstall: numpy\n",
            "    Found existing installation: numpy 1.19.5\n",
            "    Uninstalling numpy-1.19.5:\n",
            "      Successfully uninstalled numpy-1.19.5\n",
            "  Attempting uninstall: torch\n",
            "    Found existing installation: torch 1.9.0+cu102\n",
            "    Uninstalling torch-1.9.0+cu102:\n",
            "      Successfully uninstalled torch-1.9.0+cu102\n",
            "  Attempting uninstall: pillow\n",
            "    Found existing installation: Pillow 7.1.2\n",
            "    Uninstalling Pillow-7.1.2:\n",
            "      Successfully uninstalled Pillow-7.1.2\n",
            "  Attempting uninstall: torchvision\n",
            "    Found existing installation: torchvision 0.10.0+cu102\n",
            "    Uninstalling torchvision-0.10.0+cu102:\n",
            "      Successfully uninstalled torchvision-0.10.0+cu102\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "torchtext 0.10.0 requires torch==1.9.0, but you have torch 1.7.1+cu110 which is incompatible.\n",
            "tensorflow 2.5.0 requires numpy~=1.19.2, but you have numpy 1.21.2 which is incompatible.\n",
            "tensorflow 2.5.0 requires typing-extensions~=3.7.4, but you have typing-extensions 3.10.0.0 which is incompatible.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\n",
            "albumentations 0.1.12 requires imgaug<0.2.7,>=0.2.5, but you have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Successfully installed numpy-1.21.2 pillow-8.3.1 torch-1.7.1+cu110 torchaudio-0.7.2 torchvision-0.8.2+cu110 typing-extensions-3.10.0.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "PIL",
                  "numpy"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tVy7bimIi_fh",
        "outputId": "0f4b09db-fae3-4089-9404-f38c2e28e226"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "!pip install tensorflow &> log.txt\n",
        "!pip install git+https://github.com/huggingface/transformers/ > log.txt\n",
        "!pip install datasets > log.txt\n",
        "!pip install deepspeed==0.4.3 > log.txt\n",
        "!pip install mpi4py > log.txt\n",
        "import os\n",
        "import sys; print('Python:',sys.version)\n",
        "import torch; print('Pytorch:',torch.__version__)\n",
        "import tensorflow as tf; print('Tensorflow:',tf.__version__)\n",
        "import datasets; print('Datasets:',datasets.__version__)\n",
        "import transformers; print('Transformers:',transformers.__version__)\n",
        "import deepspeed; print('DeepSpeed:',deepspeed.__version__)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n",
            "  Running command git clone -q https://github.com/huggingface/transformers/ /tmp/pip-req-build-_o4j9vmv\n",
            "Python: 3.7.11 (default, Jul  3 2021, 18:01:19) \n",
            "[GCC 7.5.0]\n",
            "Pytorch: 1.7.1+cu110\n",
            "Tensorflow: 2.5.0\n",
            "Datasets: 1.11.0\n",
            "Transformers: 4.10.0.dev0\n",
            "DeepSpeed: 0.4.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7QY8xyxUq-a8",
        "outputId": "8e9a0207-f530-4780-ad04-02b2bc4c2cbf"
      },
      "source": [
        "MODEL_NAME = 'DeepESP/gpt2-spanish'\n",
        "CORPUS_PATH = 'https://raw.githubusercontent.com/gramagrass/estorrente/main/db/consolidado.txt' # AQUI PON LA URL A TU CORPUS\n",
        "\n",
        "path_base = '/content/drive/MyDrive/FineTuningGPT2'\n",
        "path_corpus = path_base + '/Corpus/'\n",
        "path_dataset = path_base + '/Dataset'\n",
        "path_tokenized_dataset = path_base + '/DatasetTokenized'\n",
        "path_ready_dataset = path_base + '/DatasetReady'\n",
        "path_model_save = path_base + '/ModelTrained'\n",
        "\n",
        "if not os.path.exists(path_base):\n",
        "  os.makedirs(path_base)\n",
        "  os.makedirs(path_corpus)\n",
        "  os.makedirs(path_dataset)\n",
        "  os.makedirs(path_tokenized_dataset)\n",
        "  os.makedirs(path_ready_dataset)\n",
        "  os.makedirs(path_model_save)\n",
        "\n",
        "!wget -P \"{path_corpus}\" \"{CORPUS_PATH}\"\n",
        "!ls \"{path_corpus}\""
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-08-17 00:27:29--  https://raw.githubusercontent.com/gramagrass/estorrente/main/db/consolidado.txt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 135659 (132K) [text/plain]\n",
            "Saving to: ‘/content/drive/MyDrive/FineTuningGPT2/Corpus/consolidado.txt.8’\n",
            "\n",
            "consolidado.txt.8   100%[===================>] 132.48K  --.-KB/s    in 0.01s   \n",
            "\n",
            "2021-08-17 00:27:29 (11.5 MB/s) - ‘/content/drive/MyDrive/FineTuningGPT2/Corpus/consolidado.txt.8’ saved [135659/135659]\n",
            "\n",
            "consolidado.txt    consolidado.txt.3  consolidado.txt.6\n",
            "consolidado.txt.1  consolidado.txt.4  consolidado.txt.7\n",
            "consolidado.txt.2  consolidado.txt.5  consolidado.txt.8\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A4RKmahCrybw"
      },
      "source": [
        "## Prepare Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HIp5yE-DlUhj"
      },
      "source": [
        "### Split Corpus"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "fhw2Q0BNiGTX",
        "outputId": "54025c33-8da7-44b2-9452-c6d9ae1994b6"
      },
      "source": [
        "import glob\n",
        "import csv\n",
        "import pandas as pd\n",
        "from datasets import load_dataset\n",
        "file_txt = glob.glob(path_corpus+ '/*.txt')[0]\n",
        "csv_file = path_dataset+'/dataset.csv'\n",
        "file_txt"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content/drive/MyDrive/FineTuningGPT2/Corpus/consolidado.txt'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 171,
          "referenced_widgets": [
            "e243c0f3792644c68fd750d5b4aea237",
            "8f0c2825d2bf49ada7efaed9a64844a5",
            "53a9953d40314e47ae18a8d027cbc2c5",
            "234c28476b2846b881a6ca675d0fdc23",
            "1fe3c29b4a9747db9075e77df98729ba",
            "adaf5e3e2f1e4ee1a4f698b1334a6d3a",
            "cb1fdca6acda410fa30a27dfa9da93c2",
            "3d6a6e25fd414415be5cad5668528b96",
            "ca38a529c24f481b817df7f3441c396c",
            "1a61c779ee6b4d8695a692bbf5e2b82c",
            "7b86a0c7d49a4a17ac4bc82d82b0e12b"
          ]
        },
        "id": "ghp1qc9MxhUI",
        "outputId": "cb0146b7-bc4e-4002-ce13-831a97f647fd"
      },
      "source": [
        "txt = ''\n",
        "df = pd.DataFrame(None, columns=[\"text\"])\n",
        "df.to_csv(csv_file, index=False, mode='w', sep='\\t', header=True)\n",
        "\n",
        "with open(file_txt) as infile:\n",
        "  lines_list = infile.readlines()\n",
        "  for l in lines_list:\n",
        "    txt += l.replace('   ', '\\n')\n",
        "\n",
        "  txt = txt.replace(' \\n', '\\n').replace('\\n ', '\\n')  \n",
        "  df = pd.DataFrame([txt], columns=[\"text\"])\n",
        "  df.to_csv(csv_file, index=False, mode='w', sep='\\t', header=True)\n",
        "\n",
        "dataset = load_dataset('csv', data_files=csv_file, split='train', delimiter='\\t')\n",
        "if not os.path.exists(path_dataset):\n",
        "  os.makedirs(path_dataset)\n",
        "dataset.save_to_disk(path_dataset)\n",
        "print('Saved file in dataset')\n",
        "dataset"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using custom data configuration default-9435d4283a48edf3\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading and preparing dataset csv/default (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/csv/default-9435d4283a48edf3/0.0.0/9144e0a4e8435090117cea53e6c7537173ef2304525df4a077c435d8ee7828ff...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e243c0f3792644c68fd750d5b4aea237",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "0 tables [00:00, ? tables/s]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Dataset csv downloaded and prepared to /root/.cache/huggingface/datasets/csv/default-9435d4283a48edf3/0.0.0/9144e0a4e8435090117cea53e6c7537173ef2304525df4a077c435d8ee7828ff. Subsequent calls will reuse this data.\n",
            "Saved file in dataset\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['text'],\n",
              "    num_rows: 1\n",
              "})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KUI4WpPrbWeE"
      },
      "source": [
        "## Tokenizing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pGS8uAKbtzRf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177,
          "referenced_widgets": [
            "3b6969623d7d47d182032547e187c5a9",
            "6309e0de39ec4dd4a175bbc43760beb7",
            "f632eea6f5b44fae81592262c04aedb4",
            "a232fea5da2d459281af7917efce2f4c",
            "4f17d23a24c34eb088b0378ff4974ffd",
            "827a3ba2fa3340e2a9fa83431ae11a1b",
            "fcde75531ec74dcfbfb250fd20ad8151",
            "33407f7c0f1a43088fe5f385ac048576",
            "41414473e5c7412b82c2fb91656741dd",
            "49a432f2f312457cad83e7c0e6c43015",
            "5585509c4eae449fa2e30645d3ec4591",
            "eed0101232404cacba582b0178fd7901",
            "db8be632c5b64b18aa541683968cf803",
            "db2987c730a746d5b5b7c13c9103668a",
            "e59a5e5c5dd54e8da1cdca3640ce2c24",
            "7ccd176a23794736b37bd5776c2aca06",
            "7a875b9d28f041f4b2fa2411ff983c4f",
            "ce1f548a89da478da6af9f75ea965ed4",
            "dc89967e1c4a43f3a915557e89c838b4",
            "719c82095616487b904e0febb724e0ad",
            "9117bc4290eb495db52d1fa870cd582f",
            "1d5a6280ebba42fa933f8f4522de24c0",
            "bcef9e3b2447466fa0ef50851728d41a",
            "3f2647e850eb4bf2b9ac5c3af893f763",
            "69f579dca8284590af10c1ac5662ea6f",
            "a43c34aeaa2848f3a73e6bede3369108",
            "326693a8a77f4705a6e2aefb8b94fb6f",
            "8ef937f54e0648a9be8a4f159e5015f8",
            "0619728ea1b64cbfbd1f68781b5d8fdc",
            "ff853b181336432f9246f3d08092332b",
            "9c0e3ea2c03d451ebf4f89441376938a",
            "521916cd9d4a4ca89a9284283ef0db6a",
            "eb1288534e1f4d71b736999deece5a39",
            "eeb0b08452694019a801c19bdaba283b",
            "265e1368bd214bdb9454f555bd676f4b",
            "44dea4f943a64cd0935847818e1bd390",
            "f997d367354b4153b42c85556e2955a4",
            "452af706e8f843b5acecbdf16665c71e",
            "0c000e55ffaf46c59093e5689021c6d1",
            "22c7e8b8c9be4716aa883d89d80960af",
            "b9668fb49b4d4344a8dda2407fadb3d4",
            "4a30d67b29d3400294f61358d6b5b63b",
            "a836d1d6070845fb97d0af603aff011b",
            "39116dc131ab4cc89170d79ac40d68ed",
            "49c34597114d4dad913bd44627345e3c",
            "03467909530a403095c008d17de4d364",
            "00dc4491e4a547098b98f27641d1cb50",
            "0356baccbebf495a9a17a28c87abe681",
            "c7f65289cfa24ba9aa0e9a441d5a4a79",
            "4713b15948ca4fb8b3221091a5bfbdce",
            "9290772521ae4ba0a6d1b68a67b39e07",
            "e7773eecba314e7bb6494d0e978b205a",
            "aef79b311de1407d992ec15e8b1a3161",
            "fec7bf69e28946129ac59b586054d1fc",
            "6decf01d245744a58ecdbd7b0ae8ba68"
          ]
        },
        "outputId": "9a2442b8-c7e2-4fd1-9355-ed23e0dd36f0"
      },
      "source": [
        "from transformers import GPT2Tokenizer\n",
        "tokenizer = GPT2Tokenizer.from_pretrained(MODEL_NAME)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3b6969623d7d47d182032547e187c5a9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/840k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "eed0101232404cacba582b0178fd7901",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/499k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bcef9e3b2447466fa0ef50851728d41a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/262 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "eeb0b08452694019a801c19bdaba283b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/115 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "49c34597114d4dad913bd44627345e3c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/914 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 133,
          "referenced_widgets": [
            "fc427444d03241748487470cb2d33693",
            "b012174d4b6d4bfba723c6c07361aeb2",
            "198f555587324367b047f18b0975f231",
            "84c8604672f3440b9241b4dc47cae279",
            "aa5781bb9d9c4f83ad41950b3c10468a",
            "d9c7b558918d4843b2b78d21a5caa0f7",
            "9e5fa69a27104b17891b2d7a3e89c96c",
            "a46dab8d60dd4565879701e379e91c2d",
            "88b6475161ad4c9798b5baff0d232429",
            "a214a23777da41588313eaa9bfb002e8",
            "5e3d9022398c4ef398a75e998c405e3e"
          ]
        },
        "id": "2-nNf59RbWJf",
        "outputId": "22f6207e-867f-4043-d33f-54b4e838c186"
      },
      "source": [
        "from datasets import load_from_disk\n",
        "\n",
        "dataset = load_from_disk(path_dataset)\n",
        "column_names = dataset.column_names\n",
        "dataset = dataset.map(lambda x: tokenizer(x['text']),\n",
        "          batch_size = 1,\n",
        "          batched=True,\n",
        "          remove_columns=column_names,\n",
        "          writer_batch_size = 1,\n",
        ")\n",
        "dataset.cleanup_cache_files()\n",
        "if not os.path.exists(path_tokenized_dataset):\n",
        "  os.makedirs(path_tokenized_dataset)\n",
        "dataset.save_to_disk(path_tokenized_dataset)\n",
        "print('Tokenized file dataset')\n",
        "dataset"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fc427444d03241748487470cb2d33693",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Tokenized file dataset\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['attention_mask', 'input_ids'],\n",
              "    num_rows: 1\n",
              "})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4zU-j7za8WpA"
      },
      "source": [
        "### Group tokenized dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0VLebG7wXgf6",
        "outputId": "7042d316-6f58-49ea-aa41-e17bc588d8d3"
      },
      "source": [
        "BLOCK_SIZE = tokenizer.max_model_input_sizes['gpt2']\n",
        "\n",
        "def create_blocks(listIds, blockSize=1024):\n",
        "  total_length = (len(listIds) // blockSize) * blockSize\n",
        "  return [listIds[i:i + blockSize] for i in range(0, total_length, blockSize)]\n",
        " \n",
        "def create_blocks_dataset(examples):\n",
        "    result = {\n",
        "      \"input_ids\": create_blocks(examples[\"input_ids\"][0], BLOCK_SIZE)\n",
        "    }\n",
        "    result[\"labels\"] = result[\"input_ids\"].copy()\n",
        "    return result\n",
        "\n",
        "BLOCK_SIZE"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1024"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 133,
          "referenced_widgets": [
            "5b136f4e60e9491fa55c695e6cfbefb8",
            "f966cc741c6e49938255964dd9e9325d",
            "90e3a560099644169b6cfe92f1ec4615",
            "bac09ab44fe6449993d2a0fad4eb99fa",
            "97bc9b6ae6dd4d278b9f5f64c1c382bc",
            "2ae3de5e2c044b88bd7698defd90c402",
            "535a9a6be3b2438f87421a88a07c9939",
            "c7705508d954476e84bee80c87402305",
            "43203d5ffd53490796ba3efdf65e5195",
            "bc73dd08bbab416f8680b74144e2ee3b",
            "1d16330089a74da5a84e3bd4defe73ea"
          ]
        },
        "id": "H8V42HGBEDfv",
        "outputId": "6228afd1-5622-4521-9fdb-8fd4734803df"
      },
      "source": [
        "from datasets import load_from_disk\n",
        "\n",
        "dataset = load_from_disk(path_tokenized_dataset)\n",
        "column_names = dataset.column_names\n",
        "\n",
        "dataset = dataset.map(\n",
        "        create_blocks_dataset,\n",
        "        batched=True,\n",
        "        batch_size = 1,\n",
        "        remove_columns=column_names,\n",
        "        writer_batch_size = 1,\n",
        ")\n",
        "dataset = dataset.shuffle()\n",
        "dataset.cleanup_cache_files()\n",
        "\n",
        "if not os.path.exists(path_ready_dataset):\n",
        "  os.makedirs(path_ready_dataset)\n",
        "dataset.save_to_disk(path_ready_dataset)\n",
        "\n",
        "print('Ready file dataset')\n",
        "dataset"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5b136f4e60e9491fa55c695e6cfbefb8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Ready file dataset\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['input_ids', 'labels'],\n",
              "    num_rows: 34\n",
              "})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IBW_X818Ha3M"
      },
      "source": [
        "# Train: Finetuning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JECZ-dCmM4Wb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "cdd68dc55e034bca9b3b33fc48b17f2f",
            "9e15f650de3747d6b0bb09dbc9837d03",
            "116e4d0df6084b6d950963e161983721",
            "8de304bfd8bf4b339108818d03d72178",
            "fecee17e58e24c52aae43dad7761cd31",
            "14e44a89fa2d453984f1b3082d5a4d93",
            "34564151bf7944a4bd5817c5666c2fe0",
            "8b0957916bce44bb9545746d1ad8100e",
            "2c0e51e8ac7e4ce98db042911302b8ee",
            "9258b1acae7b4fc688b9fd8ae7c35552",
            "ee1ce94145a34bf89585df74efd4349c"
          ]
        },
        "outputId": "22b44874-086f-48bb-8db1-4acbcf1d9495"
      },
      "source": [
        "from transformers import GPT2Tokenizer, GPT2LMHeadModel\n",
        "from datasets import load_from_disk\n",
        "\n",
        "model = GPT2LMHeadModel.from_pretrained(MODEL_NAME)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cdd68dc55e034bca9b3b33fc48b17f2f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/261M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U3JL9m7rKuW6"
      },
      "source": [
        "%%bash\n",
        "cat <<'EOT' > ds_config.json\n",
        "{\n",
        "    \"fp16\": {\n",
        "        \"enabled\": true,\n",
        "        \"loss_scale\": 0,\n",
        "        \"loss_scale_window\": 1000,\n",
        "        \"hysteresis\": 2,\n",
        "        \"min_loss_scale\": 1\n",
        "    },\n",
        "    \"zero_optimization\": {\n",
        "        \"stage\": 2,\n",
        "        \"allgather_partitions\": true,\n",
        "        \"allgather_bucket_size\": 2e8,\n",
        "        \"overlap_comm\": true,\n",
        "        \"reduce_scatter\": true,\n",
        "        \"reduce_bucket_size\": 2e8,\n",
        "        \"contiguous_gradients\": true,\n",
        "        \"cpu_offload\": false\n",
        "    },\n",
        "  \"zero_allow_untested_optimizer\": true,\n",
        "  \"optimizer\": {\n",
        "    \"type\": \"AdamW\",\n",
        "    \"params\": {\n",
        "      \"lr\": 3e-4,\n",
        "      \"betas\": [0.9, 0.999],\n",
        "      \"eps\": 1e-6\n",
        "    }\n",
        "  },\n",
        "  \"scheduler\": {\n",
        "    \"type\": \"WarmupLR\",\n",
        "    \"params\": {\n",
        "      \"warmup_min_lr\": 0,\n",
        "      \"warmup_max_lr\": 3e-4,\n",
        "      \"warmup_num_steps\": 100\n",
        "    }\n",
        "  },\n",
        "    \"gradient_accumulation_steps\": \"auto\",\n",
        "    \"train_batch_size\": \"auto\",\n",
        "    \"wall_clock_breakdown\": false\n",
        "}\n",
        "EOT"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V-BrI_MUkwwi"
      },
      "source": [
        "# Load Train Dataset \n",
        "from torch.utils.data import random_split\n",
        "import gc\n",
        "dataset = load_from_disk(path_ready_dataset)\n",
        "\n",
        "total_size = len(dataset)\n",
        "train_size = int(0.9 * total_size)\n",
        "train_dataset, val_dataset = random_split(dataset, [train_size, total_size - train_size])\n",
        "\n",
        "for i in range(10):\n",
        "  gc.collect()\n",
        "  torch.cuda.empty_cache()"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e5U-puIZLskw",
        "outputId": "6c609dd5-2f87-4fca-b4af-eaaf6d524398"
      },
      "source": [
        "# Prepare Training\n",
        "from transformers import Trainer, TrainingArguments, default_data_collator\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "                                  output_dir=path_model_save+'/checkpoints',\n",
        "                                  num_train_epochs=4, \n",
        "                                  logging_steps=500, \n",
        "                                  save_steps=1000,\n",
        "                                  per_device_train_batch_size = 2,\n",
        "                                  per_device_eval_batch_size=2,\n",
        "                                  gradient_accumulation_steps = 2, \n",
        "                                  learning_rate=3e-4,\n",
        "                                  adam_epsilon=1e-06,\n",
        "                                  warmup_steps=100,\n",
        "                                  weight_decay=0.01, \n",
        "                                  fp16=True,\n",
        "                                  logging_dir='path_model_save/logs', \n",
        "                                  deepspeed='./ds_config.json')\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "trainer = Trainer(model=model, \n",
        "        args=training_args, \n",
        "        train_dataset=train_dataset,\n",
        "        eval_dataset=val_dataset, \n",
        "        data_collator=default_data_collator)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[2021-08-17 00:28:29,973] [INFO] [distributed.py:37:init_distributed] Not using the DeepSpeed or torch.distributed launchers, attempting to detect MPI environment...\n",
            "[2021-08-17 00:28:30,433] [INFO] [distributed.py:89:mpi_discovery] Discovered MPI settings of world_rank=0, local_rank=0, world_size=1, master_addr=172.28.0.2, master_port=29500\n",
            "[2021-08-17 00:28:30,434] [INFO] [distributed.py:47:init_distributed] Initializing torch distributed with backend: nccl\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Using amp fp16 backend\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "nDHT7a-FecSZ",
        "outputId": "d4528a58-b723-4d1d-cbaf-41a9c52e8695"
      },
      "source": [
        "# Train and Save\n",
        "trainer.train()\n",
        "model.save_pretrained(path_model_save)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[2021-08-17 00:30:26,141] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.4.3, git-hash=unknown, git-branch=unknown\n",
            "[2021-08-17 00:30:26,464] [INFO] [utils.py:13:_initialize_parameter_parallel_groups] data_parallel_size: 1, parameter_parallel_size: 1\n",
            "[2021-08-17 00:30:26,527] [INFO] [engine.py:180:__init__] DeepSpeed Flops Profiler Enabled: False\n",
            "Using /root/.cache/torch_extensions as PyTorch extensions root...\n",
            "Creating extension directory /root/.cache/torch_extensions/fused_adam...\n",
            "Detected CUDA files, patching ldflags\n",
            "Emitting ninja build file /root/.cache/torch_extensions/fused_adam/build.ninja...\n",
            "Building extension module fused_adam...\n",
            "Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)\n",
            "Loading extension module fused_adam...\n",
            "Time to load fused_adam op: 25.990983486175537 seconds\n",
            "[2021-08-17 00:30:53,708] [INFO] [engine.py:710:_configure_optimizer] Using DeepSpeed Optimizer param name adamw as basic optimizer\n",
            "[2021-08-17 00:30:53,710] [INFO] [engine.py:714:_configure_optimizer] DeepSpeed Basic Optimizer = FusedAdam\n",
            "[2021-08-17 00:30:53,715] [INFO] [utils.py:44:is_zero_supported_optimizer] Checking ZeRO support for optimizer=FusedAdam type=<class 'deepspeed.ops.adam.fused_adam.FusedAdam'>\n",
            "[2021-08-17 00:30:53,718] [INFO] [logging.py:68:log_dist] [Rank 0] Creating fp16 ZeRO stage 2 optimizer\n",
            "[2021-08-17 00:30:53,719] [INFO] [stage2.py:105:__init__] Reduce bucket size 200000000.0\n",
            "[2021-08-17 00:30:53,724] [INFO] [stage2.py:106:__init__] Allgather bucket size 200000000.0\n",
            "[2021-08-17 00:30:53,725] [INFO] [stage2.py:107:__init__] CPU Offload: False\n",
            "Using /root/.cache/torch_extensions as PyTorch extensions root...\n",
            "Creating extension directory /root/.cache/torch_extensions/utils...\n",
            "Emitting ninja build file /root/.cache/torch_extensions/utils/build.ninja...\n",
            "Building extension module utils...\n",
            "Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)\n",
            "Loading extension module utils...\n",
            "Time to load utils op: 12.892651319503784 seconds\n",
            "[2021-08-17 00:31:06,935] [INFO] [stage2.py:409:__init__] optimizer state initialized\n",
            "[2021-08-17 00:31:06,936] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = adamw\n",
            "[2021-08-17 00:31:06,937] [INFO] [engine.py:512:_configure_lr_scheduler] DeepSpeed using configured LR scheduler = WarmupLR\n",
            "[2021-08-17 00:31:06,941] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = <deepspeed.runtime.lr_schedules.WarmupLR object at 0x7f31f49050d0>\n",
            "[2021-08-17 00:31:06,943] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0003], mom=[[0.9, 0.999]]\n",
            "[2021-08-17 00:31:06,946] [INFO] [config.py:900:print] DeepSpeedEngine configuration:\n",
            "[2021-08-17 00:31:06,951] [INFO] [config.py:904:print]   activation_checkpointing_config  {\n",
            "    \"partition_activations\": false, \n",
            "    \"contiguous_memory_optimization\": false, \n",
            "    \"cpu_checkpointing\": false, \n",
            "    \"number_checkpoints\": null, \n",
            "    \"synchronize_checkpoint_boundary\": false, \n",
            "    \"profile\": false\n",
            "}\n",
            "[2021-08-17 00:31:06,955] [INFO] [config.py:904:print]   aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True}\n",
            "[2021-08-17 00:31:06,958] [INFO] [config.py:904:print]   allreduce_always_fp32 ........ False\n",
            "[2021-08-17 00:31:06,959] [INFO] [config.py:904:print]   amp_enabled .................. False\n",
            "[2021-08-17 00:31:06,961] [INFO] [config.py:904:print]   amp_params ................... False\n",
            "[2021-08-17 00:31:06,964] [INFO] [config.py:904:print]   checkpoint_tag_validation_enabled  True\n",
            "[2021-08-17 00:31:06,966] [INFO] [config.py:904:print]   checkpoint_tag_validation_fail  False\n",
            "[2021-08-17 00:31:06,968] [INFO] [config.py:904:print]   disable_allgather ............ False\n",
            "[2021-08-17 00:31:06,972] [INFO] [config.py:904:print]   dump_state ................... False\n",
            "[2021-08-17 00:31:06,974] [INFO] [config.py:904:print]   dynamic_loss_scale_args ...... {'init_scale': 4294967296, 'scale_window': 1000, 'delayed_shift': 2, 'min_scale': 1}\n",
            "[2021-08-17 00:31:06,976] [INFO] [config.py:904:print]   eigenvalue_enabled ........... False\n",
            "[2021-08-17 00:31:06,978] [INFO] [config.py:904:print]   eigenvalue_gas_boundary_resolution  1\n",
            "[2021-08-17 00:31:06,980] [INFO] [config.py:904:print]   eigenvalue_layer_name ........ bert.encoder.layer\n",
            "[2021-08-17 00:31:06,983] [INFO] [config.py:904:print]   eigenvalue_layer_num ......... 0\n",
            "[2021-08-17 00:31:06,985] [INFO] [config.py:904:print]   eigenvalue_max_iter .......... 100\n",
            "[2021-08-17 00:31:06,987] [INFO] [config.py:904:print]   eigenvalue_stability ......... 1e-06\n",
            "[2021-08-17 00:31:06,990] [INFO] [config.py:904:print]   eigenvalue_tol ............... 0.01\n",
            "[2021-08-17 00:31:06,993] [INFO] [config.py:904:print]   eigenvalue_verbose ........... False\n",
            "[2021-08-17 00:31:06,999] [INFO] [config.py:904:print]   elasticity_enabled ........... False\n",
            "[2021-08-17 00:31:07,002] [INFO] [config.py:904:print]   flops_profiler_config ........ {\n",
            "    \"enabled\": false, \n",
            "    \"profile_step\": 1, \n",
            "    \"module_depth\": -1, \n",
            "    \"top_modules\": 1, \n",
            "    \"detailed\": true, \n",
            "    \"output_file\": null\n",
            "}\n",
            "[2021-08-17 00:31:07,003] [INFO] [config.py:904:print]   fp16_enabled ................. True\n",
            "[2021-08-17 00:31:07,004] [INFO] [config.py:904:print]   fp16_mixed_quantize .......... False\n",
            "[2021-08-17 00:31:07,006] [INFO] [config.py:904:print]   global_rank .................. 0\n",
            "[2021-08-17 00:31:07,007] [INFO] [config.py:904:print]   gradient_accumulation_steps .. 2\n",
            "[2021-08-17 00:31:07,008] [INFO] [config.py:904:print]   gradient_clipping ............ 0.0\n",
            "[2021-08-17 00:31:07,012] [INFO] [config.py:904:print]   gradient_predivide_factor .... 1.0\n",
            "[2021-08-17 00:31:07,013] [INFO] [config.py:904:print]   initial_dynamic_scale ........ 4294967296\n",
            "[2021-08-17 00:31:07,015] [INFO] [config.py:904:print]   loss_scale ................... 0\n",
            "[2021-08-17 00:31:07,016] [INFO] [config.py:904:print]   memory_breakdown ............. False\n",
            "[2021-08-17 00:31:07,018] [INFO] [config.py:904:print]   optimizer_legacy_fusion ...... False\n",
            "[2021-08-17 00:31:07,019] [INFO] [config.py:904:print]   optimizer_name ............... adamw\n",
            "[2021-08-17 00:31:07,021] [INFO] [config.py:904:print]   optimizer_params ............. {'lr': 0.0003, 'betas': [0.9, 0.999], 'eps': 1e-06}\n",
            "[2021-08-17 00:31:07,022] [INFO] [config.py:904:print]   pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0}\n",
            "[2021-08-17 00:31:07,023] [INFO] [config.py:904:print]   pld_enabled .................. False\n",
            "[2021-08-17 00:31:07,025] [INFO] [config.py:904:print]   pld_params ................... False\n",
            "[2021-08-17 00:31:07,026] [INFO] [config.py:904:print]   prescale_gradients ........... False\n",
            "[2021-08-17 00:31:07,027] [INFO] [config.py:904:print]   quantize_change_rate ......... 0.001\n",
            "[2021-08-17 00:31:07,029] [INFO] [config.py:904:print]   quantize_groups .............. 1\n",
            "[2021-08-17 00:31:07,031] [INFO] [config.py:904:print]   quantize_offset .............. 1000\n",
            "[2021-08-17 00:31:07,032] [INFO] [config.py:904:print]   quantize_period .............. 1000\n",
            "[2021-08-17 00:31:07,034] [INFO] [config.py:904:print]   quantize_rounding ............ 0\n",
            "[2021-08-17 00:31:07,035] [INFO] [config.py:904:print]   quantize_start_bits .......... 16\n",
            "[2021-08-17 00:31:07,037] [INFO] [config.py:904:print]   quantize_target_bits ......... 8\n",
            "[2021-08-17 00:31:07,038] [INFO] [config.py:904:print]   quantize_training_enabled .... False\n",
            "[2021-08-17 00:31:07,040] [INFO] [config.py:904:print]   quantize_type ................ 0\n",
            "[2021-08-17 00:31:07,041] [INFO] [config.py:904:print]   quantize_verbose ............. False\n",
            "[2021-08-17 00:31:07,042] [INFO] [config.py:904:print]   scheduler_name ............... WarmupLR\n",
            "[2021-08-17 00:31:07,043] [INFO] [config.py:904:print]   scheduler_params ............. {'warmup_min_lr': 0, 'warmup_max_lr': 0.0003, 'warmup_num_steps': 100}\n",
            "[2021-08-17 00:31:07,045] [INFO] [config.py:904:print]   sparse_attention ............. None\n",
            "[2021-08-17 00:31:07,046] [INFO] [config.py:904:print]   sparse_gradients_enabled ..... False\n",
            "[2021-08-17 00:31:07,048] [INFO] [config.py:904:print]   steps_per_print .............. 10\n",
            "[2021-08-17 00:31:07,049] [INFO] [config.py:904:print]   tensorboard_enabled .......... False\n",
            "[2021-08-17 00:31:07,051] [INFO] [config.py:904:print]   tensorboard_job_name ......... DeepSpeedJobName\n",
            "[2021-08-17 00:31:07,052] [INFO] [config.py:904:print]   tensorboard_output_path ...... \n",
            "[2021-08-17 00:31:07,053] [INFO] [config.py:904:print]   train_batch_size ............. 4\n",
            "[2021-08-17 00:31:07,054] [INFO] [config.py:904:print]   train_micro_batch_size_per_gpu  2\n",
            "[2021-08-17 00:31:07,056] [INFO] [config.py:904:print]   use_quantizer_kernel ......... False\n",
            "[2021-08-17 00:31:07,057] [INFO] [config.py:904:print]   wall_clock_breakdown ......... False\n",
            "[2021-08-17 00:31:07,058] [INFO] [config.py:904:print]   world_size ................... 1\n",
            "[2021-08-17 00:31:07,060] [INFO] [config.py:904:print]   zero_allow_untested_optimizer  True\n",
            "[2021-08-17 00:31:07,061] [INFO] [config.py:904:print]   zero_config .................. {\n",
            "    \"stage\": 2, \n",
            "    \"contiguous_gradients\": true, \n",
            "    \"reduce_scatter\": true, \n",
            "    \"reduce_bucket_size\": 2.000000e+08, \n",
            "    \"allgather_partitions\": true, \n",
            "    \"allgather_bucket_size\": 2.000000e+08, \n",
            "    \"overlap_comm\": true, \n",
            "    \"load_from_fp32_weights\": true, \n",
            "    \"elastic_checkpoint\": true, \n",
            "    \"offload_param\": null, \n",
            "    \"offload_optimizer\": null, \n",
            "    \"sub_group_size\": 1.000000e+09, \n",
            "    \"prefetch_bucket_size\": 5.000000e+07, \n",
            "    \"param_persistence_threshold\": 1.000000e+05, \n",
            "    \"max_live_parameters\": 1.000000e+09, \n",
            "    \"max_reuse_distance\": 1.000000e+09, \n",
            "    \"gather_fp16_weights_on_model_save\": false, \n",
            "    \"ignore_unused_parameters\": true, \n",
            "    \"legacy_stage1\": false\n",
            "}\n",
            "[2021-08-17 00:31:07,063] [INFO] [config.py:904:print]   zero_enabled ................. True\n",
            "[2021-08-17 00:31:07,065] [INFO] [config.py:904:print]   zero_optimization_stage ...... 2\n",
            "[2021-08-17 00:31:07,066] [INFO] [config.py:911:print]   json = {\n",
            "    \"fp16\": {\n",
            "        \"enabled\": true, \n",
            "        \"loss_scale\": 0, \n",
            "        \"loss_scale_window\": 1000, \n",
            "        \"hysteresis\": 2, \n",
            "        \"min_loss_scale\": 1\n",
            "    }, \n",
            "    \"zero_optimization\": {\n",
            "        \"stage\": 2, \n",
            "        \"allgather_partitions\": true, \n",
            "        \"allgather_bucket_size\": 2.000000e+08, \n",
            "        \"overlap_comm\": true, \n",
            "        \"reduce_scatter\": true, \n",
            "        \"reduce_bucket_size\": 2.000000e+08, \n",
            "        \"contiguous_gradients\": true, \n",
            "        \"cpu_offload\": false\n",
            "    }, \n",
            "    \"zero_allow_untested_optimizer\": true, \n",
            "    \"optimizer\": {\n",
            "        \"type\": \"AdamW\", \n",
            "        \"params\": {\n",
            "            \"lr\": 0.0003, \n",
            "            \"betas\": [0.9, 0.999], \n",
            "            \"eps\": 1e-06\n",
            "        }\n",
            "    }, \n",
            "    \"scheduler\": {\n",
            "        \"type\": \"WarmupLR\", \n",
            "        \"params\": {\n",
            "            \"warmup_min_lr\": 0, \n",
            "            \"warmup_max_lr\": 0.0003, \n",
            "            \"warmup_num_steps\": 100\n",
            "        }\n",
            "    }, \n",
            "    \"gradient_accumulation_steps\": 2, \n",
            "    \"train_batch_size\": 4, \n",
            "    \"wall_clock_breakdown\": false\n",
            "}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "***** Running training *****\n",
            "  Num examples = 30\n",
            "  Num Epochs = 4\n",
            "  Instantaneous batch size per device = 2\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 4\n",
            "  Gradient Accumulation steps = 2\n",
            "  Total optimization steps = 28\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Using /root/.cache/torch_extensions as PyTorch extensions root...\n",
            "No modifications detected for re-loaded extension module utils, skipping build step...\n",
            "Loading extension module utils...\n",
            "Time to load utils op: 0.0009441375732421875 seconds\n",
            "[2021-08-17 00:31:07,840] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 4294967296, reducing to 4294967296\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='28' max='28' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [28/28 00:13, Epoch 3/4]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "[2021-08-17 00:31:08,300] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 4294967296, reducing to 2147483648.0\n",
            "[2021-08-17 00:31:08,755] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 2147483648.0, reducing to 1073741824.0\n",
            "[2021-08-17 00:31:09,206] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 1073741824.0, reducing to 536870912.0\n",
            "[2021-08-17 00:31:09,661] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 536870912.0, reducing to 268435456.0\n",
            "[2021-08-17 00:31:09,662] [INFO] [timer.py:160:stop] 0/10, SamplesPerSec=9.08656894095607\n",
            "[2021-08-17 00:31:10,118] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 268435456.0, reducing to 134217728.0\n",
            "[2021-08-17 00:31:10,572] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 134217728.0, reducing to 67108864.0\n",
            "[2021-08-17 00:31:11,030] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 67108864.0, reducing to 33554432.0\n",
            "[2021-08-17 00:31:11,486] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 33554432.0, reducing to 16777216.0\n",
            "[2021-08-17 00:31:11,940] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 16777216.0, reducing to 8388608.0\n",
            "[2021-08-17 00:31:11,942] [INFO] [logging.py:68:log_dist] [Rank 0] step=10, skipped=10, lr=[0.0003], mom=[[0.9, 0.999]]\n",
            "[2021-08-17 00:31:11,945] [INFO] [timer.py:160:stop] 0/20, SamplesPerSec=9.075699457826007\n",
            "[2021-08-17 00:31:12,401] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 8388608.0, reducing to 4194304.0\n",
            "[2021-08-17 00:31:12,861] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 4194304.0, reducing to 2097152.0\n",
            "[2021-08-17 00:31:13,318] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 2097152.0, reducing to 1048576.0\n",
            "[2021-08-17 00:31:13,773] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 1048576.0, reducing to 524288.0\n",
            "[2021-08-17 00:31:14,225] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 524288.0, reducing to 262144.0\n",
            "[2021-08-17 00:31:14,228] [INFO] [timer.py:160:stop] 0/30, SamplesPerSec=9.070129076549193\n",
            "[2021-08-17 00:31:14,683] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 262144.0, reducing to 131072.0\n",
            "[2021-08-17 00:31:15,136] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 131072.0, reducing to 65536.0\n",
            "[2021-08-17 00:31:15,593] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 65536.0, reducing to 32768.0\n",
            "[2021-08-17 00:31:16,051] [INFO] [stage2.py:1506:step] [deepspeed] fp16 dynamic loss scale overflow! Rank 0 Skipping step. Attempted loss scale: 32768.0, reducing to 16384.0\n",
            "[2021-08-17 00:31:16,542] [INFO] [logging.py:68:log_dist] [Rank 0] step=20, skipped=19, lr=[0.0], mom=[[0.9, 0.999]]\n",
            "[2021-08-17 00:31:16,576] [INFO] [timer.py:160:stop] 0/40, SamplesPerSec=8.990843060315942\n",
            "[2021-08-17 00:31:19,177] [INFO] [timer.py:160:stop] 0/50, SamplesPerSec=8.729064237188467\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Configuration saved in /content/drive/MyDrive/FineTuningGPT2/ModelTrained/config.json\n",
            "Model weights saved in /content/drive/MyDrive/FineTuningGPT2/ModelTrained/pytorch_model.bin\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rbqfKrPCAZ2I"
      },
      "source": [
        "# Generar"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8VOs8KFZ_i7D"
      },
      "source": [
        "# Instalar y Cargar Dependencias"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8-ENEQ2k_Yos",
        "outputId": "fdd2cc1d-ab71-42b2-93f6-492022f681c4"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "!pip install tensorflow &> log.txt\n",
        "!pip install transformers > log.txt\n",
        "import os\n",
        "import sys; print('Python:',sys.version)\n",
        "import torch; print('Pytorch:',torch.__version__)\n",
        "import tensorflow as tf; print('Tensorflow:',tf.__version__)\n",
        "import transformers; print('Transformers:',transformers.__version__)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "Python: 3.7.11 (default, Jul  3 2021, 18:01:19) \n",
            "[GCC 7.5.0]\n",
            "Pytorch: 1.7.1+cu110\n",
            "Tensorflow: 2.5.0\n",
            "Transformers: 4.10.0.dev0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8h3-3fGf_nhc"
      },
      "source": [
        "# Cargar Tokenizador y Modelo"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "faSJRYNCBcTn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0735fb2c-f176-454a-fc19-55b1bda35a84"
      },
      "source": [
        "import torch\n",
        "from transformers import GPT2Tokenizer, GPT2LMHeadModel\n",
        "\n",
        "tokenizer_es = GPT2Tokenizer.from_pretrained('DeepESP/gpt2-spanish')\n",
        "model_es = GPT2LMHeadModel.from_pretrained(path_model_save)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loading file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/vocab.json from cache at /root/.cache/huggingface/transformers/2b38e375a606e5f8f63dbc97cf8579eda0be824f6e74ed001fb5e9d11fb80774.9654cf615b595dbd278d7f33c84b3f4565133f4c3874d094bf360e7ea285dfaf\n",
            "loading file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/merges.txt from cache at /root/.cache/huggingface/transformers/fb80110197f1b5c12472df03799ca4826aecf5fc97f788664c17035ea4ea004e.356a983865c1a96482df9e7e88858dbaa53473df869b0c13eb3c1d3c83a5e1fd\n",
            "loading file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/added_tokens.json from cache at None\n",
            "loading file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/special_tokens_map.json from cache at /root/.cache/huggingface/transformers/33f6ff51b9dee7e39734d410c27efdddda2751cd489da15522ac58d7a9dc32bf.0d87709f2545d45784d5ff8d35dc319de93e4087b3c680e0e20b305fb949d2a7\n",
            "loading file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/tokenizer_config.json from cache at /root/.cache/huggingface/transformers/257f58cef29abe2d3f559ce937e1c22360462b97e6096362a7b4e3249e0fbf65.9fc0c0f00754f9b14f9e59f344d88bc631c9b4bd7eeb23abcaf4de04945114d5\n",
            "loading file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/tokenizer.json from cache at None\n",
            "loading configuration file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/80f30f43ed5e5159d1924e70134443c131f059187f562435a148107dbf002fec.05892ce96bf9f5c8bfde40968186593b5d0feec27999103dbf7ea1d3ca7d11e1\n",
            "Model config GPT2Config {\n",
            "  \"_name_or_path\": \"/content/drive/Shareddrives/Deep-ESP/Jorge-Alex/Modelos_GPT2_publicados/gpt2-spanish/\",\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 12,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"scale_attn_weights\": true,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.10.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "loading configuration file /content/drive/MyDrive/FineTuningGPT2/ModelTrained/config.json\n",
            "Model config GPT2Config {\n",
            "  \"_name_or_path\": \"DeepESP/gpt2-spanish\",\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 12,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"scale_attn_weights\": true,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"torch_dtype\": \"float16\",\n",
            "  \"transformers_version\": \"4.10.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "loading weights file /content/drive/MyDrive/FineTuningGPT2/ModelTrained/pytorch_model.bin\n",
            "All model checkpoint weights were used when initializing GPT2LMHeadModel.\n",
            "\n",
            "All the weights of GPT2LMHeadModel were initialized from the model checkpoint at /content/drive/MyDrive/FineTuningGPT2/ModelTrained.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use GPT2LMHeadModel for predictions without further training.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rsKR2m80_zGE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ef5f907f-1c95-4df0-b82a-4c7352284754"
      },
      "source": [
        "# Tokenizador\n",
        "from transformers import GPT2Tokenizer\n",
        "tokenizer = GPT2Tokenizer.from_pretrained('DeepESP/gpt2-spanish')"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loading file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/vocab.json from cache at /root/.cache/huggingface/transformers/2b38e375a606e5f8f63dbc97cf8579eda0be824f6e74ed001fb5e9d11fb80774.9654cf615b595dbd278d7f33c84b3f4565133f4c3874d094bf360e7ea285dfaf\n",
            "loading file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/merges.txt from cache at /root/.cache/huggingface/transformers/fb80110197f1b5c12472df03799ca4826aecf5fc97f788664c17035ea4ea004e.356a983865c1a96482df9e7e88858dbaa53473df869b0c13eb3c1d3c83a5e1fd\n",
            "loading file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/added_tokens.json from cache at None\n",
            "loading file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/special_tokens_map.json from cache at /root/.cache/huggingface/transformers/33f6ff51b9dee7e39734d410c27efdddda2751cd489da15522ac58d7a9dc32bf.0d87709f2545d45784d5ff8d35dc319de93e4087b3c680e0e20b305fb949d2a7\n",
            "loading file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/tokenizer_config.json from cache at /root/.cache/huggingface/transformers/257f58cef29abe2d3f559ce937e1c22360462b97e6096362a7b4e3249e0fbf65.9fc0c0f00754f9b14f9e59f344d88bc631c9b4bd7eeb23abcaf4de04945114d5\n",
            "loading file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/tokenizer.json from cache at None\n",
            "loading configuration file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/80f30f43ed5e5159d1924e70134443c131f059187f562435a148107dbf002fec.05892ce96bf9f5c8bfde40968186593b5d0feec27999103dbf7ea1d3ca7d11e1\n",
            "Model config GPT2Config {\n",
            "  \"_name_or_path\": \"/content/drive/Shareddrives/Deep-ESP/Jorge-Alex/Modelos_GPT2_publicados/gpt2-spanish/\",\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 12,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"scale_attn_weights\": true,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.10.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Fi0aHWxa_5ED",
        "outputId": "e1a6cd96-0007-4a5b-f45e-07389fe9fe07"
      },
      "source": [
        "#Tokenizamos una frase\n",
        "inputs = tokenizer(\"Es torrente y un hilo\", return_tensors=\"pt\")\n",
        "inputs.input_ids"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 1329, 24084,   324,   304, 11812]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CCdXMhWlAHlV",
        "outputId": "f7ae7308-192f-474f-c0d9-7fee1b43446a"
      },
      "source": [
        "# Modelo\n",
        "from transformers import GPT2LMHeadModel\n",
        "\n",
        "model = GPT2LMHeadModel.from_pretrained('DeepESP/gpt2-spanish')\n",
        "model.config"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loading configuration file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/80f30f43ed5e5159d1924e70134443c131f059187f562435a148107dbf002fec.05892ce96bf9f5c8bfde40968186593b5d0feec27999103dbf7ea1d3ca7d11e1\n",
            "Model config GPT2Config {\n",
            "  \"_name_or_path\": \"/content/drive/Shareddrives/Deep-ESP/Jorge-Alex/Modelos_GPT2_publicados/gpt2-spanish/\",\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 12,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"scale_attn_weights\": true,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.10.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "loading weights file https://huggingface.co/DeepESP/gpt2-spanish/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/90a26ac08cbade5ec286472021c6965825a1b312caa7e003f948f2bc66ce7822.e3fcaa3ba58207141a5bbed65da898cffdda13dfa1f48580351532ef31c0f3e1\n",
            "All model checkpoint weights were used when initializing GPT2LMHeadModel.\n",
            "\n",
            "All the weights of GPT2LMHeadModel were initialized from the model checkpoint at DeepESP/gpt2-spanish.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use GPT2LMHeadModel for predictions without further training.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GPT2Config {\n",
              "  \"_name_or_path\": \"DeepESP/gpt2-spanish\",\n",
              "  \"activation_function\": \"gelu_new\",\n",
              "  \"architectures\": [\n",
              "    \"GPT2LMHeadModel\"\n",
              "  ],\n",
              "  \"attn_pdrop\": 0.1,\n",
              "  \"bos_token_id\": 50256,\n",
              "  \"embd_pdrop\": 0.1,\n",
              "  \"eos_token_id\": 50256,\n",
              "  \"gradient_checkpointing\": false,\n",
              "  \"initializer_range\": 0.02,\n",
              "  \"layer_norm_epsilon\": 1e-05,\n",
              "  \"model_type\": \"gpt2\",\n",
              "  \"n_ctx\": 1024,\n",
              "  \"n_embd\": 768,\n",
              "  \"n_head\": 12,\n",
              "  \"n_inner\": null,\n",
              "  \"n_layer\": 12,\n",
              "  \"n_positions\": 1024,\n",
              "  \"resid_pdrop\": 0.1,\n",
              "  \"scale_attn_weights\": true,\n",
              "  \"summary_activation\": null,\n",
              "  \"summary_first_dropout\": 0.1,\n",
              "  \"summary_proj_to_labels\": true,\n",
              "  \"summary_type\": \"cls_index\",\n",
              "  \"summary_use_proj\": true,\n",
              "  \"task_specific_params\": {\n",
              "    \"text-generation\": {\n",
              "      \"do_sample\": true,\n",
              "      \"max_length\": 50\n",
              "    }\n",
              "  },\n",
              "  \"transformers_version\": \"4.10.0.dev0\",\n",
              "  \"use_cache\": true,\n",
              "  \"vocab_size\": 50257\n",
              "}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8LqETTtNAiGM"
      },
      "source": [
        "# Creamos un piline que une el modelo y el tokenizador\n",
        "from transformers import pipeline\n",
        "\n",
        "text_generator = pipeline(\n",
        "    \"text-generation\",\n",
        "    model=model,\n",
        "    tokenizer=tokenizer\n",
        "    )"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bg27WY68Ap19",
        "outputId": "ef62d098-73fa-4534-b4aa-95fea600c233"
      },
      "source": [
        "# Generamos\n",
        "generated = text_generator(\"El jardín se estremece por dentro \", max_length=128)\n",
        "print(generated[0]['generated_text'].rsplit('.', 2)[0], '.')\n",
        "#print(generated[0]['generated_text'])"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "El jardín se estremece por dentro ominoso, como si acabara de morir. Los perros de la cocina se han llevado a sus mascotas, que se han posado cerca, porque se han ido y sus mascotas están sin la madre y no hay ninguna. Pero las hermanas, que se han llevado también sus mascotas, se han quedado atrás en ese paraíso, con su perro, sus hermanos, y se han ido. Se han ido. Por eso nos alegra haber hecho lo mismo que nosotros .\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}